fix: honor renamePart's PathNotFound (#21378)

fixes #21364

.github/workflows/depsreview.yaml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v1
+        uses: actions/dependency-review-action@v4

.github/workflows/go-cross.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -21,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/go-fips.yml

@@ -1,60 +0,0 @@
-name: FIPS Build Test
-
-on:
-  pull_request:
-    branches:
-    - master
-    - next
-
-# This ensures that previous jobs for the PR are canceled when the PR is
-# updated.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Go BoringCrypto ${{ matrix.go-version }} on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        go-version: [1.21.x]
-        os: [ubuntu-latest]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go-version }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Setup dockerfile for build test
-        run: |
-          GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
-          echo Detected go version $GO_VERSION
-          cat > Dockerfile.fips.test <<EOF
-          FROM golang:${GO_VERSION}
-          COPY . /minio
-          WORKDIR /minio
-          ENV GOEXPERIMENT=boringcrypto
-          RUN make
-          EOF
-
-      - name: Build
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: Dockerfile.fips.test
-          push: false
-          load: true
-          tags: minio/fips-test:latest
-
-      # This should fail if grep returns non-zero exit
-      - name: Test binary
-        run: |
-          docker run --rm minio/fips-test:latest ./minio --version
-          docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'

.github/workflows/go-healing.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -21,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/go-lint.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -21,23 +20,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - name: Build on ${{ matrix.os }}
-        if: matrix.os == 'windows-latest'
-        env:
-          CGO_ENABLED: 0
-          GO111MODULE: on
-        run: |
-          netsh int ipv4 set dynamicport tcp start=60000 num=61000
-          go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
-          go test -v --timeout 50m ./...
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/go-resiliency.yml

@@ -0,0 +1,39 @@
+name: Resiliency Functional Tests
+
+on:
+  pull_request:
+    branches:
+      - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.24.x]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-resiliency

.github/workflows/go.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -21,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
@@ -40,3 +39,4 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify
+          make test-timeout

.github/workflows/helm-lint.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -20,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
 
       - name: Run helm lint
         run: |

.github/workflows/iam-integrations.yaml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -62,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -76,8 +75,8 @@ jobs:
             openid: "http://127.0.0.1:5556/dex"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
@@ -112,6 +111,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           go run docs/iam/access-manager-plugin.go &
           make test-iam
+      - name: Test MinIO Old Version data to IAM import current version
+        if: matrix.ldap == 'ldaphost:389'
+        env:
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+        run: |
+          make test-iam-ldap-upgrade-import
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |
@@ -120,3 +125,37 @@ jobs:
         if: matrix.openid == 'http://127.0.0.1:5556/dex'
         run: |
           make test-site-replication-oidc
+  iam-import-with-missing-entities:
+    name: Test IAM import in new cluster with missing entities
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Checkout minio-iam-testing
+        uses: actions/checkout@v4
+        with:
+          repository: minio/minio-iam-testing
+          path: minio-iam-testing
+      - name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
+        run: |
+          make test-iam-import-with-missing-entities
+  iam-import-with-openid:
+    name: Test IAM import in new cluster with opendid configurations
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Checkout minio-iam-testing
+        uses: actions/checkout@v4
+        with:
+          repository: minio/minio-iam-testing
+          path: minio-iam-testing
+      - name: Test import of IAM artifacts when in fresh cluster with openid configurations
+        run: |
+          make test-iam-import-with-openid

.github/workflows/issues.yaml

@@ -0,0 +1,18 @@
+# @format
+
+name: Issue Workflow
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v0.5.0
+        with:
+          project-url: https://github.com/orgs/miniohq/projects/2
+          github-token: ${{ secrets.BOT_PAT }}

.github/workflows/mint.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-      - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -25,12 +24,12 @@ jobs:
           sudo -S rm -rf ${GITHUB_WORKSPACE}
           mkdir ${GITHUB_WORKSPACE}
       - name: checkout-step
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: setup-go-step
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.x
+          go-version: 1.24.x
 
       - name: github sha short
         id: vars
@@ -56,6 +55,11 @@ jobs:
         run: |
           ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
 
+      # FIXME: renable this back when we have a valid way to add deadlines for PUT()s (internode CreateFile)
+      # - name: resiliency
+      #   run: |
+      #     ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
       - name: The job must cleanup
         if: ${{ always() }}
         run: |

.github/workflows/mint/minio-resiliency.yaml

@@ -0,0 +1,78 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/rdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+    MINIO_DRIVE_MAX_TIMEOUT: "5s"
+  healthcheck:
+    test: ["CMD", "mc", "ready", "local"]
+    interval: 5s
+    timeout: 5s
+    retries: 5
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - rdata1-1:/rdata1
+      - rdata1-2:/rdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - rdata2-1:/rdata1
+      - rdata2-2:/rdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - rdata3-1:/rdata1
+      - rdata3-2:/rdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - rdata4-1:/rdata1
+      - rdata4-2:/rdata2
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  rdata1-1:
+  rdata1-2:
+  rdata2-1:
+  rdata2-2:
+  rdata3-1:
+  rdata3-2:
+  rdata4-1:
+  rdata4-2:

.github/workflows/mint/nginx-4-node.conf

@@ -23,10 +23,9 @@ http {
     # include /etc/nginx/conf.d/*.conf;
 
     upstream minio {
-        server minio1:9000;
+        server minio1:9000 max_fails=1 fail_timeout=10s;
-        server minio2:9000;
+        server minio2:9000 max_fails=1 fail_timeout=10s;
-        server minio3:9000;
+        server minio3:9000 max_fails=1 fail_timeout=10s;
-        server minio4:9000;
     }
 
     upstream console {

.github/workflows/mint/nginx-8-node.conf

@@ -23,14 +23,14 @@ http {
     # include /etc/nginx/conf.d/*.conf;
 
     upstream minio {
-        server minio1:9000;
+        server minio1:9000 max_fails=1 fail_timeout=10s;
-        server minio2:9000;
+        server minio2:9000 max_fails=1 fail_timeout=10s;
-        server minio3:9000;
+        server minio3:9000 max_fails=1 fail_timeout=10s;
-        server minio4:9000;
+        server minio4:9000 max_fails=1 fail_timeout=10s;
-        server minio5:9000;
+        server minio5:9000 max_fails=1 fail_timeout=10s;
-        server minio6:9000;
+        server minio6:9000 max_fails=1 fail_timeout=10s;
-        server minio7:9000;
+        server minio7:9000 max_fails=1 fail_timeout=10s;
-        server minio8:9000;
+        server minio8:9000 max_fails=1 fail_timeout=10s;
     }
 
     upstream console {

.github/workflows/mint/nginx.conf

@@ -23,10 +23,10 @@ http {
     # include /etc/nginx/conf.d/*.conf;
 
     upstream minio {
-        server minio1:9000;
+        server minio1:9000 max_fails=1 fail_timeout=10s;
-        server minio2:9000;
+        server minio2:9000 max_fails=1 fail_timeout=10s;
-        server minio3:9000;
+        server minio3:9000 max_fails=1 fail_timeout=10s;
-        server minio4:9000;
+        server minio4:9000 max_fails=1 fail_timeout=10s;
     }
 
     upstream console {

.github/workflows/multipart/migrate.sh

@@ -5,22 +5,25 @@ set -x
 ## change working directory
 cd .github/workflows/multipart/
 
-docker-compose -f docker-compose-site1.yaml rm -s -f
+function cleanup() {
-docker-compose -f docker-compose-site2.yaml rm -s -f
+	docker-compose -f docker-compose-site1.yaml rm -s -f || true
+	docker-compose -f docker-compose-site2.yaml rm -s -f || true
 	for volume in $(docker volume ls -q | grep minio); do
-	docker volume rm ${volume}
+		docker volume rm ${volume} || true
 	done
 
+	docker system prune -f || true
+	docker volume prune -f || true
+	docker volume rm $(docker volume ls -q -f dangling=true) || true
+}
+
+cleanup
+
 if [ ! -f ./mc ]; then
 	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
 		chmod +x mc
 fi
 
-(
-	cd /tmp
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
-)
-
 export RELEASE=RELEASE.2023-08-29T23-07-35Z
 
 docker-compose -f docker-compose-site1.yaml up -d
@@ -40,10 +43,10 @@ sleep 30s
 
 sleep 5
 
-s3-check-md5 -h
+./s3-check-md5 -h
 
-failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
-failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
 
 if [ $failed_count_site1 -ne 0 ]; then
 	echo "failed with multipart on site1 uploads"
@@ -59,8 +62,8 @@ fi
 
 sleep 5
 
-failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
-failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
 
 ## we do not need to fail here, since we are going to test
 ## upgrading to master, healing and being able to recover
@@ -88,8 +91,8 @@ for i in $(seq 1 10); do
 	./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
 done
 
-failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
-failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
 
 if [ $failed_count_site1 -ne 0 ]; then
 	echo "failed with multipart on site1 uploads"
@@ -101,15 +104,44 @@ if [ $failed_count_site2 -ne 0 ]; then
 	exit 1
 fi
 
-docker-compose -f docker-compose-site1.yaml rm -s -f
+# Add user group test
-docker-compose -f docker-compose-site2.yaml rm -s -f
+./mc admin user add site1 site-replication-issue-user site-replication-issue-password
-for volume in $(docker volume ls -q | grep minio); do
+./mc admin group add site1 site-replication-issue-group site-replication-issue-user
-	docker volume rm ${volume}
+
+max_wait_attempts=30
+wait_interval=5
+
+attempt=1
+while true; do
+	diff <(./mc admin group info site1 site-replication-issue-group) <(./mc admin group info site2 site-replication-issue-group)
+
+	if [[ $? -eq 0 ]]; then
+		echo "Outputs are consistent."
+		break
+	fi
+
+	remaining_attempts=$((max_wait_attempts - attempt))
+	if ((attempt >= max_wait_attempts)); then
+		echo "Outputs remain inconsistent after $max_wait_attempts attempts. Exiting with error."
+		exit 1
+	else
+		echo "Outputs are inconsistent. Waiting for $wait_interval seconds (attempt $attempt/$max_wait_attempts)."
+		sleep $wait_interval
+	fi
+
+	((attempt++))
 done
 
-docker system prune -f || true
+status=$(./mc admin group info site1 site-replication-issue-group --json | jq .groupStatus | tr -d '"')
-docker volume prune -f || true
+
-docker volume rm $(docker volume ls -q -f dangling=true) || true
+if [[ $status == "enabled" ]]; then
+	echo "Success"
+else
+	echo "Expected status: enabled, actual status: $status"
+	exit 1
+fi
+
+cleanup
 
 ## change working directory
 cd ../../../

.github/workflows/replication.yaml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -22,11 +21,11 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
@@ -36,6 +35,25 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-decom
 
+      - name: Test ILM
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-ilm
+          make test-ilm-transition
+
+      - name: Test PBAC
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-pbac
+
+      - name: Test Config File
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-configfile
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
@@ -48,3 +66,14 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-site-replication-minio
 
+      - name: Test Versioning
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-versioning
+
+      - name: Test Multipart upload with failures
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-multipart

.github/workflows/root-disable.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -21,12 +20,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/run-mint.sh

@@ -15,8 +15,11 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 ## change working directory
 cd .github/workflows/mint
 
+## always pull latest
+docker pull docker.io/minio/mint:edge
+
 docker-compose -f minio-${MODE}.yaml up -d
-sleep 30s
+sleep 1m
 
 docker system prune -f || true
 docker volume prune -f || true
@@ -26,6 +29,9 @@ docker volume rm $(docker volume ls -q -f dangling=true) || true
 [ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2
 [ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6
 
+# Pause one node, to check that all S3 calls work while one node goes wrong
+[ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml pause minio4
+
 docker run --rm --net=mint_default \
 	--name="mint-${MODE}-${JOB_NAME}" \
 	-e SERVER_ENDPOINT="nginx:9000" \
@@ -35,6 +41,18 @@ docker run --rm --net=mint_default \
 	-e MINT_MODE="${MINT_MODE}" \
 	docker.io/minio/mint:edge
 
+# FIXME: enable this after fixing aws-sdk-java-v2 tests
+# # unpause the node, to check that all S3 calls work while one node goes wrong
+# [ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml unpause minio4
+# [ "${MODE}" == "resiliency" ] && docker run --rm --net=mint_default \
+# 	--name="mint-${MODE}-${JOB_NAME}" \
+# 	-e SERVER_ENDPOINT="nginx:9000" \
+# 	-e ACCESS_KEY="${ACCESS_KEY}" \
+# 	-e SECRET_KEY="${SECRET_KEY}" \
+# 	-e ENABLE_HTTPS=0 \
+# 	-e MINT_MODE="${MINT_MODE}" \
+# 	docker.io/minio/mint:edge
+
 docker-compose -f minio-${MODE}.yaml down || true
 sleep 10s
 

.github/workflows/shfmt.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 permissions:
   contents: read
@@ -14,7 +13,7 @@ jobs:
     name: runner / shfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: luizm/action-sh-checker@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/typos.yml

@@ -0,0 +1,15 @@
+---
+name: Spelling
+on: [pull_request]
+
+jobs:
+  run:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Actions Repository
+      uses: actions/checkout@v4
+
+    - name: Check spelling of repo
+      uses: crate-ci/typos@master
+

.github/workflows/upgrade-ci-cd.yaml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -21,12 +20,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.24.x]
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/vulncheck.yml

@@ -3,11 +3,10 @@ on:
   pull_request:
     branches:
       - master
-      - next
+
   push:
     branches:
       - master
-      - next
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -18,15 +17,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.3
+          go-version: 1.24.x
-          check-latest: true
+          cached: false
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
         shell: bash
       - name: Run govulncheck
-        run: govulncheck ./...
+        run: govulncheck -show verbose ./...
         shell: bash

.gitignore

@@ -43,3 +43,13 @@ docs/debugging/inspect/inspect
 docs/debugging/pprofgoparser/pprofgoparser
 docs/debugging/reorder-disks/reorder-disks
 docs/debugging/populate-hard-links/populate-hardlinks
+docs/debugging/xattr/xattr
+hash-set
+healing-bin
+inspect
+pprofgoparser
+reorder-disks
+s3-check-md5
+s3-verify
+xattr
+xl-meta

.golangci.yml

@@ -1,33 +1,64 @@
-linters-settings:
+version: "2"
-  gofumpt:
-    simplify: true
-
-  misspell:
-    locale: US
-
-  staticcheck:
-    checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
-
 linters:
-  disable-all: true
+  default: none
   enable:
     - durationcheck
+    - forcetypeassert
     - gocritic
-    - gofumpt
-    - goimports
     - gomodguard
     - govet
     - ineffassign
     - misspell
     - revive
     - staticcheck
-    - tenv
-    - typecheck
     - unconvert
     - unused
-
+    - usetesting
+    - whitespace
+  settings:
+    misspell:
+      locale: US
+    staticcheck:
+      checks:
+        - all
+        - -SA1008
+        - -SA1019
+        - -SA4000
+        - -SA9004
+        - -ST1000
+        - -ST1005
+        - -ST1016
+        - -U1000
+  exclusions:
+    generated: lax
+    rules:
+      - linters:
+          - forcetypeassert
+        path: _test\.go
+      - path: (.+)\.go$
+        text: 'empty-block:'
+      - path: (.+)\.go$
+        text: 'unused-parameter:'
+      - path: (.+)\.go$
+        text: 'dot-imports:'
+      - path: (.+)\.go$
+        text: should have a package comment
+      - path: (.+)\.go$
+        text: error strings should not be capitalized or end with punctuation or a newline
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
 issues:
-  exclude-use-default: false
+  max-issues-per-linter: 100
-  exclude:
+  max-same-issues: 100
-    - should have a package comment
+formatters:
-    - error strings should not be capitalized or end with punctuation or a newline
+  enable:
+    - gofumpt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.typos.toml

@@ -0,0 +1,45 @@
+[files]
+extend-exclude = [".git/", "docs/", "CREDITS", "go.mod", "go.sum"]
+ignore-hidden = false
+
+[default]
+extend-ignore-re = [
+    "Patrick Collison",
+    "Copyright 2014 Unknwon",
+    "[0-9A-Za-z/+=]{64}",
+    "ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF",
+    "eyJmb28iOiJiYXIifQ",
+    "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.*",
+    "MIIDBTCCAe2gAwIBAgIQWHw7h.*",
+    'http\.Header\{"X-Amz-Server-Side-Encryptio":',
+    "ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge",
+    "ERRO:",
+    "(?Rm)^.*(#|//)\\s*spellchecker:disable-line$", # ignore line
+]
+
+[default.extend-words]
+"encrypter" = "encrypter"
+"kms" = "kms"
+"requestor" = "requestor"
+
+[default.extend-identifiers]
+"HashiCorp" = "HashiCorp"
+
+[type.go.extend-identifiers]
+"bui" = "bui"
+"dm2nd" = "dm2nd"
+"ot" = "ot"
+"ParseND" = "ParseND"
+"ParseNDStream" = "ParseNDStream"
+"pn" = "pn"
+"TestGetPartialObjectMisAligned" = "TestGetPartialObjectMisAligned"
+"thr" = "thr"
+"toi" = "toi"
+
+[type.go]
+extend-ignore-identifiers-re = [
+    # Variants of `typ` used to mean `type` in golang as it is otherwise a
+    # keyword - some of these (like typ1 -> type1) can be fixed, but probably
+    # not worth the effort.
+    "[tT]yp[0-9]*",
+]

CONTRIBUTING.md

@@ -12,8 +12,9 @@ Fork [MinIO upstream](https://github.com/minio/minio/fork) source repository to
 
 ```sh
 git clone https://github.com/minio/minio
+cd minio
 go install -v
-ls /go/bin/minio
+ls $(go env GOPATH)/bin/minio
 ```
 
 ### Set up git remote as ``upstream``

Dockerfile

@@ -1,5 +1,7 @@
 FROM minio/minio:latest
 
+RUN chmod -R 777 /usr/bin
+
 COPY ./minio /usr/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 

Dockerfile.dev

@@ -1,12 +0,0 @@
-FROM minio/minio:latest
-
-ENV PATH=/opt/bin:$PATH
-
-COPY ./minio /opt/bin/minio
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.hotfix

@@ -1,31 +1,38 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.24-alpine as build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
+       chmod +x /go/bin/curl; \
+    fi
+
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
 
-FROM registry.access.redhat.com/ubi9/ubi-micro:9.2
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
 ARG RELEASE
 
@@ -46,9 +53,12 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/mc* /usr/bin/
+COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Dockerfile.release

@@ -1,30 +1,39 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.24-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
+
+WORKDIR /build
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    apk add -U --no-cache bash && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
 
+COPY dockerscripts/download-static-curl.sh /build/download-static-curl
+RUN chmod +x /build/download-static-curl && \
+    /build/download-static-curl
+
 FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
 ARG RELEASE
@@ -46,9 +55,12 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/mc* /usr/bin/
+COPY --from=build /go/bin/curl* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Dockerfile.release.fips

@@ -1,53 +0,0 @@
-FROM golang:1.21-alpine as build
-
-ARG TARGETARCH
-ARG RELEASE
-
-ENV GOPATH /go
-ENV CGO_ENABLED 0
-
-# Install curl and minisign
-RUN apk add -U --no-cache ca-certificates && \
-    apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
-
-# Download minio binary and signature file
-RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
-    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
-    chmod +x /go/bin/minio
-
-# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
-RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
-
-FROM registry.access.redhat.com/ubi9/ubi-micro:9.2
-
-ARG RELEASE
-
-LABEL name="MinIO" \
-      vendor="MinIO Inc <dev@min.io>" \
-      maintainer="MinIO Inc <dev@min.io>" \
-      version="${RELEASE}" \
-      release="${RELEASE}" \
-      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
-      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_ROOT_USER_FILE=access_key \
-    MINIO_ROOT_PASSWORD_FILE=secret_key \
-    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
-
-COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
-
-COPY CREDITS /licenses/CREDITS
-COPY LICENSE /licenses/LICENSE
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-
-EXPOSE 9000
-VOLUME ["/data"]
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-CMD ["minio"]

Dockerfile.release.old_cpu

@@ -1,26 +1,33 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.24-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
+       chmod +x /go/bin/curl; \
+    fi
+
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
@@ -46,9 +53,12 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/mc* /usr/bin/
+COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Makefile

@@ -2,13 +2,13 @@ PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
-GOARCH := $(shell go env GOARCH)
+GOOS ?= $(shell go env GOOS)
-GOOS := $(shell go env GOOS)
+GOARCH ?= $(shell go env GOARCH)
 
 VERSION ?= $(shell git describe --tags)
-TAG ?= "quay.io/minio/minio:$(VERSION)"
+REPO ?= quay.io/minio
+TAG ?= $(REPO)/minio:$(VERSION)
 
-GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
 GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
 
@@ -23,9 +23,7 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
-	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -34,31 +32,54 @@ verifiers: lint check-gen
 
 check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
+	@go mod tidy -compat=1.21
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
+	@(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false)
 
 lint: getdeps ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+	@command typos && typos ./ || echo "typos binary is not found.. skipping.."
 
 lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
-test: verifiers build build-debugging ## builds minio, runs linters, tests
+test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue,dev ./...
 
 test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
 	@env bash $(PWD)/buildscripts/disable-root.sh
 
+test-ilm: install-race
+	@echo "Running ILM tests"
+	@env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh
+
+test-ilm-transition: install-race
+	@echo "Running ILM tiering tests with healing"
+	@env bash $(PWD)/docs/bucket/lifecycle/setup_ilm_transition.sh
+
+test-pbac: install-race
+	@echo "Running bucket policies tests"
+	@env bash $(PWD)/docs/iam/policies/pbac-tests.sh
+
 test-decom: install-race
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
 	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted-kes.sh
+
+test-versioning: install-race
+	@echo "Running minio versioning tests"
+	@env bash $(PWD)/docs/bucket/versioning/versioning-tests.sh
+
+test-configfile: install-race
+	@env bash $(PWD)/docs/distributed/distributed-from-config-file.sh
 
 test-upgrade: install-race
 	@echo "Running minio upgrade tests"
@@ -68,11 +89,23 @@ test-race: verifiers build ## builds minio, runs linters, tests (race)
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-test-iam: build ## verify IAM (external IDP, etcd backends)
+test-iam: install-race ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -timeout 15m -tags kqueue,dev -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 15m -race -tags kqueue,dev -v -run TestIAM* ./cmd
+
+test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP)
+	@echo "Running upgrade tests for IAM (LDAP backend)"
+	@env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh
+
+test-iam-import-with-missing-entities: install-race ## test import of external iam config withg missing entities
+	@echo "Test IAM import configurations with missing entities"
+	@env bash $(PWD)/docs/distributed/iam-import-with-missing-entities.sh
+
+test-iam-import-with-openid: install-race
+	@echo "Test IAM import configurations with openid"
+	@env bash $(PWD)/docs/distributed/iam-import-with-openid.sh
 
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
@@ -86,7 +119,10 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
+test-delete-marker-proxying:
+	@(env bash $(PWD)/docs/bucket/replication/test_del_marker_proxying.sh)
+
+test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error test-delete-marker-proxying ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
 test-site-replication-ldap: install-race ## verify automatic site replication
@@ -100,56 +136,73 @@ test-site-replication-oidc: install-race ## verify automatic site replication
 test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects"
+	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects with SSE-KMS enabled for bucket"
+	@(env bash $(PWD)/docs/site-replication/run-sse-kms-object-replication.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site"
+	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh)
 
-verify: ## verify minio various setups
+test-multipart: install-race ## test multipart
+	@echo "Test multipart behavior when part files are missing"
+	@(env bash $(PWD)/buildscripts/multipart-quorum-test.sh)
+
+test-timeout: install-race ## test multipart
+	@echo "Test server timeout"
+	@(env bash $(PWD)/buildscripts/test-timeout.sh)
+
+verify: install-race ## verify minio various setups
 	@echo "Verifying build with race"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-verify-healing: ## verify healing and replacing disks with minio binary
+verify-healing: install-race ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
-	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
+	@(env bash $(PWD)/buildscripts/verify-healing-empty-erasure-set.sh)
 	@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
 
-verify-healing-with-root-disks: ## verify healing root disks
+verify-healing-with-root-disks: install-race ## verify healing root disks
 	@echo "Verify healing with root drives"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
 
-verify-healing-with-rewrite: ## verify healing to rewrite old xl.meta -> new xl.meta
+verify-healing-with-rewrite: install-race ## verify healing to rewrite old xl.meta -> new xl.meta
 	@echo "Verify healing with rewrite"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/rewrite-old-new.sh)
 
-verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
+verify-healing-inconsistent-versions: install-race ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
 
 build-debugging:
 	@(env bash $(PWD)/docs/debugging/build.sh)
 
-build: checks ## builds minio to $(PWD)
+build: checks build-debugging ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
-	@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
 	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
-	$(eval TAG := "minio/minio:$(VERSION)")
 
-hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
-	@mv -f ./minio ./minio.$(VERSION)
+	@wget -q -c https://github.com/minio/pkger/releases/download/v2.3.11/pkger_2.3.11_linux_amd64.deb
-	@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.1.1/linux-systemd/distributed/minio.service
-	@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
+	@sudo apt install ./pkger_2.3.11_linux_amd64.deb --yes
+	@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
+	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
+	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
+	@minisign -qQSm minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@sha256sum < minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION).sha256sum
+	@cp -af minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)* minio-release/$(GOOS)-$(GOARCH)/archive/
+	@pkger -r $(VERSION) --ignore
 
 hotfix-push: hotfix
-	@scp -q -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/
-	@scp -q -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive
-	@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive
+	@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-$(GOOS)/archive/minio.$(VERSION)"
 
 docker-hotfix-push: docker-hotfix
 	@docker push -q $(TAG) && echo "Published new container $(TAG)"
@@ -162,15 +215,19 @@ docker: build ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -q --no-cache -t $(TAG) . -f Dockerfile
 
-install-race: checks ## builds minio to $(PWD)
+test-resiliency: build
-	@echo "Building minio binary to './minio'"
+	@echo "Running resiliency tests"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(DOCKER_COMPOSE_FILE=$(PWD)/docs/resiliency/docker-compose.yaml env bash $(PWD)/docs/resiliency/resiliency-tests.sh)
-	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
+
-	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
+install-race: checks build-debugging ## builds minio to $(PWD)
+	@echo "Building minio binary with -race to './minio'"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue,dev -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@echo "Installing minio binary with -race to '$(GOPATH)/bin/minio'"
+	@mkdir -p $(GOPATH)/bin && cp -af $(PWD)/minio $(GOPATH)/bin/minio
 
 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp -af $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean: ## cleanup all generated assets
@@ -183,3 +240,6 @@ clean: ## cleanup all generated assets
 	@rm -rvf build
 	@rm -rvf release
 	@rm -rvf .verify*
+	@rm -rvf minio-release
+	@rm -rvf minio.RELEASE*.hotfix.*
+	@rm -rvf pkger_*.deb

PULL_REQUESTS_ETIQUETTE.md

@@ -0,0 +1,93 @@
+# MinIO Pull Request Guidelines
+
+These guidelines ensure high-quality commits in MinIO’s GitHub repositories, maintaining 
+a clear, valuable commit history for our open-source projects. They apply to all contributors, 
+fostering efficient reviews and robust code.
+
+## Why Pull Requests?
+
+Pull Requests (PRs) drive quality in MinIO’s codebase by:
+- Enabling peer review without pair programming.
+- Documenting changes for future reference.
+- Ensuring commits tell a clear story of development.
+
+**A poor commit lasts forever, even if code is refactored.**
+
+## Crafting a Quality PR
+
+A strong MinIO PR:
+- Delivers a complete, valuable change (feature, bug fix, or improvement).
+- Has a concise title (e.g., `[S3] Fix bucket policy parsing #1234`) and a summary with context, referencing issues (e.g., `#1234`).
+- Contains well-written, logical commits explaining *why* changes were made (e.g., “Add S3 bucket tagging support so that users can organize resources efficiently”).
+- Is small, focused, and easy to review—ideally one commit, unless multiple commits better narrate complex work.
+- Adheres to MinIO’s coding standards (e.g., Go style, error handling, testing).
+
+PRs must flow smoothly through review to reach production. Large PRs should be split into smaller, manageable ones.
+
+## Submitting PRs
+
+1. **Title and Summary**:
+   - Use a scannable title: `[Subsystem] Action Description #Issue` (e.g., `[IAM] Add role-based access control #567`).
+   - Include context in the summary: what changed, why, and any issue references.
+   - Use `[WIP]` for in-progress PRs to avoid premature merging or choose GitHub draft PRs.
+
+2. **Commits**:
+   - Write clear messages: what changed and why (e.g., “Refactor S3 API handler to reduce latency so that requests process 20% faster”).
+   - Rebase to tidy commits before submitting (e.g., `git rebase -i main` to squash typos or reword messages), unless multiple contributors worked on the branch.
+   - Keep PRs focused—one feature or fix. Split large changes into multiple PRs.
+
+3. **Testing**:
+   - Include unit tests for new functionality or bug fixes.
+   - Ensure existing tests pass (`make test`).
+   - Document testing steps in the PR summary if manual testing was performed.
+
+4. **Before Submitting**:
+   - Run `make verify` to check formatting, linting, and tests.
+   - Reference related issues (e.g., “Closes #1234”).
+   - Notify team members via GitHub `@mentions` if urgent or complex.
+
+## Reviewing PRs
+
+Reviewers ensure MinIO’s commit history remains a clear, reliable record. Responsibilities include:
+
+1. **Commit Quality**:
+   - Verify each commit explains *why* the change was made (e.g., “So that…”).
+   - Request rebasing if commits are unclear, redundant, or lack context (e.g., “Please squash typo fixes into the parent commit”).
+
+2. **Code Quality**:
+   - Check adherence to MinIO’s Go standards (e.g., error handling, documentation).
+   - Ensure tests cover new code and pass CI.
+   - Flag bugs or critical issues for immediate fixes; suggest non-blocking improvements as follow-up issues.
+
+3. **Flow**:
+   - Review promptly to avoid blocking progress.
+   - Balance quality and speed—minor issues can be addressed later via issues, not PR blocks.
+   - If unable to complete the review, tag another reviewer (e.g., `@username please take over`).
+
+4. **Shared Responsibility**:
+   - All MinIO contributors are reviewers. The first commenter on a PR owns the review unless they delegate.
+   - Multiple reviewers are encouraged for complex PRs.
+
+5. **No Self-Edits**:
+   - Don’t modify the PR directly (e.g., fixing bugs). Request changes from the submitter or create a follow-up PR.
+   - If you edit, you’re a collaborator, not a reviewer, and cannot merge.
+
+6. **Testing**:
+   - Assume the submitter tested the code. If testing is unclear, ask for details (e.g., “How was this tested?”).
+   - Reject untested PRs unless testing is infeasible, then assist with test setup.
+
+## Tips for Success
+
+- **Small PRs**: Easier to review, faster to merge. Split large changes logically.
+- **Clear Commits**: Use `git rebase -i` to refine history before submitting.
+- **Engage Early**: Discuss complex changes in issues or Slack (https://slack.min.io) before coding.
+- **Be Responsive**: Address reviewer feedback promptly to keep PRs moving.
+- **Learn from Reviews**: Use feedback to improve future contributions.
+
+## Resources
+
+- [MinIO Coding Standards](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
+- [Effective Commit Messages](https://mislav.net/2014/02/hidden-documentation/)
+- [GitHub PR Tips](https://github.com/blog/1943-how-to-write-the-perfect-pull-request)
+
+By following these guidelines, we ensure MinIO’s codebase remains high-quality, maintainable, and a joy to contribute to. Happy coding!

README.fips.md

@@ -1,7 +0,0 @@
-# MinIO FIPS Builds
-
-MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring).
-
-MinIO FIPS executables are available at <http://dl.min.io> - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
-
-We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes.

README.md

@@ -4,7 +4,13 @@
 
 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
-MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
+MinIO is a high-performance, S3-compatible object storage solution released under the GNU AGPL v3.0 license. Designed for speed and scalability, it powers AI/ML, analytics, and data-intensive workloads with industry-leading performance.
+
+🔹 S3 API Compatible – Seamless integration with existing S3 tools
+🔹 Built for AI & Analytics – Optimized for large-scale data pipelines
+🔹 High Performance – Ideal for demanding storage workloads.
+
+AI storage documentation  (https://min.io/solutions/object-storage-for-ai).
 
 This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
 
@@ -34,7 +40,9 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
+> [!NOTE]
+> To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option.
+> For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
 
 ## macOS
 
@@ -51,7 +59,8 @@ brew install minio/stable/minio
 minio server /data
 ```
 
-> NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
+> [!NOTE]
+> If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
 
 ```sh
 brew uninstall minio
@@ -93,13 +102,13 @@ The following table lists supported architectures. Replace the `wget` URL with t
 | 64-bit Intel/AMD               | <https://dl.min.io/server/minio/release/linux-amd64/minio>   |
 | 64-bit ARM                     | <https://dl.min.io/server/minio/release/linux-arm64/minio>   |
 | 64-bit PowerPC LE (ppc64le)    | <https://dl.min.io/server/minio/release/linux-ppc64le/minio> |
-| IBM Z-Series (S390X)           | <https://dl.min.io/server/minio/release/linux-s390x/minio>   |
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
 You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
+> [!NOTE]
+> Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
 
 ## Microsoft Windows
 
@@ -119,11 +128,12 @@ The MinIO deployment starts using default root credentials `minioadmin:minioadmi
 
 You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
+> [!NOTE]
+> Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.19](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.24](https://golang.org/dl/#stable)
 
 ```sh
 go install github.com/minio/minio@latest
@@ -133,7 +143,8 @@ The MinIO deployment starts using default root credentials `minioadmin:minioadmi
 
 You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
+> [!NOTE]
+> Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
 
 MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
 
@@ -171,7 +182,8 @@ This command gets the active zone(s). Now, apply port rules to the relevant zone
 firewall-cmd --zone=public --add-port=9000/tcp --permanent
 ```
 
-Note that `permanent` makes sure the rules are persistent across firewall start, restart or reload. Finally reload the firewall for changes to take effect.
+> [!NOTE]
+> `permanent` makes sure the rules are persistent across firewall start, restart or reload. Finally reload the firewall for changes to take effect.
 
 ```sh
 firewall-cmd --reload
@@ -200,7 +212,8 @@ service iptables restart
 
 MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
 
-> NOTE: MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
+> [!NOTE]
+> MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
 
 ### Things to consider
 
@@ -210,10 +223,6 @@ For deployments behind a load balancer, proxy, or ingress rule where the MinIO h
 
 For example, consider a MinIO deployment behind a proxy `https://minio.example.net`, `https://console.minio.example.net` with rules for forwarding traffic on port :9000 and :9001 to MinIO and the MinIO Console respectively on the internal network. Set `MINIO_BROWSER_REDIRECT_URL` to `https://console.minio.example.net` to ensure the browser receives a valid reachable URL.
 
-Similarly, if your TLS certificates do not have the IP SAN for the MinIO server host, the MinIO Console may fail to validate the connection to the server. Use the `MINIO_SERVER_URL` environment variable  and specify the proxy-accessible hostname of the MinIO server to allow the Console to use the MinIO server API using the TLS certificate.
-
-For example: `export MINIO_SERVER_URL="https://minio.example.net"`
-
 | Dashboard                                                                                   | Creating a bucket                                                                           |
 | -------------                                                                               | -------------                                                                               |
 | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic1.png?raw=true) | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic2.png?raw=true) |
@@ -226,7 +235,8 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
 
 Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all transactions on MinIO are atomic. So upgrading all the servers simultaneously is the recommended way to upgrade MinIO.
 
-> NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
+> [!NOTE]
+> requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
 
 - For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://min.io/docs/minio/linux/reference/minio-mc-admin/mc-admin-update.html)
 

buildscripts/checkdeps.sh

@@ -74,11 +74,11 @@ check_minimum_version() {
 
 assert_is_supported_arch() {
 	case "${ARCH}" in
-	x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64)
+	x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64 | riscv64)
 		return
 		;;
 	*)
-		echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]"
+		echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64, riscv64]"
 		exit 1
 		;;
 	esac

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
 	export CGO_ENABLED=0
 
 	## List of architectures and OS to test coss compilation.
-	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
+	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64 linux/riscv64"
 }
 
 function _build() {

buildscripts/disable-root.sh

@@ -32,6 +32,7 @@ fi
 set +e
 
 export MC_HOST_minioadm=http://minioadmin:minioadmin@localhost:9100/
+./mc ready minioadm
 
 ./mc ls minioadm/
 
@@ -56,7 +57,7 @@ done
 
 set +e
 
-sleep 10
+./mc ready minioadm/
 
 ./mc ls minioadm/
 if [ $? -ne 0 ]; then
@@ -81,11 +82,12 @@ minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data
 minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
 
-sleep 20s
-
 export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001
 export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004
 
+./mc ready sitea
+./mc ready siteb
+
 ./mc admin replicate add sitea siteb
 
 ./mc admin user add sitea foobar foo12345
@@ -109,11 +111,12 @@ minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data
 minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
 
-sleep 20s
-
 export MC_HOST_sitea=http://foobar:foo12345@127.0.0.1:9001
 export MC_HOST_siteb=http://foobar:foo12345@127.0.0.1:9004
 
+./mc ready sitea
+./mc ready siteb
+
 ./mc admin user add sitea foobar-admin foo12345
 
 sleep 2s

buildscripts/heal-manual.go

@@ -44,7 +44,6 @@ func main() {
 	opts := madmin.HealOpts{
 		Recursive: true,                  // recursively heal all objects at 'prefix'
 		Remove:    true,                  // remove content that has lost quorum and not recoverable
-		Recreate:  true,                  // rewrite all old non-inlined xl.meta to new xl.meta
 		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
 	}
 

buildscripts/minio-iam-ldap-upgrade-import-test.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# This script is used to test the migration of IAM content from old minio
+# instance to new minio instance.
+#
+# To run it locally, start the LDAP server in github.com/minio/minio-iam-testing
+# repo (e.g. make podman-run), and then run this script.
+#
+# This script assumes that LDAP server is at:
+#
+#   `localhost:389`
+#
+# if this is not the case, set the environment variable
+# `_MINIO_LDAP_TEST_SERVER`.
+
+OLD_VERSION=RELEASE.2024-03-26T22-10-45Z
+OLD_BINARY_LINK=https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${OLD_VERSION}
+
+__init__() {
+	if which curl &>/dev/null; then
+		echo "curl is already installed"
+	else
+		echo "Installing curl:"
+		sudo apt install curl -y
+	fi
+
+	export GOPATH=/tmp/gopath
+	export PATH="${PATH}":"${GOPATH}"/bin
+
+	if which mc &>/dev/null; then
+		echo "mc is already installed"
+	else
+		echo "Installing mc:"
+		go install github.com/minio/mc@latest
+	fi
+
+	if [ ! -x ./minio.${OLD_VERSION} ]; then
+		echo "Downloading minio.${OLD_VERSION} binary"
+		curl -o minio.${OLD_VERSION} ${OLD_BINARY_LINK}
+		chmod +x minio.${OLD_VERSION}
+	fi
+
+	if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then
+		export _MINIO_LDAP_TEST_SERVER=localhost:389
+		echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER"
+	fi
+
+	rm -rf /tmp/data
+}
+
+create_iam_content_in_old_minio() {
+	echo "Creating IAM content in old minio instance."
+
+	MINIO_CI_CD=1 ./minio.${OLD_VERSION} server /tmp/data/{1...4} &
+	sleep 5
+
+	set -x
+	mc alias set old-minio http://localhost:9000 minioadmin minioadmin
+	mc ready old-minio
+	mc idp ldap add old-minio \
+		server_addr=localhost:389 \
+		server_insecure=on \
+		lookup_bind_dn=cn=admin,dc=min,dc=io \
+		lookup_bind_password=admin \
+		user_dn_search_base_dn=dc=min,dc=io \
+		user_dn_search_filter="(uid=%s)" \
+		group_search_base_dn=ou=swengg,dc=min,dc=io \
+		group_search_filter="(&(objectclass=groupOfNames)(member=%d))"
+	mc admin service restart old-minio
+
+	mc idp ldap policy attach old-minio readwrite --user=UID=dillon,ou=people,ou=swengg,dc=min,dc=io
+	mc idp ldap policy attach old-minio readwrite --group=CN=project.c,ou=groups,ou=swengg,dc=min,dc=io
+
+	mc idp ldap policy entities old-minio
+
+	mc admin cluster iam export old-minio
+	set +x
+
+	mc admin service stop old-minio
+}
+
+import_iam_content_in_new_minio() {
+	echo "Importing IAM content in new minio instance."
+	# Assume current minio binary exists.
+	MINIO_CI_CD=1 ./minio server /tmp/data/{1...4} &
+	sleep 5
+
+	set -x
+	mc alias set new-minio http://localhost:9000 minioadmin minioadmin
+	echo "BEFORE IMPORT mappings:"
+	mc ready new-minio
+	mc idp ldap policy entities new-minio
+	mc admin cluster iam import new-minio ./old-minio-iam-info.zip
+	echo "AFTER IMPORT mappings:"
+	mc idp ldap policy entities new-minio
+	set +x
+
+	# mc admin service stop new-minio
+}
+
+verify_iam_content_in_new_minio() {
+	output=$(mc idp ldap policy entities new-minio --json)
+
+	groups=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .groups[]')
+	if [ "$groups" != "cn=project.c,ou=groups,ou=swengg,dc=min,dc=io" ]; then
+		echo "Failed to verify groups: $groups"
+		exit 1
+	fi
+
+	users=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .users[]')
+	if [ "$users" != "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" ]; then
+		echo "Failed to verify users: $users"
+		exit 1
+	fi
+
+	mc admin service stop new-minio
+}
+
+main() {
+	create_iam_content_in_old_minio
+
+	import_iam_content_in_new_minio
+
+	verify_iam_content_in_new_minio
+}
+
+(__init__ "$@" && main "$@")

buildscripts/minio-upgrade.sh

@@ -4,10 +4,22 @@ trap 'cleanup $LINENO' ERR
 
 # shellcheck disable=SC2120
 cleanup() {
-	MINIO_VERSION=dev docker-compose \
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose \
 		-f "buildscripts/upgrade-tests/compose.yml" \
-		rm -s -f
+		down || true
+
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose \
+		-f "buildscripts/upgrade-tests/compose.yml" \
+		rm || true
+
+	for volume in $(docker volume ls -q | grep upgrade); do
+		docker volume rm ${volume} || true
+	done
+
 	docker volume prune -f
+	docker system prune -f || true
+	docker volume prune -f || true
+	docker volume rm $(docker volume ls -q -f dangling=true) || true
 }
 
 verify_checksum_after_heal() {
@@ -55,6 +67,15 @@ __init__() {
 
 	go install github.com/minio/mc@latest
 
+	## this is needed because github actions don't have
+	## docker-compose on all runners
+	COMPOSE_VERSION=v2.35.1
+	mkdir -p /tmp/gopath/bin/
+	wget -O /tmp/gopath/bin/docker-compose https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64
+	chmod +x /tmp/gopath/bin/docker-compose
+
+	cleanup
+
 	TAG=minio/minio:dev make docker
 
 	MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
@@ -72,11 +93,11 @@ __init__() {
 
 	curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
 
-	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
 }
 
 main() {
-	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
 
 	add_alias
 

buildscripts/multipart-quorum-test.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+	set -x
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+trap 'catch $LINENO' ERR
+
+function purge() {
+	rm -rf "$1"
+}
+
+# shellcheck disable=SC2120
+catch() {
+	if [ $# -ne 0 ]; then
+		echo "error on line $1"
+	fi
+
+	echo "Cleaning up instances of MinIO"
+	pkill minio || true
+	pkill -9 minio || true
+	purge "$WORK_DIR"
+	if [ $# -ne 0 ]; then
+		exit $#
+	fi
+}
+
+catch
+
+function start_minio_10drive() {
+	start_port=$1
+
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	export MINIO_CI_CD=1
+
+	mkdir ${WORK_DIR}
+	C_PWD=${PWD}
+	if [ ! -x "$PWD/mc" ]; then
+		MC_BUILD_DIR="mc-$RANDOM"
+		if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+			echo "failed to download https://github.com/minio/mc"
+			purge "${MC_BUILD_DIR}"
+			exit 1
+		fi
+
+		(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+		# remove mc source.
+		purge "${MC_BUILD_DIR}"
+	fi
+
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...10}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 5
+
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	"${PWD}/mc" mb --with-versioning minio/bucket
+
+	export AWS_ACCESS_KEY_ID=minio
+	export AWS_SECRET_ACCESS_KEY=minio123
+	aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
+	uploadId=$(jq -r '.UploadId' upload-id.json)
+
+	truncate -s 5MiB file-5mib
+	for i in {1..2}; do
+		aws --endpoint-url http://localhost:"$start_port" s3api upload-part \
+			--upload-id "$uploadId" --bucket bucket --key obj-1 \
+			--part-number "$i" --body ./file-5mib
+	done
+	for i in {1..6}; do
+		find ${WORK_DIR}/disk${i}/.minio.sys/multipart/ -type f -name "part.1" -delete
+	done
+	cat <<EOF >parts.json
+{
+    "Parts": [
+        {
+            "PartNumber": 1,
+            "ETag": "5f363e0e58a95f06cbe9bbc662c5dfb6"
+        },
+        {
+            "PartNumber": 2,
+            "ETag": "5f363e0e58a95f06cbe9bbc662c5dfb6"
+        }
+    ]
+}
+EOF
+	err=$(aws --endpoint-url http://localhost:"$start_port" s3api complete-multipart-upload --upload-id "$uploadId" --bucket bucket --key obj-1 --multipart-upload file://./parts.json 2>&1)
+	rv=$?
+	if [ $rv -eq 0 ]; then
+		echo "Failed to receive an error"
+		exit 1
+	fi
+	echo "Received an error during complete-multipart as expected: $err"
+}
+
+function main() {
+	start_port=$(shuf -i 10000-65000 -n 1)
+	start_minio_10drive ${start_port}
+}
+
+main "$@"

buildscripts/rewrite-old-new.sh

@@ -45,7 +45,8 @@ function verify_rewrite() {
 	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
 	pid=$!
 	disown $pid
-	sleep 10
+
+	"${WORK_DIR}/mc" ready minio/
 
 	if ! ps -p ${pid} 1>&2 >/dev/null; then
 		echo "server1 log:"
@@ -77,7 +78,8 @@ function verify_rewrite() {
 	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
 	pid=$!
 	disown $pid
-	sleep 10
+
+	"${WORK_DIR}/mc" ready minio/
 
 	if ! ps -p ${pid} 1>&2 >/dev/null; then
 		echo "server1 log:"
@@ -87,13 +89,12 @@ function verify_rewrite() {
 		exit 1
 	fi
 
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	if ! ./s3-check-md5 \
-	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \
 		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+		-endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
@@ -113,7 +114,7 @@ function verify_rewrite() {
 	go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
 	sleep 1
 
-	if ! s3-check-md5 \
+	if ! ./s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \

buildscripts/test-timeout.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+	set -x
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+trap 'catch $LINENO' ERR
+
+function purge() {
+	rm -rf "$1"
+}
+
+# shellcheck disable=SC2120
+catch() {
+	if [ $# -ne 0 ]; then
+		echo "error on line $1"
+	fi
+
+	echo "Cleaning up instances of MinIO"
+	pkill minio || true
+	pkill -9 minio || true
+	purge "$WORK_DIR"
+	if [ $# -ne 0 ]; then
+		exit $#
+	fi
+}
+
+catch
+
+function gen_put_request() {
+	hdr_sleep=$1
+	body_sleep=$2
+
+	echo "PUT /testbucket/testobject HTTP/1.1"
+	sleep $hdr_sleep
+	echo "Host: foo-header"
+	echo "User-Agent: curl/8.2.1"
+	echo "Accept: */*"
+	echo "Content-Length: 30"
+	echo ""
+
+	sleep $body_sleep
+	echo "random line 0"
+	echo "random line 1"
+	echo ""
+	echo ""
+}
+
+function send_put_object_request() {
+	hdr_timeout=$1
+	body_timeout=$2
+
+	start=$(date +%s)
+	timeout 5m bash -c "gen_put_request $hdr_timeout $body_timeout | netcat 127.0.0.1 $start_port | read" || return -1
+	[ $(($(date +%s) - start)) -gt $((srv_hdr_timeout + srv_idle_timeout + 1)) ] && return -1
+	return 0
+}
+
+function test_minio_with_timeout() {
+	start_port=$1
+
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	export MINIO_CI_CD=1
+
+	mkdir ${WORK_DIR}
+	C_PWD=${PWD}
+	if [ ! -x "$PWD/mc" ]; then
+		MC_BUILD_DIR="mc-$RANDOM"
+		if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+			echo "failed to download https://github.com/minio/mc"
+			purge "${MC_BUILD_DIR}"
+			exit 1
+		fi
+
+		(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+		# remove mc source.
+		purge "${MC_BUILD_DIR}"
+	fi
+
+	"${MINIO[@]}" --address ":$start_port" --read-header-timeout ${srv_hdr_timeout}s --idle-timeout ${srv_idle_timeout}s "${WORK_DIR}/disk/" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 1
+
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	set -e
+
+	"${PWD}/mc" mb minio/testbucket
+	"${PWD}/mc" anonymous set public minio/testbucket
+
+	# slow header writing
+	send_put_object_request 20 0 && exit -1
+	"${PWD}/mc" stat minio/testbucket/testobject && exit -1
+
+	# quick header write and slow bodywrite
+	send_put_object_request 0 40 && exit -1
+	"${PWD}/mc" stat minio/testbucket/testobject && exit -1
+
+	# quick header and body write
+	send_put_object_request 1 1 || exit -1
+	"${PWD}/mc" stat minio/testbucket/testobject || exit -1
+}
+
+function main() {
+	export start_port=$(shuf -i 10000-65000 -n 1)
+	export srv_hdr_timeout=5
+	export srv_idle_timeout=5
+	export -f gen_put_request
+
+	test_minio_with_timeout ${start_port}
+}
+
+main "$@"

buildscripts/unaligned-healing.sh

@@ -1,177 +0,0 @@
-#!/bin/bash -e
-#
-
-set -E
-set -o pipefail
-set -x
-
-if [ ! -x "$PWD/minio" ]; then
-	echo "minio executable binary not found in current directory"
-	exit 1
-fi
-
-WORK_DIR="$PWD/.verify-$RANDOM"
-MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=("$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server)
-MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
-
-function download_old_release() {
-	if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
-		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
-		chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
-	fi
-}
-
-function start_minio_16drive() {
-	start_port=$1
-
-	export MINIO_ROOT_USER=minio
-	export MINIO_ROOT_PASSWORD=minio123
-	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-	unset MINIO_KMS_AUTO_ENCRYPTION         # do not auto-encrypt objects
-	export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
-	export MINIO_CI_CD=1
-
-	MC_BUILD_DIR="mc-$RANDOM"
-	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-		echo "failed to download https://github.com/minio/mc"
-		purge "${MC_BUILD_DIR}"
-		exit 1
-	fi
-
-	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
-
-	# remove mc source.
-	purge "${MC_BUILD_DIR}"
-
-	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-	pid=$!
-	disown $pid
-	sleep 30
-
-	if ! ps -p ${pid} 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
-	"${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
-	"${WORK_DIR}/mc" cp \
-		"${WORK_DIR}/unaligned" \
-		minio/healing-shard-bucket/unaligned \
-		--disable-multipart --quiet
-
-	## "unaligned" object name gets consistently distributed
-	## to disks in following distribution order
-	##
-	## NOTE: if you change the name make sure to change the
-	## distribution order present here
-	##
-	## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
-
-	## make sure to remove the "last" data shard
-	rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
-	sleep 10
-	## Heal the shard
-	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-	## then remove any other data shard let's pick first disk
-	## - 1st data shard.
-	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
-	sleep 10
-
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	pkill minio
-	sleep 3
-
-	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-	pid=$!
-	disown $pid
-	sleep 30
-
-	if ! ps -p ${pid} 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		mkdir -p inspects
-		(
-			cd inspects
-			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
-		)
-
-		"${WORK_DIR}/mc" mb play/inspects
-		"${WORK_DIR}/mc" mirror inspects play/inspects
-
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		mkdir -p inspects
-		(
-			cd inspects
-			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
-		)
-
-		"${WORK_DIR}/mc" mb play/inspects
-		"${WORK_DIR}/mc" mirror inspects play/inspects
-
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	pkill minio
-	sleep 3
-}
-
-function main() {
-	download_old_release
-
-	start_port=$(shuf -i 10000-65000 -n 1)
-
-	start_minio_16drive ${start_port}
-}
-
-function purge() {
-	rm -rf "$1"
-}
-
-(main "$@")
-rv=$?
-purge "$WORK_DIR"
-exit "$rv"

buildscripts/upgrade-tests/compose.yml

@@ -1,5 +1,3 @@
-version: '3.7'
-
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
   image: minio/minio:${MINIO_VERSION}

buildscripts/verify-build.sh

@@ -15,13 +15,14 @@ WORK_DIR="$PWD/.verify-$RANDOM"
 export MINT_MODE=core
 export MINT_DATA_DIR="$WORK_DIR/data"
 export SERVER_ENDPOINT="127.0.0.1:9000"
+export MC_HOST_verify="http://minio:minio123@${SERVER_ENDPOINT}/"
+export MC_HOST_verify_ipv6="http://minio:minio123@[::1]:9000/"
 export ACCESS_KEY="minio"
 export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
 export GOGC=25
 export ENABLE_ADMIN=1
-
 export MINIO_CI_CD=1
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
@@ -36,18 +37,21 @@ function start_minio_fs() {
 	export MINIO_ROOT_USER=$ACCESS_KEY
 	export MINIO_ROOT_PASSWORD=$SECRET_KEY
 	"${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-	sleep 10
+
+	"${WORK_DIR}/mc" ready verify
 }
 
 function start_minio_erasure() {
 	"${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
-	sleep 15
+
+	"${WORK_DIR}/mc" ready verify
 }
 
 function start_minio_erasure_sets() {
 	export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
 	"${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
-	sleep 15
+
+	"${WORK_DIR}/mc" ready verify
 }
 
 function start_minio_pool_erasure_sets() {
@@ -57,7 +61,7 @@ function start_minio_pool_erasure_sets() {
 	"${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 &
 	"${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 &
 
-	sleep 40
+	"${WORK_DIR}/mc" ready verify
 }
 
 function start_minio_pool_erasure_sets_ipv6() {
@@ -67,7 +71,7 @@ function start_minio_pool_erasure_sets_ipv6() {
 	"${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
 	"${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 
-	sleep 40
+	"${WORK_DIR}/mc" ready verify_ipv6
 }
 
 function start_minio_dist_erasure() {
@@ -78,7 +82,7 @@ function start_minio_dist_erasure() {
 		"${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
 	done
 
-	sleep 40
+	"${WORK_DIR}/mc" ready verify
 }
 
 function run_test_fs() {
@@ -222,7 +226,7 @@ function __init__() {
 		exit 1
 	fi
 
-	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	(cd "${MC_BUILD_DIR}" && go build -o "${WORK_DIR}/mc")
 
 	# remove mc source.
 	purge "${MC_BUILD_DIR}"

buildscripts/verify-healing-empty-erasure-set.sh

@@ -0,0 +1,151 @@
+#!/bin/bash -e
+#
+
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+
+function start_minio_3_node() {
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ERASURE_SET_DRIVE_COUNT=6
+	export MINIO_CI_CD=1
+
+	start_port=$1
+	args=""
+	for i in $(seq 1 3); do
+		args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
+	done
+
+	"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+	pid1=$!
+	disown ${pid1}
+
+	"${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+	pid2=$!
+	disown $pid2
+
+	"${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+	pid3=$!
+	disown $pid3
+
+	export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
+
+	timeout 15m /tmp/mc ready myminio || fail
+
+	# Wait for all drives to be online and formatted
+	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].state | select(. != "ok")' | wc -l) -gt 0 ]; do sleep 1; done
+	# Wait for all drives to be healed
+	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].healing | select(. != null) | select(. == true)' | wc -l) -gt 0 ]; do sleep 1; done
+
+	# Wait for Status: in MinIO output
+	while true; do
+		rv=$(check_online)
+		if [ "$rv" != "1" ]; then
+			# success
+			break
+		fi
+
+		# Check if we should retry
+		retry=$((retry + 1))
+		if [ $retry -le 20 ]; then
+			sleep 5
+			continue
+		fi
+
+		# Failure
+		fail
+	done
+
+	if ! ps -p $pid1 1>&2 >/dev/null; then
+		echo "minio-server-1 is not running." && fail
+	fi
+
+	if ! ps -p $pid2 1>&2 >/dev/null; then
+		echo "minio-server-2 is not running." && fail
+	fi
+
+	if ! ps -p $pid3 1>&2 >/dev/null; then
+		echo "minio-server-3 is not running." && fail
+	fi
+
+	if ! pkill minio; then
+		fail
+	fi
+
+	sleep 1
+	if pgrep minio; then
+		# forcibly killing, to proceed further properly.
+		if ! pkill -9 minio; then
+			echo "no minio process running anymore, proceed."
+		fi
+	fi
+}
+
+function fail() {
+	for i in $(seq 1 3); do
+		echo "server$i log:"
+		cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+}
+
+function check_online() {
+	if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
+		echo "1"
+	fi
+}
+
+function purge() {
+	echo rm -rf "$1"
+}
+
+function __init__() {
+	echo "Initializing environment"
+	mkdir -p "$WORK_DIR"
+	mkdir -p "$MINIO_CONFIG_DIR"
+
+	## version is purposefully set to '3' for minio to migrate configuration file
+	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
+
+	if [ ! -f /tmp/mc ]; then
+		wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+			chmod +x /tmp/mc
+	fi
+}
+
+function perform_test() {
+	start_minio_3_node $2
+
+	echo "Testing Distributed Erasure setup healing of drives"
+	echo "Remove the contents of the disks belonging to '${1}' erasure set"
+
+	rm -rf ${WORK_DIR}/${1}/*/
+
+	set -x
+	start_minio_3_node $2
+}
+
+function main() {
+	# use same ports for all tests
+	start_port=$(shuf -i 10000-65000 -n 1)
+
+	perform_test "2" ${start_port}
+	perform_test "1" ${start_port}
+	perform_test "3" ${start_port}
+}
+
+(__init__ "$@" && main "$@")
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/verify-healing.sh

@@ -12,17 +12,26 @@ fi
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+GOPATH=/tmp/gopath
 
 function start_minio_3_node() {
+	for i in $(seq 1 3); do
+		rm "${WORK_DIR}/dist-minio-server$i.log"
+	done
+
 	export MINIO_ROOT_USER=minio
 	export MINIO_ROOT_PASSWORD=minio123
 	export MINIO_ERASURE_SET_DRIVE_COUNT=6
 	export MINIO_CI_CD=1
 
-	start_port=$2
+	first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l)
+
+	start_port=$1
 	args=""
-	for i in $(seq 1 3); do
+	for d in $(seq 1 3 5); do
-		args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
+		args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
+		d=$((d + 1))
+		args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
 	done
 
 	"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
@@ -37,40 +46,26 @@ function start_minio_3_node() {
 	pid3=$!
 	disown $pid3
 
-	sleep "$1"
+	export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
+	timeout 15m /tmp/mc ready myminio || fail
+
+	[ ${first_time} -eq 0 ] && upload_objects
+	[ ${first_time} -ne 0 ] && sleep 120
 
 	if ! ps -p $pid1 1>&2 >/dev/null; then
-		echo "server1 log:"
+		echo "minio server 1 is not running" && fail
-		cat "${WORK_DIR}/dist-minio-server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
 	fi
 
 	if ! ps -p $pid2 1>&2 >/dev/null; then
-		echo "server2 log:"
+		echo "minio server 2 is not running" && fail
-		cat "${WORK_DIR}/dist-minio-server2.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
 	fi
 
 	if ! ps -p $pid3 1>&2 >/dev/null; then
-		echo "server3 log:"
+		echo "minio server 3 is not running" && fail
-		cat "${WORK_DIR}/dist-minio-server3.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
 	fi
 
 	if ! pkill minio; then
-		for i in $(seq 1 3); do
+		fail
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
 	fi
 
 	sleep 1
@@ -82,16 +77,40 @@ function start_minio_3_node() {
 	fi
 }
 
-function check_online() {
+function check_heal() {
-	if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
-		echo "1"
+		return 1
 	fi
+
+	for ((i = 0; i < 20; i++)); do
+		test -f ${WORK_DIR}/$1/1/.minio.sys/format.json
+		v1=$?
+		nextInES=$(($1 + 1)) && [ $nextInES -gt 3 ] && nextInES=1
+		foundFiles1=$(find ${WORK_DIR}/$1/1/ | grep -v .minio.sys | grep xl.meta | wc -l)
+		foundFiles2=$(find ${WORK_DIR}/$nextInES/1/ | grep -v .minio.sys | grep xl.meta | wc -l)
+		test $foundFiles1 -eq $foundFiles2
+		v2=$?
+		[ $v1 == 0 -a $v2 == 0 ] && return 0
+		sleep 10
+	done
+	return 1
 }
 
 function purge() {
 	rm -rf "$1"
 }
 
+function fail() {
+	for i in $(seq 1 3); do
+		echo "server$i log:"
+		cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	pkill -9 minio
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+}
+
 function __init__() {
 	echo "Initializing environment"
 	mkdir -p "$WORK_DIR"
@@ -99,28 +118,37 @@ function __init__() {
 
 	## version is purposefully set to '3' for minio to migrate configuration file
 	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
+
+	if [ ! -f /tmp/mc ]; then
+		wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+			chmod +x /tmp/mc
+	fi
+}
+
+function upload_objects() {
+	/tmp/mc mb myminio/testbucket/
+	for ((i = 0; i < 20; i++)); do
+		echo "my content" | /tmp/mc pipe myminio/testbucket/file-$i
+	done
 }
 
 function perform_test() {
-	start_minio_3_node 120 $2
+	start_port=$2
+
+	start_minio_3_node $start_port
 
 	echo "Testing Distributed Erasure setup healing of drives"
-	echo "Remove the contents of the disks belonging to '${1}' erasure set"
+	echo "Remove the contents of the disks belonging to '${1}' node"
 
 	rm -rf ${WORK_DIR}/${1}/*/
 
-	start_minio_3_node 120 $2
+	set -x
+	start_minio_3_node $start_port
 
-	rv=$(check_online)
+	check_heal ${1}
+	rv=$?
 	if [ "$rv" == "1" ]; then
-		for i in $(seq 1 3); do
+		fail
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		pkill -9 minio
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
 	fi
 }
 

cmd/acl-handlers.go

@@ -25,7 +25,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // Data types used for returning dummy access control

cmd/admin-bucket-handlers.go

@@ -31,17 +31,17 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/event"
+	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 const (
@@ -94,12 +94,12 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		Quota:     data,
 		UpdatedAt: updatedAt,
 	}
-	if quotaConfig.Size == 0 || quotaConfig.Quota == 0 {
+	if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -428,10 +428,25 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			cfgPath := pathJoin(bi.Name, cfgFile)
 			bucket := bi.Name
 			switch cfgFile {
+			case bucketPolicyConfig:
+				config, _, err := globalBucketMetadataSys.GetBucketPolicy(bucket)
+				if err != nil {
+					if errors.Is(err, BucketPolicyNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := json.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketNotificationConfig:
 				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					adminLogIf(ctx, err)
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
@@ -447,7 +462,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
 						continue
 					}
-					logger.LogIf(ctx, err)
+					adminLogIf(ctx, err)
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
@@ -680,24 +695,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			if _, ok := bucketMap[bucket]; !ok {
 				opts := MakeBucketOptions{
 					LockEnabled: config.Enabled(),
+					ForceCreate: true, // ignore if it already exists
 				}
 				err = objectAPI.MakeBucket(ctx, bucket, opts)
 				if err != nil {
-					if _, ok := err.(BucketExists); !ok {
 					rpt.SetStatus(bucket, fileName, err)
 					continue
 				}
-				}
+				v, _ := globalBucketMetadataSys.Get(bucket)
-				v := newBucketMetadata(bucket)
 				bucketMap[bucket] = &v
 			}
 
-			// Deny object locking configuration settings on existing buckets without object lock enabled.
-			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-
 			bucketMap[bucket].ObjectLockConfigXML = configData
 			bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
@@ -724,13 +732,13 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
+				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
-					if _, ok := err.(BucketExists); !ok {
+					ForceCreate: true, // ignore if it already exists
+				}); err != nil {
 					rpt.SetStatus(bucket, fileName, err)
 					continue
 				}
-				}
+				v, _ := globalBucketMetadataSys.Get(bucket)
-				v := newBucketMetadata(bucket)
 				bucketMap[bucket] = &v
 			}
 
@@ -743,7 +751,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
 				continue
 			}
-			if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+			if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && v.Suspended() {
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
 				continue
 			}
@@ -776,14 +784,14 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
+			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
+				ForceCreate: true, // ignore if it already exists
+			})
 			if err != nil {
-				if _, ok := err.(BucketExists); !ok {
 				rpt.SetStatus(bucket, "", err)
 				continue
 			}
-			}
+			v, _ := globalBucketMetadataSys.Get(bucket)
-			v := newBucketMetadata(bucket)
 			bucketMap[bucket] = &v
 		}
 		if _, ok := bucketMap[bucket]; !ok {
@@ -791,7 +799,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		switch fileName {
 		case bucketNotificationConfig:
-			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalEventNotifier.targetList)
+			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region(), globalEventNotifier.targetList)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
 				continue
@@ -804,11 +812,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 
 			bucketMap[bucket].NotificationConfigXML = configData
+			bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
 			if sz > maxBucketPolicySize {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
+				rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyTooLarge.String()))
 				continue
 			}
 
@@ -826,7 +835,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Version in policy must not be empty
 			if bucketPolicy.Version == "" {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyInvalidVersion.String()))
+				rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyInvalidVersion.String()))
 				continue
 			}
 
@@ -845,9 +854,13 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				rpt.SetStatus(bucket, fileName, err)
 				continue
 			}
-
+			rcfg, err := globalBucketObjectLockSys.Get(bucket)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
 			// Validate the received bucket policy document
-			if err = bucketLifecycle.Validate(); err != nil {
+			if err = bucketLifecycle.Validate(rcfg); err != nil {
 				rpt.SetStatus(bucket, fileName, err)
 				continue
 			}
@@ -882,8 +895,10 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 			kmsKey := encConfig.KeyID()
 			if kmsKey != "" {
-				kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
+				_, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
-				_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
+					Name:           kmsKey,
+					AssociatedData: kms.Context{"MinIO admin API": "ServerInfoHandler"}, // Context for a test key operation
+				})
 				if err != nil {
 					if errors.Is(err, kes.ErrKeyNotFound) {
 						rpt.SetStatus(bucket, fileName, errKMSKeyNotFound)
@@ -966,7 +981,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			rpt.SetStatus(bucket, "", err)
 			continue
 		}
-
 	}
 
 	rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
@@ -1025,7 +1039,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 			}
 			if len(diffCh) == 0 {
 				// Flush if nothing is queued
-				w.(http.Flusher).Flush()
+				xhttp.Flush(w)
 			}
 		case <-keepAliveTicker.C:
 			if len(diffCh) > 0 {
@@ -1034,7 +1048,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return
 			}
-			w.(http.Flusher).Flush()
+			xhttp.Flush(w)
 		case <-ctx.Done():
 			return
 		}
@@ -1084,7 +1098,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
 			}
 			if len(mrfCh) == 0 {
 				// Flush if nothing is queued
-				w.(http.Flusher).Flush()
+				xhttp.Flush(w)
 			}
 		case <-keepAliveTicker.C:
 			if len(mrfCh) > 0 {
@@ -1093,7 +1107,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return
 			}
-			w.(http.Flusher).Flush()
+			xhttp.Flush(w)
 		case <-ctx.Done():
 			return
 		}

cmd/admin-handler-utils.go

@@ -23,11 +23,11 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // validateAdminReq will validate request against and return whether it is allowed.
@@ -216,6 +216,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case errors.Is(err, errTierInvalidConfig):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInvalidConfig",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		default:
 			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}

cmd/admin-handlers-config-kv.go

@@ -37,7 +37,7 @@ import (
 	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
@@ -58,7 +58,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -162,7 +162,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -443,7 +443,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-config.go

@@ -31,10 +31,9 @@ import (
 	"github.com/minio/minio/internal/config"
 	cfgldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/ldap"
+	"github.com/minio/pkg/v3/ldap"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
@@ -60,7 +59,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -126,7 +125,6 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	if err = validateConfig(ctx, cfg, subSys); err != nil {
-
 		var validationErr ldap.Validation
 		if errors.As(err, &validationErr) {
 			// If we got an LDAP validation error, we need to send appropriate
@@ -417,7 +415,6 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		return
 	}
 	if err = validateConfig(ctx, cfg, subSys); err != nil {
-
 		var validationErr ldap.Validation
 		if errors.As(err, &validationErr) {
 			// If we got an LDAP validation error, we need to send appropriate

cmd/admin-handlers-idp-ldap.go

@@ -19,13 +19,17 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/policy"
+	xldap "github.com/minio/pkg/v3/ldap"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
@@ -101,6 +105,12 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 		return
 	}
 
+	// fail if ldap is not enabled
+	if !globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL)
+		return
+	}
+
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
@@ -128,7 +138,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -175,3 +185,469 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 
 	writeSuccessResponseJSON(w, encryptedData)
 }
+
+// AddServiceAccountLDAP adds a new service account for provided LDAP username or DN
+//
+// PUT /minio/admin/v3/idp/ldap/add-service-account
+func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, true)
+	if APIError.Code != "" {
+		writeErrorResponseJSON(ctx, w, APIError, r.URL)
+		return
+	}
+
+	// fail if ldap is not enabled
+	if !globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL)
+		return
+	}
+
+	// Find the user for the request sender (as it may be sent via a service
+	// account or STS account):
+	requestorUser := cred.AccessKey
+	requestorParentUser := cred.AccessKey
+	requestorGroups := cred.Groups
+	requestorIsDerivedCredential := false
+	if cred.IsServiceAccount() || cred.IsTemp() {
+		requestorParentUser = cred.ParentUser
+		requestorIsDerivedCredential = true
+	}
+
+	// Check if we are creating svc account for request sender.
+	isSvcAccForRequestor := targetUser == requestorUser || targetUser == requestorParentUser
+
+	var (
+		targetGroups []string
+		err          error
+	)
+
+	// If we are creating svc account for request sender, ensure that targetUser
+	// is a real user (i.e. not derived credentials).
+	if isSvcAccForRequestor {
+		if requestorIsDerivedCredential {
+			if requestorParentUser == "" {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
+					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
+				return
+			}
+			targetUser = requestorParentUser
+		}
+		targetGroups = requestorGroups
+
+		// Deny if the target user is not LDAP
+		foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if foundResult == nil {
+			err := errors.New("Specified user does not exist on LDAP server")
+			APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err)
+			writeErrorResponseJSON(ctx, w, APIErr, r.URL)
+			return
+		}
+
+		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
+		// it is associated with the LDAP/OIDC user properly.
+		for k, v := range cred.Claims {
+			if k == expClaim {
+				continue
+			}
+			opts.claims[k] = v
+		}
+	} else {
+		// We still need to ensure that the target user is a valid LDAP user.
+		//
+		// The target user may be supplied as a (short) username or a DN.
+		// However, for now, we only support using the short username.
+
+		isDN := globalIAMSys.LDAPConfig.ParsesAsDN(targetUser)
+		opts.claims[ldapUserN] = targetUser // simple username
+		var lookupResult *xldap.DNSearchResult
+		lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
+		if err != nil {
+			// if not found, check if DN
+			if strings.Contains(err.Error(), "User DN not found for:") {
+				if isDN {
+					// warn user that DNs are not allowed
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminLDAPExpectedLoginName, err), r.URL)
+				} else {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL)
+				}
+			}
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		targetUser = lookupResult.NormDN
+		opts.claims[ldapUser] = targetUser // DN
+		opts.claims[ldapActualUser] = lookupResult.ActualDN
+
+		// Check if this user or their groups have a policy applied.
+		ldapPolicies, err := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if len(ldapPolicies) == 0 {
+			err = fmt.Errorf("No policy set for user `%s` or any of their groups: `%s`", opts.claims[ldapActualUser], strings.Join(targetGroups, "`,`"))
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL)
+			return
+		}
+
+		// Add LDAP attributes that were looked up into the claims.
+		for attribKey, attribValue := range lookupResult.Attributes {
+			opts.claims[ldapAttribPrefix+attribKey] = attribValue
+		}
+	}
+
+	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	createResp := madmin.AddServiceAccountResp{
+		Credentials: madmin.Credentials{
+			AccessKey:  newCred.AccessKey,
+			SecretKey:  newCred.SecretKey,
+			Expiration: newCred.Expiration,
+		},
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+
+	// Call hook for cluster-replication if the service account is not for a
+	// root user.
+	if newCred.ParentUser != globalActiveCred.AccessKey {
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Name:          newCred.Name,
+					Description:   newCred.Description,
+					Claims:        opts.claims,
+					SessionPolicy: madmin.SRSessionPolicy(createReq.Policy),
+					Status:        auth.AccountOn,
+					Expiration:    createReq.Expiration,
+				},
+			},
+			UpdatedAt: updatedAt,
+		}))
+	}
+}
+
+// ListAccessKeysLDAP - GET /minio/admin/v3/idp/ldap/list-access-keys
+func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	userDN := r.Form.Get("userDN")
+
+	// If listing is requested for a specific user (who is not the request
+	// sender), check that the user has permissions.
+	if userDN != "" && userDN != cred.ParentUser {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListServiceAccountsAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListServiceAccountsAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+			DenyOnly:        true,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+		userDN = cred.AccessKey
+		if cred.ParentUser != "" {
+			userDN = cred.ParentUser
+		}
+	}
+
+	dnResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(userDN)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if dnResult == nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+		return
+	}
+	targetAccount := dnResult.NormDN
+
+	listType := r.Form.Get("listType")
+	if listType != "sts-only" && listType != "svcacc-only" && listType != "" {
+		// default to both
+		listType = ""
+	}
+
+	var serviceAccounts []auth.Credentials
+	var stsKeys []auth.Credentials
+
+	if listType == "" || listType == "sts-only" {
+		stsKeys, err = globalIAMSys.ListSTSAccounts(ctx, targetAccount)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+	if listType == "" || listType == "svcacc-only" {
+		serviceAccounts, err = globalIAMSys.ListServiceAccounts(ctx, targetAccount)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	var serviceAccountList []madmin.ServiceAccountInfo
+	var stsKeyList []madmin.ServiceAccountInfo
+
+	for _, svc := range serviceAccounts {
+		expiryTime := svc.Expiration
+		serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
+			AccessKey:  svc.AccessKey,
+			Expiration: &expiryTime,
+		})
+	}
+	for _, sts := range stsKeys {
+		expiryTime := sts.Expiration
+		stsKeyList = append(stsKeyList, madmin.ServiceAccountInfo{
+			AccessKey:  sts.AccessKey,
+			Expiration: &expiryTime,
+		})
+	}
+
+	listResp := madmin.ListAccessKeysLDAPResp{
+		ServiceAccounts: serviceAccountList,
+		STSKeys:         stsKeyList,
+	}
+
+	data, err := json.Marshal(listResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}
+
+// ListAccessKeysLDAPBulk - GET /minio/admin/v3/idp/ldap/list-access-keys-bulk
+func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	dnList := r.Form["userDNs"]
+	isAll := r.Form.Get("all") == "true"
+	selfOnly := !isAll && len(dnList) == 0
+
+	if isAll && len(dnList) > 0 {
+		// This should be checked on client side, so return generic error
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	// Empty DN list and not self, list access keys for all users
+	if isAll {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListUsersAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else if len(dnList) == 1 {
+		var dn string
+		foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(dnList[0])
+		if err == nil {
+			dn = foundResult.NormDN
+		}
+		if dn == cred.ParentUser || dnList[0] == cred.ParentUser {
+			selfOnly = true
+		}
+	}
+
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.ListServiceAccountsAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        selfOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if selfOnly && len(dnList) == 0 {
+		selfDN := cred.AccessKey
+		if cred.ParentUser != "" {
+			selfDN = cred.ParentUser
+		}
+		dnList = append(dnList, selfDN)
+	}
+
+	var ldapUserList []string
+	if isAll {
+		ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		for user := range ldapUsers {
+			ldapUserList = append(ldapUserList, user)
+		}
+	} else {
+		for _, userDN := range dnList {
+			// Validate the userDN
+			foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(userDN)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			if foundResult == nil {
+				continue
+			}
+			ldapUserList = append(ldapUserList, foundResult.NormDN)
+		}
+	}
+
+	listType := r.Form.Get("listType")
+	var listSTSKeys, listServiceAccounts bool
+	switch listType {
+	case madmin.AccessKeyListUsersOnly:
+		listSTSKeys = false
+		listServiceAccounts = false
+	case madmin.AccessKeyListSTSOnly:
+		listSTSKeys = true
+		listServiceAccounts = false
+	case madmin.AccessKeyListSvcaccOnly:
+		listSTSKeys = false
+		listServiceAccounts = true
+	case madmin.AccessKeyListAll:
+		listSTSKeys = true
+		listServiceAccounts = true
+	default:
+		err := errors.New("invalid list type")
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+
+	accessKeyMap := make(map[string]madmin.ListAccessKeysLDAPResp)
+	for _, internalDN := range ldapUserList {
+		externalDN := globalIAMSys.LDAPConfig.DecodeDN(internalDN)
+		accessKeys := madmin.ListAccessKeysLDAPResp{}
+		if listSTSKeys {
+			stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, internalDN)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, sts := range stsKeys {
+				accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
+					AccessKey:  sts.AccessKey,
+					Expiration: &sts.Expiration,
+				})
+			}
+			// if only STS keys, skip if user has no STS keys
+			if !listServiceAccounts && len(stsKeys) == 0 {
+				continue
+			}
+		}
+
+		if listServiceAccounts {
+			serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, internalDN)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, svc := range serviceAccounts {
+				accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
+					AccessKey:  svc.AccessKey,
+					Expiration: &svc.Expiration,
+				})
+			}
+			// if only service accounts, skip if user has no service accounts
+			if !listSTSKeys && len(serviceAccounts) == 0 {
+				continue
+			}
+		}
+		accessKeyMap[externalDN] = accessKeys
+	}
+
+	data, err := json.Marshal(accessKeyMap)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}

cmd/admin-handlers-idp-openid.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2015-2025 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"sort"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/pkg/v3/policy"
+)
+
+const dummyRoleARN = "dummy-internal"
+
+// ListAccessKeysOpenIDBulk - GET /minio/admin/v3/idp/openid/list-access-keys-bulk
+func (a adminAPIHandlers) ListAccessKeysOpenIDBulk(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	if !globalIAMSys.OpenIDConfig.Enabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminOpenIDNotEnabled), r.URL)
+		return
+	}
+
+	userList := r.Form["users"]
+	isAll := r.Form.Get("all") == "true"
+	selfOnly := !isAll && len(userList) == 0
+	cfgName := r.Form.Get("configName")
+	allConfigs := r.Form.Get("allConfigs") == "true"
+	if cfgName == "" && !allConfigs {
+		cfgName = madmin.Default
+	}
+
+	if isAll && len(userList) > 0 {
+		// This should be checked on client side, so return generic error
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	// Empty DN list and not self, list access keys for all users
+	if isAll {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListUsersAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else if len(userList) == 1 && userList[0] == cred.ParentUser {
+		selfOnly = true
+	}
+
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.ListServiceAccountsAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        selfOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if selfOnly && len(userList) == 0 {
+		selfDN := cred.AccessKey
+		if cred.ParentUser != "" {
+			selfDN = cred.ParentUser
+		}
+		userList = append(userList, selfDN)
+	}
+
+	listType := r.Form.Get("listType")
+	var listSTSKeys, listServiceAccounts bool
+	switch listType {
+	case madmin.AccessKeyListUsersOnly:
+		listSTSKeys = false
+		listServiceAccounts = false
+	case madmin.AccessKeyListSTSOnly:
+		listSTSKeys = true
+		listServiceAccounts = false
+	case madmin.AccessKeyListSvcaccOnly:
+		listSTSKeys = false
+		listServiceAccounts = true
+	case madmin.AccessKeyListAll:
+		listSTSKeys = true
+		listServiceAccounts = true
+	default:
+		err := errors.New("invalid list type")
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+
+	s := globalServerConfig.Clone()
+	roleArnMap := make(map[string]string)
+	// Map of configs to a map of users to their access keys
+	cfgToUsersMap := make(map[string]map[string]madmin.OpenIDUserAccessKeys)
+	configs, err := globalIAMSys.OpenIDConfig.GetConfigList(s)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	for _, config := range configs {
+		if !allConfigs && cfgName != config.Name {
+			continue
+		}
+		arn := dummyRoleARN
+		if config.RoleARN != "" {
+			arn = config.RoleARN
+		}
+		roleArnMap[arn] = config.Name
+		newResp := make(map[string]madmin.OpenIDUserAccessKeys)
+		cfgToUsersMap[config.Name] = newResp
+	}
+	if len(roleArnMap) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+		return
+	}
+
+	userSet := set.CreateStringSet(userList...)
+	accessKeys, err := globalIAMSys.ListAllAccessKeys(ctx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	for _, accessKey := range accessKeys {
+		// Filter out any disqualifying access keys
+		_, ok := accessKey.Claims[subClaim]
+		if !ok {
+			continue // OpenID access keys must have a sub claim
+		}
+		if (!listSTSKeys && !accessKey.IsServiceAccount()) || (!listServiceAccounts && accessKey.IsServiceAccount()) {
+			continue // skip if not the type we want
+		}
+		arn, ok := accessKey.Claims[roleArnClaim].(string)
+		if !ok {
+			if _, ok := accessKey.Claims[iamPolicyClaimNameOpenID()]; !ok {
+				continue // skip if no roleArn and no policy claim
+			}
+		}
+		matchingCfgName, ok := roleArnMap[arn]
+		if !ok {
+			continue // skip if not part of the target config
+		}
+		var id string
+		if idClaim := globalIAMSys.OpenIDConfig.GetUserIDClaim(matchingCfgName); idClaim != "" {
+			id, _ = accessKey.Claims[idClaim].(string)
+		}
+		if !userSet.IsEmpty() && !userSet.Contains(accessKey.ParentUser) && !userSet.Contains(id) {
+			continue // skip if not in the user list
+		}
+		openIDUserAccessKeys, ok := cfgToUsersMap[matchingCfgName][accessKey.ParentUser]
+
+		// Add new user to map if not already present
+		if !ok {
+			var readableClaim string
+			if rc := globalIAMSys.OpenIDConfig.GetUserReadableClaim(matchingCfgName); rc != "" {
+				readableClaim, _ = accessKey.Claims[rc].(string)
+			}
+			openIDUserAccessKeys = madmin.OpenIDUserAccessKeys{
+				MinioAccessKey: accessKey.ParentUser,
+				ID:             id,
+				ReadableName:   readableClaim,
+			}
+		}
+		svcAccInfo := madmin.ServiceAccountInfo{
+			AccessKey:  accessKey.AccessKey,
+			Expiration: &accessKey.Expiration,
+		}
+		if accessKey.IsServiceAccount() {
+			openIDUserAccessKeys.ServiceAccounts = append(openIDUserAccessKeys.ServiceAccounts, svcAccInfo)
+		} else {
+			openIDUserAccessKeys.STSKeys = append(openIDUserAccessKeys.STSKeys, svcAccInfo)
+		}
+		cfgToUsersMap[matchingCfgName][accessKey.ParentUser] = openIDUserAccessKeys
+	}
+
+	// Convert map to slice and sort
+	resp := make([]madmin.ListAccessKeysOpenIDResp, 0, len(cfgToUsersMap))
+	for cfgName, usersMap := range cfgToUsersMap {
+		users := make([]madmin.OpenIDUserAccessKeys, 0, len(usersMap))
+		for _, user := range usersMap {
+			users = append(users, user)
+		}
+		sort.Slice(users, func(i, j int) bool {
+			return users[i].MinioAccessKey < users[j].MinioAccessKey
+		})
+		resp = append(resp, madmin.ListAccessKeysOpenIDResp{
+			ConfigName: cfgName,
+			Users:      users,
+		})
+	}
+	sort.Slice(resp, func(i, j int) bool {
+		return resp[i].ConfigName < resp[j].ConfigName
+	})
+
+	data, err := json.Marshal(resp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}

cmd/admin-handlers-pools.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,15 +18,17 @@
 package cmd
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/env"
+	"github.com/minio/pkg/v3/policy"
 )
 
 var (
@@ -66,17 +68,29 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
 
 	pools := strings.Split(v, ",")
 	poolIndices := make([]int, 0, len(pools))
 
 	for _, pool := range pools {
-		idx := globalEndpoints.GetPoolIdx(pool)
+		var idx int
+		if byID {
+			var err error
+			idx, err = strconv.Atoi(pool)
+			if err != nil {
+				// We didn't find any matching pools, invalid input
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+				return
+			}
+		} else {
+			idx = globalEndpoints.GetPoolIdx(pool)
 			if idx == -1 {
 				// We didn't find any matching pools, invalid input
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
 				return
 			}
+		}
 		var pool *erasureSets
 		for pidx := range z.serverPools {
 			if pidx == idx {
@@ -93,22 +107,13 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		poolIndices = append(poolIndices, idx)
 	}
 
-	if len(poolIndices) > 0 && !globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+	if len(poolIndices) == 0 || !proxyDecommissionRequest(ctx, globalEndpoints[poolIndices[0]].Endpoints[0], w, r) {
-		ep := globalEndpoints[poolIndices[0]].Endpoints[0]
-		for nodeIdx, proxyEp := range globalProxyEndpoints {
-			if proxyEp.Endpoint.Host == ep.Host {
-				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
-					return
-				}
-			}
-		}
-	}
-
 		if err := z.Decommission(r.Context(), poolIndices...); err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 	}
+}
 
 func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -132,29 +137,30 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		// We didn't find any matching pools, invalid input
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
 		return
 	}
 
-	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
+	if !proxyDecommissionRequest(ctx, globalEndpoints[idx].Endpoints[0], w, r) {
-		for nodeIdx, proxyEp := range globalProxyEndpoints {
-			if proxyEp.Endpoint.Host == ep.Host {
-				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
-					return
-				}
-			}
-		}
-	}
-
 		if err := pools.DecommissionCancel(ctx, idx); err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 	}
+}
 
 func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -178,8 +184,17 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
 		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
@@ -194,7 +209,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(&status))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(&status))
 }
 
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
@@ -227,7 +242,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 		poolsStatus[idx] = status
 	}
 
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
 }
 
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
@@ -243,8 +258,8 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
 	// concurrent rebalance-start commands.
 	if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
 		for nodeIdx, proxyEp := range globalProxyEndpoints {
-			if proxyEp.Endpoint.Host == ep.Host {
+			if proxyEp.Host == ep.Host {
-				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+				if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
 					return
 				}
 			}
@@ -314,8 +329,8 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 	// pools may temporarily have out of date info on the others.
 	if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
 		for nodeIdx, proxyEp := range globalProxyEndpoints {
-			if proxyEp.Endpoint.Host == ep.Host {
+			if proxyEp.Host == ep.Host {
-				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+				if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
 					return
 				}
 			}
@@ -334,11 +349,11 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
 			return
 		}
-		logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
+		adminLogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(rs))
 }
 
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
@@ -358,5 +373,21 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	// Cancel any ongoing rebalance operation
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
-	logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	globalNotificationSys.LoadRebalanceMeta(ctx, false)
+}
+
+func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w http.ResponseWriter, r *http.Request) (proxy bool) {
+	host := env.Get("_MINIO_DECOM_ENDPOINT_HOST", defaultEndPoint.Host)
+	if host == "" {
+		return
+	}
+	for nodeIdx, proxyEp := range globalProxyEndpoints {
+		if proxyEp.Host == host && !proxyEp.IsLocal {
+			if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success {
+				return true
+			}
+		}
+	}
+	return
 }

cmd/admin-handlers-site-replication.go

@@ -32,9 +32,8 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
@@ -52,9 +51,10 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	opts := getSRAddOptions(r)
+	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -68,6 +68,11 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, body)
 }
 
+func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) {
+	opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true"
+	return
+}
+
 // SRPeerJoin - PUT /minio/admin/v3/site-replication/join
 //
 // used internally to tell current cluster to enable SR with
@@ -87,7 +92,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -134,7 +139,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -186,13 +191,13 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
 
-// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta
 func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -253,9 +258,11 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeSSEConfig:
 		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
+	case madmin.SRBucketMetaLCConfig:
+		err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -308,7 +315,6 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
-			logger.LogIf(ctx, err)
 			return SRError{
 				Cause: err,
 				Code:  ErrSiteReplicationInvalidRequest,
@@ -334,12 +340,25 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
 		opts.Users = true
 		opts.Policies = true
 		opts.Groups = true
+		opts.ILMExpiryRules = true
 	}
 	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+	// Report the ILMExpiryStats only if at least one site has replication of ILM expiry enabled
+	var replicateILMExpiry bool
+	for _, site := range info.Sites {
+		if site.ReplicateILMExpiry {
+			replicateILMExpiry = true
+			break
+		}
+	}
+	if !replicateILMExpiry {
+		// explicitly send nil for ILMExpiryStats
+		info.ILMExpiryStats = nil
+	}
 
 	if err = json.NewEncoder(w).Encode(info); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -383,9 +402,11 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site)
+
+	opts := getSREditOptions(r)
+	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -398,6 +419,12 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, body)
 }
 
+func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) {
+	opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true"
+	opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true"
+	return
+}
+
 // SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
 //
 // used internally to tell current cluster to update endpoint for peer
@@ -416,7 +443,30 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit
+//
+// used internally to tell current cluster to update site replication state
+func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var state madmin.SRStateEditReq
+	if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -428,6 +478,8 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	opts.Policies = q.Get("policies") == "true"
 	opts.Groups = q.Get("groups") == "true"
 	opts.Users = q.Get("users") == "true"
+	opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true"
+	opts.PeerState = q.Get("peer-state") == "true"
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
 	opts.ShowDeleted = q.Get("showDeleted") == "true"
@@ -451,7 +503,7 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
 	}
 	status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -482,7 +534,7 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -544,7 +596,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.
 			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 			}
 		}
 		if err != nil {
@@ -567,5 +619,5 @@ func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.
 		duration = globalNetPerfMinDuration
 	}
 	result := siteNetperf(r.Context(), duration)
-	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+	adminLogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }

cmd/admin-handlers-users-race_test.go

@@ -32,7 +32,7 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	minio "github.com/minio/minio-go/v7"
-	"github.com/minio/pkg/v2/sync/errgroup"
+	"github.com/minio/pkg/v3/sync/errgroup"
 )
 
 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
@@ -120,9 +120,12 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
 			c.Fatalf("Unable to set user: %v", err)
 		}
 
-		err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+		userReq := madmin.PolicyAssociationReq{
-		if err != nil {
+			Policies: []string{policy},
-			c.Fatalf("Unable to set policy: %v", err)
+			User:     accessKey,
+		}
+		if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil {
+			c.Fatalf("Unable to attach policy: %v", err)
 		}
 
 		accessKeys[i] = accessKey

cmd/admin-handlers-users_test.go

@@ -28,6 +28,7 @@ import (
 	"net/http"
 	"net/url"
 	"runtime"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -39,7 +40,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v3/env"
 )
 
 const (
@@ -159,7 +160,7 @@ func (s *TestSuiteIAM) SetUpSuite(c *check) {
 }
 
 func (s *TestSuiteIAM) RestartIAMSuite(c *check) {
-	s.TestSuiteCommon.RestartTestServer(c)
+	s.RestartTestServer(c)
 
 	s.iamSetup(c)
 }
@@ -206,7 +207,9 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
 				suite.TestCannedPolicies(c)
 				suite.TestGroupAddRemove(c)
 				suite.TestServiceAccountOpsByAdmin(c)
+				suite.TestServiceAccountPrivilegeEscalationBug(c)
 				suite.TestServiceAccountOpsByUser(c)
+				suite.TestServiceAccountDurationSecondsCondition(c)
 				suite.TestAddServiceAccountPerms(c)
 				suite.TearDownSuite(c)
 			},
@@ -237,9 +240,12 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	c.Assert(v.Status, madmin.AccountEnabled)
 
 	// 3. Associate policy and check that user can access
-	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{"readwrite"},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	client := s.getUserClient(c, accessKey, secretKey, "")
@@ -332,23 +338,34 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
 	}
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 2.3 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -434,7 +451,7 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
     "s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::testbucket/*"
+    "arn:aws:s3:::testbucket"
    ]
   }
  ]
@@ -468,9 +485,12 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 	c.mustNotListObjects(ctx, uClient, "testbucket")
 
 	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy1, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy1},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	admClnt := s.getAdminClient(c, accessKey, secretKey, "")
@@ -488,10 +508,22 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 		c.Fatalf("policy was missing!")
 	}
 
-	// 3.2 associate policy to user
+	// Detach policy1 to set up for policy2
-	err = s.adm.SetPolicy(ctx, policy2, accessKey, false)
+	_, err = s.adm.DetachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy1},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to detach policy: %v", err)
+	}
+
+	// 3.2 associate policy to user
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy2},
+		User:     accessKey,
+	})
+	if err != nil {
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// 3.3 check user can create service account implicitly.
@@ -536,16 +568,24 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -569,9 +609,12 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
 	c.mustNotListObjects(ctx, uClient, bucket)
 
 	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 3.3 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -643,16 +686,24 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 
 	// Check that default policies can be overwritten.
 	err = s.adm.AddCannedPolicy(ctx, "readwrite", policyBytes)
@@ -665,6 +716,12 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
 		c.Fatalf("policy info err: %v", err)
 	}
 
+	// Check that policy with comma is rejected.
+	err = s.adm.AddCannedPolicy(ctx, "invalid,policy", policyBytes)
+	if err == nil {
+		c.Fatalf("invalid policy created successfully")
+	}
+
 	infoStr := string(info)
 	if !strings.Contains(infoStr, `"s3:PutObject"`) || !strings.Contains(infoStr, ":"+bucket+"/") {
 		c.Fatalf("policy contains unexpected content!")
@@ -688,16 +745,24 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -724,9 +789,12 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	c.mustNotListObjects(ctx, uClient, bucket)
 
 	// 3. Associate policy to group and check user got access.
-	err = s.adm.SetPolicy(ctx, policy, group, true)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		Group:    group,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 3.1 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -741,8 +809,9 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	if err != nil {
 		c.Fatalf("group list err: %v", err)
 	}
-	if !set.CreateStringSet(groups...).Contains(group) {
+	expected := []string{group}
-		c.Fatalf("created group not present!")
+	if !slices.Equal(groups, expected) {
+		c.Fatalf("expected group listing: %v, got: %v", expected, groups)
 	}
 	groupInfo, err := s.adm.GetGroupDescription(ctx, group)
 	if err != nil {
@@ -848,16 +917,24 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -869,9 +946,12 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// Create an madmin client with user creds
@@ -896,14 +976,6 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	// 3. Check S3 access
 	c.assertSvcAccS3Access(ctx, s, cr, bucket)
 
-	// 4. Check that svc account can restrict the policy, and that the
-	// session policy can be updated.
-	c.assertSvcAccSessionPolicyUpdate(ctx, s, userAdmClient, accessKey, bucket)
-
-	// 4. Check that service account's secret key and account status can be
-	// updated.
-	c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket)
-
 	// 5. Check that service account can be deleted.
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
 
@@ -911,6 +983,104 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
 }
 
+func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create policy, user and associate policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Deny",
+   "Action": [
+     "admin:CreateServiceAccount",
+     "admin:UpdateServiceAccount"
+   ],
+   "Condition": {"NumericGreaterThan": {"svc:DurationSeconds": "3600"}}
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
+	if err != nil {
+		c.Fatalf("unable to attach policy: %v", err)
+	}
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	distantExpiration := time.Now().Add(30 * time.Minute)
+	cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  "svc-accesskey",
+		SecretKey:  "svc-secretkey",
+		Expiration: &distantExpiration,
+	})
+	if err != nil {
+		c.Fatalf("Unable to create svc acc: %v", err)
+	}
+
+	c.assertSvcAccS3Access(ctx, s, cr, bucket)
+
+	closeExpiration := time.Now().Add(2 * time.Hour)
+	_, err = userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  "svc-accesskey",
+		SecretKey:  "svc-secretkey",
+		Expiration: &closeExpiration,
+	})
+	if err == nil {
+		c.Fatalf("Creating a svc acc with distant expiration should fail")
+	}
+}
+
 func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -929,16 +1099,24 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -950,9 +1128,12 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// 1. Create a service account for the user
@@ -979,6 +1160,95 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (s *TestSuiteIAM) TestServiceAccountPrivilegeEscalationBug(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	err := s.client.MakeBucket(ctx, "public", minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	err = s.client.MakeBucket(ctx, "private", minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	pubPolicyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:*"
+   ],
+   "Resource": [
+    "arn:aws:s3:::public",
+    "arn:aws:s3:::public/*"
+   ]
+  }
+ ]
+}`)
+
+	fullS3PolicyBytes := []byte(`{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:*"
+      ],
+      "Resource": [
+        "arn:aws:s3:::*"
+      ]
+    }
+  ]
+}
+`)
+
+	// Create a service account for the root user.
+	cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: globalActiveCred.AccessKey,
+		Policy:     pubPolicyBytes,
+	})
+	if err != nil {
+		c.Fatalf("admin should be able to create service account for themselves %s", err)
+	}
+
+	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+
+	// Check that the service account can access the public bucket.
+	buckets, err := svcClient.ListBuckets(ctx)
+	if err != nil {
+		c.Fatalf("err fetching buckets %s", err)
+	}
+	if len(buckets) != 1 || buckets[0].Name != "public" {
+		c.Fatalf("service account should only have access to public bucket")
+	}
+
+	// Create an madmin client with the service account creds.
+	svcAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  credentials.NewStaticV4(cr.AccessKey, cr.SecretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating svcacct admin client: %v", err)
+	}
+	svcAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Attempt to update the policy on the service account.
+	err = svcAdmClient.UpdateServiceAccount(ctx, cr.AccessKey,
+		madmin.UpdateServiceAccountReq{
+			NewPolicy: fullS3PolicyBytes,
+		})
+
+	if err == nil {
+		c.Fatalf("service account should not be able to update policy on itself")
+	} else if !strings.Contains(err.Error(), "Access Denied") {
+		c.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -1394,7 +1664,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
     "s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::%s/*"
+    "arn:aws:s3:::%s"
    ]
   }
  ]

cmd/admin-handlers_test.go

@@ -168,6 +168,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
 	queryVal := url.Values{}
 	queryVal.Set("action", string(cmd.toServiceAction()))
+	queryVal.Set("type", "2")
 	resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode()
 	req, err := newTestRequest(http.MethodPost, resource, 0, nil)
 	if err != nil {
@@ -220,12 +221,18 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	rec := httptest.NewRecorder()
 	adminTestBed.router.ServeHTTP(rec, req)
 
-	if rec.Code != http.StatusOK {
 	resp, _ := io.ReadAll(rec.Body)
+	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}
 
+	result := &serviceResult{}
+	if err := json.Unmarshal(resp, result); err != nil {
+		t.Error(err)
+	}
+	_ = result
+
 	// Wait until testServiceSignalReceiver() called in a goroutine quits.
 	wg.Wait()
 }
@@ -256,7 +263,7 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 }
 
 func TestAdminServerInfo(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
 	adminTestBed, err := prepareAdminErasureTestBed(ctx)
@@ -341,7 +348,7 @@ func TestExtractHealInitParams(t *testing.T) {
 		}
 		return v
 	}
-	qParmsArr := []url.Values{
+	qParamsArr := []url.Values{
 		// Invalid cases
 		mkParams("", true, true),
 		mkParams("111", true, true),
@@ -366,9 +373,9 @@ func TestExtractHealInitParams(t *testing.T) {
 	body := `{"recursive": false, "dryRun": true, "remove": false, "scanMode": 0}`
 
 	// Test all combinations!
-	for pIdx, parms := range qParmsArr {
+	for pIdx, params := range qParamsArr {
 		for vIdx, vars := range varsArr {
-			_, err := extractHealInitParams(vars, parms, bytes.NewReader([]byte(body)))
+			_, err := extractHealInitParams(vars, params, bytes.NewReader([]byte(body)))
 			isErrCase := false
 			if pIdx < 4 || vIdx < 1 {
 				isErrCase = true
@@ -456,6 +463,7 @@ func TestTopLockEntries(t *testing.T) {
 			Owner:      lri.Owner,
 			ID:         lri.UID,
 			Quorum:     lri.Quorum,
+			Timestamp:  time.Unix(0, lri.Timestamp),
 		})
 	}
 

cmd/admin-heal-ops.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -62,8 +63,8 @@ const (
 )
 
 var (
-	errHealIdleTimeout   = fmt.Errorf("healing results were not consumed for too long")
+	errHealIdleTimeout   = errors.New("healing results were not consumed for too long")
-	errHealStopSignalled = fmt.Errorf("heal stop signaled")
+	errHealStopSignalled = errors.New("heal stop signaled")
 
 	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
 		apiErr := toAdminAPIErr(ctx, err)
@@ -132,7 +133,13 @@ func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
 func (ahs *allHealState) updateHealStatus(tracker *healingTracker) {
 	ahs.Lock()
 	defer ahs.Unlock()
-	ahs.healStatus[tracker.ID] = *tracker
+
+	tracker.mu.RLock()
+	t := *tracker
+	t.QueuedBuckets = append(make([]string, 0, len(tracker.QueuedBuckets)), tracker.QueuedBuckets...)
+	t.HealedBuckets = append(make([]string, 0, len(tracker.HealedBuckets)), tracker.HealedBuckets...)
+	ahs.healStatus[tracker.ID] = t
+	tracker.mu.RUnlock()
 }
 
 // Sort by zone, set and disk index
@@ -253,7 +260,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	} else {
 		clientToken := he.clientToken
 		if globalIsDistErasure {
-			clientToken = fmt.Sprintf("%s:%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+			clientToken = fmt.Sprintf("%s%s%d", he.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 		}
 
 		hsp = madmin.HealStopSuccess{
@@ -322,12 +329,16 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 	// Add heal state and start sequence
 	ahs.healSeqMap[hpath] = h
 
-	// Launch top-level background heal go-routine
-	go h.healSequenceStart(objAPI)
-
 	clientToken := h.clientToken
 	if globalIsDistErasure {
-		clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+		clientToken = fmt.Sprintf("%s%s%d", h.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
+	}
+
+	if h.clientToken == bgHealingUUID {
+		// For background heal do nothing, do not spawn an unnecessary goroutine.
+	} else {
+		// Launch top-level background heal go-routine
+		go h.healSequenceStart(objAPI)
 	}
 
 	b, err := json.Marshal(madmin.HealStartSuccess{
@@ -336,7 +347,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		logger.LogIf(h.ctx, err)
+		bugLogIf(h.ctx, err)
 		return nil, toAdminAPIErr(h.ctx, err), ""
 	}
 	return b, noError, ""
@@ -383,7 +394,7 @@ func (ahs *allHealState) PopHealStatusJSON(hpath string,
 	if err != nil {
 		h.currentStatus.Items = nil
 
-		logger.LogIf(h.ctx, err)
+		bugLogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
@@ -444,8 +455,8 @@ type healSequence struct {
 	// Number of total items healed against item type
 	healedItemsMap map[madmin.HealItemType]int64
 
-	// Number of total items where healing failed against endpoint and drive state
+	// Number of total items where healing failed against item type
-	healFailedItemsMap map[string]int64
+	healFailedItemsMap map[madmin.HealItemType]int64
 
 	// The time of the last scan/heal activity
 	lastHealActivity time.Time
@@ -486,7 +497,7 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 		ctx:                   ctx,
 		scannedItemsMap:       make(map[madmin.HealItemType]int64),
 		healedItemsMap:        make(map[madmin.HealItemType]int64),
-		healFailedItemsMap:    make(map[string]int64),
+		healFailedItemsMap:    make(map[madmin.HealItemType]int64),
 	}
 }
 
@@ -530,14 +541,14 @@ func (h *healSequence) getHealedItemsMap() map[madmin.HealItemType]int64 {
 	return retMap
 }
 
-// gethealFailedItemsMap - returns map of all items where heal failed against
+// getHealFailedItemsMap - returns map of all items where heal failed against
 // drive endpoint and status
-func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
+func (h *healSequence) getHealFailedItemsMap() map[madmin.HealItemType]int64 {
 	h.mutex.RLock()
 	defer h.mutex.RUnlock()
 
 	// Make a copy before returning the value
-	retMap := make(map[string]int64, len(h.healFailedItemsMap))
+	retMap := make(map[madmin.HealItemType]int64, len(h.healFailedItemsMap))
 	for k, v := range h.healFailedItemsMap {
 		retMap[k] = v
 	}
@@ -545,6 +556,30 @@ func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
 	return retMap
 }
 
+func (h *healSequence) countFailed(healType madmin.HealItemType) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	h.healFailedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+}
+
+func (h *healSequence) countScanned(healType madmin.HealItemType) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	h.scannedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+}
+
+func (h *healSequence) countHealed(healType madmin.HealItemType) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	h.healedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+}
+
 // isQuitting - determines if the heal sequence is quitting (due to an
 // external signal)
 func (h *healSequence) isQuitting() bool {
@@ -697,16 +732,13 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		task.opts.ScanMode = madmin.HealNormalScan
 	}
 
-	h.mutex.Lock()
+	h.countScanned(healType)
-	h.scannedItemsMap[healType]++
-	h.lastHealActivity = UTCNow()
-	h.mutex.Unlock()
 
 	if source.noWait {
 		select {
 		case globalBackgroundHealRoutine.tasks <- task:
 			if serverDebugLog {
-				logger.Info("Task in the queue: %#v", task)
+				fmt.Printf("Task in the queue: %#v\n", task)
 			}
 		default:
 			// task queue is full, no more workers, we shall move on and heal later.
@@ -723,48 +755,46 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	select {
 	case globalBackgroundHealRoutine.tasks <- task:
 		if serverDebugLog {
-			logger.Info("Task in the queue: %#v", task)
+			fmt.Printf("Task in the queue: %#v\n", task)
 		}
 	case <-h.ctx.Done():
 		return nil
 	}
 
+	countOKDrives := func(drives []madmin.HealDriveInfo) (count int) {
+		for _, drive := range drives {
+			if drive.State == madmin.DriveStateOk {
+				count++
+			}
+		}
+		return count
+	}
+
 	// task queued, now wait for the response.
 	select {
 	case res := <-task.respCh:
+		if res.err == nil {
+			h.countHealed(healType)
+		} else {
+			h.countFailed(healType)
+		}
 		if !h.reportProgress {
 			if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
 				return nil
 			}
 
-			h.mutex.Lock()
-			defer h.mutex.Unlock()
-
-			// Progress is not reported in case of background heal processing.
-			// Instead we increment relevant counter based on the heal result
-			// for prometheus reporting.
-			if res.err != nil {
-				for _, d := range res.result.After.Drives {
-					// For failed items we report the endpoint and drive state
-					// This will help users take corrective actions for drives
-					h.healFailedItemsMap[d.Endpoint+","+d.State]++
-				}
-			} else {
-				// Only object type reported for successful healing
-				h.healedItemsMap[res.result.Type]++
-			}
-
 			// Report caller of any failure
 			return res.err
 		}
 		res.result.Type = healType
 		if res.err != nil {
-			// Only report object error
-			if healType != madmin.HealItemObject {
-				return res.err
-			}
 			res.result.Detail = res.err.Error()
 		}
+		if res.result.ParityBlocks > 0 && res.result.DataBlocks > 0 && res.result.DataBlocks > res.result.ParityBlocks {
+			if got := countOKDrives(res.result.After.Drives); got < res.result.ParityBlocks {
+				res.result.Detail = fmt.Sprintf("quorum loss - expected %d minimum, got drive states in OK %d", res.result.ParityBlocks, got)
+			}
+		}
 		return h.pushHealResultItem(res.result)
 	case <-h.ctx.Done():
 		return nil
@@ -776,18 +806,20 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
 }
 
-func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+func (h *healSequence) healItems(objAPI ObjectLayer) error {
 	if h.clientToken == bgHealingUUID {
 		// For background heal do nothing.
 		return nil
 	}
 
+	if h.bucket == "" { // heal internal meta only during a site-wide heal
 		if err := h.healDiskMeta(objAPI); err != nil {
 			return err
 		}
+	}
 
 	// Heal buckets and objects
-	return h.healBuckets(objAPI, bucketsOnly)
+	return h.healBuckets(objAPI)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -798,9 +830,8 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
-	bucketsOnly := false // Heals buckets and objects also.
+	h.traverseAndHealDoneCh <- h.healItems(objAPI)
-	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
+	xioutil.SafeClose(h.traverseAndHealDoneCh)
-	close(h.traverseAndHealDoneCh)
 }
 
 // healMinioSysMeta - heals all files under a given meta prefix, returns a function
@@ -810,7 +841,8 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		h.settings.Recursive = true
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -826,14 +858,14 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
+func (h *healSequence) healBuckets(objAPI ObjectLayer) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(objAPI, h.bucket, bucketsOnly)
+		return h.healBucket(objAPI, h.bucket, false)
 	}
 
 	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
@@ -847,7 +879,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	})
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
+		if err = h.healBucket(objAPI, bucket.Name, false); err != nil {
 			return err
 		}
 	}
@@ -865,16 +897,6 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 		return nil
 	}
 
-	if !h.settings.Recursive {
-		if h.object != "" {
-			if err := h.healObject(bucket, h.object, ""); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
 	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
@@ -882,7 +904,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 }
 
 // healObject - heal the given object and record result
-func (h *healSequence) healObject(bucket, object, versionID string) error {
+func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-router.go

@@ -19,9 +19,6 @@ package cmd
 
 import (
 	"net/http"
-	"reflect"
-	"runtime"
-	"strings"
 
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
@@ -63,20 +60,12 @@ const (
 	noObjLayerFlag
 )
 
-// Has checks if the the given flag is enabled in `h`.
+// Has checks if the given flag is enabled in `h`.
 func (h hFlag) Has(flag hFlag) bool {
 	// Use bitwise-AND and check if the result is non-zero.
 	return h&flag != 0
 }
 
-func getHandlerName(f http.HandlerFunc) string {
-	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
-	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.")
-	name = strings.TrimSuffix(name, "Handler-fm")
-	name = strings.TrimSuffix(name, "-fm")
-	return name
-}
-
 // adminMiddleware performs some common admin handler functionality for all
 // handlers:
 //
@@ -86,9 +75,13 @@ func getHandlerName(f http.HandlerFunc) string {
 //
 // - sets up call to send AuditLog
 //
-// Note that, while this is a middleware function (i.e. it takes a handler
+// While this is a middleware function (i.e. it takes a handler function and
-// function and returns one), due to flags being passed based on required
+// returns one), due to flags being passed based on required conditions, it is
-// conditions, it is done per-"handler function registration" in the router.
+// done per-"handler function registration" in the router.
+//
+// The passed in handler function must be a method of `adminAPIHandlers` for the
+// name displayed in logs and trace to be accurate. The name is extracted via
+// reflection.
 //
 // When no flags are passed, gzip compression, http tracing of headers and
 // checking of object layer availability are all enabled. Use flags to modify
@@ -100,10 +93,8 @@ func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc {
 		handlerFlags |= flag
 	}
 
-	// Get name of the handler using reflection. NOTE: The passed in handler
+	// Get name of the handler using reflection.
-	// function must be a method of `adminAPIHandlers` for this extraction to
+	handlerName := getHandlerName(f, "adminAPIHandlers")
-	// work as expected.
-	handlerName := getHandlerName(f)
 
 	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
 		// Update request context with `logger.ReqInfo`.
@@ -154,21 +145,28 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 	}
 
 	for _, adminVersion := range adminVersions {
-		// Restart and stop MinIO service.
+		// Restart and stop MinIO service type=2
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceV2Handler, traceAllFlag)).Queries("action", "{action:.*}", "type", "2")
+
+		// Deprecated: Restart and stop MinIO service.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceHandler, traceAllFlag)).Queries("action", "{action:.*}")
-		// Update MinIO servers.
+
+		// Update all MinIO servers type=2
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateV2Handler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}", "type", "2")
+
+		// Deprecated: Update MinIO servers.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateHandler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}")
 
 		// Info operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(adminMiddleware(adminAPI.ServerInfoHandler, traceAllFlag, noObjLayerFlag))
-		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceAllFlag))
+		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceHdrsS3HFlag))
 
 		// StorageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
 		// Metrics operation
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceAllFlag))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceHdrsS3HFlag))
 
 		if globalIsDistErasure || globalIsErasure {
 			// Heal operations
@@ -195,9 +193,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// Profiling operations - deprecated API
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(adminMiddleware(adminAPI.StartProfilingHandler, traceAllFlag, noObjLayerFlag)).
 			Queries("profilerType", "{profilerType:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceAllFlag, noObjLayerFlag))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceHdrsS3HFlag, noObjLayerFlag))
 		// Profiling operations
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceAllFlag, noObjLayerFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceHdrsS3HFlag, noObjLayerFlag))
 
 		// Config KV operations.
 		if enableConfigOps {
@@ -246,6 +244,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// STS accounts ops
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}")
 
+		// Access key (service account/STS) operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-access-keys-bulk").HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysBulk)).Queries("listType", "{listType:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-access-key").HandlerFunc(adminMiddleware(adminAPI.InfoAccessKey)).Queries("accessKey", "{accessKey:.*}")
+
 		// Info policy IAM latest
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 		// List policies latest
@@ -292,6 +294,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// Import IAM info
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam-v2").HandlerFunc(adminMiddleware(adminAPI.ImportIAMV2, noGZFlag))
 
 		// IDentity Provider configuration APIs
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg))
@@ -300,9 +303,21 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg))
 
+		// LDAP specific service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys").
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys-bulk").
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAPBulk)).Queries("listType", "{listType:.*}")
+
 		// LDAP IAM operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))
+
+		// OpenID specific service accounts ops
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/openid/list-access-keys-bulk").
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysOpenIDBulk)).Queries("listType", "{listType:.*}")
+
 		// -- END IAM APIs --
 
 		// GetBucketQuotaConfig
@@ -336,6 +351,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
 			adminMiddleware(adminAPI.ListBatchJobs))
 
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/status-job").HandlerFunc(
+			adminMiddleware(adminAPI.BatchJobStatus))
+
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
 			adminMiddleware(adminAPI.DescribeBatchJob))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(
@@ -376,6 +394,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(adminMiddleware(adminAPI.SRPeerEdit))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(adminMiddleware(adminAPI.SRPeerRemove))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationResyncOp)).Queries("operation", "{operation:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/state/edit").HandlerFunc(adminMiddleware(adminAPI.SRStateEdit))
 
 		if globalIsDistErasure {
 			// Top locks
@@ -411,6 +430,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// -- Health API --
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
 			HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler))
+
+		// STS Revocation
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/revoke-tokens/{userProvider}").HandlerFunc(adminMiddleware(adminAPI.RevokeTokens))
 	}
 
 	// If none of the routes match add default error handler routes

cmd/admin-server-info.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,23 +18,24 @@
 package cmd
 
 import (
-	"context"
+	"math"
 	"net/http"
 	"os"
 	"runtime"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
+	xnet "github.com/minio/pkg/v3/net"
 )
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request, metrics bool) madmin.ServerProperties {
 	addr := globalLocalNodeName
 	if r != nil {
 		addr = r.Host
@@ -42,9 +43,13 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	if globalIsDistErasure {
 		addr = globalLocalNodeName
 	}
+	poolNumbers := make(map[int]struct{})
 	network := make(map[string]string)
 	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				poolNumbers[endpoint.PoolIdx+1] = struct{}{}
+			}
 			nodeName := endpoint.Host
 			if nodeName == "" {
 				nodeName = addr
@@ -59,9 +64,11 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 				if err := isServerResolvable(endpoint, 5*time.Second); err == nil {
 					network[nodeName] = string(madmin.ItemOnline)
 				} else {
+					if xnet.IsNetworkOrHostDown(err, false) {
 						network[nodeName] = string(madmin.ItemOffline)
-					// log once the error
+					} else if xnet.IsNetworkOrHostDown(err, true) {
-					logger.LogOnceIf(context.Background(), err, nodeName)
+						network[nodeName] = "connection attempt timedout"
+					}
 				}
 			}
 		}
@@ -112,6 +119,17 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		MinioEnvVars: make(map[string]string, 10),
 	}
 
+	for poolNumber := range poolNumbers {
+		props.PoolNumbers = append(props.PoolNumbers, poolNumber)
+	}
+	sort.Ints(props.PoolNumbers)
+	props.PoolNumber = func() int {
+		if len(props.PoolNumbers) == 1 {
+			return props.PoolNumbers[0]
+		}
+		return math.MaxInt // this indicates that its unset.
+	}()
+
 	sensitive := map[string]struct{}{
 		config.EnvAccessKey:         {},
 		config.EnvSecretKey:         {},
@@ -141,7 +159,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 
 	objLayer := newObjectLayerFn()
 	if objLayer != nil {
-		storageInfo := objLayer.LocalStorageInfo(GlobalContext)
+		storageInfo := objLayer.LocalStorageInfo(GlobalContext, metrics)
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {

cmd/api-datatypes.go

@@ -32,6 +32,8 @@ type DeletedObject struct {
 	DeleteMarkerMTime DeleteMarkerMTime `xml:"-"`
 	// MinIO extensions to support delete marker replication
 	ReplicationState ReplicationState `xml:"-"`
+
+	found bool // the object was found during deletion
 }
 
 // DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
@@ -42,10 +44,10 @@ type DeleteMarkerMTime struct {
 // MarshalXML encodes expiration date if it is non-zero and encodes
 // empty string otherwise
 func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
-	if t.Time.IsZero() {
+	if t.IsZero() {
 		return nil
 	}
-	return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
+	return e.EncodeElement(t.Format(time.RFC3339), startElement)
 }
 
 // ObjectV object version key/versionId
@@ -67,7 +69,7 @@ type ObjectToDelete struct {
 	ReplicateDecisionStr string `xml:"-"`
 }
 
-// createBucketConfiguration container for bucket configuration request from client.
+// createBucketLocationConfiguration container for bucket configuration request from client.
 // Used for parsing the location from the request body for Makebucket.
 type createBucketLocationConfiguration struct {
 	XMLName  xml.Name `xml:"CreateBucketConfiguration" json:"-"`

cmd/api-errors.go

@@ -24,10 +24,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/minio/minio/internal/ioutil"
 	"google.golang.org/api/googleapi"
 
@@ -47,7 +48,7 @@ import (
 	levent "github.com/minio/minio/internal/config/lambda/event"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // APIError structure
@@ -55,6 +56,8 @@ type APIError struct {
 	Code           string
 	Description    string
 	HTTPStatusCode int
+	ObjectSize     string
+	RangeRequested string
 }
 
 // APIErrorResponse - error response format
@@ -68,6 +71,8 @@ type APIErrorResponse struct {
 	Region           string `xml:"Region,omitempty" json:"Region,omitempty"`
 	RequestID        string `xml:"RequestId" json:"RequestId"`
 	HostID           string `xml:"HostId" json:"HostId"`
+	ActualObjectSize string `xml:"ActualObjectSize,omitempty" json:"ActualObjectSize,omitempty"`
+	RangeRequested   string `xml:"RangeRequested,omitempty" json:"RangeRequested,omitempty"`
 }
 
 // APIErrorCode type of error status.
@@ -208,6 +213,10 @@ const (
 	ErrPolicyAlreadyAttached
 	ErrPolicyNotAttached
 	ErrExcessData
+	ErrPolicyInvalidName
+	ErrNoTokenRevokeType
+	ErrAdminOpenIDNotEnabled
+	ErrAdminNoSuchAccessKey
 	// Add new error codes here.
 
 	// SSE-S3/SSE-KMS related API errors
@@ -262,6 +271,7 @@ const (
 	ErrInvalidResourceName
 	ErrInvalidLifecycleQueryParameter
 	ErrServerNotInitialized
+	ErrBucketMetadataNotInitialized
 	ErrRequestTimedout
 	ErrClientDisconnected
 	ErrTooManyRequests
@@ -277,9 +287,11 @@ const (
 	ErrMalformedJSON
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchUserLDAPWarn
+	ErrAdminLDAPExpectedLoginName
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
 	ErrAdminGroupDisabled
+	ErrAdminInvalidGroupName
 	ErrAdminNoSuchJob
 	ErrAdminNoSuchPolicy
 	ErrAdminPolicyChangeAlreadyApplied
@@ -297,9 +309,9 @@ const (
 	ErrAdminConfigLDAPValidation
 	ErrAdminConfigIDPCfgNameAlreadyExists
 	ErrAdminConfigIDPCfgNameDoesNotExist
-	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
+	ErrAdminLDAPNotEnabled
 
 	// Site-Replication errors
 	ErrSiteReplicationInvalidRequest
@@ -390,7 +402,7 @@ const (
 	ErrParseExpectedIdentForGroupName
 	ErrParseExpectedIdentForAlias
 	ErrParseUnsupportedCallWithStar
-	ErrParseNonUnaryAgregateFunctionCall
+	ErrParseNonUnaryAggregateFunctionCall
 	ErrParseMalformedJoin
 	ErrParseExpectedIdentForAt
 	ErrParseAsteriskIsNotAloneInSelectList
@@ -418,6 +430,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrAddUserValidUTF
 	ErrAdminResourceInvalidArgument
 	ErrAdminAccountNotEligible
 	ErrAccountNotEligible
@@ -430,6 +443,14 @@ const (
 	ErrLambdaARNInvalid
 	ErrLambdaARNNotFound
 
+	// New Codes for GetObjectAttributes and GetObjectVersionAttributes
+	ErrInvalidAttributeName
+
+	ErrAdminNoAccessKey
+	ErrAdminNoSecretKey
+
+	ErrIAMNotInitialized
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -443,9 +464,9 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
 	if err != nil {
 		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
-	if globalSite.Region != "" {
+	if region := globalSite.Region(); region != "" {
 		if errCode == ErrAuthorizationHeaderMalformed {
-			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
+			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", region)
 			return apiErr
 		}
 	}
@@ -544,6 +565,16 @@ var errorCodes = errorCodeMap{
 		Description:    "More data provided than indicated content length",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrPolicyInvalidName: {
+		Code:           "PolicyInvalidName",
+		Description:    "Policy name may not contain comma",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminOpenIDNotEnabled: {
+		Code:           "OpenIDNotEnabled",
+		Description:    "No enabled OpenID Connect identity providers",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrPolicyTooLarge: {
 		Code:           "PolicyTooLarge",
 		Description:    "Policy exceeds the maximum allowed document size.",
@@ -606,7 +637,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrMissingContentMD5: {
 		Code:           "MissingContentMD5",
-		Description:    "Missing required header for this request: Content-Md5.",
+		Description:    "Missing or invalid required header for this request: Content-Md5 or Amz-Content-Checksum",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSecurityHeader: {
@@ -952,7 +983,7 @@ var errorCodes = errorCodeMap{
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
 		Description:    "Remote service connection error",
-		HTTPStatusCode: http.StatusNotFound,
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrReplicationBandwidthLimitError: {
 		Code:           "XMinioAdminReplicationBandwidthLimitError",
@@ -961,7 +992,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationNoExistingObjects: {
 		Code:           "XMinioReplicationNoExistingObjects",
-		Description:    "No matching ExistingsObjects rule enabled",
+		Description:    "No matching ExistingObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrRemoteTargetDenyAddError: {
@@ -1241,6 +1272,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The security token included in the request is invalid",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrNoTokenRevokeType: {
+		Code:           "InvalidArgument",
+		Description:    "No token revoke type specified and one could not be inferred from the request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminNoSuchAccessKey: {
+		Code:           "XMinioAdminNoSuchAccessKey",
+		Description:    "The specified access key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 
 	// S3 extensions.
 	ErrContentSHA256Mismatch: {
@@ -1287,7 +1328,17 @@ var errorCodes = errorCodeMap{
 	},
 	ErrServerNotInitialized: {
 		Code:           "XMinioServerNotInitialized",
-		Description:    "Server not initialized, please try again.",
+		Description:    "Server not initialized yet, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrIAMNotInitialized: {
+		Code:           "XMinioIAMNotInitialized",
+		Description:    "IAM sub-system not initialized yet, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrBucketMetadataNotInitialized: {
+		Code:           "XMinioBucketMetadataNotInitialized",
+		Description:    "Bucket metadata not initialized yet, please try again.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrMalformedJSON: {
@@ -1356,6 +1407,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The secret key is invalid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoAccessKey: {
+		Code:           "XMinioAdminNoAccessKey",
+		Description:    "No access key was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminNoSecretKey: {
+		Code:           "XMinioAdminNoSecretKey",
+		Description:    "No secret key was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigNoQuorum: {
 		Code:           "XMinioAdminConfigNoQuorum",
 		Description:    "Configuration update failed because server quorum was not met",
@@ -1422,11 +1483,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Unable to perform the requested operation because profiling is not enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrAdminCredentialsMismatch: {
-		Code:           "XMinioAdminCredentialsMismatch",
-		Description:    "Credentials in config mismatch with server environment variables",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
 	ErrAdminBucketQuotaExceeded: {
 		Code:           "XMinioAdminBucketQuotaExceeded",
 		Description:    "Bucket quota exceeded",
@@ -1454,8 +1510,8 @@ var errorCodes = errorCodeMap{
 	},
 	ErrTooManyRequests: {
 		Code:           "TooManyRequests",
-		Description:    "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
+		Description:    "Please reduce your request rate",
-		HTTPStatusCode: http.StatusServiceUnavailable,
+		HTTPStatusCode: http.StatusTooManyRequests,
 	},
 	ErrUnsupportedMetadata: {
 		Code:           "InvalidArgument",
@@ -1888,8 +1944,8 @@ var errorCodes = errorCodeMap{
 		Description:    "Only COUNT with (*) as a parameter is supported in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrParseNonUnaryAgregateFunctionCall: {
+	ErrParseNonUnaryAggregateFunctionCall: {
-		Code:           "ParseNonUnaryAgregateFunctionCall",
+		Code:           "ParseNonUnaryAggregateFunctionCall",
 		Description:    "Only one argument is supported for aggregate functions in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -2010,7 +2066,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAddUserInvalidArgument: {
 		Code:           "XMinioInvalidIAMCredentials",
-		Description:    "User is not allowed to be same as admin access key",
+		Description:    "Credential is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrAdminResourceInvalidArgument: {
@@ -2063,7 +2119,31 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified policy is not found.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	// Add your error structure here.
+	ErrInvalidAttributeName: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminLDAPNotEnabled: {
+		Code:           "XMinioLDAPNotEnabled",
+		Description:    "LDAP is not enabled. LDAP must be enabled to make LDAP requests.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrAdminLDAPExpectedLoginName: {
+		Code:           "XMinioLDAPExpectedLoginName",
+		Description:    "Expected LDAP short username but was given full DN.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminInvalidGroupName: {
+		Code:           "XMinioInvalidGroupName",
+		Description:    "The group name is invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAddUserValidUTF: {
+		Code:           "XMinioInvalidUTF",
+		Description:    "Invalid UTF-8 character detected.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // toAPIErrorCode - Converts embedded errors. Convenience
@@ -2074,6 +2154,11 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Errors that are generated by net.Conn and any context errors must be handled here.
+	if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) {
+		return ErrRequestTimedout
+	}
+
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrRequestTimedout
 	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
@@ -2094,10 +2179,14 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchUserLDAPWarn
 	case errNoSuchServiceAccount:
 		apiErr = ErrAdminServiceAccountNotFound
+	case errNoSuchAccessKey:
+		apiErr = ErrAdminNoSuchAccessKey
 	case errNoSuchGroup:
 		apiErr = ErrAdminNoSuchGroup
 	case errGroupNotEmpty:
 		apiErr = ErrAdminGroupNotEmpty
+	case errGroupNameContainsReservedChars:
+		apiErr = ErrAdminInvalidGroupName
 	case errNoSuchJob:
 		apiErr = ErrAdminNoSuchJob
 	case errNoPolicyToAttachOrDetach:
@@ -2112,10 +2201,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case errAuthentication:
 		apiErr = ErrAccessDenied
+	case auth.ErrContainsReservedChars:
+		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidAccessKeyLength:
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	case auth.ErrNoAccessKeyWithSecretKey:
+		apiErr = ErrAdminNoAccessKey
+	case auth.ErrNoSecretKeyWithAccessKey:
+		apiErr = ErrAdminNoSecretKey
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
 	case errErasureReadQuorum:
@@ -2175,6 +2270,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidMaxParts
 	case ioutil.ErrOverread:
 		apiErr = ErrExcessData
+	case errServerNotInitialized:
+		apiErr = ErrServerNotInitialized
+	case errBucketMetadataNotInitialized:
+		apiErr = ErrBucketMetadataNotInitialized
+	case hash.ErrInvalidChecksum:
+		apiErr = ErrInvalidChecksum
 	}
 
 	// Compression errors
@@ -2343,31 +2444,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case dns.ErrBucketConflict:
 		apiErr = ErrBucketAlreadyExists
 	default:
-		if _, ok := err.(tags.Error); ok {
+		if strings.Contains(err.Error(), "request declared a Content-Length") {
-			// tag errors are not exported, so we check their custom interface to avoid logging.
-			// The correct type is inserted by toAPIError.
-			apiErr = ErrInternalError
-			break
-		}
-		var ie, iw int
-		// This work-around is to handle the issue golang/go#30648
-		//nolint:gocritic
-		if _, ferr := fmt.Fscanf(strings.NewReader(err.Error()),
-			"request declared a Content-Length of %d but only wrote %d bytes",
-			&ie, &iw); ferr != nil {
-			apiErr = ErrInternalError
-			// Make sure to log the errors which we cannot translate
-			// to a meaningful S3 API errors. This is added to aid in
-			// debugging unexpected/unhandled errors.
-			logger.LogIf(ctx, err)
-		} else if ie > iw {
 			apiErr = ErrIncompleteBody
 		} else {
 			apiErr = ErrInternalError
-			// Make sure to log the errors which we cannot translate
-			// to a meaningful S3 API errors. This is added to aid in
-			// debugging unexpected/unhandled errors.
-			logger.LogIf(ctx, err)
 		}
 	}
 
@@ -2387,10 +2467,9 @@ func toAPIError(ctx context.Context, err error) APIError {
 	apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
 	switch apiErr.Code {
 	case "NotImplemented":
-		desc := fmt.Sprintf("%s (%v)", apiErr.Description, err)
 		apiErr = APIError{
 			Code:           apiErr.Code,
-			Description:    desc,
+			Description:    fmt.Sprintf("%s (%v)", apiErr.Description, err),
 			HTTPStatusCode: apiErr.HTTPStatusCode,
 		}
 	case "XMinioBackendDown":
@@ -2402,12 +2481,24 @@ func toAPIError(ctx context.Context, err error) APIError {
 		switch e := err.(type) {
 		case kms.Error:
 			apiErr = APIError{
-				Description:    e.Err.Error(),
 				Code:           e.APICode,
-				HTTPStatusCode: e.HTTPStatusCode,
+				Description:    e.Err,
+				HTTPStatusCode: e.Code,
 			}
 		case batchReplicationJobError:
-			apiErr = APIError(e)
+			apiErr = APIError{
+				Description:    e.Description,
+				Code:           e.Code,
+				HTTPStatusCode: e.HTTPStatusCode,
+			}
+		case InvalidRange:
+			apiErr = APIError{
+				Code:           "InvalidRange",
+				Description:    e.Error(),
+				HTTPStatusCode: errorCodes[ErrInvalidRange].HTTPStatusCode,
+				ObjectSize:     strconv.FormatInt(e.ResourceSize, 10),
+				RangeRequested: fmt.Sprintf("%d-%d", e.OffsetBegin, e.OffsetEnd),
+			}
 		case InvalidArgument:
 			apiErr = APIError{
 				Code:           "InvalidArgument",
@@ -2486,11 +2577,11 @@ func toAPIError(ctx context.Context, err error) APIError {
 			if len(e.Errors) >= 1 {
 				apiErr.Code = e.Errors[0].Reason
 			}
-		case azblob.StorageError:
+		case *azcore.ResponseError:
 			apiErr = APIError{
-				Code:           string(e.ServiceCode()),
+				Code:           e.ErrorCode,
 				Description:    e.Error(),
-				HTTPStatusCode: e.Response().StatusCode,
+				HTTPStatusCode: e.StatusCode,
 			}
 			// Add more other SDK related errors here if any in future.
 		default:
@@ -2511,6 +2602,13 @@ func toAPIError(ctx context.Context, err error) APIError {
 		}
 	}
 
+	if apiErr.Code == "InternalError" {
+		// Make sure to log the errors which we cannot translate
+		// to a meaningful S3 API errors. This is added to aid in
+		// debugging unexpected/unhandled errors.
+		internalLogIf(ctx, err)
+	}
+
 	return apiErr
 }
 
@@ -2522,7 +2620,7 @@ func getAPIError(code APIErrorCode) APIError {
 	return errorCodes.ToAPIErr(ErrInternalError)
 }
 
-// getErrorResponse gets in standard error and resource value and
+// getAPIErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
 func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID, hostID string) APIErrorResponse {
 	reqInfo := logger.GetReqInfo(ctx)
@@ -2532,8 +2630,10 @@ func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID,
 		BucketName:       reqInfo.BucketName,
 		Key:              reqInfo.ObjectName,
 		Resource:         resource,
-		Region:     globalSite.Region,
+		Region:           globalSite.Region(),
 		RequestID:        requestID,
 		HostID:           hostID,
+		ActualObjectSize: err.ObjectSize,
+		RangeRequested:   err.RangeRequested,
 	}
 }

cmd/api-errors_test.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"testing"
 
@@ -64,7 +63,7 @@ var toAPIErrorTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
-	ctx := context.Background()
+	ctx := t.Context()
 	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)
 		if errCode != testCase.errCode {

cmd/api-headers.go

@@ -19,9 +19,11 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"mime"
 	"net/http"
 	"strconv"
 	"strings"
@@ -30,7 +32,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	xxml "github.com/minio/xxml"
 )
 
@@ -50,11 +51,11 @@ func setEventStreamHeaders(w http.ResponseWriter) {
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set the "Server" http header.
-	w.Header().Set(xhttp.ServerInfo, "MinIO")
+	w.Header().Set(xhttp.ServerInfo, MinioStoreName)
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
-	if region := globalSite.Region; region != "" {
+	if region := globalSite.Region(); region != "" {
 		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
 	w.Header().Set(xhttp.AcceptRanges, "bytes")
@@ -68,7 +69,7 @@ func encodeResponse(response interface{}) []byte {
 	var buf bytes.Buffer
 	buf.WriteString(xml.Header)
 	if err := xml.NewEncoder(&buf).Encode(response); err != nil {
-		logger.LogIf(GlobalContext, err)
+		bugLogIf(GlobalContext, err)
 		return nil
 	}
 	return buf.Bytes()
@@ -86,7 +87,7 @@ func encodeResponseList(response interface{}) []byte {
 	var buf bytes.Buffer
 	buf.WriteString(xxml.Header)
 	if err := xxml.NewEncoder(&buf).Encode(response); err != nil {
-		logger.LogIf(GlobalContext, err)
+		bugLogIf(GlobalContext, err)
 		return nil
 	}
 	return buf.Bytes()
@@ -108,7 +109,7 @@ func setPartsCountHeaders(w http.ResponseWriter, objInfo ObjectInfo) {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec, opts ObjectOptions) (err error) {
+func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec, opts ObjectOptions) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
@@ -136,7 +137,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	// Set tag count if object has tags
 	if len(objInfo.UserTags) > 0 {
 		tags, _ := tags.ParseObjectTags(objInfo.UserTags)
-		if tags.Count() > 0 {
+		if tags != nil && tags.Count() > 0 {
 			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())}
 			if opts.Tagging {
 				// This is MinIO only extension to return back tags along with the count.
@@ -168,6 +169,32 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 			if !stringsHasPrefixFold(k, userMetadataPrefix) {
 				continue
 			}
+			// check the doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html
+			// For metadata values like "ö", "ÄMÄZÕÑ S3", and "öha, das sollte eigentlich
+			// funktionieren", tested against a real AWS S3 bucket, S3 may encode incorrectly. For
+			// example, "ö" was encoded as =?UTF-8?B?w4PCtg==?=, producing invalid UTF-8 instead
+			// of =?UTF-8?B?w7Y=?=. This mirrors errors like the ä½ in another string.
+			//
+			// S3 uses B-encoding (Base64) for non-ASCII-heavy metadata and Q-encoding
+			// (quoted-printable) for mostly ASCII strings. Long strings are split at word
+			// boundaries to fit RFC 2047’s 75-character limit, ensuring HTTP parser
+			// compatibility.
+			//
+			// However, this splitting increases header size and can introduce errors, unlike Go’s
+			// mime package in MinIO, which correctly encodes strings with fixed B/Q encodings,
+			// avoiding S3’s heuristic-driven issues.
+			//
+			// For MinIO developers, decode S3 metadata with mime.WordDecoder, validate outputs,
+			// report encoding bugs to AWS, and use ASCII-only metadata to ensure reliable S3 API
+			// compatibility.
+			if needsMimeEncoding(v) {
+				// see https://github.com/golang/go/blob/release-branch.go1.24/src/net/mail/message.go#L325
+				if strings.ContainsAny(v, "\"#$%&'(),.:;<>@[]^`{|}~") {
+					v = mime.BEncoding.Encode("UTF-8", v)
+				} else {
+					v = mime.QEncoding.Encode("UTF-8", v)
+				}
+			}
 			w.Header()[strings.ToLower(k)] = []string{v}
 			isSet = true
 			break
@@ -213,7 +240,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if objInfo.IsRemote() {
 		// Check if object is being restored. For more information on x-amz-restore header see
 		// https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax
-		w.Header()[xhttp.AmzStorageClass] = []string{objInfo.TransitionedObject.Tier}
+		w.Header()[xhttp.AmzStorageClass] = []string{filterStorageClass(ctx, objInfo.TransitionedObject.Tier)}
 	}
 
 	if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
@@ -229,3 +256,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	return nil
 }
+
+// needsEncoding reports whether s contains any bytes that need to be encoded.
+// see mime.needsEncoding
+func needsMimeEncoding(s string) bool {
+	for _, b := range s {
+		if (b < ' ' || b > '~') && b != '\t' {
+			return true
+		}
+	}
+	return false
+}

cmd/api-headers_test.go

@@ -34,7 +34,8 @@ func TestNewRequestID(t *testing.T) {
 		e = char
 
 		// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
-		if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
+		isAlnum := ('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')
+		if !isAlnum {
 			t.Fail()
 		}
 	}

cmd/api-response.go

@@ -35,7 +35,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 	xxml "github.com/minio/xxml"
 )
 
@@ -170,6 +170,7 @@ type Part struct {
 	ChecksumCRC32C    string `xml:"ChecksumCRC32C,omitempty"`
 	ChecksumSHA1      string `xml:"ChecksumSHA1,omitempty"`
 	ChecksumSHA256    string `xml:"ChecksumSHA256,omitempty"`
+	ChecksumCRC64NVME string `xml:",omitempty"`
 }
 
 // ListPartsResponse - format for list parts response.
@@ -192,6 +193,8 @@ type ListPartsResponse struct {
 	IsTruncated          bool
 
 	ChecksumAlgorithm string
+	ChecksumType      string
+
 	// List of parts.
 	Parts []Part `xml:"Part"`
 }
@@ -417,6 +420,7 @@ type CompleteMultipartUploadResponse struct {
 	ChecksumCRC32C    string `xml:"ChecksumCRC32C,omitempty"`
 	ChecksumSHA1      string `xml:"ChecksumSHA1,omitempty"`
 	ChecksumSHA256    string `xml:"ChecksumSHA256,omitempty"`
+	ChecksumCRC64NVME string `xml:",omitempty"`
 }
 
 // DeleteError structure.
@@ -502,8 +506,48 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	return data
 }
 
+func cleanReservedKeys(metadata map[string]string) map[string]string {
+	m := cloneMSS(metadata)
+
+	switch kind, _ := crypto.IsEncrypted(metadata); kind {
+	case crypto.S3:
+		m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+	case crypto.S3KMS:
+		m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+		m[xhttp.AmzServerSideEncryptionKmsID] = kmsKeyIDFromMetadata(metadata)
+		if kmsCtx, ok := metadata[crypto.MetaContext]; ok {
+			m[xhttp.AmzServerSideEncryptionKmsContext] = kmsCtx
+		}
+	case crypto.SSEC:
+		m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+	}
+
+	var toRemove []string
+	for k := range cleanMinioInternalMetadataKeys(m) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+			// Do not need to send any internal metadata
+			// values to client.
+			toRemove = append(toRemove, k)
+			continue
+		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+			toRemove = append(toRemove, k)
+			continue
+		}
+	}
+
+	for _, k := range toRemove {
+		delete(m, k)
+		delete(m, strings.ToLower(k))
+	}
+
+	return m
+}
+
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
-func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
+func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
 
 	owner := &Owner{
@@ -532,7 +576,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		}
 		content.Size = object.Size
 		if object.StorageClass != "" {
-			content.StorageClass = object.StorageClass
+			content.StorageClass = filterStorageClass(ctx, object.StorageClass)
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
@@ -549,16 +593,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 			case crypto.SSEC:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
-			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+			for k, v := range cleanReservedKeys(object.UserDefined) {
-				if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-					// Do not need to send any internal metadata
-					// values to client.
-					continue
-				}
-				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-					continue
-				}
 				content.UserMetadata.Set(k, v)
 			}
 			content.Internal = &ObjectInternalInfo{
@@ -600,7 +635,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 }
 
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
-func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
+func generateListObjectsV1Response(ctx context.Context, bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	contents := make([]Object, 0, len(resp.Objects))
 	owner := &Owner{
 		ID:          globalMinioDefaultOwnerID,
@@ -620,7 +655,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 		}
 		content.Size = object.Size
 		if object.StorageClass != "" {
-			content.StorageClass = object.StorageClass
+			content.StorageClass = filterStorageClass(ctx, object.StorageClass)
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
@@ -649,7 +684,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
+func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
 	var owner *Owner
 	if fetchOwner {
@@ -673,7 +708,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		}
 		content.Size = object.Size
 		if object.StorageClass != "" {
-			content.StorageClass = object.StorageClass
+			content.StorageClass = filterStorageClass(ctx, object.StorageClass)
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
@@ -692,16 +727,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 				case crypto.SSEC:
 					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
-				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+				for k, v := range cleanReservedKeys(object.UserDefined) {
-					if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-						// Do not need to send any internal metadata
-						// values to client.
-						continue
-					}
-					// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-					if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-						continue
-					}
 					content.UserMetadata.Set(k, v)
 				}
 				content.Internal = &ObjectInternalInfo{
@@ -763,8 +789,8 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 }
 
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
-func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
+func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo, h http.Header) CompleteMultipartUploadResponse {
-	cs := oi.decryptChecksums(0)
+	cs, _ := oi.decryptChecksums(0, h)
 	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,
@@ -775,6 +801,7 @@ func generateCompleteMultpartUploadResponse(bucket, key, location string, oi Obj
 		ChecksumSHA256:    cs[hash.ChecksumSHA256.String()],
 		ChecksumCRC32:     cs[hash.ChecksumCRC32.String()],
 		ChecksumCRC32C:    cs[hash.ChecksumCRC32C.String()],
+		ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
 	}
 	return c
 }
@@ -802,6 +829,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 	listPartsResponse.IsTruncated = partsInfo.IsTruncated
 	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
 	listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
+	listPartsResponse.ChecksumType = partsInfo.ChecksumType
 
 	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
 	for index, part := range partsInfo.Parts {
@@ -814,6 +842,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 		newPart.ChecksumCRC32C = part.ChecksumCRC32C
 		newPart.ChecksumSHA1 = part.ChecksumSHA1
 		newPart.ChecksumSHA256 = part.ChecksumSHA256
+		newPart.ChecksumCRC64NVME = part.ChecksumCRC64NVME
 		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse
@@ -865,7 +894,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	}
 	// Similar check to http.checkWriteHeaderCode
 	if statusCode < 100 || statusCode > 999 {
-		logger.Error(fmt.Sprintf("invalid WriteHeader code %v", statusCode))
+		bugLogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode))
 		statusCode = http.StatusInternalServerError
 	}
 	setCommonHeaders(w)
@@ -918,22 +947,25 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
-// writeErrorRespone writes error headers
+// writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
-	switch err.Code {
+	switch err.HTTPStatusCode {
-	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
+	case http.StatusServiceUnavailable, http.StatusTooManyRequests:
-		// Set retry-after header to indicate user-agents to retry request after 120secs.
+		// Set retry-after header to indicate user-agents to retry request after 60 seconds.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set(xhttp.RetryAfter, "120")
+		w.Header().Set(xhttp.RetryAfter, "60")
+	}
+
+	switch err.Code {
 	case "InvalidRegion":
-		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region)
+		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region())
 	case "AuthorizationHeaderMalformed":
-		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
+		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region())
 	}
 
 	// Similar check to http.checkWriteHeaderCode
 	if err.HTTPStatusCode < 100 || err.HTTPStatusCode > 999 {
-		logger.Error(fmt.Sprintf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
+		bugLogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
 		err.HTTPStatusCode = http.StatusInternalServerError
 	}
 

cmd/api-router.go

@@ -18,16 +18,13 @@
 package cmd
 
 import (
-	"compress/gzip"
 	"net"
 	"net/http"
 
-	"github.com/klauspost/compress/gzhttp"
+	consoleapi "github.com/minio/console/api"
-	"github.com/minio/console/restapi"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/v2/wildcard"
+	"github.com/minio/pkg/v3/wildcard"
 	"github.com/rs/cors"
 )
 
@@ -43,13 +40,13 @@ func setHTTPServer(h *xhttp.Server) {
 	globalObjLayerMutex.Unlock()
 }
 
-func newConsoleServerFn() *restapi.Server {
+func newConsoleServerFn() *consoleapi.Server {
 	globalObjLayerMutex.RLock()
 	defer globalObjLayerMutex.RUnlock()
 	return globalConsoleSrv
 }
 
-func setConsoleSrv(srv *restapi.Server) {
+func setConsoleSrv(srv *consoleapi.Server) {
 	globalObjLayerMutex.Lock()
 	globalConsoleSrv = srv
 	globalObjLayerMutex.Unlock()
@@ -67,7 +64,7 @@ func setObjectLayer(o ObjectLayer) {
 	globalObjLayerMutex.Unlock()
 }
 
-// objectAPIHandler implements and provides http handlers for S3 API.
+// objectAPIHandlers implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
 }
@@ -171,6 +168,87 @@ var rejectedBucketAPIs = []rejectedAPI{
 	},
 }
 
+// Set of s3 handler options as bit flags.
+type s3HFlag uint8
+
+const (
+	// when provided, disables Gzip compression.
+	noGZS3HFlag = 1 << iota
+
+	// when provided, enables only tracing of headers. Otherwise, both headers
+	// and body are traced.
+	traceHdrsS3HFlag
+
+	// when provided, disables throttling via the `maxClients` middleware.
+	noThrottleS3HFlag
+)
+
+func (h s3HFlag) has(flag s3HFlag) bool {
+	// Use bitwise-AND and check if the result is non-zero.
+	return h&flag != 0
+}
+
+// s3APIMiddleware - performs some common handler functionality for S3 API
+// handlers.
+//
+// It is set per-"handler function registration" in the router to allow for
+// behavior modification via flags.
+//
+// This middleware always calls `collectAPIStats` to collect API stats.
+//
+// The passed in handler function must be a method of `objectAPIHandlers` for
+// the name displayed in logs and trace to be accurate. The name is extracted
+// via reflection.
+//
+// When **no** flags are passed, the behavior is to trace both headers and body,
+// gzip the response and throttle the handler via `maxClients`. Each of these
+// can be disabled via the corresponding `s3HFlag`.
+//
+// CAUTION: for requests involving large req/resp bodies ensure to pass the
+// `traceHdrsS3HFlag`, otherwise both headers and body will be traced, causing
+// high memory usage!
+func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc {
+	// Collect all flags with bitwise-OR and assign operator
+	var handlerFlags s3HFlag
+	for _, flag := range flags {
+		handlerFlags |= flag
+	}
+
+	// Get name of the handler using reflection.
+	handlerName := getHandlerName(f, "objectAPIHandlers")
+
+	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		// Wrap the actual handler with the appropriate tracing middleware.
+		var tracedHandler http.HandlerFunc
+		if handlerFlags.has(traceHdrsS3HFlag) {
+			tracedHandler = httpTraceHdrs(f)
+		} else {
+			tracedHandler = httpTraceAll(f)
+		}
+
+		// Skip wrapping with the gzip middleware if specified.
+		gzippedHandler := tracedHandler
+		if !handlerFlags.has(noGZS3HFlag) {
+			gzippedHandler = gzipHandler(gzippedHandler)
+		}
+
+		// Skip wrapping with throttling middleware if specified.
+		throttledHandler := gzippedHandler
+		if !handlerFlags.has(noThrottleS3HFlag) {
+			throttledHandler = maxClients(throttledHandler)
+		}
+
+		// Collect API stats using the API name got from reflection in
+		// `getHandlerName`.
+		statsCollectedHandler := collectAPIStats(handlerName, throttledHandler)
+
+		// Call the final handler.
+		statsCollectedHandler(w, r)
+	}
+
+	return handler
+}
+
 // registerAPIRouter - registers S3 compatible APIs.
 func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
@@ -208,12 +286,6 @@ func registerAPIRouter(router *mux.Router) {
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
-	if err != nil {
-		// Static params, so this is very unlikely.
-		logger.Fatal(err, "Unable to initialize server")
-	}
-
 	for _, router := range routers {
 		// Register all rejected object APIs
 		for _, r := range rejectedObjAPIs {
@@ -225,237 +297,321 @@ func registerAPIRouter(router *mux.Router) {
 
 		// Object operations
 		// HeadObject
-		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodHead).Path("/{object:.+}").
-			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
+			HandlerFunc(s3APIMiddleware(api.HeadObjectHandler))
+
+		// GetObjectAttributes
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectAttributesHandler, traceHdrsS3HFlag)).
+			Queries("attributes", "")
+
 		// CopyObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
+			HandlerFunc(s3APIMiddleware(api.CopyObjectPartHandler)).
 			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
+			HandlerFunc(s3APIMiddleware(api.PutObjectPartHandler, traceHdrsS3HFlag)).
+			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
+			HandlerFunc(s3APIMiddleware(api.ListObjectPartsHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPost).Path("/{object:.+}").
-			collectAPIStats("completemultipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+			HandlerFunc(s3APIMiddleware(api.CompleteMultipartUploadHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPost).Path("/{object:.+}").
-			collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "")
+			HandlerFunc(s3APIMiddleware(api.NewMultipartUploadHandler)).
+			Queries("uploads", "")
 		// AbortMultipartUpload
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
-			collectAPIStats("abortmultipartupload", maxClients(gz(httpTraceAll(api.AbortMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+			HandlerFunc(s3APIMiddleware(api.AbortMultipartUploadHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("getobjectacl", maxClients(gz(httpTraceHdrs(api.GetObjectACLHandler))))).Queries("acl", "")
+			HandlerFunc(s3APIMiddleware(api.GetObjectACLHandler, traceHdrsS3HFlag)).
+			Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobjectacl", maxClients(gz(httpTraceHdrs(api.PutObjectACLHandler))))).Queries("acl", "")
+			HandlerFunc(s3APIMiddleware(api.PutObjectACLHandler, traceHdrsS3HFlag)).
+			Queries("acl", "")
 		// GetObjectTagging
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("getobjecttagging", maxClients(gz(httpTraceHdrs(api.GetObjectTaggingHandler))))).Queries("tagging", "")
+			HandlerFunc(s3APIMiddleware(api.GetObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// PutObjectTagging
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobjecttagging", maxClients(gz(httpTraceHdrs(api.PutObjectTaggingHandler))))).Queries("tagging", "")
+			HandlerFunc(s3APIMiddleware(api.PutObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// DeleteObjectTagging
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
-			collectAPIStats("deleteobjecttagging", maxClients(gz(httpTraceHdrs(api.DeleteObjectTaggingHandler))))).Queries("tagging", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// SelectObjectContent
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPost).Path("/{object:.+}").
-			collectAPIStats("selectobjectcontent", maxClients(gz(httpTraceHdrs(api.SelectObjectContentHandler))))).Queries("select", "").Queries("select-type", "2")
+			HandlerFunc(s3APIMiddleware(api.SelectObjectContentHandler, traceHdrsS3HFlag)).
+			Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("getobjectretention", maxClients(gz(httpTraceAll(api.GetObjectRetentionHandler))))).Queries("retention", "")
+			HandlerFunc(s3APIMiddleware(api.GetObjectRetentionHandler)).
+			Queries("retention", "")
 		// GetObjectLegalHold
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
+			HandlerFunc(s3APIMiddleware(api.GetObjectLegalHoldHandler)).
+			Queries("legal-hold", "")
 		// GetObject with lambda ARNs
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
+			HandlerFunc(s3APIMiddleware(api.GetObjectLambdaHandler, traceHdrsS3HFlag)).
+			Queries("lambdaArn", "{lambdaArn:.*}")
 		// GetObject
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodGet).Path("/{object:.+}").
-			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
+			HandlerFunc(s3APIMiddleware(api.GetObjectHandler, traceHdrsS3HFlag))
 		// CopyObject
-		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler)))))
+			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(s3APIMiddleware(api.CopyObjectHandler))
 		// PutObjectRetention
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobjectretention", maxClients(gz(httpTraceAll(api.PutObjectRetentionHandler))))).Queries("retention", "")
+			HandlerFunc(s3APIMiddleware(api.PutObjectRetentionHandler)).
+			Queries("retention", "")
 		// PutObjectLegalHold
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobjectlegalhold", maxClients(gz(httpTraceAll(api.PutObjectLegalHoldHandler))))).Queries("legal-hold", "")
+			HandlerFunc(s3APIMiddleware(api.PutObjectLegalHoldHandler)).
+			Queries("legal-hold", "")
 
 		// PutObject with auto-extract support for zip
-		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler)))))
+			HeadersRegexp(xhttp.AmzSnowballExtract, "true").
+			HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag))
+
+		// AppendObject to be rejected
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzWriteOffsetBytes, "").
+			HandlerFunc(s3APIMiddleware(errorResponseHandler))
 
 		// PutObject
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPut).Path("/{object:.+}").
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler)))))
+			HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag))
 
 		// DeleteObject
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
-			collectAPIStats("deleteobject", maxClients(gz(httpTraceAll(api.DeleteObjectHandler)))))
+			HandlerFunc(s3APIMiddleware(api.DeleteObjectHandler))
 
 		// PostRestoreObject
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+		router.Methods(http.MethodPost).Path("/{object:.+}").
-			collectAPIStats("restoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "")
+			HandlerFunc(s3APIMiddleware(api.PostRestoreObjectHandler)).
+			Queries("restore", "")
 
 		// Bucket operations
 
 		// GetBucketLocation
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketLocationHandler)).
+			Queries("location", "")
 		// GetBucketPolicy
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketpolicy", maxClients(gz(httpTraceAll(api.GetBucketPolicyHandler))))).Queries("policy", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketPolicyHandler)).
+			Queries("policy", "")
 		// GetBucketLifecycle
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketlifecycle", maxClients(gz(httpTraceAll(api.GetBucketLifecycleHandler))))).Queries("lifecycle", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// GetBucketEncryption
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketencryption", maxClients(gz(httpTraceAll(api.GetBucketEncryptionHandler))))).Queries("encryption", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketEncryptionHandler)).
+			Queries("encryption", "")
 		// GetBucketObjectLockConfig
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketobjectlockconfiguration", maxClients(gz(httpTraceAll(api.GetBucketObjectLockConfigHandler))))).Queries("object-lock", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketObjectLockConfigHandler)).
+			Queries("object-lock", "")
 		// GetBucketReplicationConfig
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.GetBucketReplicationConfigHandler))))).Queries("replication", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// GetBucketVersioning
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketversioning", maxClients(gz(httpTraceAll(api.GetBucketVersioningHandler))))).Queries("versioning", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketVersioningHandler)).
+			Queries("versioning", "")
 		// GetBucketNotification
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketNotificationHandler)).
+			Queries("notification", "")
 		// ListenNotification
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+			HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag, traceHdrsS3HFlag)).
+			Queries("events", "{events:.*}")
 		// ResetBucketReplicationStatus - MinIO extension API
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "")
+			HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStatusHandler)).
+			Queries("replication-reset-status", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketacl", maxClients(gz(httpTraceAll(api.GetBucketACLHandler))))).Queries("acl", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketACLHandler)).
+			Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketacl", maxClients(gz(httpTraceAll(api.PutBucketACLHandler))))).Queries("acl", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
+			Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketcors", maxClients(gz(httpTraceAll(api.GetBucketCorsHandler))))).Queries("cors", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
+			Queries("cors", "")
+		// PutBucketCors - this is a dummy call.
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
+			Queries("cors", "")
+		// DeleteBucketCors - this is a dummy call.
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
+			Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketwebsite", maxClients(gz(httpTraceAll(api.GetBucketWebsiteHandler))))).Queries("website", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)).
+			Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketaccelerate", maxClients(gz(httpTraceAll(api.GetBucketAccelerateHandler))))).Queries("accelerate", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketAccelerateHandler)).
+			Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketrequestpayment", maxClients(gz(httpTraceAll(api.GetBucketRequestPaymentHandler))))).Queries("requestPayment", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketRequestPaymentHandler)).
+			Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketlogging", maxClients(gz(httpTraceAll(api.GetBucketLoggingHandler))))).Queries("logging", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)).
+			Queries("logging", "")
+
 		// GetBucketTaggingHandler
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbuckettagging", maxClients(gz(httpTraceAll(api.GetBucketTaggingHandler))))).Queries("tagging", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)).
+			Queries("tagging", "")
 		// DeleteBucketWebsiteHandler
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebucketwebsite", maxClients(gz(httpTraceAll(api.DeleteBucketWebsiteHandler))))).Queries("website", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketWebsiteHandler)).
+			Queries("website", "")
 		// DeleteBucketTaggingHandler
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebuckettagging", maxClients(gz(httpTraceAll(api.DeleteBucketTaggingHandler))))).Queries("tagging", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketTaggingHandler)).
+			Queries("tagging", "")
 
 		// ListMultipartUploads
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listmultipartuploads", maxClients(gz(httpTraceAll(api.ListMultipartUploadsHandler))))).Queries("uploads", "")
+			HandlerFunc(s3APIMiddleware(api.ListMultipartUploadsHandler)).
+			Queries("uploads", "")
 		// ListObjectsV2M
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listobjectsv2M", maxClients(gz(httpTraceAll(api.ListObjectsV2MHandler))))).Queries("list-type", "2", "metadata", "true")
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV2MHandler)).
+			Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV2Handler)).
+			Queries("list-type", "2")
 		// ListObjectVersions
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
+			HandlerFunc(s3APIMiddleware(api.ListObjectVersionsMHandler)).
+			Queries("versions", "", "metadata", "true")
 		// ListObjectVersions
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
+			HandlerFunc(s3APIMiddleware(api.ListObjectVersionsHandler)).
+			Queries("versions", "")
 		// GetBucketPolicyStatus
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getpolicystatus", maxClients(gz(httpTraceAll(api.GetBucketPolicyStatusHandler))))).Queries("policyStatus", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketPolicyStatusHandler)).
+			Queries("policyStatus", "")
 		// PutBucketLifecycle
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketlifecycle", maxClients(gz(httpTraceAll(api.PutBucketLifecycleHandler))))).Queries("lifecycle", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// PutBucketReplicationConfig
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.PutBucketReplicationConfigHandler))))).Queries("replication", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// PutBucketEncryption
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketencryption", maxClients(gz(httpTraceAll(api.PutBucketEncryptionHandler))))).Queries("encryption", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketEncryptionHandler)).
+			Queries("encryption", "")
 
 		// PutBucketPolicy
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketpolicy", maxClients(gz(httpTraceAll(api.PutBucketPolicyHandler))))).Queries("policy", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketPolicyHandler)).
+			Queries("policy", "")
 
 		// PutBucketObjectLockConfig
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.PutBucketObjectLockConfigHandler))))).Queries("object-lock", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketObjectLockConfigHandler)).
+			Queries("object-lock", "")
 		// PutBucketTaggingHandler
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbuckettagging", maxClients(gz(httpTraceAll(api.PutBucketTaggingHandler))))).Queries("tagging", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketTaggingHandler)).
+			Queries("tagging", "")
 		// PutBucketVersioning
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketversioning", maxClients(gz(httpTraceAll(api.PutBucketVersioningHandler))))).Queries("versioning", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketVersioningHandler)).
+			Queries("versioning", "")
 		// PutBucketNotification
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "")
+			HandlerFunc(s3APIMiddleware(api.PutBucketNotificationHandler)).
+			Queries("notification", "")
 		// ResetBucketReplicationStart - MinIO extension API
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("resetbucketreplicationstart", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStartHandler))))).Queries("replication-reset", "")
+			HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStartHandler)).
+			Queries("replication-reset", "")
 
 		// PutBucket
-		router.Methods(http.MethodPut).HandlerFunc(
+		router.Methods(http.MethodPut).
-			collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler)))))
+			HandlerFunc(s3APIMiddleware(api.PutBucketHandler))
 		// HeadBucket
-		router.Methods(http.MethodHead).HandlerFunc(
+		router.Methods(http.MethodHead).
-			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
+			HandlerFunc(s3APIMiddleware(api.HeadBucketHandler))
 		// PostPolicy
-		router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+		router.Methods(http.MethodPost).
+			MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
 				return isRequestPostPolicySignatureV4(r)
-		}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
+			}).
+			HandlerFunc(s3APIMiddleware(api.PostPolicyBucketHandler, traceHdrsS3HFlag))
 		// DeleteMultipleObjects
-		router.Methods(http.MethodPost).HandlerFunc(
+		router.Methods(http.MethodPost).
-			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteMultipleObjectsHandler)).
+			Queries("delete", "")
 		// DeleteBucketPolicy
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebucketpolicy", maxClients(gz(httpTraceAll(api.DeleteBucketPolicyHandler))))).Queries("policy", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketPolicyHandler)).
+			Queries("policy", "")
 		// DeleteBucketReplication
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.DeleteBucketReplicationConfigHandler))))).Queries("replication", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// DeleteBucketLifecycle
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebucketlifecycle", maxClients(gz(httpTraceAll(api.DeleteBucketLifecycleHandler))))).Queries("lifecycle", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// DeleteBucketEncryption
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebucketencryption", maxClients(gz(httpTraceAll(api.DeleteBucketEncryptionHandler))))).Queries("encryption", "")
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketEncryptionHandler)).
+			Queries("encryption", "")
 		// DeleteBucket
-		router.Methods(http.MethodDelete).HandlerFunc(
+		router.Methods(http.MethodDelete).
-			collectAPIStats("deletebucket", maxClients(gz(httpTraceAll(api.DeleteBucketHandler)))))
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketHandler))
 
 		// MinIO extension API for replication.
 		//
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2")
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsV2Handler)).
+			Queries("replication-metrics", "2")
 		// deprecated handler
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsHandler)).
+			Queries("replication-metrics", "")
 
 		// ValidateBucketReplicationCreds
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
+			HandlerFunc(s3APIMiddleware(api.ValidateBucketReplicationCredsHandler)).
+			Queries("replication-check", "")
 
 		// Register rejected bucket APIs
 		for _, r := range rejectedBucketAPIs {
@@ -465,24 +621,25 @@ func registerAPIRouter(router *mux.Router) {
 		}
 
 		// S3 ListObjectsV1 (Legacy)
-		router.Methods(http.MethodGet).HandlerFunc(
+		router.Methods(http.MethodGet).
-			collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler)))))
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV1Handler))
 	}
 
 	// Root operation
 
 	// ListenNotification
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
-		collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+		HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag, traceHdrsS3HFlag)).
+		Queries("events", "{events:.*}")
 
 	// ListBuckets
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
-		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
+		HandlerFunc(s3APIMiddleware(api.ListBucketsHandler))
 
 	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
 	// than failing with UnknownAPIRequest we simply handle it for now.
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).
-		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
+		HandlerFunc(s3APIMiddleware(api.ListBucketsHandler))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))

cmd/api-utils.go

@@ -18,6 +18,10 @@
 package cmd
 
 import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"runtime"
 	"strings"
 )
 
@@ -39,7 +43,7 @@ func shouldEscape(c byte) bool {
 //   - Force encoding of '~'
 func s3URLEncode(s string) string {
 	spaceCount, hexCount := 0, 0
-	for i := 0; i < len(s); i++ {
+	for i := range len(s) {
 		c := s[i]
 		if shouldEscape(c) {
 			if c == ' ' {
@@ -66,7 +70,7 @@ func s3URLEncode(s string) string {
 
 	if hexCount == 0 {
 		copy(t, s)
-		for i := 0; i < len(s); i++ {
+		for i := range len(s) {
 			if s[i] == ' ' {
 				t[i] = '+'
 			}
@@ -75,7 +79,7 @@ func s3URLEncode(s string) string {
 	}
 
 	j := 0
-	for i := 0; i < len(s); i++ {
+	for i := range len(s) {
 		switch c := s[i]; {
 		case c == ' ':
 			t[j] = '+'
@@ -100,3 +104,16 @@ func s3EncodeName(name, encodingType string) string {
 	}
 	return name
 }
+
+// getHandlerName returns the name of the handler function. It takes the type
+// name as a string to clean up the name retrieved via reflection. This function
+// only works correctly when the type is present in the cmd package.
+func getHandlerName(f http.HandlerFunc, cmdType string) string {
+	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+
+	packageName := fmt.Sprintf("github.com/minio/minio/cmd.%s.", cmdType)
+	name = strings.TrimPrefix(name, packageName)
+	name = strings.TrimSuffix(name, "Handler-fm")
+	name = strings.TrimSuffix(name, "-fm")
+	return name
+}

cmd/auth-handler.go

@@ -41,7 +41,7 @@ import (
 	xjwt "github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 // Verify if request has JWT.
@@ -96,7 +96,7 @@ func isRequestSignStreamingTrailerV4(r *http.Request) bool {
 // Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer.
 func isRequestUnsignedTrailerV4(r *http.Request) bool {
 	return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer &&
-		r.Method == http.MethodPut && strings.Contains(r.Header.Get(xhttp.ContentEncoding), streamingContentEncoding)
+		r.Method == http.MethodPut
 }
 
 // Authorization type.
@@ -126,7 +126,7 @@ func getRequestAuthType(r *http.Request) (at authType) {
 		var err error
 		r.Form, err = url.ParseQuery(r.URL.RawQuery)
 		if err != nil {
-			logger.LogIf(r.Context(), err)
+			authNLogIf(r.Context(), err)
 			return authTypeUnknown
 		}
 	}
@@ -162,7 +162,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
 		getRequestAuthType(r) == authTypeSigned {
-		// We only support admin credentials to access admin APIs.
+		// Get credential information from the request.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
 			return cred, owner, s3Err
@@ -177,7 +177,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 
 	logger.GetReqInfo(ctx).Cred = cred
 	logger.GetReqInfo(ctx).Owner = owner
-	logger.GetReqInfo(ctx).Region = globalSite.Region
+	logger.GetReqInfo(ctx).Region = globalSite.Region()
 
 	return cred, owner, ErrNone
 }
@@ -221,7 +221,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }
 
-func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
+func getClaimsFromTokenWithSecret(token, secret string) (*xjwt.MapClaims, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -243,7 +243,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 
 	// If AuthZPlugin is set, return without any further checks.
 	if newGlobalAuthZPluginFn() != nil {
-		return claims.Map(), nil
+		return claims, nil
 	}
 
 	// Check if a session policy is set. If so, decode it here.
@@ -256,18 +256,22 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(GlobalContext, err, logger.Application)
+			authNLogIf(GlobalContext, err, logger.ErrorKind)
 			return nil, errAuthentication
 		}
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
 	}
 
-	return claims.Map(), nil
+	return claims, nil
 }
 
 // Fetch claims in the security token returned by the client.
 func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+	jwtClaims, err := getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+	return jwtClaims.Map(), nil
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
@@ -293,7 +297,21 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrInvalidToken
 	}
 
+	// Expired credentials must return error right away.
+	if cred.IsTemp() && cred.IsExpired() {
+		return nil, toAPIErrorCode(r.Context(), errInvalidAccessKeyID)
+	}
 	secret := globalActiveCred.SecretKey
+	if globalSiteReplicationSys.isEnabled() && cred.AccessKey != siteReplicatorSvcAcc {
+		nsecret, err := getTokenSigningKey()
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		// sign root's temporary accounts also with site replicator creds
+		if cred.ParentUser != globalActiveCred.AccessKey || cred.IsTemp() {
+			secret = nsecret
+		}
+	}
 	if cred.IsServiceAccount() {
 		token = cred.SessionToken
 		secret = cred.SecretKey
@@ -304,7 +322,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		if err != nil {
 			return nil, toAPIErrorCode(r.Context(), err)
 		}
-		return claims, ErrNone
+		return claims.Map(), ErrNone
 	}
 
 	claims := xjwt.NewMapClaims()
@@ -338,14 +356,14 @@ func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action po
 
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
-		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
+		bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
 		return ErrAccessDenied
 	}
 
 	var cred auth.Credentials
 	var owner bool
 	switch getRequestAuthType(r) {
-	case authTypeUnknown, authTypeStreamingSigned:
+	case authTypeUnknown, authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
 		return ErrSignatureVersionNotSupported
 	case authTypePresignedV2, authTypeSignedV2:
 		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
@@ -353,7 +371,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
-		region := globalSite.Region
+		region := globalSite.Region()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
@@ -369,7 +387,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 
 	logger.GetReqInfo(ctx).Cred = cred
 	logger.GetReqInfo(ctx).Owner = owner
-	logger.GetReqInfo(ctx).Region = globalSite.Region
+	logger.GetReqInfo(ctx).Region = globalSite.Region()
 
 	// region is valid only for CreateBucketAction.
 	var region string
@@ -377,7 +395,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 		// To extract region from XML in request body, get copy of request body.
 		payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
-			logger.LogIf(ctx, err, logger.Application)
+			authZLogIf(ctx, err, logger.ErrorKind)
 			return ErrMalformedXML
 		}
 
@@ -656,32 +674,6 @@ func setAuthMiddleware(h http.Handler) http.Handler {
 	})
 }
 
-func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, APIErrorCode) {
-	var cred auth.Credentials
-	var owner bool
-	var s3Err APIErrorCode
-	switch atype {
-	case authTypeUnknown, authTypeStreamingSigned:
-		return cred, owner, ErrSignatureVersionNotSupported
-	case authTypeSignedV2, authTypePresignedV2:
-		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
-			return cred, owner, s3Err
-		}
-		cred, owner, s3Err = getReqAccessKeyV2(r)
-	case authTypePresigned, authTypeSigned:
-		region := globalSite.Region
-		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
-			return cred, owner, s3Err
-		}
-		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
-	}
-	if s3Err != ErrNone {
-		return cred, owner, s3Err
-	}
-
-	return cred, owner, ErrNone
-}
-
 func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
 	var retSet bool
 	if cred.AccessKey == "" {
@@ -730,14 +722,20 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
-	region := globalSite.Region
+	region := globalSite.Region()
 	switch atype {
 	case authTypeUnknown:
 		return ErrSignatureVersionNotSupported
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
-	case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer:
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+	case authTypeStreamingUnsignedTrailer:
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+		if s3Err == ErrMissingFields {
+			// Could be anonymous. cred + owner is zero value.
+			s3Err = ErrNone
+		}
 	}
 	if s3Err != ErrNone {
 		return s3Err

cmd/auth-handler_test.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v3/policy"
 )
 
 type nullReader struct{}
@@ -237,7 +237,7 @@ func TestIsRequestPresignedSignatureV2(t *testing.T) {
 	}
 }
 
-// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature verision v4 detection.
+// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature version v4 detection.
 func TestIsRequestPresignedSignatureV4(t *testing.T) {
 	testCases := []struct {
 		inputQueryKey   string
@@ -287,7 +287,7 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -298,7 +298,7 @@ func mustNewSignedV2Request(method string, urlStr string, contentLength int64, b
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -309,7 +309,7 @@ func mustNewPresignedV2Request(method string, urlStr string, contentLength int64
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -320,7 +320,7 @@ func mustNewPresignedRequest(method string, urlStr string, contentLength int64,
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -403,7 +403,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region(), serviceS3)
 		if s3Error != testCase.s3Error {
 			if _, err := io.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
@@ -413,7 +413,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 }
 
 func TestCheckAdminRequestAuthType(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
 	objLayer, fsDir, err := prepareFS(ctx)
@@ -443,14 +443,14 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region()); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}
 }
 
 func TestValidateAdminSignature(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
 
 	objLayer, fsDir, err := prepareFS(ctx)
@@ -491,7 +491,7 @@ func TestValidateAdminSignature(t *testing.T) {
 	for i, testCase := range testCases {
 		req := mustNewRequest(http.MethodGet, "http://localhost:9000/", 0, nil, t)
 		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
-			t.Fatalf("Unable to inititalized new signed http request %s", err)
+			t.Fatalf("Unable to initialized new signed http request %s", err)
 		}
 		_, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
 		if s3Error != testCase.ErrCode {

cmd/background-heal-ops.go

@@ -25,8 +25,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/env"
-	"github.com/minio/pkg/v2/env"
 )
 
 // healTask represents what to heal along with options
@@ -101,17 +100,17 @@ func waitForLowHTTPReq() {
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	bgSeq := newBgHealSequence()
 	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
+	for range globalBackgroundHealRoutine.workers {
-	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
+		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI, bgSeq)
-		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
 	}
 
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
+	globalBackgroundHealState.LaunchNewHealSequence(bgSeq, objAPI)
 }
 
 // Wait for heal requests and process them
-func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
+func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer, bgSeq *healSequence) {
 	for {
 		select {
 		case task, ok := <-h.tasks:
@@ -136,8 +135,18 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 
 			if task.respCh != nil {
 				task.respCh <- healResult{result: res, err: err}
+				continue
 			}
 
+			// when respCh is not set caller is not waiting but we
+			// update the relevant metrics for them
+			if bgSeq != nil {
+				if err == nil {
+					bgSeq.countHealed(res.Type)
+				} else {
+					bgSeq.countFailed(res.Type)
+				}
+			}
 		case <-ctx.Done():
 			return
 		}
@@ -149,7 +158,7 @@ func newHealRoutine() *healRoutine {
 
 	if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" {
 		if numHealers, err := strconv.Atoi(envHealWorkers); err != nil {
-			logger.LogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
+			bugLogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
 		} else {
 			workers = numHealers
 		}

cmd/background-newdisks-heal-ops.go

@@ -33,8 +33,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/env"
-	"github.com/minio/pkg/v2/env"
 )
 
 const (
@@ -75,8 +74,10 @@ type healingTracker struct {
 	// for resuming with correct numbers.
 	ResumeItemsHealed  uint64 `json:"-"`
 	ResumeItemsFailed  uint64 `json:"-"`
+	ResumeItemsSkipped uint64 `json:"-"`
 	ResumeBytesDone    uint64 `json:"-"`
 	ResumeBytesFailed  uint64 `json:"-"`
+	ResumeBytesSkipped uint64 `json:"-"`
 
 	// Filled on startup/restarts.
 	QueuedBuckets []string
@@ -87,6 +88,13 @@ type healingTracker struct {
 	// ID of the current healing operation
 	HealID string
 
+	ItemsSkipped uint64
+	BytesSkipped uint64
+
+	RetryAttempts uint64
+
+	Finished bool // finished healing, whether with errors or not
+
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -140,14 +148,34 @@ func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	return h
 }
 
-func (h healingTracker) getLastUpdate() time.Time {
+func (h *healingTracker) resetHealing() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.ItemsHealed = 0
+	h.ItemsFailed = 0
+	h.BytesDone = 0
+	h.BytesFailed = 0
+	h.ResumeItemsHealed = 0
+	h.ResumeItemsFailed = 0
+	h.ResumeBytesDone = 0
+	h.ResumeBytesFailed = 0
+	h.ItemsSkipped = 0
+	h.BytesSkipped = 0
+
+	h.HealedBuckets = nil
+	h.Object = ""
+	h.Bucket = ""
+}
+
+func (h *healingTracker) getLastUpdate() time.Time {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
 	return h.LastUpdate
 }
 
-func (h healingTracker) getBucket() string {
+func (h *healingTracker) getBucket() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
@@ -161,7 +189,7 @@ func (h *healingTracker) setBucket(bucket string) {
 	h.Bucket = bucket
 }
 
-func (h healingTracker) getObject() string {
+func (h *healingTracker) getObject() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
@@ -175,14 +203,18 @@ func (h *healingTracker) setObject(object string) {
 	h.Object = object
 }
 
-func (h *healingTracker) updateProgress(success bool, bytes uint64) {
+func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	if success {
+	switch {
+	case success:
 		h.ItemsHealed++
 		h.BytesDone += bytes
-	} else {
+	case skipped:
+		h.ItemsSkipped++
+		h.BytesSkipped += bytes
+	default:
 		h.ItemsFailed++
 		h.BytesFailed += bytes
 	}
@@ -191,9 +223,6 @@ func (h *healingTracker) updateProgress(success bool, bytes uint64) {
 // update will update the tracker on the disk.
 // If the tracker has been deleted an error is returned.
 func (h *healingTracker) update(ctx context.Context) error {
-	if h.disk.Healing() == nil {
-		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
-	}
 	h.mu.Lock()
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
@@ -232,7 +261,7 @@ func (h *healingTracker) delete(ctx context.Context) error {
 		pathJoin(bucketMetaPrefix, healingTrackerFilename),
 		DeleteOptions{
 			Recursive: false,
-			Force:     false,
+			Immediate: false,
 		},
 	)
 }
@@ -255,8 +284,10 @@ func (h *healingTracker) resume() {
 
 	h.ItemsHealed = h.ResumeItemsHealed
 	h.ItemsFailed = h.ResumeItemsFailed
+	h.ItemsSkipped = h.ResumeItemsSkipped
 	h.BytesDone = h.ResumeBytesDone
 	h.BytesFailed = h.ResumeBytesFailed
+	h.BytesSkipped = h.ResumeBytesSkipped
 }
 
 // bucketDone should be called when a bucket is done healing.
@@ -267,8 +298,10 @@ func (h *healingTracker) bucketDone(bucket string) {
 
 	h.ResumeItemsHealed = h.ItemsHealed
 	h.ResumeItemsFailed = h.ItemsFailed
+	h.ResumeItemsSkipped = h.ItemsSkipped
 	h.ResumeBytesDone = h.BytesDone
 	h.ResumeBytesFailed = h.BytesFailed
+	h.ResumeBytesSkipped = h.BytesSkipped
 	h.HealedBuckets = append(h.HealedBuckets, bucket)
 	for i, b := range h.QueuedBuckets {
 		if b == bucket {
@@ -317,19 +350,23 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		PoolIndex:         h.PoolIndex,
 		SetIndex:          h.SetIndex,
 		DiskIndex:         h.DiskIndex,
+		Finished:          h.Finished,
 		Path:              h.Path,
 		Started:           h.Started.UTC(),
 		LastUpdate:        h.LastUpdate.UTC(),
 		ObjectsTotalCount: h.ObjectsTotalCount,
 		ObjectsTotalSize:  h.ObjectsTotalSize,
 		ItemsHealed:       h.ItemsHealed,
+		ItemsSkipped:      h.ItemsSkipped,
 		ItemsFailed:       h.ItemsFailed,
 		BytesDone:         h.BytesDone,
+		BytesSkipped:      h.BytesSkipped,
 		BytesFailed:       h.BytesFailed,
 		Bucket:            h.Bucket,
 		Object:            h.Object,
 		QueuedBuckets:     h.QueuedBuckets,
 		HealedBuckets:     h.HealedBuckets,
+		RetryAttempts:     h.RetryAttempts,
 
 		ObjectsHealed: h.ItemsHealed, // Deprecated July 2021
 		ObjectsFailed: h.ItemsFailed, // Deprecated July 2021
@@ -344,25 +381,26 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	}
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
-
+	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn {
 		globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
-
-	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
+
+	go globalMRFState.startMRFPersistence()
+	go globalMRFState.healRoutine(z)
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	globalLocalDrives := globalLocalDrives
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
-	for _, disk := range globalLocalDrives {
+	for _, disk := range localDrives {
-		_, err := disk.GetDiskID()
+		_, err := disk.DiskInfo(context.Background(), DiskInfoOptions{})
 		if errors.Is(err, errUnformattedDisk) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 			continue
 		}
-		if disk.Healing() != nil {
+		if h := disk.Healing(); h != nil && !h.Finished {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 		}
 	}
@@ -376,26 +414,24 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
+var errRetryHealing = errors.New("some items failed to heal, we will retry healing this drive again")
+
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
-	disk, format, err := connectEndpoint(endpoint)
+	poolIdx, setIdx := endpoint.PoolIdx, endpoint.SetIdx
-	if err != nil {
+	disk := getStorageViaEndpoint(endpoint)
-		return fmt.Errorf("Error: %w, %s", err, endpoint)
+	if disk == nil {
-	}
+		return fmt.Errorf("Unexpected error disk must be initialized by now after formatting: %s", endpoint)
-	defer disk.Close()
-	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
-	if poolIdx < 0 {
-		return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
 	}
 
-	// Calculate the set index where the current endpoint belongs
+	_, err := disk.DiskInfo(ctx, DiskInfoOptions{})
-	z.serverPools[poolIdx].erasureDisksMu.RLock()
-	setIdx, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
-	z.serverPools[poolIdx].erasureDisksMu.RUnlock()
 	if err != nil {
+		if errors.Is(err, errDriveIsRoot) {
+			// This is a root drive, ignore and move on
+			return nil
+		}
+		if !errors.Is(err, errUnformattedDisk) {
 			return err
 		}
-	if setIdx < 0 {
-		return fmt.Errorf("unexpected set index (%d) found for  %s", setIdx, disk.Endpoint())
 	}
 
 	// Prevent parallel erasure set healing
@@ -417,11 +453,11 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		if errors.Is(err, errFileNotFound) {
 			return nil
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
+		healingLogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
 		tracker = initHealingTracker(disk, mustGetUUID())
 	}
 
-	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
+	healingLogEvent(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint)
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple pools/sets, make
@@ -441,10 +477,6 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return buckets[i].Created.After(buckets[j].Created)
 	})
 
-	if serverDebugLog {
-		logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
-	}
-
 	// Load bucket totals
 	cache := dataUsageCache{}
 	if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil {
@@ -464,23 +496,37 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 
+	// if objects have failed healing, we attempt a retry to heal the drive upto 3 times before giving up.
+	if tracker.ItemsFailed > 0 && tracker.RetryAttempts < 4 {
+		tracker.RetryAttempts++
+
+		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retrying %s time (healed: %d, skipped: %d, failed: %d).", disk,
+			humanize.Ordinal(int(tracker.RetryAttempts)), tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+
+		tracker.resetHealing()
+		bugLogIf(ctx, tracker.update(ctx))
+
+		return errRetryHealing
+	}
+
 	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retried %d times (healed: %d, skipped: %d, failed: %d).", disk,
+			tracker.RetryAttempts, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 	} else {
-		logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		if tracker.RetryAttempts > 0 {
+			healingLogEvent(ctx, "Healing of drive '%s' is complete, retried %d times (healed: %d, skipped: %d).", disk,
+				tracker.RetryAttempts-1, tracker.ItemsHealed, tracker.ItemsSkipped)
+		} else {
+			healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped)
 		}
-
-	if len(tracker.QueuedBuckets) > 0 {
-		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
 	}
-
 	if serverDebugLog {
 		tracker.printTo(os.Stdout)
-		logger.Info("\n")
+		fmt.Printf("\n")
 	}
 
 	if tracker.HealID == "" { // HealID was empty only before Feb 2023
-		logger.LogIf(ctx, tracker.delete(ctx))
+		bugLogIf(ctx, tracker.delete(ctx))
 		return nil
 	}
 
@@ -491,15 +537,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	for _, disk := range disks {
+		if disk == nil {
+			continue
+		}
+
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 			}
 			continue
 		}
 		if t.HealID == tracker.HealID {
-			t.delete(ctx)
+			t.Finished = true
+			t.update(ctx)
 		}
 	}
 
@@ -529,7 +580,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 			// Reformat disks immediately
 			_, err := z.HealFormat(context.Background(), false)
 			if err != nil && !errors.Is(err, errNoHealRequired) {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 				// Reset for next interval.
 				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 				continue
@@ -541,7 +592,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
 						timedout := OperationTimedOut{}
-						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
+						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) && !errors.Is(err, errRetryHealing) {
 							printEndpointError(disk, err, false)
 						}
 						return

cmd/background-newdisks-heal-ops_gen.go

@@ -132,6 +132,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ResumeItemsFailed")
 				return
 			}
+		case "ResumeItemsSkipped":
+			z.ResumeItemsSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeItemsSkipped")
+				return
+			}
 		case "ResumeBytesDone":
 			z.ResumeBytesDone, err = dc.ReadUint64()
 			if err != nil {
@@ -144,6 +150,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ResumeBytesFailed")
 				return
 			}
+		case "ResumeBytesSkipped":
+			z.ResumeBytesSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesSkipped")
+				return
+			}
 		case "QueuedBuckets":
 			var zb0002 uint32
 			zb0002, err = dc.ReadArrayHeader()
@@ -188,6 +200,30 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "HealID")
 				return
 			}
+		case "ItemsSkipped":
+			z.ItemsSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ItemsSkipped")
+				return
+			}
+		case "BytesSkipped":
+			z.BytesSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "BytesSkipped")
+				return
+			}
+		case "RetryAttempts":
+			z.RetryAttempts, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "RetryAttempts")
+				return
+			}
+		case "Finished":
+			z.Finished, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "Finished")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -201,9 +237,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 23
+	// map header, size 29
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -382,6 +418,16 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ResumeItemsFailed")
 		return
 	}
+	// write "ResumeItemsSkipped"
+	err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeItemsSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeItemsSkipped")
+		return
+	}
 	// write "ResumeBytesDone"
 	err = en.Append(0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
 	if err != nil {
@@ -402,6 +448,16 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ResumeBytesFailed")
 		return
 	}
+	// write "ResumeBytesSkipped"
+	err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeBytesSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeBytesSkipped")
+		return
+	}
 	// write "QueuedBuckets"
 	err = en.Append(0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
 	if err != nil {
@@ -446,15 +502,55 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "HealID")
 		return
 	}
+	// write "ItemsSkipped"
+	err = en.Append(0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ItemsSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ItemsSkipped")
+		return
+	}
+	// write "BytesSkipped"
+	err = en.Append(0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.BytesSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "BytesSkipped")
+		return
+	}
+	// write "RetryAttempts"
+	err = en.Append(0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.RetryAttempts)
+	if err != nil {
+		err = msgp.WrapError(err, "RetryAttempts")
+		return
+	}
+	// write "Finished"
+	err = en.Append(0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.Finished)
+	if err != nil {
+		err = msgp.WrapError(err, "Finished")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 23
+	// map header, size 29
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -507,12 +603,18 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ResumeItemsFailed"
 	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.ResumeItemsFailed)
+	// string "ResumeItemsSkipped"
+	o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeItemsSkipped)
 	// string "ResumeBytesDone"
 	o = append(o, 0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
 	o = msgp.AppendUint64(o, z.ResumeBytesDone)
 	// string "ResumeBytesFailed"
 	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.ResumeBytesFailed)
+	// string "ResumeBytesSkipped"
+	o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeBytesSkipped)
 	// string "QueuedBuckets"
 	o = append(o, 0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.QueuedBuckets)))
@@ -528,6 +630,18 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "HealID"
 	o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
 	o = msgp.AppendString(o, z.HealID)
+	// string "ItemsSkipped"
+	o = append(o, 0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ItemsSkipped)
+	// string "BytesSkipped"
+	o = append(o, 0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.BytesSkipped)
+	// string "RetryAttempts"
+	o = append(o, 0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	o = msgp.AppendUint64(o, z.RetryAttempts)
+	// string "Finished"
+	o = append(o, 0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.Finished)
 	return
 }
 
@@ -657,6 +771,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ResumeItemsFailed")
 				return
 			}
+		case "ResumeItemsSkipped":
+			z.ResumeItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeItemsSkipped")
+				return
+			}
 		case "ResumeBytesDone":
 			z.ResumeBytesDone, bts, err = msgp.ReadUint64Bytes(bts)
 			if err != nil {
@@ -669,6 +789,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ResumeBytesFailed")
 				return
 			}
+		case "ResumeBytesSkipped":
+			z.ResumeBytesSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesSkipped")
+				return
+			}
 		case "QueuedBuckets":
 			var zb0002 uint32
 			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -713,6 +839,30 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "HealID")
 				return
 			}
+		case "ItemsSkipped":
+			z.ItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ItemsSkipped")
+				return
+			}
+		case "BytesSkipped":
+			z.BytesSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BytesSkipped")
+				return
+			}
+		case "RetryAttempts":
+			z.RetryAttempts, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RetryAttempts")
+				return
+			}
+		case "Finished":
+			z.Finished, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Finished")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -727,7 +877,7 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *healingTracker) Msgsize() (s int) {
-	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
+	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
 	for za0001 := range z.QueuedBuckets {
 		s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001])
 	}
@@ -735,6 +885,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
-	s += 7 + msgp.StringPrefixSize + len(z.HealID)
+	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size + 9 + msgp.BoolSize
 	return
 }

cmd/batch-expire.go

@@ -0,0 +1,839 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/pkg/v3/env"
+	"github.com/minio/pkg/v3/wildcard"
+	"github.com/minio/pkg/v3/workers"
+	"github.com/minio/pkg/v3/xtime"
+	"gopkg.in/yaml.v3"
+)
+
+// expire: # Expire objects that match a condition
+//   apiVersion: v1
+//   bucket: mybucket # Bucket where this batch job will expire matching objects from
+//   prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
+//   rules:
+//     - type: object  # regular objects with zero or more older versions
+//       name: NAME # match object names that satisfy the wildcard expression.
+//       olderThan: 70h # match objects older than this value
+//       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+//       tags:
+//         - key: name
+//           value: pick* # match objects with tag 'name', all values starting with 'pick'
+//       metadata:
+//         - key: content-type
+//           value: image/* # match objects with 'content-type', all values starting with 'image/'
+//       size:
+//         lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
+//         greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
+//       purge:
+//           # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+//           # retainVersions: 5 # keep the latest 5 versions of the object.
+//
+//     - type: deleted # objects with delete marker as their latest version
+//       name: NAME # match object names that satisfy the wildcard expression.
+//       olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
+//       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+//       purge:
+//           # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+//           # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
+//
+//   notify:
+//     endpoint: https://notify.endpoint # notification endpoint to receive job completion status
+//     token: Bearer xxxxx # optional authentication token for the notification endpoint
+//
+//   retry:
+//     attempts: 10 # number of retries for the job before giving up
+//     delay: 500ms # least amount of delay between each retry
+
+//go:generate msgp -file $GOFILE
+
+// BatchJobExpirePurge type accepts non-negative versions to be retained
+type BatchJobExpirePurge struct {
+	line, col      int
+	RetainVersions int `yaml:"retainVersions" json:"retainVersions"`
+}
+
+var _ yaml.Unmarshaler = &BatchJobExpirePurge{}
+
+// UnmarshalYAML - BatchJobExpirePurge extends unmarshal to extract line, col
+func (p *BatchJobExpirePurge) UnmarshalYAML(val *yaml.Node) error {
+	type purge BatchJobExpirePurge
+	var tmp purge
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*p = BatchJobExpirePurge(tmp)
+	p.line, p.col = val.Line, val.Column
+	return nil
+}
+
+// Validate returns nil if value is valid, ie > 0.
+func (p BatchJobExpirePurge) Validate() error {
+	if p.RetainVersions < 0 {
+		return BatchJobYamlErr{
+			line: p.line,
+			col:  p.col,
+			msg:  "retainVersions must be >= 0",
+		}
+	}
+	return nil
+}
+
+// BatchJobExpireFilter holds all the filters currently supported for batch replication
+type BatchJobExpireFilter struct {
+	line, col     int
+	OlderThan     xtime.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedBefore *time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV        `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV        `yaml:"metadata,omitempty" json:"metadata"`
+	Size          BatchJobSizeFilter  `yaml:"size" json:"size"`
+	Type          string              `yaml:"type" json:"type"`
+	Name          string              `yaml:"name" json:"name"`
+	Purge         BatchJobExpirePurge `yaml:"purge" json:"purge"`
+}
+
+var _ yaml.Unmarshaler = &BatchJobExpireFilter{}
+
+// UnmarshalYAML - BatchJobExpireFilter extends unmarshal to extract line, col
+// information
+func (ef *BatchJobExpireFilter) UnmarshalYAML(value *yaml.Node) error {
+	type expFilter BatchJobExpireFilter
+	var tmp expFilter
+	err := value.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+	*ef = BatchJobExpireFilter(tmp)
+	ef.line, ef.col = value.Line, value.Column
+	return err
+}
+
+// Matches returns true if obj matches the filter conditions specified in ef.
+func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
+	switch ef.Type {
+	case BatchJobExpireObject:
+		if obj.DeleteMarker {
+			return false
+		}
+	case BatchJobExpireDeleted:
+		if !obj.DeleteMarker {
+			return false
+		}
+	default:
+		// we should never come here, Validate should have caught this.
+		batchLogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
+		return false
+	}
+
+	if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
+		return false
+	}
+	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan.D() {
+		return false
+	}
+
+	if ef.CreatedBefore != nil && !obj.ModTime.Before(*ef.CreatedBefore) {
+		return false
+	}
+
+	if len(ef.Tags) > 0 && !obj.DeleteMarker {
+		// Only parse object tags if tags filter is specified.
+		var tagMap map[string]string
+		if len(obj.UserTags) != 0 {
+			t, err := tags.ParseObjectTags(obj.UserTags)
+			if err != nil {
+				return false
+			}
+			tagMap = t.ToMap()
+		}
+
+		for _, kv := range ef.Tags {
+			// Object (version) must match all tags specified in
+			// the filter
+			var match bool
+			for t, v := range tagMap {
+				if kv.Match(BatchJobKV{Key: t, Value: v}) {
+					match = true
+				}
+			}
+			if !match {
+				return false
+			}
+		}
+	}
+
+	if len(ef.Metadata) > 0 && !obj.DeleteMarker {
+		for _, kv := range ef.Metadata {
+			// Object (version) must match all x-amz-meta and
+			// standard metadata headers
+			// specified in the filter
+			var match bool
+			for k, v := range obj.UserDefined {
+				if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
+					continue
+				}
+				// We only need to match x-amz-meta or standardHeaders
+				if kv.Match(BatchJobKV{Key: k, Value: v}) {
+					match = true
+				}
+			}
+			if !match {
+				return false
+			}
+		}
+	}
+
+	return ef.Size.InRange(obj.Size)
+}
+
+const (
+	// BatchJobExpireObject - object type
+	BatchJobExpireObject string = "object"
+	// BatchJobExpireDeleted - delete marker type
+	BatchJobExpireDeleted string = "deleted"
+)
+
+// Validate returns nil if ef has valid fields, validation error otherwise.
+func (ef BatchJobExpireFilter) Validate() error {
+	switch ef.Type {
+	case BatchJobExpireObject:
+	case BatchJobExpireDeleted:
+		if len(ef.Tags) > 0 || len(ef.Metadata) > 0 {
+			return BatchJobYamlErr{
+				line: ef.line,
+				col:  ef.col,
+				msg:  "delete type filter can't have tags or metadata",
+			}
+		}
+	default:
+		return BatchJobYamlErr{
+			line: ef.line,
+			col:  ef.col,
+			msg:  "invalid batch-expire type",
+		}
+	}
+
+	for _, tag := range ef.Tags {
+		if err := tag.Validate(); err != nil {
+			return err
+		}
+	}
+
+	for _, meta := range ef.Metadata {
+		if err := meta.Validate(); err != nil {
+			return err
+		}
+	}
+	if err := ef.Purge.Validate(); err != nil {
+		return err
+	}
+	if err := ef.Size.Validate(); err != nil {
+		return err
+	}
+	if ef.CreatedBefore != nil && !ef.CreatedBefore.Before(time.Now()) {
+		return BatchJobYamlErr{
+			line: ef.line,
+			col:  ef.col,
+			msg:  "CreatedBefore is in the future",
+		}
+	}
+	return nil
+}
+
+// BatchJobExpire represents configuration parameters for a batch expiration
+// job typically supplied in yaml form
+type BatchJobExpire struct {
+	line, col       int
+	APIVersion      string                 `yaml:"apiVersion" json:"apiVersion"`
+	Bucket          string                 `yaml:"bucket" json:"bucket"`
+	Prefix          BatchJobPrefix         `yaml:"prefix" json:"prefix"`
+	NotificationCfg BatchJobNotification   `yaml:"notify" json:"notify"`
+	Retry           BatchJobRetry          `yaml:"retry" json:"retry"`
+	Rules           []BatchJobExpireFilter `yaml:"rules" json:"rules"`
+}
+
+var _ yaml.Unmarshaler = &BatchJobExpire{}
+
+// RedactSensitive will redact any sensitive information in b.
+func (r *BatchJobExpire) RedactSensitive() {
+	if r == nil {
+		return
+	}
+	if r.NotificationCfg.Token != "" {
+		r.NotificationCfg.Token = redactedText
+	}
+}
+
+// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
+func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
+	type expireJob BatchJobExpire
+	var tmp expireJob
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*r = BatchJobExpire(tmp)
+	r.line, r.col = val.Line, val.Column
+	return nil
+}
+
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
+	if r.NotificationCfg.Endpoint == "" {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.NotificationCfg.Endpoint, body)
+	if err != nil {
+		return err
+	}
+
+	if r.NotificationCfg.Token != "" {
+		req.Header.Set("Authorization", r.NotificationCfg.Token)
+	}
+
+	clnt := http.Client{Transport: getRemoteInstanceTransport()}
+	resp, err := clnt.Do(req)
+	if err != nil {
+		return err
+	}
+
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+// Expire expires object versions which have already matched supplied filter conditions
+func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
+	opts := ObjectOptions{
+		PrefixEnabledFn:  vc.PrefixEnabled,
+		VersionSuspended: vc.Suspended(),
+	}
+
+	allErrs := make([]error, 0, len(objsToDel))
+
+	for {
+		count := len(objsToDel)
+		if count == 0 {
+			break
+		}
+		if count > maxDeleteList {
+			count = maxDeleteList
+		}
+		_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
+		allErrs = append(allErrs, errs...)
+		// Next batch of deletion
+		objsToDel = objsToDel[count:]
+	}
+
+	return allErrs
+}
+
+const (
+	batchExpireName                 = "batch-expire.bin"
+	batchExpireFormat               = 1
+	batchExpireVersionV1            = 1
+	batchExpireVersion              = batchExpireVersionV1
+	batchExpireAPIVersion           = "v1"
+	batchExpireJobDefaultRetries    = 3
+	batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
+)
+
+type objInfoCache map[string]*ObjectInfo
+
+func newObjInfoCache() objInfoCache {
+	return objInfoCache(make(map[string]*ObjectInfo))
+}
+
+func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
+	oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
+}
+
+func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
+	oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
+	return oi, ok
+}
+
+func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
+	vc, _ := globalBucketVersioningSys.Get(r.Bucket)
+	retryAttempts := job.Expire.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchExpireJobDefaultRetries
+	}
+	delay := job.Expire.Retry.Delay
+	if delay <= 0 {
+		delay = batchExpireJobDefaultRetryDelay
+	}
+
+	var i int
+	for toExpire := range expireCh {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		if i > 0 {
+			if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
+				time.Sleep(wait)
+			}
+		}
+		i++
+		wk.Take()
+		go func(toExpire []expireObjInfo) {
+			defer wk.Give()
+
+			toExpireAll := make([]expireObjInfo, 0, len(toExpire))
+			toDel := make([]ObjectToDelete, 0, len(toExpire))
+			oiCache := newObjInfoCache()
+			for _, exp := range toExpire {
+				if exp.ExpireAll {
+					toExpireAll = append(toExpireAll, exp)
+					continue
+				}
+				// Cache ObjectInfo value via pointers for
+				// subsequent use to track objects which
+				// couldn't be deleted.
+				od := ObjectToDelete{
+					ObjectV: ObjectV{
+						ObjectName: exp.Name,
+						VersionID:  exp.VersionID,
+					},
+				}
+				toDel = append(toDel, od)
+				oiCache.Add(od, &exp.ObjectInfo)
+			}
+
+			// DeleteObject(deletePrefix: true) to expire all versions of an object
+			for _, exp := range toExpireAll {
+				var success bool
+				for attempts := 1; attempts <= retryAttempts; attempts++ {
+					select {
+					case <-ctx.Done():
+						ri.trackMultipleObjectVersions(exp, success)
+						return
+					default:
+					}
+					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
+						DeletePrefix:       true,
+						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
+					})
+					if err != nil {
+						stopFn(exp, err)
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
+					} else {
+						stopFn(exp, err)
+						success = true
+						break
+					}
+				}
+				ri.trackMultipleObjectVersions(exp, success)
+			}
+
+			// DeleteMultiple objects
+			toDelCopy := make([]ObjectToDelete, len(toDel))
+			for attempts := 1; attempts <= retryAttempts; attempts++ {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+
+				stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+				// Copying toDel to select from objects whose
+				// deletion failed
+				copy(toDelCopy, toDel)
+				var failed int
+				errs := r.Expire(ctx, api, vc, toDel)
+				// reslice toDel in preparation for next retry attempt
+				toDel = toDel[:0]
+				for i, err := range errs {
+					if err != nil {
+						stopFn(toDelCopy[i], err)
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
+							err, attempts))
+						failed++
+						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
+							ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)
+						}
+						if attempts != retryAttempts {
+							// retry
+							toDel = append(toDel, toDelCopy[i])
+						}
+					} else {
+						stopFn(toDelCopy[i], nil)
+						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
+							ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts)
+						}
+					}
+				}
+
+				globalBatchJobsMetrics.save(ri.JobID, ri)
+
+				if failed == 0 {
+					break
+				}
+
+				// Add a delay between retry attempts
+				if attempts < retryAttempts {
+					time.Sleep(delay)
+				}
+			}
+		}(toExpire)
+	}
+}
+
+type expireObjInfo struct {
+	ObjectInfo
+	ExpireAll         bool
+	DeleteMarkerCount int64
+}
+
+// Start the batch expiration job, resumes if there was a pending job via "job.ID"
+func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
+		return err
+	}
+
+	globalBatchJobsMetrics.save(job.ID, ri)
+	lastObject := ri.Object
+
+	now := time.Now().UTC()
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_EXPIRATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	ctx, cancelCause := context.WithCancelCause(ctx)
+	defer cancelCause(nil)
+
+	results := make(chan itemOrErr[ObjectInfo], workerSize)
+	go func() {
+		prefixes := r.Prefix.F()
+		if len(prefixes) == 0 {
+			prefixes = []string{""}
+		}
+		for _, prefix := range prefixes {
+			prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
+			err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
+				Marker:       lastObject,
+				LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
+				VersionsSort: WalkVersionsSortDesc,
+			})
+			if err != nil {
+				cancelCause(err)
+				xioutil.SafeClose(results)
+				return
+			}
+			for result := range prefixResultCh {
+				results <- result
+			}
+		}
+		xioutil.SafeClose(results)
+	}()
+
+	// Goroutine to periodically save batch-expire job's in-memory state
+	saverQuitCh := make(chan struct{})
+	go func() {
+		saveTicker := time.NewTicker(10 * time.Second)
+		defer saveTicker.Stop()
+		quit := false
+		after := time.Minute
+		for !quit {
+			select {
+			case <-saveTicker.C:
+			case <-ctx.Done():
+				quit = true
+			case <-saverQuitCh:
+				quit = true
+			}
+
+			if quit {
+				// save immediately if we are quitting
+				after = 0
+			}
+
+			ctx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) // independent context
+			batchLogIf(ctx, ri.updateAfter(ctx, api, after, job))
+			cancel()
+		}
+	}()
+
+	expireCh := make(chan []expireObjInfo, workerSize)
+	expireDoneCh := make(chan struct{})
+	go func() {
+		defer close(expireDoneCh)
+		batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+	}()
+
+	var (
+		prevObj       ObjectInfo
+		matchedFilter BatchJobExpireFilter
+		versionsCount int
+		toDel         []expireObjInfo
+		failed        bool
+		done          bool
+	)
+	deleteMarkerCountMap := map[string]int64{}
+	pushToExpire := func() {
+		// set preObject deleteMarkerCount
+		if len(toDel) > 0 {
+			lastDelIndex := len(toDel) - 1
+			lastDel := toDel[lastDelIndex]
+			if lastDel.ExpireAll {
+				toDel[lastDelIndex].DeleteMarkerCount = deleteMarkerCountMap[lastDel.Name]
+				// delete the key
+				delete(deleteMarkerCountMap, lastDel.Name)
+			}
+		}
+		// send down filtered entries to be deleted using
+		// DeleteObjects method
+		if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously.
+			xfer := make([]expireObjInfo, len(toDel))
+			copy(xfer, toDel)
+			select {
+			case expireCh <- xfer:
+				toDel = toDel[:0] // resetting toDel
+			case <-ctx.Done():
+				done = true
+			}
+		}
+	}
+	for {
+		select {
+		case result, ok := <-results:
+			if !ok {
+				done = true
+				break
+			}
+			if result.Err != nil {
+				failed = true
+				batchLogIf(ctx, result.Err)
+				continue
+			}
+			if result.Item.DeleteMarker {
+				deleteMarkerCountMap[result.Item.Name]++
+			}
+			// Apply filter to find the matching rule to apply expiry
+			// actions accordingly.
+			// nolint:gocritic
+			if result.Item.IsLatest {
+				var match BatchJobExpireFilter
+				var found bool
+				for _, rule := range r.Rules {
+					if rule.Matches(result.Item, now) {
+						match = rule
+						found = true
+						break
+					}
+				}
+				if !found {
+					continue
+				}
+
+				if prevObj.Name != result.Item.Name {
+					// switch the object
+					pushToExpire()
+				}
+
+				prevObj = result.Item
+				matchedFilter = match
+				versionsCount = 1
+				// Include the latest version
+				if matchedFilter.Purge.RetainVersions == 0 {
+					toDel = append(toDel, expireObjInfo{
+						ObjectInfo: result.Item,
+						ExpireAll:  true,
+					})
+					continue
+				}
+			} else if prevObj.Name == result.Item.Name {
+				if matchedFilter.Purge.RetainVersions == 0 {
+					continue // including latest version in toDel suffices, skipping other versions
+				}
+				versionsCount++
+			} else {
+				// switch the object
+				pushToExpire()
+				// a file switched with no LatestVersion, logging it
+				batchLogIf(ctx, fmt.Errorf("skipping object %s, no latest version found", result.Item.Name))
+				continue
+			}
+
+			if versionsCount <= matchedFilter.Purge.RetainVersions {
+				continue // retain versions
+			}
+			toDel = append(toDel, expireObjInfo{
+				ObjectInfo: result.Item,
+			})
+			pushToExpire()
+		case <-ctx.Done():
+			done = true
+		}
+		if done {
+			break
+		}
+	}
+
+	if context.Cause(ctx) != nil {
+		xioutil.SafeClose(expireCh)
+		return context.Cause(ctx)
+	}
+	pushToExpire()
+	// Send any remaining objects downstream
+	if len(toDel) > 0 {
+		select {
+		case <-ctx.Done():
+		case expireCh <- toDel:
+		}
+	}
+	xioutil.SafeClose(expireCh)
+
+	<-expireDoneCh // waits for the expire goroutine to complete
+	wk.Wait()      // waits for all expire workers to retire
+
+	ri.Complete = !failed && ri.ObjectsFailed == 0
+	ri.Failed = failed || ri.ObjectsFailed > 0
+	globalBatchJobsMetrics.save(job.ID, ri)
+
+	// Close the saverQuitCh - this also triggers saving in-memory state
+	// immediately one last time before we exit this method.
+	xioutil.SafeClose(saverQuitCh)
+
+	// Notify expire jobs final status to the configured endpoint
+	buf, _ := json.Marshal(ri)
+	if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil {
+		batchLogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
+	}
+
+	return nil
+}
+
+//msgp:ignore batchExpireJobError
+type batchExpireJobError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+func (e batchExpireJobError) Error() string {
+	return e.Description
+}
+
+// maxBatchRules maximum number of rules a batch-expiry job supports
+const maxBatchRules = 50
+
+// Validate validates the job definition input
+func (r *BatchJobExpire) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
+	if r == nil {
+		return nil
+	}
+
+	if r.APIVersion != batchExpireAPIVersion {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    "Unsupported batch expire API version",
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	if r.Bucket == "" {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    "Bucket argument missing",
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
+		if isErrBucketNotFound(err) {
+			return batchExpireJobError{
+				Code:           "NoSuchSourceBucket",
+				Description:    "The specified source bucket does not exist",
+				HTTPStatusCode: http.StatusNotFound,
+			}
+		}
+		return err
+	}
+
+	if len(r.Rules) > maxBatchRules {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    "Too many rules. Batch expire job can't have more than 100 rules",
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	for _, rule := range r.Rules {
+		if err := rule.Validate(); err != nil {
+			return batchExpireJobError{
+				Code:           "InvalidArgument",
+				Description:    fmt.Sprintf("Invalid batch expire rule: %s", err),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		}
+	}
+
+	if err := r.Retry.Validate(); err != nil {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    fmt.Sprintf("Invalid batch expire retry configuration: %s", err),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+	return nil
+}

cmd/batch-expire_gen.go

@@ -0,0 +1,864 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"time"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobExpire) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "APIVersion":
+			z.APIVersion, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "APIVersion")
+				return
+			}
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			err = z.Prefix.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "NotificationCfg":
+			err = z.NotificationCfg.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationCfg")
+				return
+			}
+		case "Retry":
+			err = z.Retry.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Retry")
+				return
+			}
+		case "Rules":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Rules")
+				return
+			}
+			if cap(z.Rules) >= int(zb0002) {
+				z.Rules = (z.Rules)[:zb0002]
+			} else {
+				z.Rules = make([]BatchJobExpireFilter, zb0002)
+			}
+			for za0001 := range z.Rules {
+				err = z.Rules[za0001].DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Rules", za0001)
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BatchJobExpire) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "APIVersion"
+	err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.APIVersion)
+	if err != nil {
+		err = msgp.WrapError(err, "APIVersion")
+		return
+	}
+	// write "Bucket"
+	err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Prefix"
+	err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	if err != nil {
+		return
+	}
+	err = z.Prefix.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
+	// write "NotificationCfg"
+	err = en.Append(0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
+	if err != nil {
+		return
+	}
+	err = z.NotificationCfg.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationCfg")
+		return
+	}
+	// write "Retry"
+	err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	if err != nil {
+		return
+	}
+	err = z.Retry.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Retry")
+		return
+	}
+	// write "Rules"
+	err = en.Append(0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Rules)))
+	if err != nil {
+		err = msgp.WrapError(err, "Rules")
+		return
+	}
+	for za0001 := range z.Rules {
+		err = z.Rules[za0001].EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Rules", za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BatchJobExpire) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "APIVersion"
+	o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	o = msgp.AppendString(o, z.APIVersion)
+	// string "Bucket"
+	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Prefix"
+	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	o, err = z.Prefix.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
+	// string "NotificationCfg"
+	o = append(o, 0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
+	o, err = z.NotificationCfg.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationCfg")
+		return
+	}
+	// string "Retry"
+	o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	o, err = z.Retry.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Retry")
+		return
+	}
+	// string "Rules"
+	o = append(o, 0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Rules)))
+	for za0001 := range z.Rules {
+		o, err = z.Rules[za0001].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Rules", za0001)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "APIVersion":
+			z.APIVersion, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "APIVersion")
+				return
+			}
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			bts, err = z.Prefix.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "NotificationCfg":
+			bts, err = z.NotificationCfg.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationCfg")
+				return
+			}
+		case "Retry":
+			bts, err = z.Retry.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Retry")
+				return
+			}
+		case "Rules":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Rules")
+				return
+			}
+			if cap(z.Rules) >= int(zb0002) {
+				z.Rules = (z.Rules)[:zb0002]
+			} else {
+				z.Rules = make([]BatchJobExpireFilter, zb0002)
+			}
+			for za0001 := range z.Rules {
+				bts, err = z.Rules[za0001].UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Rules", za0001)
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BatchJobExpire) Msgsize() (s int) {
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + z.Prefix.Msgsize() + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize
+	for za0001 := range z.Rules {
+		s += z.Rules[za0001].Msgsize()
+	}
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "OlderThan":
+			err = z.OlderThan.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "OlderThan")
+				return
+			}
+		case "CreatedBefore":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "CreatedBefore")
+					return
+				}
+				z.CreatedBefore = nil
+			} else {
+				if z.CreatedBefore == nil {
+					z.CreatedBefore = new(time.Time)
+				}
+				*z.CreatedBefore, err = dc.ReadTime()
+				if err != nil {
+					err = msgp.WrapError(err, "CreatedBefore")
+					return
+				}
+			}
+		case "Tags":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Tags")
+				return
+			}
+			if cap(z.Tags) >= int(zb0002) {
+				z.Tags = (z.Tags)[:zb0002]
+			} else {
+				z.Tags = make([]BatchJobKV, zb0002)
+			}
+			for za0001 := range z.Tags {
+				err = z.Tags[za0001].DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Tags", za0001)
+					return
+				}
+			}
+		case "Metadata":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Metadata")
+				return
+			}
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
+			} else {
+				z.Metadata = make([]BatchJobKV, zb0003)
+			}
+			for za0002 := range z.Metadata {
+				err = z.Metadata[za0002].DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Metadata", za0002)
+					return
+				}
+			}
+		case "Size":
+			err = z.Size.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "Type":
+			z.Type, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Purge":
+			var zb0004 uint32
+			zb0004, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Purge")
+				return
+			}
+			for zb0004 > 0 {
+				zb0004--
+				field, err = dc.ReadMapKeyPtr()
+				if err != nil {
+					err = msgp.WrapError(err, "Purge")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "RetainVersions":
+					z.Purge.RetainVersions, err = dc.ReadInt()
+					if err != nil {
+						err = msgp.WrapError(err, "Purge", "RetainVersions")
+						return
+					}
+				default:
+					err = dc.Skip()
+					if err != nil {
+						err = msgp.WrapError(err, "Purge")
+						return
+					}
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 8
+	// write "OlderThan"
+	err = en.Append(0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	if err != nil {
+		return
+	}
+	err = z.OlderThan.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
+	// write "CreatedBefore"
+	err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
+	if err != nil {
+		return
+	}
+	if z.CreatedBefore == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteTime(*z.CreatedBefore)
+		if err != nil {
+			err = msgp.WrapError(err, "CreatedBefore")
+			return
+		}
+	}
+	// write "Tags"
+	err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Tags)))
+	if err != nil {
+		err = msgp.WrapError(err, "Tags")
+		return
+	}
+	for za0001 := range z.Tags {
+		err = z.Tags[za0001].EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Tags", za0001)
+			return
+		}
+	}
+	// write "Metadata"
+	err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Metadata)))
+	if err != nil {
+		err = msgp.WrapError(err, "Metadata")
+		return
+	}
+	for za0002 := range z.Metadata {
+		err = z.Metadata[za0002].EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Metadata", za0002)
+			return
+		}
+	}
+	// write "Size"
+	err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = z.Size.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	// write "Type"
+	err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Type)
+	if err != nil {
+		err = msgp.WrapError(err, "Type")
+		return
+	}
+	// write "Name"
+	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "Purge"
+	err = en.Append(0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
+	if err != nil {
+		return
+	}
+	// map header, size 1
+	// write "RetainVersions"
+	err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Purge.RetainVersions)
+	if err != nil {
+		err = msgp.WrapError(err, "Purge", "RetainVersions")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 8
+	// string "OlderThan"
+	o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
+	// string "CreatedBefore"
+	o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
+	if z.CreatedBefore == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendTime(o, *z.CreatedBefore)
+	}
+	// string "Tags"
+	o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Tags)))
+	for za0001 := range z.Tags {
+		o, err = z.Tags[za0001].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Tags", za0001)
+			return
+		}
+	}
+	// string "Metadata"
+	o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata)))
+	for za0002 := range z.Metadata {
+		o, err = z.Metadata[za0002].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Metadata", za0002)
+			return
+		}
+	}
+	// string "Size"
+	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	o, err = z.Size.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	// string "Type"
+	o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = msgp.AppendString(o, z.Type)
+	// string "Name"
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "Purge"
+	o = append(o, 0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
+	// map header, size 1
+	// string "RetainVersions"
+	o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	o = msgp.AppendInt(o, z.Purge.RetainVersions)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "OlderThan":
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "OlderThan")
+				return
+			}
+		case "CreatedBefore":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.CreatedBefore = nil
+			} else {
+				if z.CreatedBefore == nil {
+					z.CreatedBefore = new(time.Time)
+				}
+				*z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "CreatedBefore")
+					return
+				}
+			}
+		case "Tags":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Tags")
+				return
+			}
+			if cap(z.Tags) >= int(zb0002) {
+				z.Tags = (z.Tags)[:zb0002]
+			} else {
+				z.Tags = make([]BatchJobKV, zb0002)
+			}
+			for za0001 := range z.Tags {
+				bts, err = z.Tags[za0001].UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Tags", za0001)
+					return
+				}
+			}
+		case "Metadata":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Metadata")
+				return
+			}
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
+			} else {
+				z.Metadata = make([]BatchJobKV, zb0003)
+			}
+			for za0002 := range z.Metadata {
+				bts, err = z.Metadata[za0002].UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Metadata", za0002)
+					return
+				}
+			}
+		case "Size":
+			bts, err = z.Size.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "Type":
+			z.Type, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Purge":
+			var zb0004 uint32
+			zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Purge")
+				return
+			}
+			for zb0004 > 0 {
+				zb0004--
+				field, bts, err = msgp.ReadMapKeyZC(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Purge")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "RetainVersions":
+					z.Purge.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Purge", "RetainVersions")
+						return
+					}
+				default:
+					bts, err = msgp.Skip(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Purge")
+						return
+					}
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BatchJobExpireFilter) Msgsize() (s int) {
+	s = 1 + 10 + z.OlderThan.Msgsize() + 14
+	if z.CreatedBefore == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.TimeSize
+	}
+	s += 5 + msgp.ArrayHeaderSize
+	for za0001 := range z.Tags {
+		s += z.Tags[za0001].Msgsize()
+	}
+	s += 9 + msgp.ArrayHeaderSize
+	for za0002 := range z.Metadata {
+		s += z.Metadata[za0002].Msgsize()
+	}
+	s += 5 + z.Size.Msgsize() + 5 + msgp.StringPrefixSize + len(z.Type) + 5 + msgp.StringPrefixSize + len(z.Name) + 6 + 1 + 15 + msgp.IntSize
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobExpirePurge) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "RetainVersions":
+			z.RetainVersions, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "RetainVersions")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobExpirePurge) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "RetainVersions"
+	err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.RetainVersions)
+	if err != nil {
+		err = msgp.WrapError(err, "RetainVersions")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobExpirePurge) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "RetainVersions"
+	o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	o = msgp.AppendInt(o, z.RetainVersions)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobExpirePurge) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "RetainVersions":
+			z.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RetainVersions")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobExpirePurge) Msgsize() (s int) {
+	s = 1 + 15 + msgp.IntSize
+	return
+}

cmd/batch-expire_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobExpire(t *testing.T) {
+	v := BatchJobExpire{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobExpire(t *testing.T) {
+	v := BatchJobExpire{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobExpire Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobExpire{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobExpireFilter(t *testing.T) {
+	v := BatchJobExpireFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobExpireFilter(t *testing.T) {
+	v := BatchJobExpireFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobExpireFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobExpireFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobExpirePurge(t *testing.T) {
+	v := BatchJobExpirePurge{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobExpirePurge(t *testing.T) {
+	v := BatchJobExpirePurge{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobExpirePurge Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobExpirePurge{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-expire_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"slices"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestParseBatchJobExpire(t *testing.T) {
+	expireYaml := `
+expire: # Expire objects that match a condition
+  apiVersion: v1
+  bucket: mybucket # Bucket where this batch job will expire matching objects from
+  prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
+  rules:
+    - type: object  # regular objects with zero or more older versions
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 7d10h # match objects older than this value
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      tags:
+        - key: name
+          value: pick* # match objects with tag 'name', all values starting with 'pick'
+      metadata:
+        - key: content-type
+          value: image/* # match objects with 'content-type', all values starting with 'image/'
+      size:
+        lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
+        greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object.
+  
+    - type: deleted # objects with delete marker as their latest version
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
+
+  notify:
+    endpoint: https://notify.endpoint # notification endpoint to receive job completion status
+    token: Bearer xxxxx # optional authentication token for the notification endpoint
+  
+  retry:
+    attempts: 10 # number of retries for the job before giving up
+    delay: 500ms # least amount of delay between each retry
+`
+	var job BatchJobRequest
+	err := yaml.Unmarshal([]byte(expireYaml), &job)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-expire yaml", err)
+	}
+	if !slices.Equal(job.Expire.Prefix.F(), []string{"myprefix"}) {
+		t.Fatal("Failed to parse batch-job-expire yaml")
+	}
+
+	multiPrefixExpireYaml := `
+expire: # Expire objects that match a condition
+  apiVersion: v1
+  bucket: mybucket # Bucket where this batch job will expire matching objects from
+  prefix: # (Optional) Prefix under which this job will expire objects matching the rules below.
+    - myprefix
+    - myprefix1
+  rules:
+    - type: object  # regular objects with zero or more older versions
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 7d10h # match objects older than this value
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      tags:
+        - key: name
+          value: pick* # match objects with tag 'name', all values starting with 'pick'
+      metadata:
+        - key: content-type
+          value: image/* # match objects with 'content-type', all values starting with 'image/'
+      size:
+        lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
+        greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object.
+  
+    - type: deleted # objects with delete marker as their latest version
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
+
+  notify:
+    endpoint: https://notify.endpoint # notification endpoint to receive job completion status
+    token: Bearer xxxxx # optional authentication token for the notification endpoint
+  
+  retry:
+    attempts: 10 # number of retries for the job before giving up
+    delay: 500ms # least amount of delay between each retry
+`
+	var multiPrefixJob BatchJobRequest
+	err = yaml.Unmarshal([]byte(multiPrefixExpireYaml), &multiPrefixJob)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-expire yaml", err)
+	}
+	if !slices.Equal(multiPrefixJob.Expire.Prefix.F(), []string{"myprefix", "myprefix1"}) {
+		t.Fatal("Failed to parse batch-job-expire yaml")
+	}
+}

cmd/batch-handlers_gen.go

@@ -6,6 +6,89 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobPrefix) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if cap((*z)) >= int(zb0002) {
+		(*z) = (*z)[:zb0002]
+	} else {
+		(*z) = make(BatchJobPrefix, zb0002)
+	}
+	for zb0001 := range *z {
+		(*z)[zb0001], err = dc.ReadString()
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobPrefix) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(len(z)))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0003 := range z {
+		err = en.WriteString(z[zb0003])
+		if err != nil {
+			err = msgp.WrapError(err, zb0003)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobPrefix) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(len(z)))
+	for zb0003 := range z {
+		o = msgp.AppendString(o, z[zb0003])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobPrefix) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if cap((*z)) >= int(zb0002) {
+		(*z) = (*z)[:zb0002]
+	} else {
+		(*z) = make(BatchJobPrefix, zb0002)
+	}
+	for zb0001 := range *z {
+		(*z)[zb0001], bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobPrefix) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize
+	for zb0003 := range z {
+		s += msgp.StringPrefixSize + len(z[zb0003])
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -42,12 +125,6 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -84,6 +161,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "Expire":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Expire")
+					return
+				}
+				z.Expire = nil
+			} else {
+				if z.Expire == nil {
+					z.Expire = new(BatchJobExpire)
+				}
+				err = z.Expire.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Expire")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -128,16 +223,6 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Started")
 		return
 	}
-	// write "Location"
-	err = en.Append(0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Location)
-	if err != nil {
-		err = msgp.WrapError(err, "Location")
-		return
-	}
 	// write "Replicate"
 	err = en.Append(0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if err != nil {
@@ -172,6 +257,23 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "Expire"
+	err = en.Append(0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+	if err != nil {
+		return
+	}
+	if z.Expire == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.Expire.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Expire")
+			return
+		}
+	}
 	return
 }
 
@@ -188,9 +290,6 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Started"
 	o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
 	o = msgp.AppendTime(o, z.Started)
-	// string "Location"
-	o = append(o, 0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	o = msgp.AppendString(o, z.Location)
 	// string "Replicate"
 	o = append(o, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if z.Replicate == nil {
@@ -213,6 +312,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "Expire"
+	o = append(o, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+	if z.Expire == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.Expire.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Expire")
+			return
+		}
+	}
 	return
 }
 
@@ -252,12 +362,6 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -292,6 +396,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "Expire":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Expire = nil
+			} else {
+				if z.Expire == nil {
+					z.Expire = new(BatchJobExpire)
+				}
+				bts, err = z.Expire.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Expire")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -306,7 +427,7 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobRequest) Msgsize() (s int) {
-	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 9 + msgp.StringPrefixSize + len(z.Location) + 10
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 10
 	if z.Replicate == nil {
 		s += msgp.NilSize
 	} else {
@@ -318,6 +439,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.KeyRotate.Msgsize()
 	}
+	s += 7
+	if z.Expire == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.Expire.Msgsize()
+	}
 	return
 }
 
@@ -375,6 +502,12 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "RetryAttempts")
 				return
 			}
+		case "at":
+			z.Attempts, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Attempts")
+				return
+			}
 		case "cmp":
 			z.Complete, err = dc.ReadBool()
 			if err != nil {
@@ -448,9 +581,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 16
+	// map header, size 17
 	// write "v"
-	err = en.Append(0xde, 0x0, 0x10, 0xa1, 0x76)
+	err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76)
 	if err != nil {
 		return
 	}
@@ -509,6 +642,16 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "RetryAttempts")
 		return
 	}
+	// write "at"
+	err = en.Append(0xa2, 0x61, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Attempts)
+	if err != nil {
+		err = msgp.WrapError(err, "Attempts")
+		return
+	}
 	// write "cmp"
 	err = en.Append(0xa3, 0x63, 0x6d, 0x70)
 	if err != nil {
@@ -615,9 +758,9 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 16
+	// map header, size 17
 	// string "v"
-	o = append(o, 0xde, 0x0, 0x10, 0xa1, 0x76)
+	o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76)
 	o = msgp.AppendInt(o, z.Version)
 	// string "jid"
 	o = append(o, 0xa3, 0x6a, 0x69, 0x64)
@@ -634,6 +777,9 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ra"
 	o = append(o, 0xa2, 0x72, 0x61)
 	o = msgp.AppendInt(o, z.RetryAttempts)
+	// string "at"
+	o = append(o, 0xa2, 0x61, 0x74)
+	o = msgp.AppendInt(o, z.Attempts)
 	// string "cmp"
 	o = append(o, 0xa3, 0x63, 0x6d, 0x70)
 	o = msgp.AppendBool(o, z.Complete)
@@ -721,6 +867,12 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "RetryAttempts")
 				return
 			}
+		case "at":
+			z.Attempts, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Attempts")
+				return
+			}
 		case "cmp":
 			z.Complete, bts, err = msgp.ReadBoolBytes(bts)
 			if err != nil {
@@ -795,6 +947,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *batchJobInfo) Msgsize() (s int) {
-	s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
+	s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
 	return
 }

cmd/batch-handlers_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalBatchJobPrefix(t *testing.T) {
+	v := BatchJobPrefix{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobPrefix(t *testing.T) {
+	v := BatchJobPrefix{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobPrefix Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobPrefix{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalBatchJobRequest(t *testing.T) {
 	v := BatchJobRequest{}
 	bts, err := v.MarshalMsg(nil)

cmd/batch-handlers_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"slices"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestBatchJobPrefix_UnmarshalYAML(t *testing.T) {
+	type args struct {
+		yamlStr string
+	}
+	type PrefixTemp struct {
+		Prefix BatchJobPrefix `yaml:"prefix"`
+	}
+	tests := []struct {
+		name    string
+		b       PrefixTemp
+		args    args
+		want    []string
+		wantErr bool
+	}{
+		{
+			name: "test1",
+			b:    PrefixTemp{},
+			args: args{
+				yamlStr: `
+prefix: "foo"
+`,
+			},
+			want:    []string{"foo"},
+			wantErr: false,
+		},
+		{
+			name: "test2",
+			b:    PrefixTemp{},
+			args: args{
+				yamlStr: `
+prefix:
+  - "foo"
+  - "bar"
+`,
+			},
+			want: []string{"foo", "bar"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := yaml.Unmarshal([]byte(tt.args.yamlStr), &tt.b); (err != nil) != tt.wantErr {
+				t.Errorf("UnmarshalYAML() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !slices.Equal(tt.b.Prefix.F(), tt.want) {
+				t.Errorf("UnmarshalYAML() = %v, want %v", tt.b.Prefix.F(), tt.want)
+			}
+		})
+	}
+}

cmd/batch-job-common-types.go

@@ -18,24 +18,66 @@
 package cmd
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
-	"github.com/minio/pkg/v2/wildcard"
+	"github.com/dustin/go-humanize"
+	"github.com/minio/pkg/v3/wildcard"
+	"gopkg.in/yaml.v3"
 )
 
 //go:generate msgp -file $GOFILE
+//msgp:ignore BatchJobYamlErr
+
+// BatchJobYamlErr can be used to return yaml validation errors with line,
+// column information guiding user to fix syntax errors
+type BatchJobYamlErr struct {
+	line, col int
+	msg       string
+}
+
+// message returns the error message excluding line, col information.
+// Intended to be used in unit tests.
+func (b BatchJobYamlErr) message() string {
+	return b.msg
+}
+
+// Error implements Error interface
+func (b BatchJobYamlErr) Error() string {
+	return fmt.Sprintf("%s\n Hint: error near line: %d, col: %d", b.msg, b.line, b.col)
+}
 
 // BatchJobKV is a key-value data type which supports wildcard matching
 type BatchJobKV struct {
+	line, col int
 	Key       string `yaml:"key" json:"key"`
 	Value     string `yaml:"value" json:"value"`
 }
 
+var _ yaml.Unmarshaler = &BatchJobKV{}
+
+// UnmarshalYAML - BatchJobKV extends default unmarshal to extract line, col information.
+func (kv *BatchJobKV) UnmarshalYAML(val *yaml.Node) error {
+	type jobKV BatchJobKV
+	var tmp jobKV
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+	*kv = BatchJobKV(tmp)
+	kv.line, kv.col = val.Line, val.Column
+	return nil
+}
+
 // Validate returns an error if key is empty
 func (kv BatchJobKV) Validate() error {
 	if kv.Key == "" {
-		return errInvalidArgument
+		return BatchJobYamlErr{
+			line: kv.line,
+			col:  kv.col,
+			msg:  "key can't be empty",
+		}
 	}
 	return nil
 }
@@ -59,25 +101,190 @@ func (kv BatchJobKV) Match(ikv BatchJobKV) bool {
 // BatchJobNotification stores notification endpoint and token information.
 // Used by batch jobs to notify of their status.
 type BatchJobNotification struct {
+	line, col int
 	Endpoint  string `yaml:"endpoint" json:"endpoint"`
 	Token     string `yaml:"token" json:"token"`
 }
 
+var _ yaml.Unmarshaler = &BatchJobNotification{}
+
+// UnmarshalYAML - BatchJobNotification extends unmarshal to extract line, column information
+func (b *BatchJobNotification) UnmarshalYAML(val *yaml.Node) error {
+	type notification BatchJobNotification
+	var tmp notification
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*b = BatchJobNotification(tmp)
+	b.line, b.col = val.Line, val.Column
+	return nil
+}
+
 // BatchJobRetry stores retry configuration used in the event of failures.
 type BatchJobRetry struct {
+	line, col int
 	Attempts  int           `yaml:"attempts" json:"attempts"` // number of retry attempts
 	Delay     time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
 }
 
+var _ yaml.Unmarshaler = &BatchJobRetry{}
+
+// UnmarshalYAML - BatchJobRetry extends unmarshal to extract line, column information
+func (r *BatchJobRetry) UnmarshalYAML(val *yaml.Node) error {
+	type retry BatchJobRetry
+	var tmp retry
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*r = BatchJobRetry(tmp)
+	r.line, r.col = val.Line, val.Column
+	return nil
+}
+
 // Validate validates input replicate retries.
 func (r BatchJobRetry) Validate() error {
 	if r.Attempts < 0 {
-		return errInvalidArgument
+		return BatchJobYamlErr{
+			line: r.line,
+			col:  r.col,
+			msg:  "Invalid arguments specified",
+		}
 	}
 
 	if r.Delay < 0 {
-		return errInvalidArgument
+		return BatchJobYamlErr{
+			line: r.line,
+			col:  r.col,
+			msg:  "Invalid arguments specified",
+		}
 	}
 
 	return nil
 }
+
+//   # snowball based archive transfer is by default enabled when source
+//   # is local and target is remote which is also minio.
+//   snowball:
+//     disable: false # optionally turn-off snowball archive transfer
+//     batch: 100 # upto this many objects per archive
+//     inmemory: true # indicates if the archive must be staged locally or in-memory
+//     compress: true # S2/Snappy compressed archive
+//     smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+//     skipErrs: false # skips any source side read() errors
+
+// BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target
+type BatchJobSnowball struct {
+	line, col   int
+	Disable     *bool   `yaml:"disable" json:"disable"`
+	Batch       *int    `yaml:"batch" json:"batch"`
+	InMemory    *bool   `yaml:"inmemory" json:"inmemory"`
+	Compress    *bool   `yaml:"compress" json:"compress"`
+	SmallerThan *string `yaml:"smallerThan" json:"smallerThan"`
+	SkipErrs    *bool   `yaml:"skipErrs" json:"skipErrs"`
+}
+
+var _ yaml.Unmarshaler = &BatchJobSnowball{}
+
+// UnmarshalYAML - BatchJobSnowball extends unmarshal to extract line, column information
+func (b *BatchJobSnowball) UnmarshalYAML(val *yaml.Node) error {
+	type snowball BatchJobSnowball
+	var tmp snowball
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*b = BatchJobSnowball(tmp)
+	b.line, b.col = val.Line, val.Column
+	return nil
+}
+
+// Validate the snowball parameters in the job description
+func (b BatchJobSnowball) Validate() error {
+	if *b.Batch <= 0 {
+		return BatchJobYamlErr{
+			line: b.line,
+			col:  b.col,
+			msg:  "batch number should be non positive zero",
+		}
+	}
+	_, err := humanize.ParseBytes(*b.SmallerThan)
+	if err != nil {
+		return BatchJobYamlErr{
+			line: b.line,
+			col:  b.col,
+			msg:  err.Error(),
+		}
+	}
+	return nil
+}
+
+// BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan
+type BatchJobSizeFilter struct {
+	line, col  int
+	UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"`
+	LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"`
+}
+
+// UnmarshalYAML - BatchJobSizeFilter extends unmarshal to extract line, column information
+func (sf *BatchJobSizeFilter) UnmarshalYAML(val *yaml.Node) error {
+	type sizeFilter BatchJobSizeFilter
+	var tmp sizeFilter
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*sf = BatchJobSizeFilter(tmp)
+	sf.line, sf.col = val.Line, val.Column
+	return nil
+}
+
+// InRange returns true in the following cases and false otherwise,
+// - sf.LowerBound < sz, when sf.LowerBound alone is specified
+// - sz < sf.UpperBound, when sf.UpperBound alone is specified
+// - sf.LowerBound < sz < sf.UpperBound when both are specified,
+func (sf BatchJobSizeFilter) InRange(sz int64) bool {
+	if sf.UpperBound > 0 && sz > int64(sf.UpperBound) {
+		return false
+	}
+
+	if sf.LowerBound > 0 && sz < int64(sf.LowerBound) {
+		return false
+	}
+	return true
+}
+
+// Validate checks if sf is a valid batch-job size filter
+func (sf BatchJobSizeFilter) Validate() error {
+	if sf.LowerBound > 0 && sf.UpperBound > 0 && sf.LowerBound >= sf.UpperBound {
+		return BatchJobYamlErr{
+			line: sf.line,
+			col:  sf.col,
+			msg:  "invalid batch-job size filter",
+		}
+	}
+	return nil
+}
+
+// BatchJobSize supports humanized byte values in yaml files type BatchJobSize uint64
+type BatchJobSize int64
+
+// UnmarshalYAML to parse humanized byte values
+func (s *BatchJobSize) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var batchExpireSz string
+	err := unmarshal(&batchExpireSz)
+	if err != nil {
+		return err
+	}
+	sz, err := humanize.ParseBytes(batchExpireSz)
+	if err != nil {
+		return err
+	}
+	*s = BatchJobSize(sz)
+	return nil
+}

cmd/batch-job-common-types_gen.go

@@ -389,3 +389,666 @@ func (z BatchJobRetry) Msgsize() (s int) {
 	s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
 	return
 }
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobSize) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 int64
+		zb0001, err = dc.ReadInt64()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = BatchJobSize(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobSize) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteInt64(int64(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobSize) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendInt64(o, int64(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobSize) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 int64
+		zb0001, bts, err = msgp.ReadInt64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = BatchJobSize(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobSize) Msgsize() (s int) {
+	s = msgp.Int64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobSizeFilter) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UpperBound":
+			{
+				var zb0002 int64
+				zb0002, err = dc.ReadInt64()
+				if err != nil {
+					err = msgp.WrapError(err, "UpperBound")
+					return
+				}
+				z.UpperBound = BatchJobSize(zb0002)
+			}
+		case "LowerBound":
+			{
+				var zb0003 int64
+				zb0003, err = dc.ReadInt64()
+				if err != nil {
+					err = msgp.WrapError(err, "LowerBound")
+					return
+				}
+				z.LowerBound = BatchJobSize(zb0003)
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobSizeFilter) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "UpperBound"
+	err = en.Append(0x82, 0xaa, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(int64(z.UpperBound))
+	if err != nil {
+		err = msgp.WrapError(err, "UpperBound")
+		return
+	}
+	// write "LowerBound"
+	err = en.Append(0xaa, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(int64(z.LowerBound))
+	if err != nil {
+		err = msgp.WrapError(err, "LowerBound")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobSizeFilter) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "UpperBound"
+	o = append(o, 0x82, 0xaa, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	o = msgp.AppendInt64(o, int64(z.UpperBound))
+	// string "LowerBound"
+	o = append(o, 0xaa, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	o = msgp.AppendInt64(o, int64(z.LowerBound))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobSizeFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UpperBound":
+			{
+				var zb0002 int64
+				zb0002, bts, err = msgp.ReadInt64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "UpperBound")
+					return
+				}
+				z.UpperBound = BatchJobSize(zb0002)
+			}
+		case "LowerBound":
+			{
+				var zb0003 int64
+				zb0003, bts, err = msgp.ReadInt64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "LowerBound")
+					return
+				}
+				z.LowerBound = BatchJobSize(zb0003)
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobSizeFilter) Msgsize() (s int) {
+	s = 1 + 11 + msgp.Int64Size + 11 + msgp.Int64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobSnowball) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Disable":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Disable")
+					return
+				}
+				z.Disable = nil
+			} else {
+				if z.Disable == nil {
+					z.Disable = new(bool)
+				}
+				*z.Disable, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "Disable")
+					return
+				}
+			}
+		case "Batch":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Batch")
+					return
+				}
+				z.Batch = nil
+			} else {
+				if z.Batch == nil {
+					z.Batch = new(int)
+				}
+				*z.Batch, err = dc.ReadInt()
+				if err != nil {
+					err = msgp.WrapError(err, "Batch")
+					return
+				}
+			}
+		case "InMemory":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "InMemory")
+					return
+				}
+				z.InMemory = nil
+			} else {
+				if z.InMemory == nil {
+					z.InMemory = new(bool)
+				}
+				*z.InMemory, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "InMemory")
+					return
+				}
+			}
+		case "Compress":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Compress")
+					return
+				}
+				z.Compress = nil
+			} else {
+				if z.Compress == nil {
+					z.Compress = new(bool)
+				}
+				*z.Compress, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "Compress")
+					return
+				}
+			}
+		case "SmallerThan":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "SmallerThan")
+					return
+				}
+				z.SmallerThan = nil
+			} else {
+				if z.SmallerThan == nil {
+					z.SmallerThan = new(string)
+				}
+				*z.SmallerThan, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "SmallerThan")
+					return
+				}
+			}
+		case "SkipErrs":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "SkipErrs")
+					return
+				}
+				z.SkipErrs = nil
+			} else {
+				if z.SkipErrs == nil {
+					z.SkipErrs = new(bool)
+				}
+				*z.SkipErrs, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "SkipErrs")
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BatchJobSnowball) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "Disable"
+	err = en.Append(0x86, 0xa7, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65)
+	if err != nil {
+		return
+	}
+	if z.Disable == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.Disable)
+		if err != nil {
+			err = msgp.WrapError(err, "Disable")
+			return
+		}
+	}
+	// write "Batch"
+	err = en.Append(0xa5, 0x42, 0x61, 0x74, 0x63, 0x68)
+	if err != nil {
+		return
+	}
+	if z.Batch == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteInt(*z.Batch)
+		if err != nil {
+			err = msgp.WrapError(err, "Batch")
+			return
+		}
+	}
+	// write "InMemory"
+	err = en.Append(0xa8, 0x49, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79)
+	if err != nil {
+		return
+	}
+	if z.InMemory == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.InMemory)
+		if err != nil {
+			err = msgp.WrapError(err, "InMemory")
+			return
+		}
+	}
+	// write "Compress"
+	err = en.Append(0xa8, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73)
+	if err != nil {
+		return
+	}
+	if z.Compress == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.Compress)
+		if err != nil {
+			err = msgp.WrapError(err, "Compress")
+			return
+		}
+	}
+	// write "SmallerThan"
+	err = en.Append(0xab, 0x53, 0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	if err != nil {
+		return
+	}
+	if z.SmallerThan == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteString(*z.SmallerThan)
+		if err != nil {
+			err = msgp.WrapError(err, "SmallerThan")
+			return
+		}
+	}
+	// write "SkipErrs"
+	err = en.Append(0xa8, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x72, 0x72, 0x73)
+	if err != nil {
+		return
+	}
+	if z.SkipErrs == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.SkipErrs)
+		if err != nil {
+			err = msgp.WrapError(err, "SkipErrs")
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BatchJobSnowball) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "Disable"
+	o = append(o, 0x86, 0xa7, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65)
+	if z.Disable == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.Disable)
+	}
+	// string "Batch"
+	o = append(o, 0xa5, 0x42, 0x61, 0x74, 0x63, 0x68)
+	if z.Batch == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendInt(o, *z.Batch)
+	}
+	// string "InMemory"
+	o = append(o, 0xa8, 0x49, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79)
+	if z.InMemory == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.InMemory)
+	}
+	// string "Compress"
+	o = append(o, 0xa8, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73)
+	if z.Compress == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.Compress)
+	}
+	// string "SmallerThan"
+	o = append(o, 0xab, 0x53, 0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	if z.SmallerThan == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendString(o, *z.SmallerThan)
+	}
+	// string "SkipErrs"
+	o = append(o, 0xa8, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x72, 0x72, 0x73)
+	if z.SkipErrs == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.SkipErrs)
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobSnowball) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Disable":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Disable = nil
+			} else {
+				if z.Disable == nil {
+					z.Disable = new(bool)
+				}
+				*z.Disable, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Disable")
+					return
+				}
+			}
+		case "Batch":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Batch = nil
+			} else {
+				if z.Batch == nil {
+					z.Batch = new(int)
+				}
+				*z.Batch, bts, err = msgp.ReadIntBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Batch")
+					return
+				}
+			}
+		case "InMemory":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.InMemory = nil
+			} else {
+				if z.InMemory == nil {
+					z.InMemory = new(bool)
+				}
+				*z.InMemory, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "InMemory")
+					return
+				}
+			}
+		case "Compress":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Compress = nil
+			} else {
+				if z.Compress == nil {
+					z.Compress = new(bool)
+				}
+				*z.Compress, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Compress")
+					return
+				}
+			}
+		case "SmallerThan":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.SmallerThan = nil
+			} else {
+				if z.SmallerThan == nil {
+					z.SmallerThan = new(string)
+				}
+				*z.SmallerThan, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "SmallerThan")
+					return
+				}
+			}
+		case "SkipErrs":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.SkipErrs = nil
+			} else {
+				if z.SkipErrs == nil {
+					z.SkipErrs = new(bool)
+				}
+				*z.SkipErrs, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "SkipErrs")
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BatchJobSnowball) Msgsize() (s int) {
+	s = 1 + 8
+	if z.Disable == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	s += 6
+	if z.Batch == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.IntSize
+	}
+	s += 9
+	if z.InMemory == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	s += 9
+	if z.Compress == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	s += 12
+	if z.SmallerThan == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.StringPrefixSize + len(*z.SmallerThan)
+	}
+	s += 9
+	if z.SkipErrs == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	return
+}

cmd/batch-job-common-types_gen_test.go

@@ -347,3 +347,229 @@ func BenchmarkDecodeBatchJobRetry(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalBatchJobSizeFilter(t *testing.T) {
+	v := BatchJobSizeFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobSizeFilter(t *testing.T) {
+	v := BatchJobSizeFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobSizeFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobSizeFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobSnowball(t *testing.T) {
+	v := BatchJobSnowball{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobSnowball(t *testing.T) {
+	v := BatchJobSnowball{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobSnowball Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobSnowball{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-job-common-types_test.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestBatchJobSizeInRange(t *testing.T) {
+	tests := []struct {
+		objSize    int64
+		sizeFilter BatchJobSizeFilter
+		want       bool
+	}{
+		{
+			// 1Mib < 2Mib < 10MiB -> in range
+			objSize: 2 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 1 << 20,
+			},
+			want: true,
+		},
+		{
+			// 2KiB < 1 MiB -> out of range from left
+			objSize: 2 << 10,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 1 << 20,
+			},
+			want: false,
+		},
+		{
+			// 11MiB > 10 MiB -> out of range from right
+			objSize: 11 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 1 << 20,
+			},
+			want: false,
+		},
+		{
+			//  2MiB < 10MiB -> in range
+			objSize: 2 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+			},
+			want: true,
+		},
+		{
+			//  2MiB > 1MiB -> in range
+			objSize: 2 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				LowerBound: 1 << 20,
+			},
+			want: true,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
+			if got := test.sizeFilter.InRange(test.objSize); got != test.want {
+				t.Fatalf("Expected %v but got %v", test.want, got)
+			}
+		})
+	}
+}
+
+func TestBatchJobSizeValidate(t *testing.T) {
+	errInvalidBatchJobSizeFilter := BatchJobYamlErr{
+		msg: "invalid batch-job size filter",
+	}
+
+	tests := []struct {
+		sizeFilter BatchJobSizeFilter
+		err        error
+	}{
+		{
+			// Unspecified size filter is a valid filter
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 0,
+				LowerBound: 0,
+			},
+			err: nil,
+		},
+		{
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 0,
+				LowerBound: 1 << 20,
+			},
+			err: nil,
+		},
+		{
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 0,
+			},
+			err: nil,
+		},
+		{
+			// LowerBound > UpperBound -> empty range
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 1 << 20,
+				LowerBound: 10 << 20,
+			},
+			err: errInvalidBatchJobSizeFilter,
+		},
+		{
+			// LowerBound == UpperBound -> empty range
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 1 << 20,
+				LowerBound: 1 << 20,
+			},
+			err: errInvalidBatchJobSizeFilter,
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
+			err := test.sizeFilter.Validate()
+			if err != nil {
+				gotErr := err.(BatchJobYamlErr)
+				testErr := test.err.(BatchJobYamlErr)
+				if gotErr.message() != testErr.message() {
+					t.Fatalf("Expected %v but got %v", test.err, err)
+				}
+			}
+			if err == nil && test.err != nil {
+				t.Fatalf("Expected %v but got nil", test.err)
+			}
+		})
+	}
+}

cmd/batch-replicate.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
-
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/v3/xtime"
 )
 
 //go:generate msgp -file $GOFILE
@@ -65,8 +65,8 @@ import (
 
 // BatchReplicateFilter holds all the filters currently supported for batch replication
 type BatchReplicateFilter struct {
-	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	NewerThan     xtime.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	OlderThan     xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
 	CreatedAfter  time.Time      `yaml:"createdAfter,omitempty" json:"createdAfter"`
 	CreatedBefore time.Time      `yaml:"createdBefore,omitempty" json:"createdBefore"`
 	Tags          []BatchJobKV   `yaml:"tags,omitempty" json:"tags"`
@@ -151,10 +151,11 @@ func (t BatchJobReplicateTarget) ValidPath() bool {
 type BatchJobReplicateSource struct {
 	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
 	Bucket   string                        `yaml:"bucket" json:"bucket"`
-	Prefix   string                        `yaml:"prefix" json:"prefix"`
+	Prefix   BatchJobPrefix                `yaml:"prefix" json:"prefix"`
 	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
 	Path     string                        `yaml:"path" json:"path"`
 	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
+	Snowball BatchJobSnowball              `yaml:"snowball" json:"snowball"`
 }
 
 // ValidPath returns true if path is valid

cmd/batch-replicate_gen.go

@@ -411,7 +411,7 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Prefix":
-			z.Prefix, err = dc.ReadString()
+			err = z.Prefix.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Prefix")
 				return
@@ -469,6 +469,12 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "Snowball":
+			err = z.Snowball.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Snowball")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -482,9 +488,9 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 7
 	// write "Type"
-	err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	err = en.Append(0x87, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	if err != nil {
 		return
 	}
@@ -508,7 +514,7 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteString(z.Prefix)
+	err = z.Prefix.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "Prefix")
 		return
@@ -569,22 +575,36 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Creds", "SessionToken")
 		return
 	}
+	// write "Snowball"
+	err = en.Append(0xa8, 0x53, 0x6e, 0x6f, 0x77, 0x62, 0x61, 0x6c, 0x6c)
+	if err != nil {
+		return
+	}
+	err = z.Snowball.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Snowball")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
 	// string "Type"
-	o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = append(o, 0x87, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	o = msgp.AppendString(o, string(z.Type))
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
 	o = msgp.AppendString(o, z.Bucket)
 	// string "Prefix"
 	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
-	o = msgp.AppendString(o, z.Prefix)
+	o, err = z.Prefix.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
 	// string "Endpoint"
 	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
 	o = msgp.AppendString(o, z.Endpoint)
@@ -603,6 +623,13 @@ func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "SessionToken"
 	o = append(o, 0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
 	o = msgp.AppendString(o, z.Creds.SessionToken)
+	// string "Snowball"
+	o = append(o, 0xa8, 0x53, 0x6e, 0x6f, 0x77, 0x62, 0x61, 0x6c, 0x6c)
+	o, err = z.Snowball.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Snowball")
+		return
+	}
 	return
 }
 
@@ -641,7 +668,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 				return
 			}
 		case "Prefix":
-			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			bts, err = z.Prefix.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Prefix")
 				return
@@ -699,6 +726,12 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 					}
 				}
 			}
+		case "Snowball":
+			bts, err = z.Snowball.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Snowball")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -713,7 +746,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateSource) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + z.Prefix.Msgsize() + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken) + 9 + z.Snowball.Msgsize()
 	return
 }
 
@@ -1380,13 +1413,13 @@ func (z *BatchReplicateFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, err = dc.ReadDuration()
+			err = z.NewerThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1460,7 +1493,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.NewerThan)
+	err = z.NewerThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "NewerThan")
 		return
@@ -1470,7 +1503,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -1538,10 +1571,18 @@ func (z *BatchReplicateFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 6
 	// string "NewerThan"
 	o = append(o, 0x86, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.NewerThan)
+	o, err = z.NewerThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "NewerThan")
+		return
+	}
 	// string "OlderThan"
 	o = append(o, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedAfter"
 	o = append(o, 0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72)
 	o = msgp.AppendTime(o, z.CreatedAfter)
@@ -1590,13 +1631,13 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.NewerThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1665,7 +1706,7 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchReplicateFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
+	s = 1 + 10 + z.NewerThan.Msgsize() + 10 + z.OlderThan.Msgsize() + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.Tags {
 		s += z.Tags[za0001].Msgsize()
 	}

cmd/batch-replicate_test.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"slices"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestParseBatchJobReplicate(t *testing.T) {
+	replicateYaml := `
+replicate:
+  apiVersion: v1
+  # source of the objects to be replicated
+  source:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: object-prefix1 # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+#    endpoint: "http://127.0.0.1:9000"
+#    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+#    credentials:
+#      accessKey: minioadmin # Required
+#      secretKey: minioadmin # Required
+#    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+    snowball: # automatically activated if the source is local
+      disable: true # optionally turn-off snowball archive transfer
+#      batch: 100 # upto this many objects per archive
+#      inmemory: true # indicates if the archive must be staged locally or in-memory
+#      compress: false # S2/Snappy compressed archive
+#      smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+#      skipErrs: false # skips any source side read() errors
+
+  # target where the objects must be replicated
+  target:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: stage # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+    endpoint: "http://127.0.0.1:9001"
+    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+    credentials:
+      accessKey: minioadmin
+      secretKey: minioadmin
+    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+
+  # NOTE: All flags are optional
+  # - filtering criteria only applies for all source objects match the criteria
+  # - configurable notification endpoints
+  # - configurable retries for the job (each retry skips successfully previously replaced objects)
+  flags:
+    filter:
+      newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s)
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+#      createdAfter: "date" # match objects created after "date"
+#      createdBefore: "date" # match objects created before "date"
+
+      ## NOTE: tags are not supported when "source" is remote.
+      tags:
+         - key: "name"
+           value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+
+      metadata:
+         - key: "content-type"
+           value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+
+#    notify:
+#      endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+#      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+#
+#    retry:
+#      attempts: 10 # number of retries for the job before giving up
+#      delay: "500ms" # least amount of delay between each retry
+
+`
+	var job BatchJobRequest
+	err := yaml.Unmarshal([]byte(replicateYaml), &job)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+	if !slices.Equal(job.Replicate.Source.Prefix.F(), []string{"object-prefix1"}) {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+	multiPrefixReplicateYaml := `
+replicate:
+  apiVersion: v1
+  # source of the objects to be replicated
+  source:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: # 'PREFIX' is optional
+      - object-prefix1 
+      - object-prefix2
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+#    endpoint: "http://127.0.0.1:9000"
+#    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+#    credentials:
+#      accessKey: minioadmin # Required
+#      secretKey: minioadmin # Required
+#    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+    snowball: # automatically activated if the source is local
+      disable: true # optionally turn-off snowball archive transfer
+#      batch: 100 # upto this many objects per archive
+#      inmemory: true # indicates if the archive must be staged locally or in-memory
+#      compress: false # S2/Snappy compressed archive
+#      smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+#      skipErrs: false # skips any source side read() errors
+
+  # target where the objects must be replicated
+  target:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: stage # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+    endpoint: "http://127.0.0.1:9001"
+    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+    credentials:
+      accessKey: minioadmin
+      secretKey: minioadmin
+    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+
+  # NOTE: All flags are optional
+  # - filtering criteria only applies for all source objects match the criteria
+  # - configurable notification endpoints
+  # - configurable retries for the job (each retry skips successfully previously replaced objects)
+  flags:
+    filter:
+      newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s)
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+#      createdAfter: "date" # match objects created after "date"
+#      createdBefore: "date" # match objects created before "date"
+
+      ## NOTE: tags are not supported when "source" is remote.
+      tags:
+         - key: "name"
+           value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+
+      metadata:
+         - key: "content-type"
+           value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+
+#    notify:
+#      endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+#      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+#
+#    retry:
+#      attempts: 10 # number of retries for the job before giving up
+#      delay: "500ms" # least amount of delay between each retry
+
+`
+	var multiPrefixJob BatchJobRequest
+	err = yaml.Unmarshal([]byte(multiPrefixReplicateYaml), &multiPrefixJob)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+	if !slices.Equal(multiPrefixJob.Replicate.Source.Prefix.F(), []string{"object-prefix1", "object-prefix2"}) {
+		t.Fatal("Failed to parse batch-job-replicate yaml")
+	}
+}

cmd/batch-rotate.go

@@ -33,9 +33,8 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/env"
-	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v3/workers"
-	"github.com/minio/pkg/v2/workers"
 )
 
 // keyrotate:
@@ -96,6 +95,7 @@ func (e BatchJobKeyRotateEncryption) Validate() error {
 	if e.Type == ssekms && spaces {
 		return crypto.ErrInvalidEncryptionKeyID
 	}
+
 	if e.Type == ssekms && GlobalKMS != nil {
 		ctx := kms.Context{}
 		if e.Context != "" {
@@ -114,7 +114,7 @@ func (e BatchJobKeyRotateEncryption) Validate() error {
 			e.kmsContext[k] = v
 		}
 		ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
-		if _, err := GlobalKMS.GenerateKey(GlobalContext, e.Key, ctx); err != nil {
+		if _, err := GlobalKMS.GenerateKey(GlobalContext, &kms.GenerateKeyRequest{Name: e.Key, AssociatedData: ctx}); err != nil {
 			return err
 		}
 	}
@@ -154,7 +154,6 @@ type BatchJobKeyRotateV1 struct {
 	Flags      BatchJobKeyRotateFlags      `yaml:"flags" json:"flags"`
 	Bucket     string                      `yaml:"bucket" json:"bucket"`
 	Prefix     string                      `yaml:"prefix" json:"prefix"`
-	Endpoint   string                      `yaml:"endpoint" json:"endpoint"`
 	Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
 }
 
@@ -258,21 +257,28 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
+	if ri.Complete {
+		return nil
+	}
 
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
+	retryAttempts := job.KeyRotate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchKeyRotateJobDefaultRetries
+	}
 	delay := job.KeyRotate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
 
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -352,21 +358,25 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		return err
 	}
 
-	retryAttempts := ri.RetryAttempts
 	ctx, cancel := context.WithCancel(ctx)
 
-	results := make(chan ObjectInfo, 100)
+	results := make(chan itemOrErr[ObjectInfo], 100)
-	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
+	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
-		WalkMarker: lastObject,
+		Marker: lastObject,
-		WalkFilter: skip,
+		Filter: selectObj,
 	}); err != nil {
 		cancel()
 		// Do not need to retry if we can't list objects on source.
 		return err
 	}
-
+	failed := false
-	for result := range results {
+	for res := range results {
-		result := result
+		if res.Err != nil {
+			failed = true
+			batchLogIf(ctx, res.Err)
+			break
+		}
+		result := res.Item
 		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
 		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
 		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
@@ -376,21 +386,30 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		go func() {
 			defer wk.Give()
 			for attempts := 1; attempts <= retryAttempts; attempts++ {
-				attempts := attempts
+				stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
-				stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
 				success := true
 				if err := r.KeyRotate(ctx, api, result); err != nil {
-					stopFn(err)
+					stopFn(result, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
-				} else {
+					if attempts >= retryAttempts {
-					stopFn(nil)
+						auditOptions := AuditLogOptions{
+							Event:     "KeyRotate",
+							APIName:   "StartBatchJob",
+							Bucket:    result.Bucket,
+							Object:    result.Name,
+							VersionID: result.VersionID,
+							Error:     err.Error(),
 						}
-				ri.trackCurrentBucketObject(r.Bucket, result, success)
+						auditLogInternal(ctx, auditOptions)
-				ri.RetryAttempts = attempts
+					}
+				} else {
+					stopFn(result, nil)
+				}
+				ri.trackCurrentBucketObject(r.Bucket, result, success, attempts)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 				if success {
 					break
 				}
@@ -398,18 +417,22 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 					time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
 				}
 			}
+
+			if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
+				time.Sleep(wait)
+			}
 		}()
 	}
 	wk.Wait()
 
-	ri.Complete = ri.ObjectsFailed == 0
+	ri.Complete = !failed && ri.ObjectsFailed == 0
-	ri.Failed = ri.ObjectsFailed > 0
+	ri.Failed = failed || ri.ObjectsFailed > 0
 	globalBatchJobsMetrics.save(job.ID, ri)
 	// persist in-memory state to disk.
-	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+	batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 	if err := r.Notify(ctx, ri); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 	}
 
 	cancel()
@@ -470,8 +493,5 @@ func (r *BatchJobKeyRotateV1) Validate(ctx context.Context, job BatchJobRequest,
 		}
 	}
 
-	if err := r.Flags.Retry.Validate(); err != nil {
+	return r.Flags.Retry.Validate()
-		return err
-	}
-	return nil
 }

cmd/batch-rotate_gen.go

@@ -409,12 +409,6 @@ func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
-		case "Endpoint":
-			z.Endpoint, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Endpoint")
-				return
-			}
 		case "Encryption":
 			err = z.Encryption.DecodeMsg(dc)
 			if err != nil {
@@ -434,9 +428,9 @@ func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 5
 	// write "APIVersion"
-	err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	err = en.Append(0x85, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
 	if err != nil {
 		return
 	}
@@ -501,16 +495,6 @@ func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Prefix")
 		return
 	}
-	// write "Endpoint"
-	err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Endpoint)
-	if err != nil {
-		err = msgp.WrapError(err, "Endpoint")
-		return
-	}
 	// write "Encryption"
 	err = en.Append(0xaa, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e)
 	if err != nil {
@@ -527,9 +511,9 @@ func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 5
 	// string "APIVersion"
-	o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	o = append(o, 0x85, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
 	o = msgp.AppendString(o, z.APIVersion)
 	// string "Flags"
 	o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73)
@@ -561,9 +545,6 @@ func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Prefix"
 	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
 	o = msgp.AppendString(o, z.Prefix)
-	// string "Endpoint"
-	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
-	o = msgp.AppendString(o, z.Endpoint)
 	// string "Encryption"
 	o = append(o, 0xaa, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e)
 	o, err = z.Encryption.MarshalMsg(o)
@@ -651,12 +632,6 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
-		case "Endpoint":
-			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Endpoint")
-				return
-			}
 		case "Encryption":
 			bts, err = z.Encryption.UnmarshalMsg(bts)
 			if err != nil {
@@ -677,7 +652,7 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobKeyRotateV1) Msgsize() (s int) {
-	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 11 + z.Encryption.Msgsize()
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 11 + z.Encryption.Msgsize()
 	return
 }