deps: pin SILO Console 2.7.0

Pin the Console module to the v2.7.0 tag commit (4e91ae506e70), published
on 2026-11-09. It adds ten UI languages with Arabic right-to-left layout,
fixes usage-worker TLS, scheduling and shutdown, serves folder sizes over
plain HTTP, and fixes the container CA bundle and /tmp handling.

Console 2.7.0 requires silo-pkg v3.16.0 and builds with the mcli
RELEASE.2026-11-09T00-00-00Z source (pgsty/mc e1d4cc5abfe0). Go ignores
replacements declared by dependencies, so copy Console's single mc
replacement here and raise the silo-pkg requirement to match. silo-pkg
v3.16.0 changes only a policy comment and dependency floors.

Minimal version selection also raises klauspost/cpuid/v2 to v2.4.2,
go.yaml.in/yaml/v3 to v3.0.6, go-jose/v4 to v4.1.9, go-openapi/runtime to
v0.33.4 and tidwall/gjson to v1.20.1. minio-go stays at 320af050c445.
CREDITS is unchanged after regeneration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  # Dockerfile.goreleaser and Dockerfile.distroless pin their bases by digest
+  # so a release tag rebuilds from the same bytes. Without this the pins would
+  # silently rot and stop receiving base-image CVE fixes.
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: build

.github/goreleaser.yml

@@ -27,6 +27,10 @@ archives:
     ids:
       - silo
     name_template: "silo_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
+    formats: ["tar.gz"]
+    format_overrides:
+      - goos: windows
+        formats: ["zip"]
     # Explicit so the license materials cannot silently drop out of the
     # binary archives: GoReleaser's default file globs would miss NOTICE.
     # CREDITS stays out deliberately -- at 1.8MB it would dominate the

.github/workflows/docker-release.yml

@@ -140,7 +140,7 @@ jobs:
             mkdir -p "${context}/dockerscripts"
             tar -xzf "${archive}" -C "${context}" silo
             cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
-            cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
+            cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
               "${context}/dockerscripts/"
           done
 

.github/workflows/finalize-release.yml

@@ -148,7 +148,9 @@ jobs:
           for rpm_file in finalize/packages/*.rpm; do
             signature_output="$(sudo rpmkeys --checksig --verbose "${rpm_file}")"
             printf '%s\n' "${signature_output}"
-            if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q "key id ${key_id}: ok"; then
+            normalized_signature="$(printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]')"
+            full_fingerprint="$(printf '%s' "${expected_fingerprint}" | tr '[:upper:]' '[:lower:]')"
+            if ! printf '%s\n' "${normalized_signature}" | grep -Eq "key id ${key_id}: ok|key fingerprint: ${full_fingerprint}: ok"; then
               echo "Signature verification failed for ${rpm_file}" >&2
               exit 1
             fi

.github/workflows/go.yml

@@ -90,6 +90,56 @@ jobs:
       - name: Run S3 Select tests under race detector
         run: go test -race ./internal/s3select/... -count=1
 
+      - name: Run conditional PUT and multipart tests under race detector
+        run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP|PoolsMultipartCondition)' -count=1 -timeout=5m
+
+      - name: Run multi-pool delete consistency tests under race detector
+        run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
+
+      # Build tags: kqueue selects the kqueue backend of rjeczalik/notify on
+      # BSD and macOS and changes nothing on Linux. dev came with upstream's
+      # 2024-06 Makefile change for development builds; no file in this
+      # repository or its dependencies carries a dev build constraint, so it
+      # selects nothing and is kept only so that the documented command
+      # lines (make test, make install-race) stay valid.
+      - name: Run lifecycle expiry and native restore tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(PoolsLifecycle|PoolsMigrationLifecycle|ExpireRestored|NativeRestore)' -count=1 -timeout=5m
+
+      - name: Run multipart listing, cancellation and cleanup tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race ./cmd -run '^Test(ObjectCompleteMultipartUpload|Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
+
+      - name: Run listing metadata recovery, pagination and cancellation tests under race detector
+        run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m
+
+      - name: Run lock cancellation and late-grant cleanup tests under race detector
+        run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m
+
+      - name: Run CPU metrics tests under race detector
+        run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m
+
+      - name: Run tag replication tests under race detector
+        run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m
+
+      - name: Run Object Lock authorization and overwrite tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
+
+      - name: Run maintenance storage timeout and compatibility tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(StorageTimeout|StorageRollback|StoragePrerelease|MaintenancePartialFailedOverwriteAvailability|SingleRequest|WithDeadline)' -count=1 -timeout=5m
+
+      - name: Run logger target lifecycle tests under race detector
+        run: go test -race ./cmd -run '^TestDynamicWebhookOutlivesConfigContext' -count=1 -timeout=5m
+
+      - name: Compile deferred consistency diagnostics
+        run: go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'
+
   crosscompile:
     name: Cross Compile
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -70,6 +70,12 @@ jobs:
           buildscripts/verify-rebrand.sh
           dockerscripts/docker-entrypoint_test.sh
 
+      # Keep this gate aligned with the Object Lock race regression in go.yml.
+      - name: Gate release on Object Lock authorization and overwrite tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
+
       - name: Compute release variables
         env:
           # Passed through the environment, never interpolated into the script
@@ -132,9 +138,9 @@ jobs:
           cosign-release: v3.1.2
 
       - name: Build Draft release with GoReleaser
-        uses: goreleaser/goreleaser-action@v7
+        uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
-          version: "~> v2"
+          version: v2.18.3
           args: release --clean --skip=validate --config .github/goreleaser.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -150,7 +156,7 @@ jobs:
       - name: Install nFPM
         run: |
           set -euo pipefail
-          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
+          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
 
       - name: Build nFPM packages

.github/workflows/repository-cards.yml

@@ -0,0 +1,102 @@
+name: Repository Cards
+
+on:
+  schedule:
+    # 00:00 UTC = 08:00 Asia/Shanghai. GitHub may queue scheduled runs.
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - .github/workflows/repository-cards.yml
+      - .github/silo.svg
+      - buildscripts/repository-cards/**
+  pull_request:
+    branches: [main]
+    paths:
+      - .github/workflows/repository-cards.yml
+      - .github/silo.svg
+      - buildscripts/repository-cards/**
+
+permissions:
+  contents: read
+
+concurrency:
+  group: repository-cards-${{ github.event.pull_request.number || 'publish' }}
+  cancel-in-progress: false
+
+jobs:
+  check:
+    name: Validate repository cards
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - name: Use the runner's Python in an isolated environment
+        run: |
+          python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
+          "$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
+          echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
+      - run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v
+
+  publish:
+    name: Update README images
+    needs: check
+    if: github.repository == 'pgsty/silo' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: write
+      issues: read
+      pull-requests: read
+    env:
+      OUTPUT_BRANCH: codex/repository-cards
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - name: Use the runner's Python in an isolated environment
+        run: |
+          python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
+          "$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
+          echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
+
+      - name: Load the generated-assets branch
+        shell: bash
+        run: |
+          set -euo pipefail
+          artifacts="$RUNNER_TEMP/repository-cards"
+          if git ls-remote --exit-code --heads origin "$OUTPUT_BRANCH"; then
+            git fetch --depth=1 origin "$OUTPUT_BRANCH"
+            git worktree add --detach "$artifacts" FETCH_HEAD
+          else
+            status=$?
+            # Exit 2 means no matching ref; transport/auth failures must stop.
+            if [ "$status" -ne 2 ]; then exit "$status"; fi
+            git worktree add --detach "$artifacts" HEAD
+            git -C "$artifacts" checkout --orphan "$OUTPUT_BRANCH"
+            git -C "$artifacts" rm -rf .
+          fi
+
+      - name: Refresh contributor and star cards
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python buildscripts/repository-cards/update.py --output "$RUNNER_TEMP/repository-cards"
+
+      - name: Publish changed assets
+        shell: bash
+        run: |
+          set -euo pipefail
+          cd "$RUNNER_TEMP/repository-cards"
+          git add README.md history.json curated.json contributors.json \
+            contributors-light.svg contributors-dark.svg \
+            star-history-light.svg star-history-dark.svg
+          if git diff --cached --quiet; then
+            echo "Repository cards are already current."
+            exit 0
+          fi
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git commit -s -m "chore: update repository cards $(date -u +%F)"
+          # A normal push preserves history and refuses concurrent overwrites.
+          git push origin "HEAD:refs/heads/$OUTPUT_BRANCH"

.github/workflows/test-release.yml

@@ -4,13 +4,15 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
+      - "go.mod"
+      - "go.sum"
       - ".github/goreleaser.yml"
       - ".github/nfpm.yml"
       - "Dockerfile.goreleaser"
       - "Dockerfile.distroless"
       - "cmd/healthcheck-main.go"
       - "cmd/main.go"
-      - "dockerscripts/download-static-curl.sh"
+      - "dockerscripts/build-static-curl.sh"
       - "dockerscripts/docker-entrypoint.sh"
       - "dockerscripts/docker-entrypoint_test.sh"
       - "silo.service"
@@ -41,6 +43,28 @@ permissions:
   contents: read
 
 jobs:
+  curl:
+    name: Static curl (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-latest
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v7
+      - name: Build and exercise curl in an empty runtime
+        run: |
+          docker build --build-arg TARGETARCH=${{ matrix.arch }} \
+            --target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
+          docker run --rm silo-curl-test --version | tee curl-version.txt
+          grep -F 'curl 8.23.0 ' curl-version.txt
+          grep -F 'HTTP2' curl-version.txt
+          docker run --rm silo-curl-test --fail --silent --show-error \
+            --connect-timeout 15 --max-time 60 https://curl.se/robots.txt
+
   validate:
     runs-on: ubuntu-latest
     steps:
@@ -71,9 +95,9 @@ jobs:
           echo "LDFLAGS: ${LDFLAGS}"
 
       - name: GoReleaser config check
-        uses: goreleaser/goreleaser-action@v7
+        uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
-          version: "~> v2"
+          version: v2.18.3
           args: check --config .github/goreleaser.yml
 
       - name: Validate Helm chart and legacy upgrade identity
@@ -85,9 +109,9 @@ jobs:
           syft-version: v1.50.0
 
       - name: Build snapshot artifacts
-        uses: goreleaser/goreleaser-action@v7
+        uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
-          version: "~> v2"
+          version: v2.18.3
           # A pull-request snapshot has no trusted release identity. Exercise
           # the SBOM/checksum pipeline here, and reserve keyless signing for
           # the tag-triggered release workflow with GitHub OIDC.
@@ -96,13 +120,34 @@ jobs:
           LDFLAGS: ${{ env.LDFLAGS }}
           PKG_VERSION: ${{ env.PKG_VERSION }}
 
-      - name: Verify archive SBOM and checksum coverage
+      - name: Verify archive formats, documentation, SBOMs, and checksums
         run: |
           set -euo pipefail
-          mapfile -t archives < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz' | sort)
-          mapfile -t sboms < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz.sbom.json' | sort)
+          mapfile -t archives < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz' -o -name 'silo_*.zip' \) | sort)
+          mapfile -t sboms < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz.sbom.json' -o -name 'silo_*.zip.sbom.json' \) | sort)
           test "${#archives[@]}" -eq 6
           test "${#sboms[@]}" -eq 6
+          for os in linux darwin windows; do
+            format=tar.gz
+            binary=silo
+            if [ "${os}" = windows ]; then
+              format=zip
+              binary=silo.exe
+            fi
+            for arch in amd64 arm64; do
+              archive="dist/silo_${PKG_VERSION}_${os}_${arch}.${format}"
+              test -s "${archive}"
+              jq -e '.spdxVersion | startswith("SPDX-")' "${archive}.sbom.json" >/dev/null
+              if [ "${format}" = zip ]; then
+                listing="$(unzip -Z1 "${archive}")"
+              else
+                listing="$(tar -tzf "${archive}")"
+              fi
+              for member in "${binary}" LICENSE NOTICE README.md; do
+                grep -Fx "${member}" <<< "${listing}"
+              done
+            done
+          done
           manifest="dist/silo_${PKG_VERSION}_checksums.txt"
           test -s "${manifest}"
           (
@@ -119,7 +164,7 @@ jobs:
       - name: Install package validation tools
         run: |
           set -euo pipefail
-          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
+          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
           sudo apt-get update
           sudo apt-get install --yes rpm binutils
@@ -437,6 +482,17 @@ jobs:
             fi
           done
 
+          # Both variants must trust the same roots: the UBI bundle with the
+          # RHSB-2026-012 blocklist applied, not Debian's bundle.
+          tar -xOf "${ctx}/rootfs.tar" etc/ssl/certs/ca-certificates.crt > "${ctx}/ca.pem"
+          openssl crl2pkcs7 -nocrl -certfile "${ctx}/ca.pem" \
+            | openssl pkcs7 -print_certs -noout > "${ctx}/ca-subjects.txt"
+          test "$(grep -c '^subject=' "${ctx}/ca-subjects.txt")" -gt 100
+          if grep -E '^subject=.*(AAA Certificate Services|emSign ECC Root CA - G3)' "${ctx}/ca-subjects.txt"; then
+            echo "distroless CA bundle still trusts a blocklisted root"
+            exit 1
+          fi
+
           # The baked-in healthcheck must drive Docker's health state on its
           # own, the probe binary must be directly exec-able without any
           # shell, and SIGTERM must still reach PID 1 (the server binary is
@@ -490,7 +546,7 @@ jobs:
           bash -n buildscripts/package/lifecycle_test.sh
           buildscripts/package/lifecycle_test.sh
           bash -n dockerscripts/docker-entrypoint_test.sh
-          bash -n dockerscripts/download-static-curl.sh
+          bash -n dockerscripts/build-static-curl.sh
           dockerscripts/docker-entrypoint_test.sh
           go run ./buildscripts/rebrand-guard
           buildscripts/verify-rebrand.sh

.github/workflows/vulncheck.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Install govulncheck
         run: |
-          go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
+          go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
 
       - name: Run govulncheck

.gitignore

@@ -62,10 +62,10 @@ dist/
 
 .claude/
 .codex/
-AGENTS.md
-CLAUDE.md
 _bmad/
 _bmad-output/
+# Investigation and editorial working files belong outside this repository.
+docs/investigations/
 docs/security/
 docs/rebranding.md
 .release-sign/

.oss-scanner/Dockerfile

@@ -0,0 +1,34 @@
+# Copyright 2026 PGSTY contributors.
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+FROM golang:1.27.3-bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    GOTOOLCHAIN=local \
+    CGO_ENABLED=0 \
+    MINIO_API_REQUESTS_MAX=10000
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates curl git jq openssl python3 procps \
+    && rm -rf /var/lib/apt/lists/*
+
+# Cache dependencies in the final image: subsequent analysis runs offline.
+# Honor go.mod's PGSTY replacements and leave the module files unchanged.
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+COPY . /src
+
+# Keep normal Go debug information; the release Makefile strips it.
+RUN go build -tags kqueue -o /usr/local/bin/silo . \
+    && silo --version
+
+# Compile test packages while online, including the server and internal code.
+# Retain both the module and build caches so new reproducers work offline.
+RUN go test -tags kqueue,dev -run '^$' -timeout=10m ./cmd ./internal/...
+
+ENV GOPROXY=off \
+    GOSUMDB=off
+
+CMD ["bash"]

.oss-scanner/Dockerfile.dockerignore

@@ -0,0 +1,43 @@
+# Override the release .dockerignore: keep source, CI files and all test fixtures.
+# The scanner adds Git history separately after building the image.
+.git
+.bin
+.claude
+.codex
+.idea
+.release-sign
+.vscode
+_bmad
+_bmad-output
+dist
+vendor
+**/node_modules
+**/*.test
+**/.DS_Store
+default.etcd
+docs/investigations
+docs/security
+docs/rebranding.md
+
+# Local server and debugging binaries.
+minio
+silo
+hash-set
+healing-bin
+inspect
+pprofgoparser
+reorder-disks
+s3-check-md5
+s3-verify
+xattr
+xl-meta
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
+docs/debugging/reorder-disks/reorder-disks
+docs/debugging/populate-hard-links/populate-hardlinks
+docs/debugging/xattr/xattr

.oss-scanner/threat_model.md

@@ -0,0 +1,131 @@
+# PGSTY Silo security scope
+
+Silo is PGSTY's independently maintained S3-compatible object-storage server,
+derived from MinIO. It serves untrusted network requests and stores user data,
+including PostgreSQL backups. Scan the current `pgsty/silo` `main` branch.
+Read `/src/SECURITY.md`, `/src/VULNERABILITY_REPORT.md`, and `/src/CHANGELOG.md`.
+
+## Repository boundaries and lineage
+
+The Go module intentionally remains `github.com/minio/minio`. Historical
+`MINIO_*` environment variables, S3 headers, `/minio/*` routes, and `.minio.sys`
+storage names are compatibility interfaces, not evidence that upstream MinIO
+is the target. Use this checkout's code and dependency versions for reproducers.
+
+This enrollment covers the server code, including inherited code still present
+in the supported branch and its PGSTY changes. The maintained Console, client,
+and shared packages appear as dependencies in `go.mod`. Do not replace them
+with upstream releases. They are not additional independent scan targets under
+this enrollment. A server-side integration defect is in scope; clearly identify
+a dependency-only root cause rather than attributing it to this repository.
+
+## Attackers and important surfaces
+
+- Unauthenticated S3/HTTP clients, including hostile object names, headers,
+  query parameters, XML/JSON bodies, signatures, and multipart data.
+- Authenticated users, service accounts, and temporary STS credentials whose
+  authority must remain bounded by the effective IAM and bucket policies.
+- Bucket owners and delegated administrators with explicitly limited actions;
+  distinguish them from the unrestricted deployment/root administrator.
+- Inputs from configured identity providers, replication/tiering targets, and
+  webhooks where the configuration and permissions make an attack reachable.
+
+Prioritize authentication/signature verification, IAM/STS and policy
+composition, administrative API authorization, object/version isolation,
+Object Lock retention and legal holds, multipart operations, replication,
+lifecycle/tiering, and multi-pool deletion/migration. Include archive, XML,
+JSON and S3 Select parsing when remotely reachable. Investigate credential
+exposure, path traversal, unauthorized object access, SSRF across a meaningful
+boundary, and attacks that corrupt or irreversibly delete other users' data.
+
+The deployment administrator controls root credentials, storage directories,
+trusted peers, and identity/KMS/replication endpoint configuration. Host root
+access, stolen root credentials, or a malicious replacement of the server
+binary do not by themselves demonstrate a vulnerability. An unprivileged
+account gaining those capabilities does. An administrator's documented ability
+to configure an outbound endpoint is not sufficient evidence of SSRF; show
+an unauthorized actor, forbidden destination, or confused-deputy effect.
+
+Object Lock has explicit retention semantics. Do not discard a retention bypass
+just because the caller has some administrative permission: describe the mode,
+version, legal hold, and exact IAM actions, and compare with the intended rules.
+Likewise, do not assume bucket access implies access to other buckets or users.
+
+Known consistency limitations are described in `CONTRIBUTING.md` and the
+changelog. Tests tagged `consistency_strict` exercise stronger guarantees
+outside the current maintenance contract and can fail. A failing diagnostic
+alone is not a new vulnerability; demonstrate a reachable security impact.
+These limitations do not excuse an authorization or retention bypass.
+
+## Offline environment
+
+Source is in `/src`; the unstripped server is `/usr/local/bin/silo`. Go, the
+module cache, and the test build cache are retained. The scanner adds Git
+history to `/src/.git` after building this image; a plain Docker build omits it.
+The Dockerfile compiles the server and tests with the selected PGSTY dependency
+replacements. `GOTOOLCHAIN=local` and `GOPROXY=off` prevent surprise downloads.
+CGO is disabled as in the normal server/unit-test CI; enable it explicitly for
+race tests when needed (the Go base image includes a C compiler).
+
+Useful offline checks:
+
+```sh
+cd /src
+silo --version
+go test -tags kqueue,dev ./internal/auth ./internal/config/identity/... ./internal/s3select/... -count=1 -timeout=10m
+go test -tags kqueue,dev ./cmd -run '^(TestGetRequestAuthType|TestS3SupportedAuthType|TestIsReqAuthenticated|TestCheckAdminRequestAuthType|TestValidateAdminSignature|TestDoesPolicySignatureMatch|TestDoesPresignedSignatureMatch|TestDoesPresignedV2SignatureMatch|TestSTSLDAPLogin.*|TestGetSTSLDAPLoginSourceIP.*)$' -count=1 -timeout=10m
+```
+
+For HTTP reproducers, start a disposable loopback server with synthetic
+credentials and a fresh data directory:
+
+```sh
+MINIO_ROOT_USER=scanneradmin MINIO_ROOT_PASSWORD=scanner-local-test-only \
+  silo server /tmp/silo-scanner-data \
+  --address 127.0.0.1:9000 --console-address 127.0.0.1:9001
+```
+
+The broader suites are `go test -tags kqueue,dev ./internal/... -timeout=15m`
+and `go test -tags kqueue,dev ./cmd -timeout=30m`. Some integration suites need
+LDAP, OIDC, etcd, KMS, multiple servers, or IPv6; those services are not started
+by this Dockerfile. Do not treat a missing external service as a security
+finding. The focused commands above do not require those providers.
+
+## Reports and severity
+
+Include the commit, exact attacker privileges/policy, relevant deployment
+settings, API requests or a standalone test, observed impact, and affected
+versions where established. For data-loss or cross-pool reports, include the
+object/version state and a deterministic sequence. Deduplicate by root cause
+and distinguish a new flaw from an already documented limitation or fixed bug.
+
+Unauthenticated code execution or broad credential/authorization compromise
+can be critical. Cross-user object access, unauthorized destructive operations,
+and retention bypass can be high or critical according to their demonstrated
+reach. Rate availability issues by practical cost, scope and recovery; a
+single malformed request and an authorized full-disk upload are different
+threats. Do not claim exploitability solely from a static pattern.
+
+Prefer minimal patches and regression tests that preserve S3/storage format
+compatibility. Use only synthetic data and local test instances; do not test
+against public Silo/MinIO deployments or real backup repositories.
+
+## Maintaining the scanner image
+
+Build from the repository root, then run the checks above without networking:
+
+```sh
+docker build -f .oss-scanner/Dockerfile -t silo-scanner:local .
+docker run --rm -it --network=none silo-scanner:local bash
+```
+
+Keep the Go image version aligned with `go.mod`. The adjacent
+`Dockerfile.dockerignore` preserves compressed test fixtures that the release
+image's root `.dockerignore` excludes. Local build outputs, editor state, and
+private working directories are excluded from the scanner context.
+
+Enrollment lives in `anthropics/oss-scanner`, under `projects/silo/project.yaml`.
+Once these files are published on `main`, that configuration can use
+`dockerfile: .oss-scanner/Dockerfile` and
+`threat_model: .oss-scanner/threat_model.md`; remove its adjacent copies in the
+same update. Do not switch the enrollment paths before the files are public.

AGENTS.md

@@ -0,0 +1,84 @@
+# SILO Repository Guide
+
+## Project map
+
+- Server: `pgsty/silo` (this checkout, default branch `main`)
+- Console: `pgsty/silo-console` (usual sibling checkout `../silo-console`)
+- Client: `pgsty/mc`, shipped as `mcli` (usual sibling checkout `../mc`)
+- Shared packages: `pgsty/silo-pkg` (usual sibling checkout `../silo-pkg`)
+- Documentation: `pgsty/silo.pgsty.com` (usual sibling checkout `../silo.pgsty.com`),
+  published at <https://silo.pgsty.com/>
+
+The server executable, package, systemd service, and container image use `silo`;
+the public Docker image is `docker.io/pgsty/silo`. The server module remains
+`github.com/minio/minio`. Consult the current `go.mod` and release configuration
+for selected component versions and compatibility replacements.
+
+## Supported stack and compatibility policy
+
+The maintained, release-gating product graph is the coordinated PGSTY stack:
+`silo` + `silo-console` + `mc` + `silo-pkg`.
+
+Keep compatibility with upstream MinIO/MC on a best-effort basis. Preserve
+inexpensive wire, configuration, CLI, migration, and import compatibility when
+it helps users, and document known differences. Do not infer from source
+lineage, MinIO-compatible protocols, retained `MINIO_*`/`MC_*` names, or an
+inherited upstream test that unmodified upstream MinIO is a supported release
+target.
+
+Upstream-only compatibility checks may remain as advisory evidence, but they
+must not force a downgrade of a maintained SILO component, a fork-only API
+workaround, or a release block. Make upstream compatibility a hard gate only
+when the user explicitly requests that scope.
+
+## Dependency selection
+
+- When PGSTY maintains a component, import and require it directly under its own
+  module path. In particular, prefer `github.com/pgsty/silo-pkg/v3` over
+  `github.com/minio/pkg/v3` in maintained SILO source.
+- Do not use an upstream package merely to make unmodified upstream MinIO or MC
+  compile. Unavoidable transitive upstream modules should be documented and
+  kept separate from the maintained product dependency.
+- `github.com/minio/minio-go/v7` is the explicit exception: use the verified
+  upstream module/commit while it contains the required fixes; do not recreate
+  a SILO fork without a concrete functional divergence.
+- Check the current `go.mod` before changing versions. Coordinate breaking
+  import-path changes across package, client, Console, and server releases.
+
+## Documentation and delivery
+
+The companion site owns product, operations, migration, security-advisory,
+design, and release documentation. Maintain English and Chinese content in
+that repository and link to its canonical URLs from this one. The site's
+`content/docs/` is the documentation entry point; detailed pages live under
+`content/operations/`, `administration/`, `reference/`, `compatibility/`,
+`about/`, and `blog/`, following the existing structure.
+
+Keep repository entry points and contributor instructions here. Retained
+upstream documents, examples, and test fixtures under `docs/` may be updated
+when a code or tooling change requires it; do not build a second documentation
+site in this tree.
+
+Investigation plans, prompts, AI session transcripts, execution logs, temporary
+reports, and private security material belong outside the checkout, for example
+in a task-specific directory under `~/tmp/`. Do not recreate
+`docs/investigations/`, `docs/security/`, or `docs/rebranding.md`, or move their
+working material to another repository directory. Extract reusable, verified
+knowledge into the companion site; keep private evidence outside public Git.
+
+Compatibility pages may continue to describe similarities with upstream, but
+must label that compatibility as best effort. The supported and tested server
+for Console and mcli administration is `pgsty/silo`.
+
+Before removing or moving documentation, check repository links and scripts,
+the companion site's links and anchors, and references from this guide. Use
+fixed commit URLs for historical evidence that should survive deletion from
+the current tree. Run the site's `make check` for site changes and this
+repository's `make rebrand-guard` for documentation cleanup that changes the
+identifier inventory. When new public URLs are involved, publish the site
+before publishing source changes that depend on those URLs.
+
+Keep changes in the repository that owns the affected surface. Treat every
+repository commit, tag, release, image, documentation update, and deployment as
+a separate deliverable. Follow `CONTRIBUTING.md`, including DCO sign-off
+(`git commit -s`). `CLAUDE.md` imports this guide so both agents use one policy.

CHANGELOG.md

@@ -0,0 +1,174 @@
+# Changelog
+
+## RELEASE.2026-10-26T00-00-00Z
+
+Package version: `20261026000000.0.0`; published from source `c3ad836aed0d`.
+Artifacts are listed in the [GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z).
+The coordinated [English release notes](https://silo.pgsty.com/blog/release/silo-20261026/)
+and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261026/)
+cover all components, upgrade guidance, and validation scope.
+
+- Fix LDAP login for long normalized DNs without hiding unreadable IAM
+  revocation records. Only length errors for valid keys with an ASCII path
+  component over 255 bytes are treated as absent; long-DN writes still fail.
+  Thanks to [@910patin](https://github.com/910patin) for the original fix and
+  reproduction in [#250](https://github.com/pgsty/silo/pull/250).
+- Build the server and standalone debugging tools with Go 1.27.3 and refresh
+  Go dependencies, including silo-pkg v3.15.2, mcli 20261026
+  and upstream minio-go `320af050c445` for multipart, read/seek,
+  batch-delete and bucket-replication removal fixes. Retain output-format and NetBSD compatibility selections.
+- Embed Console v2.5.1 source `fff1618d9abb`, including its GUI update,
+  diagnostic WebSocket closing handshake, complete health-report decoding,
+  group-membership preservation, OIDC group permissions, TLS-proxy session
+  cookies, credential exports and accessibility fixes.
+- Bundle mcli 20261026 in container builds, the verified client installer and
+  Helm client jobs. Include client identity isolation, resync status and rate
+  fixes, and bounded batch-status fallback. Scripts must handle the additive
+  batch `pending` state and nonzero exits for partial resync mutations.
+- Read current bucket inventory in account-info responses so Console refreshes
+  immediately reflect successful creation and deletion. This management endpoint
+  now performs a live list instead of returning a cached inventory.
+- Restore the native built-in policy immediately after resetting an override.
+  Peer and etcd notifications reload committed state, so delayed delete events
+  preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
+  custom-policy deletion behavior; storage failures retain the previous policy.
+- Reload policies without holding the authentication-cache lock, retain IAM
+  write ordering and publish replacements only after a complete successful load.
+  Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
+  the caller's policy-query deadline.
+- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
+  choose an available pool for a new delete marker, and keep tier data referenced
+  by surviving copies. This does not introduce strict cross-pool read recovery.
+- Preserve remote tier data when lifecycle expiry only adds a delete marker,
+  including non-null versions in suspended buckets. Forward `FULL_OBJECT`
+  checksum type for multipart replicas. Thanks to
+  [@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
+  and multi-pool deletion contributions.
+- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
+  fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
+- Forward storage RPC delete options, including the previous data directory
+  needed by overwrite rollback. Preserve existing drive/RPC timeouts, request
+  cancellation, object locking, read-quorum and healing behavior.
+- Update bundled curl to 8.23.0 and require patched static libraries. Apply
+  [Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
+  the root blocklist also rejects legitimate chains under the affected roots.
+- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
+
+Storage timeouts retain their existing meaning: a returned timeout does not
+prove that the underlying disk operation stopped or that a write did not
+commit. Late commits or delayed rollback can affect newer writes; stale reads
+remain possible after partial failures, lease loss or network partitions. This
+release does not provide distributed fencing or repair previously inconsistent
+objects.
+
+These source changes are not included in the published 20261018 artifacts.
+
+## RELEASE.2026-10-18T00-00-00Z
+
+Package version: `20261018000000.0.0`; published on 2026-10-18 from source
+`0cd388c666f8211f552604ae73448a066e9c25ce`. This section covers the changes
+after the public 20260916 source, `2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
+
+Full [English release notes](https://silo.pgsty.com/blog/release/silo-20261018/)
+and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261018/)
+cover the upgrade contract, evidence and remaining limits.
+
+- **Object Lock (#234, #235):** require `s3:PutObjectRetention` with request-body
+  conditions before object lookup; weakening active GOVERNANCE additionally
+  requires bypass intent and permission. Protect caller-addressed write versions,
+  directory null versions and persisted multipart targets under the write lock;
+  fail closed on unreadable lock state. See SN-2026-015/016 in the
+  [advisory ledger](https://silo.pgsty.com/about/security-advisories/).
+- **IAM Admin authorization (#242, `995439260`):** enforce explicit self denies,
+  session restrictions and plugin decisions; prevent restricted delegation and
+  non-root mutation of root-derived credentials under built-in authorization.
+  Preserve empty-policy inheritance and damaged-key cleanup. Normalize bulk
+  self-list aliases and allow ordinary policyless self inspection. See
+  [SN-2026-017](https://silo.pgsty.com/blog/security/20261018-release-hardening/#sn-2026-017).
+- **Scoped STS revocation (`0cd388c66`):** enumerate durable parent/type-selected
+  tokens, propagate each committed tombstone to sibling nodes and sites, and
+  recover missed notifications through existing refresh/healing. Tokens issued
+  only at an isolated site are outside the requesting cluster's snapshot;
+  propagation is eventual and newly issued keys remain valid.
+- **Pool migration (#248, `60c7179dd`, `1748f62c7`):** prevent stale moved
+  payloads from overwriting newer copies, preserve null/UUID delete markers,
+  recheck source versions and clean only confirmed moved versions under the
+  object lock. Added per-pool reads and per-version cleanup have maintenance
+  costs; earlier data loss is not reconstructed.
+
+- **LIST recovery (#239, `3dd53eb81`, `2080b3d03`):** confirm eligible uncertain
+  latest-object groups under the appropriate read locks, distinguish absence
+  from failure, preserve directory/common-prefix entries and terminal pagination
+  errors, and make lock backoff cancelable. Resolved entries keep their fast path;
+  version history, quorum and disk formats retain their contracts. The strict
+  sub-5% performance target remains unconfirmed in the recorded comparison.
+- **Multipart listing (`4733e1423`):** withdraw #243's default persistent scan.
+  Public listing uses node-local cache or exact-key lookup; restart omissions,
+  prefix and cache-pagination limits remain. The retired strict switch is ignored
+  with a startup warning. **Former strict users lose durable listing and strict
+  cancellation.** Keep identity metadata, path/wrong-target safety, default
+  best-effort Abort and peer notification. Diagnostic `multipart-preflight`
+  reports `mode=legacy`; its budgets do not gate public operations. #79 remains
+  open, including delayed creation after cancellation.
+- **Multipart cleanup (#236, `0f9199e0c`):** parse native timestamps before the
+  Base64 fallback and remove empty hash parents without a second expiry wait.
+  Keep active/unreadable uploads; default expiry/check intervals stay 24h/6h.
+- **Commit errors (#232):** missing staged `xl.meta` at final commit becomes
+  HTTP 503 `SlowDownWrite`; preserve this classification across storage RPC and
+  healing. Real incomplete bodies retain their existing errors.
+- **Reused metadata (#233):** clear absent legacy `EcM`/`Signature` and locally
+  reset mutually exclusive scanner response fields without changing disk/RPC formats.
+- **Checksum and Console errors (#225, #229, #238):** align Content-MD5 and
+  algorithm-specific messages, retaining the generic unknown-algorithm message;
+  embed Console's wrapped-error and `InvalidBucketState` handling.
+- **Lambda and erasure documentation (#220, `2fde3cf53`):** fix inverted
+  `TargetIDSet.IsEmpty`; document parity/quorum per object and erasure set.
+- **Coordinated dependencies:** Console v2.4.2 (`ef2dd2c41d21`), mcli 20261018
+  (`fa3b8bf72458`), silo-pkg v3.14.3, upstream minio-go v7.4.1 and Go 1.27.2.
+  Standalone Console/mcli still declare pkg v3.14.2. The Server package update
+  speeds admin-action classification without changing policy behavior. Refresh
+  Go x/*, go-jose v4.1.6, LDAP v3.4.15, compress v1.20.3, msgp v1.6.5, etcd v3.7.2,
+  fwd v1.2.1 and cpuid v2.4.1. OIDC RSA JWKs above 4096 bits are rejected.
+  Keep the NetBSD go-systemd v22.6.0 replacement; parallel SDK GET/Compose remain opt-in.
+- **Build and delivery:** gate releases on Object Lock regressions; use
+  Git for rebrand scanning without a ripgrep dependency (`7007c164a`). Refresh UBI 9 pins,
+  GoReleaser v2.18.3, nFPM v2.48.0, golangci-lint v2.15.0 and govulncheck v1.9.0.
+  Update bundled client hashes/helper and the Helm client image to mcli 20261018.
+  The chart's Server default stays at the public 20260916 image until publication.
+
+Upgrade every writer and replication site; follow the
+[IAM recovery guide](https://silo.pgsty.com/operations/replication/iam-upgrade/).
+Old strict multipart users must review the capability loss before upgrading.
+#217's multi-site delete-marker late creation and crash residue remain unresolved.
+Source tests, draft artifacts, public images and production rollout are separate
+acceptance boundaries; historical 20261015 test counts do not certify this source.
+
+## RELEASE.2026-09-16T00-00-00Z
+
+Package version: `20260916000000.0.0`; published source:
+`2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
+
+This release includes signed-header and payload-policy hardening (SN-2026-011/012),
+durable IAM revocations (SN-2026-013), the bounded Console sharing proxy
+(SN-2026-014), password-permission separation, multi-pool conditions/deletion/tag
+repairs, encrypted/federated copy repairs, delete-marker retry fixes, opt-in strict
+multipart discovery, Go-default TLS key exchange, absolute HTTP/1 header deadlines
+and synchronized CPU metrics. It embeds Console v2.4.1, mcli 20260916 and pkg v3.14.1.
+
+The complete release, contributor credits, historical artifact acceptance and
+upgrade requirements are preserved in the canonical
+[English](https://silo.pgsty.com/blog/release/silo-20260916/) and
+[Chinese](https://silo.pgsty.com/zh/blog/release/silo-20260916/) release records.
+
+## RELEASE.2026-09-03T13-18-01Z
+
+Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
+[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
+[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
+
+This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
+mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
+Installing the newer standalone mcli or Console does not replace components
+inside this existing Server binary or image.
+
+Earlier releases: [release archive](https://github.com/pgsty/silo/releases).

CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

CONTRIBUTING.md

@@ -53,6 +53,20 @@ Before opening a pull request:
 - Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
   `.minio.sys`, ARN, module/import-path, or serialized compatibility name.
 
+The maintenance branch retains existing storage timeouts, cancellation,
+object locking, read-quorum and healing behavior. The normal suite checks local
+and RPC timeout/cancellation, remote rollback options, and readability after a
+failed partial overwrite. A timeout does not prove that a disk worker stopped
+or that a write did not commit.
+
+Stronger mutation lifetime, rollback ownership, generation ordering, strict-read
+and cross-pool recovery diagnostics are preserved behind `consistency_strict`.
+Compile them with `go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'`
+and run individual scenarios explicitly. They describe guarantees outside the
+maintenance contract and may fail; report their results separately from the
+normal suite. They must not be used to claim distributed fencing or repair of
+previously inconsistent objects.
+
 ### Commit changes
 
 After verification, commit your changes with a concise message and a DCO
@@ -77,6 +91,16 @@ test evidence, compatibility notes, and documentation impact. Public product
 documentation is owned by the separate
 [`pgsty/silo.pgsty.com`](https://github.com/pgsty/silo.pgsty.com) repository.
 
+## Contributor recognition
+
+Every human issue or pull-request author is recognized in [CONTRIBUTORS.md](CONTRIBUTORS.md),
+including open issues, draft PRs, and PRs closed without merging. Merged fixes,
+adopted proposals, and reports that lead to fixes receive greater prominence;
+first participation guides the remaining order. Work incorporated through a
+later PR retains credit without changing the original PR's recorded status.
+Security disclosures are credited with the reporter's agreement. DCO sign-off
+applies to code commits, not to opening an issue.
+
 ## Licensing of Contributions
 
 Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the

Dockerfile.distroless

@@ -8,10 +8,30 @@
 # seeds fresh volumes from the image-layer mountpoint, no entrypoint
 # exists to repair ownership at runtime, and 0777 is what keeps every
 # privilege mode working, --user included.
-FROM busybox:1.37.0 AS prep
+FROM busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec AS prep
 RUN mkdir -p /prep/data && chmod 0777 /prep/data
 
-FROM gcr.io/distroless/static-debian12:latest
+# Debian's bundle in distroless still trusts the roots the classic image
+# blocklists. Build the same bundle as Dockerfile.goreleaser so both variants
+# trust the same roots; it is architecture-neutral, so build it natively.
+FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
+# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
+RUN dnf -y install ca-certificates && \
+    if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
+      blocklist=/etc/pki/ca-trust/source/blocklist; \
+    else \
+      blocklist=/etc/pki/ca-trust/source/blacklist; \
+    fi && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
+      "$blocklist/Comodo_AAA_Services_root.pem" && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
+      "$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
+    update-ca-trust && \
+    cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt
+
+FROM gcr.io/distroless/static-debian12:latest@sha256:d75cdd72874d4790092fcb1b058493ecf6bb5bf2b2b897045b00ff01d91843f2
 
 LABEL org.opencontainers.image.title="Silo" \
       org.opencontainers.image.description="S3-Interface Libre Object Storage (distroless)" \
@@ -28,6 +48,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     HOME=/tmp
 
+# The only file under /etc/ssl/certs, so replacing it is sufficient.
+COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --chmod=0755 silo /usr/bin/silo
 # COPY of a directory copies its contents, not the directory entry, so an
 # empty /prep/data would arrive as a default root:0755 /data and non-root

Dockerfile.goreleaser

@@ -1,4 +1,15 @@
-FROM golang:1.27.1-alpine AS build
+FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build
+ARG TARGETARCH
+COPY dockerscripts/build-static-curl.sh /build/build-static-curl
+RUN /bin/sh /build/build-static-curl
+
+# Exercise the exact shipped curl without a dynamic loader or shared libraries.
+FROM scratch AS curl-runtime
+COPY --from=curl-build /go/bin/curl /curl
+COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ENTRYPOINT ["/curl"]
+
+FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build
 
 ARG TARGETARCH
 
@@ -6,9 +17,9 @@ ENV GOPATH=/go
 ENV CGO_ENABLED=0
 
 ARG MC_REPO=pgsty/mc
-ARG MC_VERSION=RELEASE.2026-09-03T07-13-05Z
-ARG MC_AMD64_SHA256=cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8
-ARG MC_ARM64_SHA256=7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8
+ARG MC_VERSION=RELEASE.2026-10-26T00-00-00Z
+ARG MC_AMD64_SHA256=e86295bb11ed59369c0560f030d59abd42fb64a376b1d7e9799ec2e8596ebf64
+ARG MC_ARM64_SHA256=47b1a558f766f26bef5beeedd5300e082bcdec07a5e7d4a16f5f9d168f5275d1
 
 RUN apk add -U --no-cache \
     ca-certificates \
@@ -54,20 +65,30 @@ RUN apk add -U --no-cache \
       echo "No mc or mcli binary found in archive:"; ls -la /tmp/mcli-extract/; exit 1; \
     fi && \
     chmod +x /go/bin/mcli && \
-    ln -sf mcli /go/bin/mc
+    ln -sf mcli /go/bin/mc && \
+    mkdir -p /go/share/mcli && \
+    cp /tmp/mcli-extract/LICENSE /tmp/mcli-extract/NOTICE /tmp/mcli-extract/CREDITS /go/share/mcli/
 
-COPY dockerscripts/download-static-curl.sh /build/download-static-curl
-RUN chmod +x /build/download-static-curl && \
-    /build/download-static-curl
-
-FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
+FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
+# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
 RUN dnf -y install ca-certificates && \
+    if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
+      blocklist=/etc/pki/ca-trust/source/blocklist; \
+    else \
+      blocklist=/etc/pki/ca-trust/source/blacklist; \
+    fi && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
+      "$blocklist/Comodo_AAA_Services_root.pem" && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
+      "$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
     update-ca-trust && \
     cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
-FROM registry.access.redhat.com/ubi9/ubi-micro:latest
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
 
 LABEL org.opencontainers.image.title="Silo" \
       org.opencontainers.image.description="S3-Interface Libre Object Storage" \
@@ -88,7 +109,10 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY silo /usr/bin/silo
 COPY --from=build /go/bin/mcli /usr/bin/mcli
-COPY --from=build /go/bin/curl* /usr/bin/
+# mcli's own legal files, from the same verified archive as the binary.
+COPY --from=build /go/share/mcli /licenses/mcli
+COPY --from=curl-build /go/bin/curl /usr/bin/curl
+COPY --from=curl-build /go/share/curl /licenses/curl
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY LICENSE /licenses/LICENSE
 COPY NOTICE /licenses/NOTICE

Makefile

@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
 GOOS ?= $(shell go env GOOS)
 GOARCH ?= $(shell go env GOARCH)
-GOLANGCI_VERSION ?= v2.13.1
+GOLANGCI_VERSION ?= v2.15.0
 
 VERSION ?= $(shell git describe --tags)
 REPO ?= docker.io/pgsty
@@ -73,6 +73,11 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
+# -tags kqueue,dev: kqueue selects the kqueue backend of rjeczalik/notify on
+# BSD and macOS (no-op on Linux); dev came with upstream's 2024-06 change for
+# development builds and no file in this repository or its dependencies
+# carries a dev build constraint, so it selects nothing. Kept for the
+# documented command lines; .github/workflows/go.yml carries the same note.
 test: verifiers build ## builds Silo, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue,dev ./...
@@ -216,7 +221,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
 			--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
 		mkdir -p "$$context/dockerscripts"; \
 		cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
-		cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
+		cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
 			"$$context/dockerscripts/"; \
 		docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
 			-f "$$context/Dockerfile.goreleaser" "$$context"

README.md

@@ -24,7 +24,7 @@
 <p align="center">
   <a href="https://silo.pgsty.com/"><img alt="Website" src="https://img.shields.io/badge/Website-silo.pgsty.com-1d588c"></a>
   <a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
-  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
+  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
   <a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
   <a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
 </p>
@@ -35,6 +35,14 @@
 > [!NOTE]
 > Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
 
+## SILO 20261026000000
+
+[SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) is available as platform archives, Linux packages, and standard or distroless Docker images for amd64 and arm64. The release tag is `RELEASE.2026-10-26T00-00-00Z`; the package version is `20261026000000.0.0`, built from source `c3ad836aed0d`.
+
+This release fixes LDAP long-DN login, IAM policy refresh, multi-pool deletion safety, tiered-object expiry, multipart replication checksums, and remote rollback-option propagation. It embeds Console v2.5.1 (`fff1618d9abb`) and uses silo-pkg v3.15.2, Go 1.27.3, and upstream minio-go `v7.4.2-0.20261025165926-320af050c445`. Use mcli 20261026 for administration; the container images bundle it as `mcli`.
+
+The Server IAM cache repair enables Console's **Reset to default** action. Existing storage timeouts and cancellation behavior remain: a timeout does not prove that a disk operation stopped or a write did not commit. Read the [coordinated release notes](https://silo.pgsty.com/blog/release/silo-20261026/) and [remaining storage limits](CHANGELOG.md#release2026-10-26t00-00-00z) before upgrading. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
+
 ## Overview
 
 PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
@@ -54,7 +62,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
   -e MINIO_ROOT_USER=minioadmin \
   -e MINIO_ROOT_PASSWORD=change-me-long-password \
   -v "$PWD/data:/data" \
-  docker.io/pgsty/silo:latest server /data --console-address ":9001"
+  docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
 ```
 
 <p align="center">
@@ -85,12 +93,15 @@ Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub
 
 ## Compatibility
 
-The S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, the `github.com/minio/*` import paths, and the on-disk format (including `.minio.sys`) are preserved and held in place by a CI compatibility check. Only Silo-owned delivery surfaces change: the `silo` executable, package, service, Helm chart, and container image — no `minio` binary alias is installed.
+Silo preserves S3 and storage-format compatibility, including existing `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and `.minio.sys` data. CI guards selected compatibility identifiers; release notes document intentional security and behavior changes. Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and container image; no `minio` server binary alias is installed.
+
+The supported release stack is `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`; compatibility with unmodified upstream MinIO/MC is best effort. The Server, Console, and client retain their historical module paths where needed, while maintained code imports `github.com/pgsty/silo-pkg/v3` directly. The SDK `github.com/minio/minio-go/v7` is an explicit upstream dependency. See the current [go.mod](go.mod) for versions and replacements.
 
 Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
 
 ### TLS and Go upgrades
 
+Server 20260916 restored Go's default key exchange; Server 20261018 retains it with Go 1.27.2.
 TLS key exchange follows Go's defaults across the S3 listener, node links,
 replication, identity providers, etcd, and external HTTP services. If an endpoint
 cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
@@ -107,7 +118,16 @@ values to restore Keychain trust. Explicit certificates in the configured `CAs`
 directory remain additive to the selected root pool.
 Go 1.27 binaries require macOS 13 or later. See the
 [Go release notes](https://go.dev/doc/go1.27) and the
-[SILO stack investigation](docs/investigations/go127-stack.md).
+[Go 1.27 TLS and OIDC discovery guide](https://silo.pgsty.com/blog/design/go127-tls-oidc-discovery/).
+
+## Documentation ownership
+
+User documentation is maintained at [silo.pgsty.com](https://silo.pgsty.com/docs/),
+with source in [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com).
+The remaining `docs/` tree contains inherited references, examples, and tooling
+fixtures. Investigation logs, AI work records, and temporary reports are kept
+outside this repository; reusable findings belong in the companion site.
+See [AGENTS.md](AGENTS.md) for repository ownership and maintenance rules.
 
 ## Security & Contributing
 
@@ -115,53 +135,25 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e
 
 ## Contributors
 
-**41 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.
+<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
 
-<p align="center">
-<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
-<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — Implemented per-bucket CORS configuration and enforcement"></a>
-<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — Fixed effective bucket quota metrics; proposed access-frequency ILM"></a>
-<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — Added ChecksumType to multipart completion responses"></a>
-<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — Fixed missing-bucket ListObjects semantics"></a>
-<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — Repaired bucket notification streaming"></a>
-<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — Patched CVE-2026-34986 in go-jose"></a>
-<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — Patched CVE-2026-39883 in OpenTelemetry"></a>
-<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — Repointed documentation links to the SILO portal"></a>
-<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — Contributed the replicated SSE-C plaintext part-size fix"></a>
-<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
-<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
-<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
-<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — Proposed robustness and goroutine improvements"></a>
-<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — Proposed dependency updates"></a>
-<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — Reported multipart and streaming checksum defects and missing-bucket semantics"></a>
-<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — Reported LDAP TLS and Console login regressions"></a>
-<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — Reported the unsigned-header CopyObject cross-object read (SN-2026-011)"></a>
-<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — Reported bucket quota metrics reading a deprecated field"></a>
-<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — Reported missing user, group, and defaults in Debian packages"></a>
-<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — Reported missing RPM package signatures"></a>
-<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — Reported the missing client in release tarballs"></a>
-<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — Reported the missing client in the container image"></a>
-<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — Requested the migration guide from upstream MinIO"></a>
-<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — Reported NATS JWT credentials and target reload issues"></a>
-<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — Reported ListMultipartUploads prefix and pagination semantics"></a>
-<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — Reported indeterminate progress for prefix downloads"></a>
-<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — Reported Console lifecycle management and file preview gaps"></a>
-<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — Requested cpuv1 support and reported a workflow token failure"></a>
-<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — Reported inter-node I/O timeouts in ReadFileStreamHandler"></a>
-<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — Requested KES-compatible external KMS and OpenBao support"></a>
-<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — Reported the missing documentation navigation"></a>
-<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — Requested a maintained Helm chart"></a>
-<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — Reported missing Console tiering and site replication"></a>
-<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — Reported the unusable profiling option"></a>
-<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — Reported Chinese documentation availability"></a>
-<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — Requested Windows build guidance"></a>
-<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — Asked for clarity on Helm chart and operator options"></a>
-<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — Proposed making the SILO Operator easier to discover"></a>
-<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — Proposed CNCF Sandbox governance"></a>
-<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — Asked about community support and image maintenance"></a>
-</p>
+Every human issue or pull-request author is part of the SILO community, including open and unmerged work. We also recognize security reports, including reports sent by email, with the reporter's permission. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
 
-[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports.
+<a href="CONTRIBUTORS.md">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-dark.svg">
+  <img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-light.svg" alt="SILO community contributors">
+</picture>
+</a>
+
+[View contribution notes and actual PR status](CONTRIBUTORS.md).
+
+## Star History
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-dark.svg">
+  <img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-light.svg" alt="SILO GitHub star history">
+</picture>
 
 ## Background
 

README_ZH.md

@@ -24,7 +24,7 @@
 <p align="center">
   <a href="https://silo.pgsty.com/zh/"><img alt="官网" src="https://img.shields.io/badge/%E5%AE%98%E7%BD%91-silo.pgsty.com%2Fzh-1d588c"></a>
   <a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
-  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
+  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
   <a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
   <a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
 </p>
@@ -35,6 +35,14 @@
 > [!NOTE]
 > 2026-08-06，本仓库由 `pgsty/minio` 更名为 `pgsty/silo`，默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支，以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
 
+## SILO 20261026000000
+
+[SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) 已正式发布，提供各平台归档包、Linux 软件包，以及 amd64／arm64 标准版和 distroless Docker 镜像。版本标签为 `RELEASE.2026-10-26T00-00-00Z`，包版本为 `20261026000000.0.0`，构建源码为 `c3ad836aed0d`。
+
+本版修复 LDAP 长 DN 登录、IAM 策略刷新、多池删除安全、分层对象过期、分片复制校验和及远端回滚参数透传。内置 Console v2.5.1（`fff1618d9abb`），使用 silo-pkg v3.15.2、Go 1.27.3 与上游 minio-go `v7.4.2-0.20261025165926-320af050c445`。配套管理客户端为 mcli 20261026，容器镜像内置该版本的 `mcli`。
+
+Server IAM 缓存修复使 Console **Reset to default（恢复默认）** 操作即时生效。存储超时与取消行为保持既有语义：超时不代表底层磁盘操作已停止，也不代表写入未提交。升级前请阅读[联合发布说明](https://silo.pgsty.com/zh/blog/release/silo-20261026/)与[仍存的存储限制](CHANGELOG.md#release2026-10-26t00-00-00z)。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
+
 ## 概述
 
 上游停止社区发行后，Silo 为开源 MinIO 服务端维护一条持续可用的版本线：构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
@@ -54,7 +62,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
   -e MINIO_ROOT_USER=minioadmin \
   -e MINIO_ROOT_PASSWORD=change-me-long-password \
   -v "$PWD/data:/data" \
-  docker.io/pgsty/silo:latest server /data --console-address ":9001"
+  docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
 ```
 
 <p align="center">
@@ -85,63 +93,50 @@ docker exec silo mcli mb local/demo && docker exec silo mcli ls local
 
 ## 兼容性
 
-S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由、`github.com/minio/*` 导入路径与磁盘格式（含 `.minio.sys`）原样保留，并由 CI 兼容性门禁冻结。只有 Silo 自有交付面改名：`silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像 —— 原生交付物不会安装 `minio` 二进制别名。
+Silo 保留 S3 与存储格式兼容性，包括既有 `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与 `.minio.sys` 数据。CI 守卫检查选定的兼容性标识，有意的安全与行为变化在发布说明中记录。Silo 自有交付面使用 `silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像；原生交付物不会安装 `minio` 服务端二进制别名。
+
+正式支持和发布验收的组合为 `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`，对未修改的上游 MinIO/MC 尽最大努力保持兼容。Server、Console 与客户端按需保留历史模块路径，维护源码直接导入 `github.com/pgsty/silo-pkg/v3`；SDK `github.com/minio/minio-go/v7` 是明确保留的上游依赖。具体版本与 replace 以当前 [go.mod](go.mod) 为准。
 
 与上游的全部分歧，以逐项核验代码的[兼容性审计](https://silo.pgsty.com/zh/compatibility/server/)形式维护。每个版本仍应视为下游升级：锁定版本，阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/)，并保留回滚路径。
 
+### TLS 与 Go 升级
+
+Server 20260916 已恢复 Go 默认密钥交换策略，20261018 在 Go 1.27.2 下保留该行为。
+`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化，
+以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
+
+## 文档归属
+
+用户文档统一维护在 [silo.pgsty.com](https://silo.pgsty.com/zh/docs/)，源码位于
+[pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com)。本仓库保留的 `docs/`
+主要是继承的参考材料、示例和工具测试夹具。调查日志、AI 工作记录与临时报告放在仓库外；
+可复用的结论应整理进伴生文档站。仓库职责与维护规则见 [AGENTS.md](AGENTS.md)。
+
 ## 安全与贡献
 
 请按照 [`SECURITY.md`](SECURITY.md) 私密报告漏洞；每项修复都会发布公开[安全公告](https://silo.pgsty.com/zh/blog/security/)。本项目不要求签署 CLA：贡献按 AGPL-3.0-or-later（inbound=outbound）接收，只需 DCO 签署（`git commit -s`），详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
 
 ## 贡献者
 
-**41 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者，以及所有提出 Issue 或 PR 的真人作者；按已合并 PR、其他 PR、Issue 报告排序，黄圈标记显著贡献。
+<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
 
-<p align="center">
-<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — 维护 SILO、Console、mcli、公共包、发行与文档"></a>
-<a href="https://github.com/h5vx"><img src="https://silo.pgsty.com/images/contributors/h5vx.svg" width="60" height="60" alt="@h5vx" title="@h5vx — 实现单桶 CORS 配置与请求执行"></a>
-<a href="https://github.com/mrjavadseydi"><img src="https://silo.pgsty.com/images/contributors/mrjavadseydi.svg" width="60" height="60" alt="@mrjavadseydi" title="@mrjavadseydi — 修复有效桶配额指标，并提交按访问频率分层的 ILM 方案"></a>
-<a href="https://github.com/Dansyuqri"><img src="https://silo.pgsty.com/images/contributors/Dansyuqri.svg" width="60" height="60" alt="@Dansyuqri" title="@Dansyuqri — 为分片上传完成响应补充 ChecksumType"></a>
-<a href="https://github.com/ycjlin"><img src="https://silo.pgsty.com/images/contributors/ycjlin.svg" width="60" height="60" alt="@ycjlin" title="@ycjlin — 修复缺失桶的 ListObjects 语义"></a>
-<a href="https://github.com/pinginfo"><img src="https://silo.pgsty.com/images/contributors/pinginfo.svg" width="60" height="60" alt="@pinginfo" title="@pinginfo — 修复桶通知的流式输出"></a>
-<a href="https://github.com/ZouhairCharef"><img src="https://silo.pgsty.com/images/contributors/ZouhairCharef.svg" width="60" height="60" alt="@ZouhairCharef" title="@ZouhairCharef — 修复 go-jose 中的 CVE-2026-34986"></a>
-<a href="https://github.com/mfredenhagen"><img src="https://silo.pgsty.com/images/contributors/mfredenhagen.svg" width="60" height="60" alt="@mfredenhagen" title="@mfredenhagen — 修复 OpenTelemetry 中的 CVE-2026-39883"></a>
-<a href="https://github.com/waterkip"><img src="https://silo.pgsty.com/images/contributors/waterkip.svg" width="60" height="60" alt="@waterkip" title="@waterkip — 将文档链接指向 SILO 门户"></a>
-<a href="https://github.com/mikemikimike"><img src="https://silo.pgsty.com/images/contributors/mikemikimike.svg" width="60" height="60" alt="@mikemikimike" title="@mikemikimike — 提交 SSE-C 复制分片明文尺寸修复"></a>
-<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — 报告并提交显式版本删除鉴权方案"></a>
-<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — 报告并提交 DELETE If-Match 条件请求支持方案"></a>
-<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — 提交 distroless 容器镜像与依赖自动更新方案"></a>
-<a href="https://github.com/lem21h"><img src="https://silo.pgsty.com/images/contributors/lem21h.svg" width="48" height="48" alt="@lem21h" title="@lem21h — 提交健壮性与 goroutine 改进"></a>
-<a href="https://github.com/sulin37392"><img src="https://silo.pgsty.com/images/contributors/sulin37392.svg" width="48" height="48" alt="@sulin37392" title="@sulin37392 — 提交依赖更新"></a>
-<a href="https://github.com/cbornet"><img src="https://silo.pgsty.com/images/contributors/cbornet.svg" width="60" height="60" alt="@cbornet" title="@cbornet — 报告分片与流式校验和缺陷及缺失桶语义问题"></a>
-<a href="https://github.com/vampywiz17"><img src="https://silo.pgsty.com/images/contributors/vampywiz17.svg" width="60" height="60" alt="@vampywiz17" title="@vampywiz17 — 报告 LDAP TLS 与 Console 登录回归"></a>
-<a href="https://github.com/orenyomtov"><img src="https://silo.pgsty.com/images/contributors/orenyomtov.svg" width="60" height="60" alt="@orenyomtov" title="@orenyomtov — 报告未签名头导致的 CopyObject 跨对象读取（SN-2026-011）"></a>
-<a href="https://github.com/mumu-lab"><img src="https://silo.pgsty.com/images/contributors/mumu-lab.svg" width="48" height="48" alt="@mumu-lab" title="@mumu-lab — 报告桶配额指标读取已弃用字段的问题"></a>
-<a href="https://github.com/jvasile"><img src="https://silo.pgsty.com/images/contributors/jvasile.svg" width="48" height="48" alt="@jvasile" title="@jvasile — 报告 Debian 包缺少用户、用户组与默认配置"></a>
-<a href="https://github.com/pmezhuev"><img src="https://silo.pgsty.com/images/contributors/pmezhuev.svg" width="48" height="48" alt="@pmezhuev" title="@pmezhuev — 报告 RPM 包缺少 GPG 签名"></a>
-<a href="https://github.com/TLINDEN"><img src="https://silo.pgsty.com/images/contributors/TLINDEN.svg" width="48" height="48" alt="@TLINDEN" title="@TLINDEN — 报告发布压缩包缺少客户端"></a>
-<a href="https://github.com/makinikm"><img src="https://silo.pgsty.com/images/contributors/makinikm.svg" width="48" height="48" alt="@makinikm" title="@makinikm — 报告容器镜像缺少客户端"></a>
-<a href="https://github.com/meesudzu"><img src="https://silo.pgsty.com/images/contributors/meesudzu.svg" width="48" height="48" alt="@meesudzu" title="@meesudzu — 提出从上游 MinIO 迁移的指南需求"></a>
-<a href="https://github.com/kuldeep-link11"><img src="https://silo.pgsty.com/images/contributors/kuldeep-link11.svg" width="48" height="48" alt="@kuldeep-link11" title="@kuldeep-link11 — 报告 NATS JWT 凭据与通知目标重载问题"></a>
-<a href="https://github.com/sargarass"><img src="https://silo.pgsty.com/images/contributors/sargarass.svg" width="48" height="48" alt="@sargarass" title="@sargarass — 报告 ListMultipartUploads 前缀与分页语义问题"></a>
-<a href="https://github.com/liuhaodongliu990-cmyk"><img src="https://silo.pgsty.com/images/contributors/liuhaodongliu990-cmyk.svg" width="48" height="48" alt="@liuhaodongliu990-cmyk" title="@liuhaodongliu990-cmyk — 报告前缀下载进度显示异常"></a>
-<a href="https://github.com/Xavier-777"><img src="https://silo.pgsty.com/images/contributors/Xavier-777.svg" width="48" height="48" alt="@Xavier-777" title="@Xavier-777 — 报告 Console 生命周期管理与文件预览缺失"></a>
-<a href="https://github.com/spaceg00se-r"><img src="https://silo.pgsty.com/images/contributors/spaceg00se-r.svg" width="48" height="48" alt="@spaceg00se-r" title="@spaceg00se-r — 提出 cpuv1 支持需求并报告工作流令牌错误"></a>
-<a href="https://github.com/kh0mka"><img src="https://silo.pgsty.com/images/contributors/kh0mka.svg" width="48" height="48" alt="@kh0mka" title="@kh0mka — 报告 ReadFileStreamHandler 节点间 I/O 超时"></a>
-<a href="https://github.com/bagutzu"><img src="https://silo.pgsty.com/images/contributors/bagutzu.svg" width="48" height="48" alt="@bagutzu" title="@bagutzu — 提出兼容 KES 的外部 KMS 与 OpenBao 支持需求"></a>
-<a href="https://github.com/DestroyLee"><img src="https://silo.pgsty.com/images/contributors/DestroyLee.svg" width="48" height="48" alt="@DestroyLee" title="@DestroyLee — 报告文档目录导航缺失"></a>
-<a href="https://github.com/mosesdd"><img src="https://silo.pgsty.com/images/contributors/mosesdd.svg" width="48" height="48" alt="@mosesdd" title="@mosesdd — 提出维护 Helm Chart 的需求"></a>
-<a href="https://github.com/zylpsrs"><img src="https://silo.pgsty.com/images/contributors/zylpsrs.svg" width="48" height="48" alt="@zylpsrs" title="@zylpsrs — 报告 Console 缺少分层与站点复制"></a>
-<a href="https://github.com/heroes1412"><img src="https://silo.pgsty.com/images/contributors/heroes1412.svg" width="48" height="48" alt="@heroes1412" title="@heroes1412 — 报告性能分析选项不可用"></a>
-<a href="https://github.com/redfoxfox"><img src="https://silo.pgsty.com/images/contributors/redfoxfox.svg" width="48" height="48" alt="@redfoxfox" title="@redfoxfox — 报告中文文档站点不可用"></a>
-<a href="https://github.com/jiadzh"><img src="https://silo.pgsty.com/images/contributors/jiadzh.svg" width="48" height="48" alt="@jiadzh" title="@jiadzh — 提出 Windows 构建指导需求"></a>
-<a href="https://github.com/AntonOfTheWoods"><img src="https://silo.pgsty.com/images/contributors/AntonOfTheWoods.svg" width="48" height="48" alt="@AntonOfTheWoods" title="@AntonOfTheWoods — 提出明确 Helm Chart 与 Operator 选项的需求"></a>
-<a href="https://github.com/chalukyaj"><img src="https://silo.pgsty.com/images/contributors/chalukyaj.svg" width="48" height="48" alt="@chalukyaj" title="@chalukyaj — 提出改善 SILO Operator 可发现性的建议"></a>
-<a href="https://github.com/nsanitate"><img src="https://silo.pgsty.com/images/contributors/nsanitate.svg" width="48" height="48" alt="@nsanitate" title="@nsanitate — 提出加入 CNCF Sandbox 的治理建议"></a>
-<a href="https://github.com/Kesavaambati"><img src="https://silo.pgsty.com/images/contributors/Kesavaambati.svg" width="48" height="48" alt="@Kesavaambati" title="@Kesavaambati — 提出社区支持与容器镜像维护问题"></a>
-</p>
+每位 issue 或 PR 的作者，以及同意公开署名的安全报告者，都是 SILO 社区的一员，包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示，其余参考首次参与时间；金色圆环突出经过审核的显著贡献。
 
-[查看完整贡献记录](CONTRIBUTORS.md)，了解每位贡献者的提案、修复与问题报告。
+<a href="CONTRIBUTORS.md">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-dark.svg">
+  <img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-light.svg" alt="SILO 社区贡献者">
+</picture>
+</a>
+
+[查看贡献记录与实际 PR 状态](CONTRIBUTORS.md)。
+
+## Star History
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-dark.svg">
+  <img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-light.svg" alt="SILO GitHub 星标历史">
+</picture>
 
 ## 背景
 

SECURITY.md

@@ -7,7 +7,7 @@ Silo-specific fixes or release notes.
 ## Supported Versions
 
 Security fixes are tracked on the active development branch and summarized in
-[docs/security/advisories.md](docs/security/advisories.md). Only the current
+[the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). Only the current
 Silo release line is supported unless an advisory says otherwise.
 
 ## Inherited Fix Evidence
@@ -26,20 +26,29 @@ separately even when the fork preserves the original commit object and SHA.
   The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
   and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
   regression groups remain part of `go test ./cmd`; see the
-  [canonical ledger](docs/security/advisories.md#inherited-upstream-advisory-baseline)
+  [canonical ledger](https://silo.pgsty.com/about/security-advisories/#inherited)
   for the operator-facing record.
 
 ## Reporting a Vulnerability
 
+The security contact for Silo is [ron@pgsty.com](mailto:ron@pgsty.com).
+This address also receives automated security reports for this repository.
+
 For vulnerabilities in this fork:
 
 1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
 2. Prefer this repository's [private GitHub security advisory](https://github.com/pgsty/silo/security/advisories/new) workflow.
-3. If private reporting is unavailable, contact the maintainers through the
-   repository without publishing exploit details until a private channel is
-   established.
+3. If you cannot use GitHub private reporting, email [ron@pgsty.com](mailto:ron@pgsty.com)
+   directly. Do not publish vulnerability details or exploit code in public
+   issues or pull requests.
 4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
 
+## Scanner Configuration
+
+The [OSS Scanner build environment](.oss-scanner/Dockerfile) and
+[scan scope and offline checks](.oss-scanner/threat_model.md) are maintained
+with the source. Update them when dependencies or security boundaries change.
+
 ## Disclosure Process
 
-Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
+Fork-specific fixes and user-visible upgrade notes are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).

VULNERABILITY_REPORT.md

@@ -34,4 +34,4 @@ Based on the report, the Silo maintainers investigate:
 If the vulnerability exists in this fork itself, the maintainers will, when
 feasible, fix the issue or implement reasonable countermeasures such that the
 vulnerability can no longer be exploited. Fork-specific upgrade notes and
-security advisories are published in `docs/security/advisories.md`.
+security advisories are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/).

buildscripts/checkdeps.sh

@@ -7,7 +7,7 @@ _init() {
 
 	## Minimum required versions for build dependencies
 	GIT_VERSION="1.0"
-	GO_VERSION="1.27.1"
+	GO_VERSION="1.27.3"
 	OSX_VERSION="10.8"
 	KNAME=$(uname -s)
 	ARCH=$(uname -m)

buildscripts/install-mcli.sh

@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
 	exit 0
 fi
 
-release=${MCLI_RELEASE:-RELEASE.2026-09-03T07-13-05Z}
+release=${MCLI_RELEASE:-RELEASE.2026-10-26T00-00-00Z}
 version_hyphen=${release#RELEASE.}
 package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
 if [ "${package_version}" = "${version_hyphen}" ]; then

buildscripts/rebrand-guard/compat-baseline.json

@@ -132,8 +132,8 @@
     "MINIO_API_DELETE_CLEANUP_INTERVAL",
     "MINIO_API_DISABLE_ODIRECT",
     "MINIO_API_GZIP_OBJECTS",
-    "MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH",
     "MINIO_API_LIST_QUORUM",
+    "MINIO_API_MULTIPART_LISTING",
     "MINIO_API_OBJECT_MAX_VERSIONS",
     "MINIO_API_ODIRECT",
     "MINIO_API_REMOTE_TRANSPORT_DEADLINE",
@@ -152,6 +152,7 @@
     "MINIO_API_TRANSITION_WORKERS",
     "MINIO_API_TRUSTED_PROXIES",
     "MINIO_ARGS",
+    "MINIO_AUDIT_",
     "MINIO_AUDIT_KAFKA_BROKERS",
     "MINIO_AUDIT_KAFKA_CLIENT_TLS_CERT",
     "MINIO_AUDIT_KAFKA_CLIENT_TLS_KEY",
@@ -289,16 +290,6 @@
     "MINIO_IDENTITY_TLS_SKIP_VERIFY",
     "MINIO_IDENTITY_TLS_STS_EXPIRY",
     "MINIO_IDLE_TIMEOUT",
-    "MINIO_ILM_ACCESS_BINS",
-    "MINIO_ILM_ACCESS_BIN_WIDTH",
-    "MINIO_ILM_ACCESS_FLUSH",
-    "MINIO_ILM_ACCESS_MAX_SIZE",
-    "MINIO_ILM_ACCESS_MAX_TRACKED",
-    "MINIO_ILM_ACCESS_MIN_RESIDENCY",
-    "MINIO_ILM_ACCESS_POOLS",
-    "MINIO_ILM_ACCESS_PROMOTE_WATERMARK",
-    "MINIO_ILM_ACCESS_TIERING",
-    "MINIO_ILM_ACCESS_WORKERS",
     "MINIO_ILM_EXPIRATION_WORKERS",
     "MINIO_ILM_TRANSITION_WORKERS",
     "MINIO_INTERFACE",
@@ -323,6 +314,7 @@
     "MINIO_LAMBDA_WEBHOOK_ENABLE",
     "MINIO_LAMBDA_WEBHOOK_ENDPOINT",
     "MINIO_LISTENERS",
+    "MINIO_LOGGER_",
     "MINIO_LOGGER_HTTP_ENDPOINT",
     "MINIO_LOGGER_WEBHOOK_AUTH_TOKEN",
     "MINIO_LOGGER_WEBHOOK_BATCH_SIZE",
@@ -581,6 +573,14 @@
     "minio_resource_stats",
     "minio_s3",
     "minio_stats",
+    "minio_system_cpu_avg_idle",
+    "minio_system_cpu_avg_iowait",
+    "minio_system_cpu_load",
+    "minio_system_cpu_load_perc",
+    "minio_system_cpu_nice",
+    "minio_system_cpu_steal",
+    "minio_system_cpu_system",
+    "minio_system_cpu_user",
     "minio_test"
   ],
   "headers": [
@@ -596,11 +596,13 @@
     "x-minio-from-peer",
     "x-minio-healing-drives",
     "x-minio-internal",
+    "x-minio-internal-a",
     "x-minio-internal-actual-object-size",
     "x-minio-internal-actual-size",
     "x-minio-internal-archive-info",
     "x-minio-internal-archive-type",
     "x-minio-internal-b",
+    "x-minio-internal-c",
     "x-minio-internal-compression",
     "x-minio-internal-encrypted-multipart",
     "x-minio-internal-encryptedmultipart",
@@ -613,6 +615,7 @@
     "x-minio-internal-replication-reset-arn",
     "x-minio-internal-replication-status",
     "x-minio-internal-replication-timestamp",
+    "x-minio-internal-restore-attempt",
     "x-minio-internal-server-side-encryption",
     "x-minio-internal-server-side-encryption-context",
     "x-minio-internal-server-side-encryption-iv",
@@ -637,7 +640,6 @@
     "x-minio-origin-endpoint",
     "x-minio-prefixes-total",
     "x-minio-read-quorum",
-    "x-minio-replication",
     "x-minio-replication-actual-object-size",
     "x-minio-replication-delete-status",
     "x-minio-replication-deletemarker-status",
@@ -745,7 +747,6 @@
     "/idp/ldap/policy/{operation}",
     "/idp/openid/list-access-keys-bulk",
     "/ilm",
-    "/ilm/access",
     "/import-bucket-metadata",
     "/import-iam",
     "/import-iam-v2",
@@ -777,6 +778,7 @@
     "/metrics/v3",
     "/minio/grid/",
     "/minio/grid/lock/",
+    "/multipart-preflight",
     "/netperf",
     "/notification",
     "/oauth2/callback",
@@ -840,6 +842,7 @@
     "/site-replication/peer/bucket-ops",
     "/site-replication/peer/edit",
     "/site-replication/peer/iam-item",
+    "/site-replication/peer/iam-revisions",
     "/site-replication/peer/idp-settings",
     "/site-replication/peer/join",
     "/site-replication/peer/remove",
@@ -888,6 +891,7 @@
     "/v2/metrics/cluster",
     "/v2/metrics/node",
     "/v2/metrics/resource",
+    "/v3/site-replication/peer/iam-revisions",
     "/var/vcap/bosh",
     "/verifybinary",
     "/version",
@@ -915,6 +919,7 @@
     ".minio.sys/config/config.json",
     ".minio.sys/config/hello.txt",
     ".minio.sys/config/iam/${username}/identity.json",
+    ".minio.sys/config/ilm/access",
     ".minio.sys/format.json",
     ".minio.sys/multipart",
     ".minio.sys/multipart/bucket/object/uploads.json",
@@ -942,6 +947,7 @@
     "arn:minio:kms:::this-is-disregarded",
     "arn:minio:kms:::xyz-test-key",
     "arn:minio:replication:",
+    "arn:minio:replication::",
     "arn:minio:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket",
     "arn:minio:replication:::",
     "arn:minio:replication:::dest-bucket",

buildscripts/rebrand-guard/main.go

@@ -115,9 +115,8 @@ func collect(repo string) (manifest, error) {
 	fset := token.NewFileSet()
 
 	for _, rel := range files {
-		// Investigation artifacts contain synthetic routes and archived configurations.
+		// Migration notes and guard fixtures contain archived identifiers.
 		if rel == "SILO_REBRANDING_MIGRATION.md" ||
-			strings.HasPrefix(rel, "docs/investigations/") ||
 			strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
 			strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
 			continue

buildscripts/repository-cards/.gitignore

@@ -0,0 +1 @@
+__pycache__/

buildscripts/repository-cards/render.py

@@ -0,0 +1,163 @@
+# Copyright (c) 2026 Feng Ruohang
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Pure, self-contained SVG rendering for SILO's README cards."""
+from datetime import date, timedelta
+from html import escape
+import math
+import xml.etree.ElementTree as ET
+
+themes = {
+    'light': dict(bg='#ffffff', wash='#f2f7fc', edge='#d9e3ee', ink='#16222e',
+                  muted='#62758a', blue='#1d588c', copper='#b4762e', grid='#e5edf5',
+                  line='#2b6ca3', ring='#dce5ef', field='#f7f9fc', label='#3d4e61'),
+    'dark': dict(bg='#101923', wash='#152738', edge='#2b3c50', ink='#e8eef6',
+                 muted='#93a3b8', blue='#7fb8e8', copper='#e0a35c', grid='#263749',
+                 line='#5da2dd', ring='#3a4e63', field='#0b1119', label='#b6c2d2'),
+}
+
+def read_emblem(path):
+    emblem = ET.parse(path).getroot()
+    body = ''.join(ET.tostring(child, encoding='unicode') for child in emblem
+                   if child.tag.rsplit('}', 1)[-1] in ('defs', 'g'))
+    return '\n'.join(line.rstrip() for line in body.splitlines()).strip()
+
+
+def txt(x, y, value, size=14, color=None, weight=400, anchor='start', mono=False, spacing=None):
+    family = 'Menlo,Consolas,monospace' if mono else 'Arial,Helvetica,sans-serif'
+    extra = f' letter-spacing="{spacing}"' if spacing is not None else ''
+    return (f'<text x="{x}" y="{y}" font-family="{family}" font-size="{size}" '
+            f'font-weight="{weight}" fill="{color}" text-anchor="{anchor}"{extra}>'
+            f'{escape(str(value))}</text>')
+
+
+def start(height, theme, title, description, emblem_body):
+    t = themes[theme]
+    return [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" '
+            f'viewBox="0 0 1000 {height}" role="img" aria-labelledby="title desc">',
+            f'<title id="title">{escape(title)}</title><desc id="desc">{escape(description)}</desc>',
+            '<defs><linearGradient id="surface" x1="0" y1="1" x2="1" y2="0">'
+            f'<stop offset="0" stop-color="{t["bg"]}"/>'
+            f'<stop offset="1" stop-color="{t["wash"]}"/></linearGradient>'
+            '<linearGradient id="accent" x1="0" y1="0" x2="1" y2="0">'
+            f'<stop offset="0" stop-color="{t["blue"]}"/>'
+            f'<stop offset="1" stop-color="{t["copper"]}"/></linearGradient>'
+            '<linearGradient id="area" x1="0" y1="0" x2="0" y2="1">'
+            f'<stop offset="0" stop-color="{t["line"]}" stop-opacity=".22"/>'
+            f'<stop offset="1" stop-color="{t["line"]}" stop-opacity=".015"/>'
+            '</linearGradient></defs>',
+            f'<rect x=".75" y=".75" width="998.5" height="{height-1.5}" rx="22" '
+            f'fill="url(#surface)" stroke="{t["edge"]}" stroke-width="1.5"/>',
+            f'<svg x="40" y="25" width="25" height="25" viewBox="230 213 570 570">{emblem_body}</svg>']
+
+
+def heading(parts, t, eyebrow, title, subtitle, value, value_label):
+    parts.extend([
+        txt(76, 43, eyebrow, 11, t['muted'], 600, mono=True, spacing=1.7),
+        txt(40, 94, title, 32, t['ink'], 700),
+        txt(41, 123, subtitle, 14, t['muted']),
+        txt(958, 89, f'{value:,}', 45, t['ink'], 700, anchor='end'),
+        txt(957, 114, value_label, 10, t['muted'], 600, anchor='end', mono=True, spacing=1.5),
+        f'<path d="M40 146 H960" stroke="{t["edge"]}"/>',
+    ])
+
+
+def contributors(theme, people, snapshot, emblem_body):
+    height = 206 + 76 * math.ceil(len(people) / 10)
+    t = themes[theme]
+    parts = start(height, theme, f'SILO community — {len(people)} contributors',
+                  f'The existing SILO community roll, including code, proposals and reports across related projects. '
+                  f'Gold rings retain the existing significant-contribution designation. Snapshot {snapshot}.', emblem_body)
+    heading(parts, t, 'SILO / COMMUNITY', 'Contributors',
+            'Code, proposals & reports across SILO and related projects', len(people), 'COMMUNITY CONTRIBUTORS')
+    for row in range(math.ceil(len(people) / 10)):
+        group = people[row * 10:(row + 1) * 10]
+        row_width = len(group) * 91
+        for col, person in enumerate(group):
+            x = (1000 - row_width) / 2 + col * 91 + 45.5
+            y = 199 + row * 76
+            identifier = f'avatar-{row}-{col}'
+            featured = bool(person.get('featured'))
+            parts.append(f'<g><title>@{escape(person["handle"])} — {escape(person["what"])}</title>')
+            parts.append(f'<defs><clipPath id="{identifier}"><circle cx="{x}" cy="{y}" r="29"/></clipPath></defs>')
+            if featured:
+                parts.append(f'<circle cx="{x}" cy="{y}" r="34" fill="{t["copper"]}" opacity=".09"/>')
+            if person.get('avatarDataUrl'):
+                parts.append(f'<image x="{x-29}" y="{y-29}" width="58" height="58" '
+                             f'clip-path="url(#{identifier})" href="{escape(person["avatarDataUrl"])}"/>')
+            else:
+                parts.append(f'<circle cx="{x}" cy="{y}" r="29" fill="{t["ring"]}"/>')
+                parts.append(txt(x, y + 9, person['handle'][0].upper(), 26, t['ink'], 700, 'middle'))
+            parts.append(f'<circle cx="{x}" cy="{y}" r="30.5" fill="none" '
+                         f'stroke="{t["copper"] if featured else t["ring"]}" stroke-width="{2 if featured else 1.25}"/></g>')
+    parts.extend([
+        f'<path d="M40 {height-42} H960" stroke="{t["edge"]}"/>',
+        f'<circle cx="47" cy="{height-21}" r="4" fill="none" stroke="{t["copper"]}" stroke-width="1.5"/>',
+        txt(61, height-17, 'Gold rings mark significant contributions', 12, t['muted']),
+        txt(959, height-17, f'AS OF {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6),
+        '</svg>',
+    ])
+    return ''.join(parts)
+
+
+def stars(theme, history, snapshot, emblem_body):
+    points = history['points']
+    star_count = points[-1]['stars']
+    t = themes[theme]
+    provenance = ('Initial history reconstructed · Daily totals since ' + history['bootstrap']['through'] + ' · UTC'
+                  if history['bootstrap']['reconstructed'] else 'Observed daily star totals · UTC')
+    parts = start(558, theme, f'SILO star history — {star_count:,} stars',
+                  f'GitHub repository pgsty/silo. {star_count:,} stars as of {snapshot}. ' +
+                  provenance, emblem_body)
+    heading(parts, t, 'SILO / GITHUB', 'Star History', 'pgsty/silo', star_count, 'GITHUB STARS')
+    left, right, top, bottom = 76, 958, 177, 440
+    begin = date.fromisoformat(points[0]['date'])
+    end = date.fromisoformat(points[-1]['date'])
+    days = max(1, (end - begin).days)
+    maximum = max(500, math.ceil(max(p['stars'] for p in points) / 500) * 500)
+    tick_step = 10 ** max(0, int(math.log10(maximum)))
+    xy = lambda day, n: (left + (right-left)*(date.fromisoformat(day)-begin).days / days,
+                         bottom-(bottom-top)*n/maximum)
+    for value in range(0, maximum+1, tick_step):
+        y = xy(points[0]['date'], value)[1]
+        parts.append(f'<path d="M{left} {y:.2f} H{right}" stroke="{t["grid"]}" stroke-dasharray="4 6"/>')
+        parts.append(txt(left-16, round(y+4, 2), f'{value / 1000:g}k' if value >= 1000 else str(value), 12, t['muted'], anchor='end'))
+    dates = sorted({begin + timedelta(days=round((end-begin).days*i/5)) for i in range(6)})
+    ticks = [(d.isoformat(), d.strftime('%b %Y') if days > 90 else d.strftime('%b %d')) for d in dates]
+    for day, label in ticks:
+        x = xy(day, 0)[0]
+        parts.append(f'<path d="M{x:.2f} {top} V{bottom}" stroke="{t["grid"]}" stroke-opacity=".65"/>')
+        anchor = 'start' if day == points[0]['date'] else 'end' if day == snapshot else 'middle'
+        parts.append(txt(round(x, 2), 466, label, 12, t['muted'], anchor=anchor))
+    coords = [xy(p['date'], p['stars']) for p in points]
+    line = 'M' + ' L'.join(f'{x:.2f} {y:.2f}' for x, y in coords)
+    area = line + f' L{coords[-1][0]:.2f} {bottom} L{left} {bottom} Z'
+    parts.extend([
+        f'<path d="{area}" fill="url(#area)"/>',
+        f'<path d="{line}" fill="none" stroke="url(#accent)" stroke-width="3" '
+        'stroke-linecap="round" stroke-linejoin="round"/>',
+        f'<path d="M{left} {bottom} H{right}" stroke="{t["edge"]}"/>',
+    ])
+    x, y = coords[-1]
+    parts.extend([
+        f'<circle cx="{x}" cy="{y}" r="10" fill="{t["copper"]}" opacity=".12"/>',
+        f'<circle cx="{x}" cy="{y}" r="5" fill="{t["copper"]}" stroke="{t["bg"]}" stroke-width="2"/>',
+        f'<path d="M40 491 H960" stroke="{t["edge"]}"/>',
+        txt(40, 516, f'{begin:%b %Y} — {end:%b %Y}'.upper(), 10, t['muted'], 500, mono=True, spacing=.7),
+        txt(959, 516, f'SNAPSHOT {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6),
+        txt(40, 539, provenance, 11, t['muted']),
+        '</svg>',
+    ])
+    return ''.join(parts)

buildscripts/repository-cards/requirements.txt

@@ -0,0 +1 @@
+PyYAML==6.0.3

buildscripts/repository-cards/test_cards.py

@@ -0,0 +1,178 @@
+# Copyright (c) 2026 Feng Ruohang
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Regression checks for historical accuracy, contributor scope and SVG safety."""
+
+import base64
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+from urllib.error import URLError
+import xml.etree.ElementTree as ET
+
+import render
+import update
+
+NS = {'s': 'http://www.w3.org/2000/svg'}
+PNG = base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a7mgAAAAASUVORK5CYII=')
+
+
+def person(handle='Alice', group='reports', featured=False):
+    return {'handle': handle, 'group': group, 'featured': featured,
+            'what': 'A reviewed contribution', 'firstContribution': '2026-09-01'}
+
+
+def history():
+    return {'repository': 'pgsty/silo',
+            'bootstrap': {'through': '2026-09-15', 'reconstructed': True},
+            'points': [{'date': '2026-09-14', 'stars': 100}, {'date': '2026-09-15', 'stars': 105}]}
+
+
+class HistoryTests(unittest.TestCase):
+    def test_new_day_preserves_old_counts_and_unstars(self):
+        before = history()
+        after = update.update_history(before, '2026-09-16', 103)
+        self.assertEqual(after['points'][:-1], before['points'])
+        self.assertEqual(after['points'][-1], {'date': '2026-09-16', 'stars': 103})
+        self.assertEqual(before, history())
+
+    def test_same_day_rerun_replaces_instead_of_appending(self):
+        first = update.update_history(history(), '2026-09-15', 107)
+        self.assertEqual(len(first['points']), 2)
+        self.assertEqual(first, update.update_history(first, '2026-09-15', 107))
+
+    def test_missing_days_are_not_invented(self):
+        result = update.update_history(history(), '2026-09-18', 106)
+        self.assertEqual([p['date'] for p in result['points']], ['2026-09-14', '2026-09-15', '2026-09-18'])
+
+    def test_rejects_wrong_repository_and_corrupt_history(self):
+        cases = []
+        wrong = history(); wrong['repository'] = 'someone/else'; cases.append(wrong)
+        duplicate = history(); duplicate['points'].append(duplicate['points'][-1]); cases.append(duplicate)
+        unordered = history(); unordered['points'].reverse(); cases.append(unordered)
+        negative = history(); negative['points'][0]['stars'] = -1; cases.append(negative)
+        future = history(); future['points'][-1]['date'] = '2026-09-20'; cases.append(future)
+        for case in cases:
+            with self.subTest(case=case), self.assertRaises(ValueError):
+                update.update_history(case, '2026-09-16', 100)
+
+    def test_first_run_has_no_fabricated_history(self):
+        result = update.update_history(None, '2026-09-16', 10)
+        self.assertFalse(result['bootstrap']['reconstructed'])
+        self.assertEqual(result['points'], [{'date': '2026-09-16', 'stars': 10}])
+
+
+class ContributorTests(unittest.TestCase):
+    def test_retries_truncated_json_before_using_it(self):
+        with patch('update.request', side_effect=[b'{"partial":', b'{"ok":true}']), patch('update.time.sleep'):
+            self.assertEqual(update.GitHub('').get('repos/pgsty/silo'), {'ok': True})
+
+    def test_paginates_past_one_full_page(self):
+        class API(update.GitHub):
+            def __init__(self): self.calls = []
+            def get(self, path):
+                self.calls.append(path)
+                return list(range(100)) if 'page=1&' in path else [100]
+        api = API()
+        self.assertEqual(len(list(api.issues('pgsty/silo'))), 101)
+        self.assertIn('state=all', api.calls[0])
+        self.assertIn('page=2&', api.calls[1])
+
+    def test_bots_deduplication_unmerged_work_and_reviewed_credit(self):
+        def issue(login, kind='issue', user_type='User'):
+            item = {'user': {'login': login, 'type': user_type, 'avatar_url': ''}, 'created_at': '2026-09-02T00:00:00Z'}
+            if kind != 'issue': item['pull_request'] = {'merged_at': None if kind == 'open' else '2026-09-03T00:00:00Z'}
+            return item
+        class API:
+            def issues(self, _repo):
+                return [issue('alice'), issue('Bob', 'open'), issue('Bob', 'merged'),
+                        issue('Carol', 'open'), issue('Copilot'), issue('robot', user_type='Bot')]
+        curated = {'repositories': ['pgsty/silo', 'pgsty/mc'], 'bots': ['Copilot'],
+                   'people': [person('Alice', featured=True), person('Reporter'), person('Copilot')]}
+        result = update.collect_people(API(), curated)
+        self.assertEqual({p['handle'] for p in result}, {'Alice', 'Bob', 'Carol', 'Reporter'})
+        self.assertEqual(result[0]['handle'], 'Bob')
+        self.assertEqual(result[1]['handle'], 'Carol')
+        self.assertTrue(next(p for p in result if p['handle'] == 'Alice')['featured'])
+        self.assertEqual(next(p for p in result if p['handle'] == 'Bob')['group'], 'code')
+        self.assertFalse(next(p for p in result if p['handle'] == 'Carol')['featured'])
+
+    def test_newer_reviewed_preview_survives_until_site_catches_up(self):
+        remote = {'updated': '2026-09-16T03:00:00+00:00'}
+        cached = {'updated': '2026-09-16T04:00:00+00:00'}
+        self.assertIs(update.select_curated(remote, cached), cached)
+        newer = {'updated': '2026-09-17T03:00:00+00:00'}
+        self.assertIs(update.select_curated(newer, cached), newer)
+
+    def test_avatar_failure_reuses_raster_cache(self):
+        previous = {**person(), 'avatarDataUrl': update.raster_data_url(PNG)}
+        with patch('update.request', side_effect=URLError('unavailable')):
+            result = update.add_avatars(None, [{**person(), 'avatarUrl': 'https://avatars.githubusercontent.com/u/1'}], [previous])
+        self.assertEqual(result[0]['avatarDataUrl'], previous['avatarDataUrl'])
+        with self.assertRaises(ValueError): update.raster_data_url(b'<svg onload="bad()"/>')
+        with self.assertRaises(ValueError): update.cached_avatar({'avatarDataUrl': 'data:image/svg+xml;base64,PHN2Zy8+'})
+
+    def test_fetch_failure_leaves_published_assets_untouched(self):
+        class API:
+            def get(self, path):
+                if path == 'repos/pgsty/silo': return {'full_name': 'pgsty/silo', 'stargazers_count': 106}
+                raise URLError('roster unavailable')
+        with tempfile.TemporaryDirectory() as directory:
+            out = Path(directory)
+            original = json.dumps(history())
+            (out / 'history.json').write_text(original)
+            (out / 'contributors-light.svg').write_text('previous image')
+            with self.assertRaises(URLError): update.refresh(out, API(), Path('.'))
+            self.assertEqual((out / 'history.json').read_text(), original)
+            self.assertEqual((out / 'contributors-light.svg').read_text(), 'previous image')
+
+
+class RenderTests(unittest.TestCase):
+    def test_real_emblem_generates_clean_xml(self):
+        emblem = render.read_emblem(Path(__file__).resolve().parents[2] / '.github/silo.svg')
+        svg = render.contributors('light', [person()], '2026-09-16', emblem)
+        ET.fromstring(svg)
+        self.assertTrue(all(line == line.rstrip() for line in svg.splitlines()))
+
+    def test_all_avatars_fit_when_the_roster_grows(self):
+        people = [{**person(f'person-{i}'), 'avatarDataUrl': update.raster_data_url(PNG)} for i in range(151)]
+        for theme in ('light', 'dark'):
+            root = ET.fromstring(render.contributors(theme, people, '2026-09-16', ''))
+            images = root.findall('.//s:image', NS)
+            self.assertEqual(len(images), 151)
+            footer = float(root.attrib['height']) - 42
+            self.assertTrue(all(float(i.attrib['y']) + float(i.attrib['height']) < footer for i in images))
+            self.assertTrue(all(i.attrib['href'].startswith('data:image/png;base64,') for i in images))
+
+    def test_untrusted_text_is_escaped(self):
+        data = [{**person(), 'what': '<script>alert("x")</script> & contributions'}]
+        svg = render.contributors('light', data, '2026-09-16', '')
+        root = ET.fromstring(svg)
+        self.assertEqual(root.findall('.//s:script', NS), [])
+        self.assertIn('&lt;script&gt;', svg)
+
+    def test_single_point_and_decreasing_star_history_render(self):
+        for data in (update.update_history(None, '2026-09-16', 0), update.update_history(history(), '2026-09-16', 90)):
+            for theme in ('light', 'dark'):
+                svg = render.stars(theme, data, '2026-09-16', '')
+                ET.fromstring(svg)
+                self.assertNotIn('nan', svg.lower())
+                self.assertNotIn('inf', svg.lower())
+
+
+if __name__ == '__main__':
+    unittest.main()

buildscripts/repository-cards/update.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 Feng Ruohang
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Refresh the generated-asset checkout; publishing is handled by the workflow."""
+
+import argparse
+import base64
+from concurrent.futures import ThreadPoolExecutor
+from datetime import date, datetime, timezone
+import json
+from http.client import IncompleteRead
+import os
+from pathlib import Path
+import re
+import sys
+import time
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
+import xml.etree.ElementTree as ET
+
+import yaml
+
+import render
+
+REPOSITORY = 'pgsty/silo'
+SOURCE = 'repos/pgsty/silo.pgsty.com/contents/data/home/contributors.yaml?ref=main'
+GROUPS = ('code', 'proposed', 'reports')
+HANDLE = re.compile(r'[A-Za-z0-9][A-Za-z0-9-]{0,38}\Z')
+
+
+def request(url, token='', limit=8 * 1024 * 1024):
+    headers = {'User-Agent': 'silo-repository-cards', 'Accept': 'application/vnd.github+json'}
+    if urlparse(url).netloc == 'api.github.com':
+        headers['X-GitHub-Api-Version'] = '2022-11-28'
+        if token:
+            headers['Authorization'] = f'Bearer {token}'
+    for attempt in range(3):
+        try:
+            with urlopen(Request(url, headers=headers), timeout=25) as response:
+                data = response.read(limit + 1)
+                if len(data) > limit:
+                    raise ValueError('Response exceeds the size limit')
+                expected = response.headers.get('Content-Length')
+                if expected is not None and len(data) != int(expected):
+                    raise URLError('Incomplete response body')
+                return data
+        except HTTPError as exc:
+            if exc.code < 500 or attempt == 2:
+                raise
+        except (URLError, TimeoutError, IncompleteRead):
+            if attempt == 2:
+                raise
+        time.sleep(attempt + 1)
+
+
+class GitHub:
+    def __init__(self, token):
+        self.token = token
+
+    def get(self, path):
+        for attempt in range(3):
+            try:
+                return json.loads(request('https://api.github.com/' + path, self.token))
+            except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+                if attempt == 2:
+                    raise ValueError(f'Incomplete or invalid GitHub JSON: {path}') from exc
+                time.sleep(attempt + 1)
+
+    def issues(self, repository):
+        page = 1
+        while True:
+            batch = self.get(f'repos/{repository}/issues?state=all&per_page=100&page={page}&sort=created&direction=asc')
+            if not isinstance(batch, list):
+                raise ValueError(f'Invalid issues response for {repository}')
+            yield from batch
+            if len(batch) < 100:
+                return
+            page += 1
+
+
+def curated_snapshot(data, revision):
+    updated = str(data['updated'])
+    datetime.fromisoformat(updated)
+    repositories = [item['repo'] for item in data['repositories']]
+    if not repositories or any(not re.fullmatch(r'pgsty/[A-Za-z0-9_.-]+', repo) for repo in repositories):
+        raise ValueError('Invalid contributor repository scope')
+    people = []
+    for group in GROUPS:
+        for entry in data[group]:
+            if not HANDLE.fullmatch(entry['handle']):
+                raise ValueError('Invalid GitHub contributor handle')
+            people.append({
+                'handle': entry['handle'], 'group': group,
+                'featured': bool(entry.get('featured')), 'what': entry['what'],
+                'firstContribution': str(entry.get('firstContribution', '9999-12-31')),
+            })
+    if not people or len({p['handle'].lower() for p in people}) != len(people):
+        raise ValueError('Empty or duplicate contributor roster')
+    return {'updated': updated, 'revision': revision, 'repositories': repositories,
+            'bots': data.get('bots', ['Copilot', 'dependabot[bot]']), 'people': people}
+
+
+def select_curated(remote, cached):
+    # The initial, approved preview can contain reviewed credit not published by
+    # the companion site yet. Keep that newer snapshot until the site catches up.
+    if cached and datetime.fromisoformat(cached['updated']) > datetime.fromisoformat(remote['updated']):
+        return cached
+    return remote
+
+
+def collect_people(api, curated):
+    bots = {name.lower() for name in curated['bots']}
+    people = {p['handle'].lower(): dict(p) for p in curated['people']
+              if p['handle'].lower() not in bots and not p['handle'].lower().endswith('[bot]')}
+    order = {p['handle'].lower(): index for index, p in enumerate(curated['people'])}
+    for repository in curated['repositories']:
+        print(f'Reading issue and PR authors: {repository}', flush=True)
+        for issue in api.issues(repository):
+            user = issue.get('user') or {}
+            handle = user.get('login', '')
+            key = handle.lower()
+            if user.get('type') != 'User' or key in bots or key.endswith('[bot]'):
+                continue
+            if not HANDLE.fullmatch(handle):
+                raise ValueError('Invalid issue author')
+            pr = issue.get('pull_request')
+            group = 'code' if pr and pr.get('merged_at') else 'proposed' if pr else 'reports'
+            first = issue['created_at'][:10]
+            date.fromisoformat(first)
+            person = people.setdefault(key, {
+                'handle': handle, 'group': group, 'featured': False,
+                'what': 'Contributed an issue or pull request to SILO and related projects',
+                'firstContribution': first,
+            })
+            person['avatarUrl'] = user.get('avatar_url', '')
+            person['firstContribution'] = min(person['firstContribution'], first)
+            if GROUPS.index(group) < GROUPS.index(person['group']):
+                person['group'] = group
+    if not people:
+        raise ValueError('No human contributors were collected')
+    return sorted(people.values(), key=lambda p: (
+        GROUPS.index(p['group']), not p['featured'],
+        order.get(p['handle'].lower(), len(order)), p['firstContribution'], p['handle'].lower()))
+
+
+def raster_data_url(data):
+    if data.startswith(b'\x89PNG\r\n\x1a\n'):
+        mime = 'image/png'
+    elif data.startswith(b'\xff\xd8\xff'):
+        mime = 'image/jpeg'
+    elif data.startswith((b'GIF87a', b'GIF89a')):
+        mime = 'image/gif'
+    elif data[:4] == b'RIFF' and data[8:12] == b'WEBP':
+        mime = 'image/webp'
+    else:
+        raise ValueError('Avatar is not a raster image')
+    return f'data:{mime};base64,' + base64.b64encode(data).decode('ascii')
+
+
+def cached_avatar(person):
+    value = person.get('avatarDataUrl', '')
+    if not value:
+        return ''
+    prefix, encoded = value.split(',', 1)
+    if prefix not in ('data:image/png;base64', 'data:image/jpeg;base64', 'data:image/gif;base64', 'data:image/webp;base64'):
+        raise ValueError('Invalid cached avatar format')
+    raw = base64.b64decode(encoded, validate=True)
+    if len(raw) > 512 * 1024 or raster_data_url(raw) != value:
+        raise ValueError('Invalid cached avatar')
+    return value
+
+
+def add_avatars(api, people, previous):
+    cached = {p['handle'].lower(): cached_avatar(p) for p in previous}
+
+    def update(person):
+        person = dict(person)
+        try:
+            url = person.pop('avatarUrl', '') or api.get('users/' + person['handle'])['avatar_url']
+            parsed = urlparse(url)
+            if parsed.scheme != 'https' or parsed.netloc != 'avatars.githubusercontent.com':
+                raise ValueError('Unexpected avatar host')
+            data = request(url + ('&' if '?' in url else '?') + 's=96', limit=512 * 1024)
+            person['avatarDataUrl'] = raster_data_url(data)
+        except (HTTPError, URLError, TimeoutError, IncompleteRead, ValueError, KeyError) as exc:
+            person.pop('avatarUrl', None)
+            person['avatarDataUrl'] = cached.get(person['handle'].lower(), '')
+            print(f'Avatar fallback for @{person["handle"]}: {type(exc).__name__}', file=sys.stderr)
+        return person
+
+    with ThreadPoolExecutor(max_workers=6) as pool:
+        return list(pool.map(update, people))
+
+
+def update_history(history, day, stars):
+    date.fromisoformat(day)
+    if type(stars) is not int or stars < 0:
+        raise ValueError('Invalid repository star count')
+    if history is None:
+        history = {'repository': REPOSITORY, 'bootstrap': {'through': day, 'reconstructed': False}, 'points': []}
+    if history['repository'] != REPOSITORY:
+        raise ValueError('Star history belongs to a different repository')
+    date.fromisoformat(history['bootstrap']['through'])
+    dates = []
+    for point in history['points']:
+        date.fromisoformat(point['date'])
+        if type(point['stars']) is not int or point['stars'] < 0:
+            raise ValueError('Invalid historical star count')
+        dates.append(point['date'])
+    if dates != sorted(set(dates)) or any(d > day for d in dates):
+        raise ValueError('History contains duplicate, unordered, or future dates')
+    # Replace today's observation, preserve previous days, and allow unstars.
+    points = [dict(p) for p in history['points'] if p['date'] != day]
+    points.append({'date': day, 'stars': stars})
+    return {**history, 'points': points}
+
+
+def read_json(path, default=None):
+    return json.loads(path.read_text()) if path.exists() else default
+
+
+def refresh(output, api, source_root):
+    day = datetime.now(timezone.utc).date().isoformat()
+    metadata = api.get('repos/' + REPOSITORY)
+    if metadata['full_name'].lower() != REPOSITORY:
+        raise ValueError('Unexpected repository metadata')
+    history = update_history(read_json(output / 'history.json'), day, metadata['stargazers_count'])
+    source = api.get(SOURCE)
+    reviewed = yaml.safe_load(base64.b64decode(source['content'], validate=False))
+    curated = select_curated(curated_snapshot(reviewed, source['sha']), read_json(output / 'curated.json'))
+    people = collect_people(api, curated)
+    previous = read_json(output / 'contributors.json', {}).get('people', [])
+    people = add_avatars(api, people, previous)
+    emblem = render.read_emblem(source_root / '.github/silo.svg')
+    payloads = {}
+    for theme in ('light', 'dark'):
+        payloads[f'contributors-{theme}.svg'] = render.contributors(theme, people, day, emblem) + '\n'
+        payloads[f'star-history-{theme}.svg'] = render.stars(theme, history, day, emblem) + '\n'
+    for svg in payloads.values():
+        ET.fromstring(svg)
+    for name, data in {
+        'history.json': history,
+        'curated.json': curated,
+        'contributors.json': {'repository': REPOSITORY, 'updated': day, 'people': people},
+    }.items():
+        payloads[name] = json.dumps(data, indent=2, ensure_ascii=False) + '\n'
+    payloads['README.md'] = f'''# SILO repository cards
+
+Generated by [Repository Cards](https://github.com/pgsty/silo/actions/workflows/repository-cards.yml)
+at 00:00 UTC daily (08:00 Asia/Shanghai). GitHub may queue scheduled runs.
+
+Snapshot: {day}. {metadata['stargazers_count']:,} stars; {len(people)} community contributors.
+
+- `contributors-light.svg` / `contributors-dark.svg`: human issue and PR authors across the SILO project scope, plus reviewed acknowledgements. Bots are excluded. Gold rings follow the reviewed companion-site roster; new authors are collected automatically.
+- `star-history-light.svg` / `star-history-dark.svg`: initial history reconstructed from the then-current stargazers; later points are daily observed totals, including decreases. Missing days are not fabricated.
+- `curated.json`: a cache of reviewed contributor credit from `pgsty/silo.pgsty.com/data/home/contributors.yaml`. The approved initial preview may be newer than the published site; a newer reviewed snapshot is retained until the site catches up.
+- `contributors.json`: generated contributor data and embedded raster avatars. Failed avatar refreshes use the previous image, or an initial when no image is available.
+- `history.json`: persistent daily totals. Keep this file when regenerating images.
+
+The SVGs are self-contained. Source and instructions live on the default branch;
+this branch contains generated assets only. Do not merge it into `main`.
+'''
+    # Collect and validate everything before touching the publication checkout.
+    output.mkdir(parents=True, exist_ok=True)
+    for filename, text in payloads.items():
+        (output / filename).write_text(text)
+    print(f'{day}: {len(people)} contributors; {metadata["stargazers_count"]:,} stars; {len(history["points"])} history points')
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--output', type=Path, required=True)
+    args = parser.parse_args()
+    configured = os.environ.get('GITHUB_REPOSITORY', REPOSITORY)
+    if configured.lower() != REPOSITORY:
+        raise SystemExit('This workflow is scoped to pgsty/silo')
+    refresh(args.output, GitHub(os.environ.get('GH_TOKEN', '')), Path(__file__).resolve().parents[2])
+
+
+if __name__ == '__main__':
+    main()

buildscripts/verify-helm-migration.sh

@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
 go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
 
 helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
-test -s "${work_dir}/silo-7.0.2.tgz"
+test -s "${work_dir}/silo-7.0.3.tgz"
 if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
 	echo "Helm packaging emitted a legacy MinIO chart name" >&2
 	exit 1

buildscripts/verify-rebrand.sh

@@ -11,6 +11,33 @@ fail() {
 	exit 1
 }
 
+# Git is already a build prerequisite. --no-index also scans untracked files
+# and exported source trees. As before, honor .gitignore only in a worktree.
+search_ignore=--no-exclude-standard
+if git rev-parse --show-toplevel >/dev/null 2>&1; then
+	search_ignore=--exclude-standard
+fi
+search_errors="$(mktemp)"
+trap 'rm -f "${search_errors}"' EXIT
+
+search_text() {
+	local status=0
+	git --no-pager grep --no-index "${search_ignore}" --no-color -nIHE "$@" 2>"${search_errors}" || status=$?
+	# Git can return 1 for both no matches and a file read error. Check stderr
+	# as well, before callers filter matches through their allowlists.
+	if [ "${status}" -gt 1 ] || [ -s "${search_errors}" ]; then
+		cat "${search_errors}" >&2
+		fail "could not scan source files"
+	fi
+}
+
+# Keep hidden files and build output out of the repository-wide scans.
+source_excludes=(
+	':(exclude,glob)**/.*'
+	':(exclude,glob)**/.*/**'
+	':(exclude)dist/**'
+)
+
 require_file() {
 	[ -f "$1" ] || fail "missing required file: $1"
 }
@@ -36,7 +63,7 @@ for file in \
 	buildscripts/verify-helm-migration.sh \
 	Dockerfile.goreleaser \
 	Dockerfile.distroless \
-	dockerscripts/download-static-curl.sh \
+	dockerscripts/build-static-curl.sh \
 	dockerscripts/docker-entrypoint.sh \
 	helm/silo/Chart.yaml \
 	helm/silo/values.yaml \
@@ -101,7 +128,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
 require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
 require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
 require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
-require_text dockerscripts/download-static-curl.sh "sha256sum -c"
+require_text dockerscripts/build-static-curl.sh "sha256sum -c"
 require_text helm/silo/Chart.yaml "name: silo"
 require_text helm/silo/values.yaml "repository: pgsty/silo"
 require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"
@@ -152,7 +179,11 @@ if grep -Ev '^[[:space:]]*(#|$)' silo.env | grep -q '='; then
 	fail "silo.env must not contain active assignments that shadow /etc/default/minio"
 fi
 
-if rg -n 'pgsty/minio:' .github/workflows Dockerfile.goreleaser helm/silo; then
+delivery_hits="$(search_text 'pgsty/minio:' -- .github/workflows Dockerfile.goreleaser helm/silo \
+	':(exclude,glob).github/workflows/**/.*' ':(exclude,glob).github/workflows/**/.*/**' \
+	':(exclude,glob)helm/silo/**/.*' ':(exclude,glob)helm/silo/**/.*/**')"
+if [ -n "${delivery_hits}" ]; then
+	printf '%s\n' "${delivery_hits}" >&2
 	fail "an active delivery surface still publishes the frozen pgsty/minio image"
 fi
 
@@ -165,10 +196,10 @@ fi
 # So two rules. First, no live URL may resolve to the old repository anywhere,
 # READMEs and CONTRIBUTORS.md included.
 old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
-stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
-	--glob '!.git/**' --glob '!dist/**' \
-	--glob '!SILO_REBRANDING_MIGRATION.md' \
-	--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
+stale_repo_url="$(search_text -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" -- . \
+	"${source_excludes[@]}" ':(exclude)SILO_REBRANDING_MIGRATION.md' \
+	':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
+stale_repo_url="$(printf '%s\n' "${stale_repo_url}" |
 	sed 's#^\./##' | grep -v '^buildscripts/verify-rebrand\.sh:' || true)"
 if [ -n "${stale_repo_url}" ]; then
 	printf '%s\n' "${stale_repo_url}" >&2
@@ -180,32 +211,32 @@ fi
 # legacy image, the two READMEs that document the rename and the archived
 # minio branch, and historical issue titles in CONTRIBUTORS.md.
 repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
-stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
-	--glob '!SILO_REBRANDING_MIGRATION.md' \
-	--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
+stale_repo="$(search_text "${old_repo_pattern}" -- . "${source_excludes[@]}" \
+	':(exclude)SILO_REBRANDING_MIGRATION.md' ':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
+stale_repo="$(printf '%s\n' "${stale_repo}" |
 	sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
 if [ -n "${stale_repo}" ]; then
 	printf '%s\n' "${stale_repo}" >&2
 	fail "a source reference still names the pre-rename pgsty/minio repository"
 fi
 
-stale_branch="$(rg -n 'pgsty/silo/(blob/|tree/|raw/)?master' \
-	--glob '!.git/**' --glob '!dist/**' . || true)"
+stale_branch="$(search_text 'pgsty/silo/(blob/|tree/|raw/)?master' -- . "${source_excludes[@]}")"
 if [ -n "${stale_branch}" ]; then
 	printf '%s\n' "${stale_branch}" >&2
 	fail "a link still targets the retired master branch; raw and Actions URLs do not follow a branch rename"
 fi
 
 for workflow in .github/workflows/go.yml .github/workflows/vulncheck.yml; do
-	if rg -q '^\s+- master$' "${workflow}"; then
+	stale_filter="$(search_text '^[[:space:]]+- master$' -- "${workflow}")"
+	if [ -n "${stale_filter}" ]; then
 		fail "${workflow} still filters on master and would go silently dormant on main"
 	fi
 	require_text "${workflow}" "      - main"
 done
 
-network_hits="$(rg -n --glob '*.go' --glob '!**/*_test.go' \
-	'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' \
-	cmd internal || true)"
+network_hits="$(search_text \
+	'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' -- \
+	':(glob)cmd/**/*.go' ':(glob)internal/**/*.go' ':(exclude,glob)**/*_test.go' "${source_excludes[@]}")"
 network_hits="$(printf '%s\n' "${network_hits}" | grep -Ev '^[^:]+:[0-9]+:[[:space:]]*//' || true)"
 if [ -n "${network_hits}" ]; then
 	printf '%s\n' "${network_hits}" >&2

cmd/abandoned-mutation_test.go

@@ -0,0 +1,184 @@
+//go:build consistency_strict
+
+// Copyright (c) 2026 Feng Ruohang
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type abandonedCleanupCommitDisk struct {
+	StorageAPI
+	entered chan<- struct{}
+	release <-chan struct{}
+}
+
+func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
+	var success int64
+	gate := &scopeStagedContext{
+		Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
+		entered: d.entered, release: d.release,
+		tracker: healthDiskCtxValue{lastSuccess: &success},
+	}
+	return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
+}
+
+// A lost namespace lease can admit cleanup while a disk transaction is between
+// moving its external shards and replacing xl.meta. Those shards are live work,
+// even though the old metadata does not refer to them yet. Cleanup must wait.
+func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
+	z, bucket := mutationPools(t)
+	const object = "cleanup-during-commit"
+	body := strings.Repeat("new payload", 200000)
+	mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
+	set := z.serverPools[0].getHashedSet(object)
+	disks, swap := installListingDiskSwitch(set)
+	defer swap(disks)
+	entered, release := make(chan struct{}, len(disks)), make(chan struct{})
+	unblock := sync.OnceFunc(func() { close(release) })
+	defer unblock()
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i, d := range disks {
+		wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
+	}
+	swap(wrapped)
+	committed := make(chan error, 1)
+	go func() {
+		_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
+		committed <- err
+	}()
+	for range disks {
+		select {
+		case <-entered:
+		case err := <-committed:
+			t.Fatalf("PUT returned before shard barrier: %v", err)
+		case <-time.After(5 * time.Second):
+			t.Fatal("PUT did not move all external shards")
+		}
+	}
+	t.Log("All eight new external shard directories exist; old xl.meta is still active")
+	cleaned := make(chan error, len(disks))
+	for _, d := range disks {
+		go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
+	}
+	early := 0
+	timer := time.NewTimer(150 * time.Millisecond)
+wait:
+	for {
+		select {
+		case err := <-cleaned:
+			early++
+			t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
+			if early == len(disks) {
+				break wait
+			}
+		case <-timer.C:
+			break wait
+		}
+	}
+	timer.Stop()
+	unblock()
+	select {
+	case err := <-committed:
+		if err != nil {
+			t.Fatalf("PUT failed: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("PUT did not finish")
+	}
+	for i := early; i < len(disks); i++ {
+		select {
+		case err := <-cleaned:
+			if err != nil {
+				t.Fatal(err)
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatal("cleanup did not finish")
+		}
+	}
+	swap(disks)
+	r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
+	}
+	defer r.Close()
+	data, err := io.ReadAll(r)
+	if err != nil || string(data) != body {
+		t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
+	}
+}
+
+// Cleanup can also arrive after a disk has committed but before its delayed
+// reply lets the coordinator decide to undo a failed quorum. The previous
+// generation's backup is still owned by that unfinished transaction.
+func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
+	z, bucket := mutationPools(t)
+	const object = "cleanup-before-undo"
+	body := strings.Repeat("last acknowledged", 150000)
+	base := time.Now().Add(-time.Minute)
+	mutationPut(t, z, bucket, object, 0, body, base)
+	set := z.serverPools[0].getHashedSet(object)
+	disks, swap := installListingDiskSwitch(set)
+	defer swap(disks)
+	entered, release := make(chan struct{}, 4), make(chan struct{})
+	unblock := sync.OnceFunc(func() { close(release) })
+	defer unblock()
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i, d := range disks {
+		wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
+	}
+	swap(wrapped)
+	committed := make(chan error, 1)
+	go func() {
+		next := strings.Repeat("failed update", 180000)
+		_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
+		committed <- err
+	}()
+	for range 4 {
+		select {
+		case <-entered:
+		case err := <-committed:
+			t.Fatalf("PUT returned before reply barrier: %v", err)
+		case <-time.After(5 * time.Second):
+			t.Fatal("disk commit did not reach reply barrier")
+		}
+	}
+	for _, d := range disks[:4] {
+		if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
+			t.Fatal(err)
+		}
+	}
+	unblock()
+	select {
+	case err := <-committed:
+		if err == nil {
+			t.Fatal("four-of-eight commit unexpectedly reached write quorum")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("failed PUT did not finish undo")
+	}
+	swap(disks)
+	for i, d := range disks {
+		fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
+		if err != nil || !fi.ModTime.Equal(base) {
+			t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
+		}
+	}
+	r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
+	}
+	defer r.Close()
+	data, err := io.ReadAll(r)
+	if err != nil || string(data) != body {
+		t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
+	}
+}

cmd/admin-account-buckets_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/cachevalue"
+)
+
+func TestAccountInfoReflectsBucketMutations(t *testing.T) {
+	ctx := t.Context()
+	bed, err := prepareAdminErasureTestBed(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer bed.TearDown()
+	previousCache := listBucketsCache
+	defer func() { listBucketsCache = previousCache }()
+
+	// Hold a real pre-mutation snapshot past the assertions so the regression
+	// cannot disappear merely because a slow test outlasted the cache TTL.
+	freezeListing := func() {
+		t.Helper()
+		buckets, err := bed.objLayer.ListBuckets(ctx, BucketOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		listBucketsCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) ([]BucketInfo, error) {
+			return buckets, nil
+		})
+		if _, err := listBucketsCache.GetWithCtx(ctx); err != nil {
+			t.Fatal(err)
+		}
+	}
+	const bucket = "account-info-freshness"
+	assertPresent := func(want bool) {
+		t.Helper()
+		req, err := buildAdminRequest(nil, http.MethodGet, "/accountinfo", 0, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		rec := httptest.NewRecorder()
+		bed.router.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("account info: %d %s", rec.Code, rec.Body.String())
+		}
+		var info madmin.AccountInfo
+		if err := json.Unmarshal(rec.Body.Bytes(), &info); err != nil {
+			t.Fatal(err)
+		}
+		found := false
+		for _, item := range info.Buckets {
+			if item.Name == bucket {
+				found = true
+			}
+		}
+		if found != want {
+			t.Errorf("account info bucket present = %v, want %v", found, want)
+		}
+	}
+
+	freezeListing()
+	if err := bed.objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	assertPresent(true)
+	freezeListing()
+	if err := bed.objLayer.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	assertPresent(false)
+}

cmd/admin-handlers-idp-ldap.go

@@ -249,8 +249,10 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 
 		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
 		// it is associated with the LDAP/OIDC user properly.
+		// The new credential's policy comes from the request, not the caller.
 		for k, v := range cred.Claims {
-			if k == expClaim {
+			switch k {
+			case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
 				continue
 			}
 			opts.claims[k] = v

cmd/admin-handlers-pools.go

@@ -373,7 +373,9 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	// Cancel any ongoing rebalance operation
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
-	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	if err := pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt); !errors.Is(err, errRebalanceNotStarted) {
+		adminLogIf(ctx, err)
+	}
 	globalNotificationSys.LoadRebalanceMeta(ctx, false)
 }
 

cmd/admin-handlers-users-authz_test.go

@@ -0,0 +1,556 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	"github.com/minio/mux"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func iamAdminFixture(t *testing.T) (context.Context, *IAMSys) {
+	t.Helper()
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = obj
+	globalObjLayerMutex.Unlock()
+	return ctx, sys
+}
+
+func iamAdminUser(ctx context.Context, t *testing.T, sys *IAMSys, name, policyName string) auth.Credentials {
+	t.Helper()
+	cred := auth.Credentials{AccessKey: name, SecretKey: "review-user-valid-secret"}
+	_, err := sys.CreateUser(ctx, name, madmin.AddOrUpdateUserReq{SecretKey: cred.SecretKey, Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	if policyName != "" {
+		_, err = sys.PolicyDBSet(ctx, name, policyName, regUser, false)
+		mustIAM(t, err)
+	}
+	return cred
+}
+
+func iamAdminServiceAccount(ctx context.Context, t *testing.T, sys *IAMSys, parent, key, document string) auth.Credentials {
+	t.Helper()
+	var sp *policy.Policy
+	var err error
+	if document != "" {
+		sp, err = policy.ParseConfig(strings.NewReader(document))
+		mustIAM(t, err)
+	}
+	cred, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "review-service-valid-secret", sessionPolicy: sp})
+	mustIAM(t, err)
+	return cred
+}
+
+func iamAdminRequest(t *testing.T, cred auth.Credentials, method, target string, payload any) *http.Request {
+	t.Helper()
+	var body []byte
+	if payload != nil {
+		raw, err := json.Marshal(payload)
+		mustIAM(t, err)
+		body, err = madmin.EncryptData(cred.SecretKey, raw)
+		mustIAM(t, err)
+	}
+	headers := map[string]string{}
+	if cred.IsTemp() && !cred.IsServiceAccount() {
+		headers["X-Amz-Security-Token"] = cred.SessionToken
+	}
+	req, err := newTestSignedRequestV4(method, "http://localhost"+target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
+	mustIAM(t, err)
+	mustIAM(t, req.ParseForm())
+	return req
+}
+
+func iamAdminCall(t *testing.T, cred auth.Credentials, method, target string, payload any, vars map[string]string, handler http.HandlerFunc) *httptest.ResponseRecorder {
+	t.Helper()
+	req := iamAdminRequest(t, cred, method, target, payload)
+	req = mux.SetURLVars(req, vars)
+	rec := httptest.NewRecorder()
+	req = req.WithContext(newContext(req, rec, "IAMAdminAuthorization"))
+	handler(rec, req)
+	return rec
+}
+
+func assertIAMAdminStatus(t *testing.T, operation string, rec *httptest.ResponseRecorder, want int) {
+	t.Helper()
+	if rec.Code != want {
+		t.Errorf("%s status=%d want=%d body=%s", operation, rec.Code, want, rec.Body.String())
+		return
+	}
+	if want == http.StatusForbidden || want == http.StatusNotFound {
+		var response APIErrorResponse
+		mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &response))
+		wantCode := errorCodes.ToAPIErr(ErrAccessDenied).Code
+		if want == http.StatusNotFound {
+			wantCode = errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound).Code
+		}
+		if response.Code != wantCode {
+			t.Errorf("%s error code=%s want=%s", operation, response.Code, wantCode)
+		}
+	}
+}
+
+func iamAdminPlugin(t *testing.T, allow bool) {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]bool{"result": allow})
+	}))
+	t.Cleanup(server.Close)
+	u, err := xnet.ParseHTTPURL(server.URL)
+	mustIAM(t, err)
+	previous := newGlobalAuthZPluginFn()
+	setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: u, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
+	t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
+}
+
+func TestIAMAdminRootEmptyPolicy(t *testing.T) {
+	for _, doc := range []string{"", "null", "{}", `{"Statement":null}`} {
+		t.Run("policy="+doc, func(t *testing.T) {
+			ctx, sys := iamAdminFixture(t)
+			caller := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-empty", doc)
+			sibling := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-sibling", "")
+			req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
+			checked, owner, code := checkKeyValid(req, caller.AccessKey)
+			if code != ErrNone {
+				t.Fatalf("checkKeyValid: %v", code)
+			}
+			active, _ := isAllowedBySessionPolicyForServiceAccount(policy.Args{Claims: checked.Claims})
+			t.Logf("owner=%v parsed_active_session_policy=%v", owner, active)
+			api := adminAPIHandlers{}
+			assertIAMAdminStatus(t, "create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-root-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 200)
+			assertIAMAdminStatus(t, "update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewName: "review"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 204)
+			assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+sibling.AccessKey, nil, map[string]string{"accessKey": sibling.AccessKey}, api.DeleteServiceAccount), 204)
+			assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
+		})
+	}
+}
+
+func TestIAMAdminEmptyPolicyDelegation(t *testing.T) {
+	for _, parentType := range []string{"root", "user"} {
+		for _, document := range []string{"null", "{}", `{"Statement":null}`} {
+			t.Run(parentType+"/"+document, func(t *testing.T) {
+				ctx, sys := iamAdminFixture(t)
+				parent := globalActiveCred
+				if parentType == "user" {
+					parent = iamAdminUser(ctx, t, sys, "empty-policy-parent", "readwrite")
+				}
+				sp, err := policy.ParseConfig(strings.NewReader(document))
+				mustIAM(t, err)
+				caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
+					accessKey: "empty-policy-caller", secretKey: "empty-policy-caller-secret", sessionPolicy: sp,
+					claims: map[string]any{"identity-attribute": "preserved"},
+				})
+				mustIAM(t, err)
+				const limitedPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:ListServiceAccounts"]}]}`
+				for _, tc := range []struct {
+					policy      string
+					listStatus  int
+					allowObject bool
+				}{
+					{"", http.StatusOK, true},
+					{"null", http.StatusOK, true},
+					{"{}", http.StatusOK, true},
+					{`{"Statement":null}`, http.StatusOK, true},
+					{limitedPolicy, http.StatusOK, false},
+					{`{"Version":"2012-10-17"}`, http.StatusForbidden, false},
+					{`{"Statement":[]}`, http.StatusForbidden, false},
+				} {
+					expires := UTCNow().Add(time.Hour)
+					req := madmin.AddServiceAccountReq{Expiration: &expires}
+					if tc.policy != "" {
+						req.Policy = json.RawMessage(tc.policy)
+					}
+					api := adminAPIHandlers{}
+					rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", req, nil, api.AddServiceAccount)
+					assertIAMAdminStatus(t, "create expiring child", rec, http.StatusOK)
+					data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
+					mustIAM(t, err)
+					var response madmin.AddServiceAccountResp
+					mustIAM(t, json.Unmarshal(data, &response))
+					child, ok := sys.store.GetUser(response.Credentials.AccessKey)
+					if !ok {
+						t.Fatal("new service account missing")
+					}
+					// Exercise the issued key through authentication and authorization.
+					assertIAMAdminStatus(t, "use expiring child policy="+tc.policy, iamAdminCall(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), tc.listStatus)
+					signed := iamAdminRequest(t, child.Credentials, "GET", "/example/object", nil)
+					checked, owner, code := checkKeyValid(signed, child.Credentials.AccessKey)
+					if code != ErrNone {
+						t.Fatalf("child authentication: %v", code)
+					}
+					if checked.Claims["identity-attribute"] != "preserved" {
+						t.Fatal("creation lost the caller's identity context")
+					}
+					allowed := sys.IsAllowed(policy.Args{AccountName: checked.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.GetObjectAction, BucketName: "example", ObjectName: "object"})
+					if allowed != tc.allowObject {
+						t.Errorf("GetObject allowed=%v with requested policy=%s", allowed, tc.policy)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestIAMAdminCorruptServiceCleanup(t *testing.T) {
+	for _, status := range []string{auth.AccountOn, auth.AccountOff, "expired"} {
+		t.Run(status, func(t *testing.T) {
+			ctx, sys := iamAdminFixture(t)
+			caller := iamAdminUser(ctx, t, sys, "review-admin", "consoleAdmin")
+			parent := iamAdminUser(ctx, t, sys, "review-parent", "readwrite")
+			svc := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-corrupt", "")
+			if status == auth.AccountOff {
+				_, err := sys.UpdateServiceAccount(ctx, svc.AccessKey, updateServiceAccountOpts{status: status})
+				mustIAM(t, err)
+			}
+			cache := sys.store.lock()
+			u := cache.iamUsersMap[svc.AccessKey]
+			u.Credentials.SessionToken = "broken-session-token"
+			if status == "expired" {
+				u.Credentials.Expiration = UTCNow().Add(-time.Minute)
+			}
+			cache.iamUsersMap[svc.AccessKey] = u
+			sys.store.unlock()
+			_, _, err := sys.GetServiceAccount(ctx, svc.AccessKey)
+			if err == nil {
+				t.Fatal("fault injection failed")
+			}
+			raw, exists := sys.store.GetUser(svc.AccessKey)
+			_, loginOK := sys.GetUser(ctx, svc.AccessKey)
+			t.Logf("metadata_exists=%v parent_known=%v GetUser_ok=%v", exists, raw.Credentials.ParentUser == parent.AccessKey, loginOK)
+			api := adminAPIHandlers{}
+			outsider := iamAdminUser(ctx, t, sys, "cleanup-outsider", "readwrite")
+			assertIAMAdminStatus(t, "unauthorized cleanup", iamAdminCall(t, outsider, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 404)
+			assertIAMAdminStatus(t, "admin cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
+			if _, ok := sys.store.GetUser(svc.AccessKey); ok {
+				t.Fatal("deleted service account remains in the IAM cache")
+			}
+		})
+	}
+}
+
+func TestIAMAdminPolicylessSelf(t *testing.T) {
+	ctx, sys := iamAdminFixture(t)
+	caller := iamAdminUser(ctx, t, sys, "review-policyless", "")
+	svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "")
+	other := iamAdminUser(ctx, t, sys, "bulk-other", "readwrite")
+	_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "bulk-other-child", "")
+	api := adminAPIHandlers{}
+	assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200)
+	rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk)
+	assertIAMAdminStatus(t, "list bulk self", rec, 200)
+	data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
+	mustIAM(t, err)
+	var listed map[string]madmin.ListAccessKeysResp
+	mustIAM(t, json.Unmarshal(data, &listed))
+	entry, ok := listed[caller.AccessKey]
+	if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 1 || entry.ServiceAccounts[0].AccessKey != svc.AccessKey || len(entry.STSKeys) != 0 {
+		t.Fatalf("policyless bulk self returned unexpected identities: %+v", listed)
+	}
+	assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200)
+	assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200)
+	assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
+	assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
+	assertIAMAdminStatus(t, "list another user", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+globalActiveCred.AccessKey, nil, nil, api.ListServiceAccounts), 403)
+	assertIAMAdminStatus(t, "revoke another user", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
+}
+
+func TestIAMAdminExpiringInheritedPolicy(t *testing.T) {
+	ctx, sys := iamAdminFixture(t)
+	parent := iamAdminUser(ctx, t, sys, "review-legacy-parent", "consoleAdmin")
+	document := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
+	// Model a credential persisted by the old handler: it copied its caller's
+	// policy claims but set the newly created service's type to inherited.
+	expires := UTCNow().Add(time.Hour)
+	caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "rv-exp-inherited", secretKey: "review-service-valid-secret", expiration: &expires,
+		claims: map[string]any{policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document))},
+	})
+	mustIAM(t, err)
+	req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
+	checked, owner, code := checkKeyValid(req, caller.AccessKey)
+	if code != ErrNone {
+		t.Fatalf("checkKeyValid: %v", code)
+	}
+	args := policy.Args{AccountName: caller.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.CreateServiceAccountAdminAction, DenyOnly: true}
+	if !sys.IsAllowed(args) {
+		t.Fatal("fixture cannot create")
+	}
+	args.Action = policy.CreateUserAdminAction
+	args.DenyOnly = false
+	if sys.IsAllowed(args) {
+		t.Fatal("fixture not constrained")
+	}
+	t.Logf("service=%v temp=%v inherited=%v; create allowed, create-user denied", checked.IsServiceAccount(), checked.IsTemp(), checked.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType)
+	api := adminAPIHandlers{}
+	rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-escaped", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount)
+	if rec.Code == 200 {
+		child, ok := sys.GetUser(ctx, "review-escaped")
+		if !ok {
+			t.Fatal("new key missing")
+		}
+		childReq := iamAdminRequest(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
+		childChecked, childOwner, childCode := checkKeyValid(childReq, child.Credentials.AccessKey)
+		if childCode != ErrNone {
+			t.Fatalf("child auth=%v", childCode)
+		}
+		allowed := sys.IsAllowed(policy.Args{AccountName: childChecked.AccessKey, Claims: childChecked.Claims, IsOwner: childOwner, Action: policy.CreateUserAdminAction})
+		t.Errorf("constrained inherited service created a new key; new key CreateUser allowed=%v", allowed)
+	} else {
+		assertIAMAdminStatus(t, "reject derived creation", rec, 403)
+	}
+	updated := iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+caller.AccessKey, madmin.UpdateServiceAccountReq{NewName: "metadata-only"}, map[string]string{"accessKey": caller.AccessKey}, api.UpdateServiceAccount)
+	if updated.Code == 204 {
+		after, ok := sys.GetUser(ctx, caller.AccessKey)
+		if !ok {
+			t.Fatal("updated key missing")
+		}
+		afterReq := iamAdminRequest(t, after.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
+		afterChecked, afterOwner, afterCode := checkKeyValid(afterReq, caller.AccessKey)
+		if afterCode != ErrNone {
+			t.Fatalf("updated auth=%v", afterCode)
+		}
+		allowed := sys.IsAllowed(policy.Args{AccountName: caller.AccessKey, Claims: afterChecked.Claims, IsOwner: afterOwner, Action: policy.CreateUserAdminAction})
+		t.Errorf("constrained inherited service updated itself; CreateUser allowed after update=%v", allowed)
+	} else {
+		assertIAMAdminStatus(t, "reject inherited self update", updated, 403)
+	}
+}
+
+func TestIAMAdminSessionPolicyCannotDelegate(t *testing.T) {
+	for _, temp := range []bool{false, true} {
+		t.Run(map[bool]string{false: "service", true: "sts"}[temp], func(t *testing.T) {
+			ctx, sys := iamAdminFixture(t)
+			parent := iamAdminUser(ctx, t, sys, "rv-parent", "consoleAdmin")
+			doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
+			var caller auth.Credentials
+			if !temp {
+				caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "review-restricted", doc)
+			} else {
+				var err error
+				caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(doc))}, globalActiveCred.SecretKey)
+				mustIAM(t, err)
+				caller.ParentUser = parent.AccessKey
+				_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
+				mustIAM(t, err)
+			}
+			sibling := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-sibling", "")
+			api := adminAPIHandlers{}
+			assertIAMAdminStatus(t, "restrict create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 403)
+			assertIAMAdminStatus(t, "restrict LDAP create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/idp/ldap/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccountLDAP), 403)
+			assertIAMAdminStatus(t, "restrict update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 403)
+			if _, ok := sys.store.GetUser("review-bad-new"); ok {
+				t.Fatal("denied creation persisted a credential")
+			}
+			u, ok := sys.store.GetUser(sibling.AccessKey)
+			if !ok || u.Credentials.SecretKey != sibling.SecretKey {
+				t.Fatal("denied update changed the target credential")
+			}
+		})
+	}
+}
+
+func TestIAMAdminSelfDenials(t *testing.T) {
+	for _, mode := range []string{"parent_deny", "group_deny", "session_implicit_deny", "session_explicit_deny", "plugin_deny"} {
+		t.Run(mode, func(t *testing.T) {
+			ctx, sys := iamAdminFixture(t)
+			parent := iamAdminUser(ctx, t, sys, "rv-self-parent", "readwrite")
+			target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-target", "")
+			caller := parent
+			doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["admin:ListServiceAccounts","admin:RemoveServiceAccount"]}]}`
+			switch mode {
+			case "parent_deny", "group_deny":
+				p, err := policy.ParseConfig(strings.NewReader(doc))
+				mustIAM(t, err)
+				_, err = sys.SetPolicy(ctx, "review-self-deny", *p)
+				mustIAM(t, err)
+				if mode == "group_deny" {
+					_, err = sys.AddUsersToGroup(ctx, "self-deny-group", []string{parent.AccessKey})
+					mustIAM(t, err)
+					_, err = sys.PolicyDBSet(ctx, "self-deny-group", "review-self-deny", regUser, true)
+				} else {
+					_, err = sys.PolicyDBSet(ctx, parent.AccessKey, "readwrite,review-self-deny", regUser, false)
+				}
+				mustIAM(t, err)
+			case "session_implicit_deny":
+				caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::example/*"]}]}`)
+			case "session_explicit_deny":
+				caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", doc)
+			case "plugin_deny":
+				iamAdminPlugin(t, false)
+			}
+			api := adminAPIHandlers{}
+			assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403)
+			assertIAMAdminStatus(t, "list bulk self", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk), 403)
+			assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403)
+			assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403)
+			assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403)
+			assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
+		})
+	}
+}
+
+func TestIAMAdminNonRootCannotManageRoot(t *testing.T) {
+	ctx, sys := iamAdminFixture(t)
+	caller := iamAdminUser(ctx, t, sys, "rv-console-admin", "consoleAdmin")
+	target := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "rv-root-target", "")
+	api := adminAPIHandlers{}
+	assertIAMAdminStatus(t, "create root", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "rv-new-root", SecretKey: "review-service-valid-secret", TargetUser: globalActiveCred.AccessKey}, nil, api.AddServiceAccount), 403)
+	assertIAMAdminStatus(t, "update root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 403)
+	assertIAMAdminStatus(t, "delete root", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
+	assertIAMAdminStatus(t, "revoke root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
+	// Restricted root-derived credentials and the site replicator are not root
+	// administrators, even when their effective policy allows these actions.
+	limited := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "limited-root", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:*"]}]}`)
+	assertIAMAdminStatus(t, "limited root delete", iamAdminCall(t, limited, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
+	assertIAMAdminStatus(t, "limited root revoke", iamAdminCall(t, limited, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
+	replicator, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{accessKey: siteReplicatorSvcAcc, secretKey: "replicator-valid-secret", allowSiteReplicatorAccount: true})
+	mustIAM(t, err)
+	assertIAMAdminStatus(t, "replicator create root", iamAdminCall(t, replicator, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 403)
+
+	// An external plugin remains authoritative, including for root targets.
+	iamAdminPlugin(t, true)
+	assertIAMAdminStatus(t, "plugin update root", iamAdminCall(t, limited, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewName: "plugin-authorized"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 204)
+	assertIAMAdminStatus(t, "plugin create root", iamAdminCall(t, limited, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 200)
+}
+
+func TestIAMAdminDeleteCorruptServiceReplicates(t *testing.T) {
+	ctx, sys := iamAdminFixture(t)
+	caller := iamAdminUser(ctx, t, sys, "cleanup-admin", "consoleAdmin")
+	parent := iamAdminUser(ctx, t, sys, "cleanup-parent", "readwrite")
+	target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "cleanup-target", "")
+	cache := sys.store.lock()
+	u := cache.iamUsersMap[target.AccessKey]
+	u.Credentials.SessionToken = "broken-session-token"
+	u.Credentials.Status = auth.AccountOff
+	cache.iamUsersMap[target.AccessKey] = u
+	sys.store.unlock()
+
+	var deletions atomic.Int32
+	peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			return
+		}
+		var batch iamRevisionBatch
+		if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+			t.Error(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if len(batch.Items) != 1 || batch.Items[0].SvcAccChange == nil || batch.Items[0].SvcAccChange.Delete == nil || batch.Items[0].SvcAccChange.Delete.AccessKey != target.AccessKey {
+			t.Error("replication did not carry the service account deletion")
+		} else {
+			deletions.Add(1)
+		}
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "peer", Instance: "peer-boot", Digest: "deleted"}})
+	}))
+	t.Cleanup(peer.Close)
+	globalSiteReplicatorCred.RLock()
+	previousSecret := globalSiteReplicatorCred.secretKey
+	globalSiteReplicatorCred.RUnlock()
+	globalSiteReplicatorCred.Set(caller.SecretKey)
+	t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSecret) })
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = true
+	globalSiteReplicationSys.state = srState{ServiceAccountAccessKey: caller.AccessKey, Peers: map[string]madmin.PeerInfo{
+		globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"},
+		"peer":               {DeploymentID: "peer", Name: "peer", Endpoint: peer.URL},
+	}}
+	globalSiteReplicationSys.Unlock()
+	t.Cleanup(func() {
+		globalSiteReplicationSys.Lock()
+		globalSiteReplicationSys.enabled = false
+		globalSiteReplicationSys.Unlock()
+	})
+	api := adminAPIHandlers{}
+	assertIAMAdminStatus(t, "replicated cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 204)
+	if deletions.Load() != 1 {
+		t.Fatalf("expected one immediate deletion notification, got %d", deletions.Load())
+	}
+}
+
+func TestIAMAdminListSelfAliases(t *testing.T) {
+	for _, kind := range []string{"service", "sts"} {
+		t.Run(kind, func(t *testing.T) {
+			ctx, sys := iamAdminFixture(t)
+			parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite")
+			target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "")
+			serviceCaller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
+			caller := serviceCaller
+			other := iamAdminUser(ctx, t, sys, "alias-other", "readwrite")
+			_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "alias-other-child", "")
+			if kind == "sts" {
+				var err error
+				caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
+				mustIAM(t, err)
+				caller.ParentUser = parent.AccessKey
+				_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
+				mustIAM(t, err)
+			}
+			api := adminAPIHandlers{}
+			for _, alias := range []string{"", caller.AccessKey, parent.AccessKey} {
+				rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+alias, nil, nil, api.ListServiceAccounts)
+				assertIAMAdminStatus(t, "list self alias", rec, 200)
+				data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
+				mustIAM(t, err)
+				var response madmin.ListServiceAccountsResp
+				mustIAM(t, json.Unmarshal(data, &response))
+				found := false
+				for _, account := range response.Accounts {
+					found = found || account.AccessKey == target.AccessKey
+				}
+				if !found {
+					t.Fatalf("self alias %q did not list the parent's service accounts", alias)
+				}
+
+				query := url.Values{"listType": {madmin.AccessKeyListAll}}
+				if alias != "" {
+					query.Set("users", alias)
+				}
+				bulk := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?"+query.Encode(), nil, nil, api.ListAccessKeysBulk)
+				assertIAMAdminStatus(t, "list bulk self alias", bulk, 200)
+				data, err = madmin.DecryptData(caller.SecretKey, bulk.Body)
+				mustIAM(t, err)
+				var listed map[string]madmin.ListAccessKeysResp
+				mustIAM(t, json.Unmarshal(data, &listed))
+				entry, ok := listed[parent.AccessKey]
+				if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 2 {
+					t.Fatalf("bulk self alias %q returned unexpected identities: %+v", alias, listed)
+				}
+				want := map[string]bool{target.AccessKey: true, serviceCaller.AccessKey: true}
+				for _, account := range entry.ServiceAccounts {
+					if !want[account.AccessKey] {
+						t.Fatalf("bulk self alias %q returned unexpected or duplicate service account %s", alias, account.AccessKey)
+					}
+					delete(want, account.AccessKey)
+				}
+				if kind == "sts" {
+					if len(entry.STSKeys) != 1 || entry.STSKeys[0].AccessKey != caller.AccessKey {
+						t.Fatalf("bulk self alias %q returned unexpected STS accounts: %+v", alias, entry.STSKeys)
+					}
+				} else if len(entry.STSKeys) != 0 {
+					t.Fatalf("service caller returned unexpected STS accounts: %+v", entry.STSKeys)
+				}
+				if bytes.Contains(data, []byte(serviceCaller.SecretKey)) || bytes.Contains(data, []byte(`"secretKey"`)) || bytes.Contains(data, []byte(`"sessionToken"`)) {
+					t.Fatal("bulk listing returned credential secrets")
+				}
+			}
+		})
+	}
+}

cmd/admin-handlers-users.go

@@ -622,7 +622,11 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 			policiesNames = policySet.ToSlice()
 		}
 
-		stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		stsAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	policyJSON, err := json.MarshalIndent(stsAccountPolicy, "", " ")
@@ -714,8 +718,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
 		// it is associated with the LDAP/OIDC user properly.
+		// The new credential's policy comes from the request, not the caller.
 		for k, v := range cred.Claims {
-			if k == expClaim {
+			switch k {
+			case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
 				continue
 			}
 			opts.claims[k] = v
@@ -855,10 +861,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Permission checks:
 	//
-	// 1. Any type of account (i.e. access keys (previously/still called service
-	// accounts), STS accounts, internal IDP accounts, etc) with the
-	// policy.UpdateServiceAccountAdminAction permission can update any service
-	// account.
+	// 1. Updating requires policy.UpdateServiceAccountAdminAction. Under built-in
+	// IAM, session-limited callers cannot update credentials, and only unrestricted
+	// root credentials can manage root's service accounts.
 	//
 	// 2. We would like to let a user update their own access keys, however it
 	// is currently blocked pending a re-design. Users are still able to delete
@@ -874,6 +879,11 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
 	}
+	if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
+		(svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
 
 	var sp *policy.Policy
 	if len(updateReq.NewPolicy) > 0 {
@@ -951,23 +961,21 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(policy.Args{
+	requestUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		requestUser = cred.ParentUser
+	}
+	if !isAllowedServiceAccountAction(cred, policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.ListServiceAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
+		DenyOnly:        requestUser == svcAccount.ParentUser,
 	}) {
-		requestUser := cred.AccessKey
-		if cred.ParentUser != "" {
-			requestUser = cred.ParentUser
-		}
-
-		if requestUser != svcAccount.ParentUser {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
 	}
 
 	// if session policy is nil or empty, then it is implied policy
@@ -983,7 +991,11 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		svcAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
@@ -1039,29 +1051,27 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	var targetAccount string
-
-	// If listing is requested for a specific user (who is not the request
-	// sender), check that the user has permissions.
+	selfUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		selfUser = cred.ParentUser
+	}
 	user := r.Form.Get("user")
-	if user != "" && user != cred.AccessKey {
-		if !globalIAMSys.IsAllowed(policy.Args{
-			AccountName:     cred.AccessKey,
-			Groups:          cred.Groups,
-			Action:          policy.ListServiceAccountsAdminAction,
-			ConditionValues: getConditionValues(r, "", cred),
-			IsOwner:         owner,
-			Claims:          cred.Claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-		targetAccount = user
-	} else {
-		targetAccount = cred.AccessKey
-		if cred.ParentUser != "" {
-			targetAccount = cred.ParentUser
-		}
+	selfOnly := user == "" || user == cred.AccessKey || user == selfUser
+	if !isAllowedServiceAccountAction(cred, policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.ListServiceAccountsAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        selfOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+	targetAccount := user
+	if selfOnly {
+		targetAccount = selfUser
 	}
 
 	serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, targetAccount)
@@ -1110,7 +1120,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil || !globalIAMSys.Initialized() {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -1131,35 +1141,41 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
 		return
 	}
-	// We do not care if service account is readable or not at this point,
-	// since this is a delete call we shall allow it to be deleted if possible.
-	svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
-	if errors.Is(err, errNoSuchServiceAccount) {
+	// Deletion needs the stored identity, not a valid token or session policy.
+	// Keep disabled, expired and damaged service accounts removable, and retain
+	// their parent for authorization and the site replication hook below.
+	u, ok := globalIAMSys.store.GetUser(serviceAccount)
+	if !ok || !u.Credentials.IsServiceAccount() {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
 		return
 	}
+	svcAccount := u.Credentials
 
-	adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
+	parentUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		parentUser = cred.ParentUser
+	}
+	isSelf := parentUser == svcAccount.ParentUser
+	if !isAllowedServiceAccountAction(cred, policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.RemoveServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-	})
-
-	if !adminPrivilege {
-		parentUser := cred.AccessKey
-		if cred.ParentUser != "" {
-			parentUser = cred.ParentUser
-		}
-		if svcAccount.ParentUser != "" && parentUser != svcAccount.ParentUser {
-			// The service account belongs to another user but return not
-			// found error to mitigate brute force attacks. or the
-			// serviceAccount doesn't exist.
+		DenyOnly:        isSelf,
+	}) {
+		if !isSelf {
+			// Hide the existence of another user's service account.
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
 			return
 		}
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+	if newGlobalAuthZPluginFn() == nil && svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
+		return
 	}
 
 	if err := globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true); err != nil {
@@ -1229,7 +1245,7 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
-	if !globalIAMSys.IsAllowed(policy.Args{
+	if !isAllowedServiceAccountAction(cred, policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.ListServiceAccountsAdminAction,
@@ -1242,12 +1258,12 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if selfOnly && len(users) == 0 {
+	if selfOnly {
 		selfUser := cred.AccessKey
 		if cred.ParentUser != "" {
 			selfUser = cred.ParentUser
 		}
-		users = append(users, selfUser)
+		users = []string{selfUser}
 	}
 
 	var checkedUserList []string
@@ -1439,7 +1455,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
+		// Console refreshes this inventory immediately after bucket mutations.
+		// A stale cached list can otherwise hide a creation or retain a deletion
+		// indefinitely in the client's snapshot.
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1455,6 +1474,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
 	policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
 	var effectivePolicy policy.Policy
+	var policyErr error
 
 	var buf []byte
 	switch {
@@ -1478,10 +1498,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return
 		}
 		policySlice := newMappedPolicy(policy).toSlice()
-		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
+		effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policySlice...)
 
 	case hasPolicyClaim:
-		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
+		effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
 
 	default:
 		policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
@@ -1490,7 +1510,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
+		effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policies...)
+	}
+	if policyErr != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, policyErr), r.URL)
+		return
 	}
 
 	buf, err = json.MarshalIndent(effectivePolicy, "", " ")
@@ -2046,26 +2070,30 @@ func (a adminAPIHandlers) RevokeTokens(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
+	parentUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		parentUser = cred.ParentUser
+	}
+	isSelfTarget := isTokenSelfRevoke || user == parentUser
+	if !isAllowedServiceAccountAction(cred, policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.RemoveServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-	})
-
-	if !adminPrivilege || isTokenSelfRevoke {
-		parentUser := cred.AccessKey
-		if cred.ParentUser != "" {
-			parentUser = cred.ParentUser
-		}
-		if !isTokenSelfRevoke && user != parentUser {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+		DenyOnly:        isSelfTarget,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+	if isSelfTarget {
 		user = parentUser
 	}
+	if newGlobalAuthZPluginFn() == nil && user == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
 
 	// Infer token revoke type from the request if requestor is STS.
 	if isTokenSelfRevoke && tokenRevokeType == "" && !fullRevoke {
@@ -2112,29 +2140,22 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
 	u, ok := globalIAMSys.GetUser(ctx, accessKey)
 	targetCred := u.Credentials
 
-	if !globalIAMSys.IsAllowed(policy.Args{
+	requestUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		requestUser = cred.ParentUser
+	}
+	isSelf := ok && targetCred.ParentUser != "" && requestUser == targetCred.ParentUser
+	if !isAllowedServiceAccountAction(cred, policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.ListServiceAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
+		DenyOnly:        isSelf,
 	}) {
-		// If requested user does not exist and requestor is not allowed to list service accounts, return access denied.
-		if !ok {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-
-		requestUser := cred.AccessKey
-		if cred.ParentUser != "" {
-			requestUser = cred.ParentUser
-		}
-
-		if requestUser != targetCred.ParentUser {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
 	}
 
 	if !ok {
@@ -2181,7 +2202,11 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		svcAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
@@ -2908,6 +2933,52 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
 	return nil
 }
 
+// hasActiveSessionPolicy follows the same credential dispatch as IAMSys.IsAllowed.
+// Expiring service accounts also satisfy IsTemp and are evaluated as STS there.
+// Callers use this only after authorization under built-in IAM.
+func hasActiveSessionPolicy(cred auth.Credentials) bool {
+	args := policy.Args{Claims: cred.Claims}
+	if cred.IsTemp() {
+		hasSP, _ := isAllowedBySessionPolicy(args)
+		return hasSP
+	}
+	if cred.IsServiceAccount() {
+		if cred.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
+			return false
+		}
+		hasSP, _ := isAllowedBySessionPolicyForServiceAccount(args)
+		return hasSP
+	}
+	return false
+}
+
+// canManageRootCredentials is an additional guard after built-in IAM authorization.
+// The signature's owner flag is insufficient: historical empty session policies
+// clear it even though the service account still inherits root's permissions.
+func canManageRootCredentials(cred auth.Credentials) bool {
+	return cred.AccessKey == globalActiveCred.AccessKey ||
+		(cred.ParentUser == globalActiveCred.AccessKey && cred.AccessKey != siteReplicatorSvcAcc && !hasActiveSessionPolicy(cred))
+}
+
+// isAllowedServiceAccountAction preserves regular users' self inspection and
+// cleanup even without a policy attachment, while enforcing explicit denies.
+// Derived credentials and external authorization plugins use full IAM evaluation.
+func isAllowedServiceAccountAction(cred auth.Credentials, args policy.Args) bool {
+	if args.DenyOnly && !args.IsOwner && !cred.IsTemp() && !cred.IsServiceAccount() && newGlobalAuthZPluginFn() == nil {
+		policies, err := globalIAMSys.PolicyDBGet(args.AccountName, args.Groups...)
+		if err != nil {
+			return false
+		}
+		combined, err := globalIAMSys.GetCombinedPolicy(policies...)
+		if err != nil {
+			iamLogIf(GlobalContext, err)
+			return false
+		}
+		return combined.IsAllowed(args)
+	}
+	return globalIAMSys.IsAllowed(args)
+}
+
 func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
 	ctx := r.Context()
 
@@ -2996,6 +3067,14 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
 	}) {
 		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
 	}
+	effectiveTarget := targetUser
+	if (targetUser == cred.AccessKey || targetUser == cred.ParentUser) && cred.ParentUser != "" {
+		effectiveTarget = cred.ParentUser
+	}
+	if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
+		(effectiveTarget == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
+	}
 
 	var sp *policy.Policy
 	if len(createReq.Policy) > 0 {
@@ -3003,6 +3082,12 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
 		if err != nil {
 			return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
 		}
+		// Empty service policies mean inheritance, including for expiring keys.
+		// Normalize the replication request as well as the local credential.
+		if sp.Version == "" && sp.Statements == nil && sp.ID == "" {
+			sp = nil
+			createReq.Policy = nil
+		}
 	}
 
 	opts.sessionPolicy = sp

cmd/admin-router.go

@@ -163,6 +163,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// StorageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/multipart-preflight").HandlerFunc(adminMiddleware(adminAPI.MultipartPreflightHandler, traceAllFlag))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
 		// Metrics operation
@@ -388,6 +389,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
+		adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))

cmd/api-errors-checksum_test.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of SILO Object Storage.
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/minio/minio/internal/hash"
+)
+
+func TestChecksumMismatchAPIMessage(t *testing.T) {
+	for _, algorithm := range []string{"CRC32", "CRC32C", "SHA1", "SHA256", "CRC64NVME", ""} {
+		t.Run(algorithm, func(t *testing.T) {
+			mismatch := hash.ChecksumMismatch{Algorithm: algorithm, Want: "expected", Got: "actual"}
+			want := errorCodes.ToAPIErr(ErrContentChecksumMismatch).Description
+			if algorithm != "" {
+				want = fmt.Sprintf("The %s you specified did not match the calculated checksum.", algorithm)
+			}
+			for _, err := range []error{mismatch, fmt.Errorf("upload: %w", mismatch)} {
+				got := toAPIError(t.Context(), err)
+				if got.HTTPStatusCode != http.StatusBadRequest || got.Code != "XAmzContentChecksumMismatch" || got.Description != want {
+					t.Fatalf("toAPIError(%v) = %+v, want 400 XAmzContentChecksumMismatch: %s", err, got, want)
+				}
+			}
+		})
+	}
+}

cmd/api-errors.go

@@ -450,6 +450,9 @@ const (
 	ErrAdminNoSecretKey
 
 	ErrIAMNotInitialized
+	ErrMultipartListingLegacy // Reserved: legacy uploads no longer block listing.
+	ErrMultipartListingIdentity
+	ErrSlowDown
 
 	apiErrCodeEnd // This is used only for the testing code
 )
@@ -547,7 +550,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrBadDigest: {
 		Code:           "BadDigest",
-		Description:    "The Content-Md5 you specified did not match what we received.",
+		Description:    "The Content-MD5 you specified did not match what we received.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrEntityTooSmall: {
@@ -1336,6 +1339,21 @@ var errorCodes = errorCodeMap{
 		Description:    "IAM sub-system not initialized yet, please try again.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrMultipartListingLegacy: {
+		Code:           "MultipartListingNotReady",
+		Description:    "Legacy multipart uploads prevent a complete listing. Upgrade all writers, drain old uploads and run the multipart preflight check.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrMultipartListingIdentity: {
+		Code:           "MultipartListingMetadataInvalid",
+		Description:    "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSlowDown: {
+		Code:           "SlowDown",
+		Description:    "Please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	ErrBucketMetadataNotInitialized: {
 		Code:           "XMinioBucketMetadataNotInitialized",
 		Description:    "Bucket metadata not initialized yet, please try again.",
@@ -2173,6 +2191,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	err = unwrapAll(err)
 
 	switch err {
+	case errMultipartListingIdentity:
+		apiErr = ErrMultipartListingIdentity
 	case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
 		apiErr = ErrBadDigest
 	case errMissingPartChecksum:
@@ -2303,6 +2323,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	}
 
 	switch err.(type) {
+	case SlowDown:
+		apiErr = ErrSlowDown
 	case StorageFull:
 		apiErr = ErrStorageFull
 	case hash.BadDigest:
@@ -2481,6 +2503,14 @@ func toAPIError(ctx context.Context, err error) APIError {
 	case errors.Is(err, errMissingPartChecksum):
 		apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
 	}
+	// PutObject/UploadPart checksum mismatches name the algorithm in the
+	// message, matching S3 (e.g. "The CRC32 you specified did not match the
+	// calculated checksum."). Falls through to the generic table description
+	// when the algorithm is unknown.
+	var checksumMismatch hash.ChecksumMismatch
+	if errors.As(err, &checksumMismatch) && checksumMismatch.Algorithm != "" {
+		apiErr.Description = fmt.Sprintf("The %s you specified did not match the calculated checksum.", checksumMismatch.Algorithm)
+	}
 	switch apiErr.Code {
 	case "NotImplemented":
 		apiErr = APIError{

cmd/api-response.go

@@ -41,7 +41,7 @@ import (
 const (
 	maxObjectList  = 1000  // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
 	maxDeleteList  = 1000  // Limit number of objects deleted in a delete call.
-	maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
+	maxUploadsList = 1000  // Limit number of uploads in a listUploadsResponse.
 	maxPartsList   = 10000 // Limit number of parts in a listPartsResponse.
 )
 

cmd/auth-handler.go

@@ -28,13 +28,10 @@ import (
 	"mime"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 	"sync/atomic"
-	"time"
 
 	"github.com/minio/minio/internal/auth"
-	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -733,48 +730,6 @@ func setAuthMiddleware(h http.Handler) http.Handler {
 	})
 }
 
-func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
-	var retSet bool
-	if cred.AccessKey == "" {
-		return ErrAccessDenied
-	}
-
-	conditions := getConditionValues(r, "", cred)
-	conditions["object-lock-mode"] = []string{string(retMode)}
-	conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
-	if retDays > 0 {
-		conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
-	}
-	if retMode == objectlock.RetGovernance && byPassSet {
-		byPassSet = globalIAMSys.IsAllowed(policy.Args{
-			AccountName:     cred.AccessKey,
-			Groups:          cred.Groups,
-			Action:          policy.BypassGovernanceRetentionAction,
-			BucketName:      bucketName,
-			ObjectName:      objectName,
-			ConditionValues: conditions,
-			IsOwner:         owner,
-			Claims:          cred.Claims,
-		})
-	}
-	if globalIAMSys.IsAllowed(policy.Args{
-		AccountName:     cred.AccessKey,
-		Groups:          cred.Groups,
-		Action:          policy.PutObjectRetentionAction,
-		BucketName:      bucketName,
-		ConditionValues: conditions,
-		ObjectName:      objectName,
-		IsOwner:         owner,
-		Claims:          cred.Claims,
-	}) {
-		retSet = true
-	}
-	if byPassSet || retSet {
-		return ErrNone
-	}
-	return ErrAccessDenied
-}
-
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.

cmd/batch-authz-review2_test.go

@@ -0,0 +1,356 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A pull into a suspended bucket replaces the null version. Its policy
+// decision must name that version even when the source is unversioned S3.
+func TestBatchReviewPullImplicitNullVersionAuthorization(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
+	mustIAM(t, err)
+	data := []byte("original null version")
+	before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: true})
+	mustIAM(t, err)
+	batchAuthzPolicy(f.ctx, t, f.sys, "null-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"null"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "null-deny-user", "null-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	job := batchPullJob(f.bucket)
+	job.Source.Type = BatchJobReplicateResourceS3
+	err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+		Name: before.Name, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
+	}, false, sub)
+	if !isBatchJobAuthzError(err) {
+		t.Errorf("null version policy must refuse the write, got %v", err)
+	}
+	after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: nullVersionID})
+	if gerr != nil || before.ETag != after.ETag {
+		t.Errorf("denied null version must remain unchanged: read=%v unchanged=%v", gerr, before.ETag == after.ETag)
+	}
+}
+
+type batchReviewPrefixWalkFailure struct{ ObjectLayer }
+
+func (a batchReviewPrefixWalkFailure) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
+	if prefix != "a/" {
+		return a.ObjectLayer.Walk(ctx, bucket, prefix, results, opts)
+	}
+	go func() {
+		defer close(results)
+		results <- itemOrErr[ObjectInfo]{Err: errors.New("test: incomplete first prefix")}
+	}()
+	return nil
+}
+
+type batchReviewWalkCutoff struct {
+	ObjectLayer
+	cutoff string
+}
+
+func (a batchReviewWalkCutoff) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
+	inner := make(chan itemOrErr[ObjectInfo], 1)
+	if err := a.ObjectLayer.Walk(ctx, bucket, prefix, inner, opts); err != nil {
+		close(results)
+		return err
+	}
+	go func() {
+		defer close(results)
+		stopped := false
+		for result := range inner {
+			if stopped {
+				continue
+			}
+			if result.Err == nil && result.Item.Name >= a.cutoff {
+				stopped = true
+				result = itemOrErr[ObjectInfo]{Err: errors.New("test: listing stopped before remaining child keys")}
+			}
+			select {
+			case results <- result:
+			case <-ctx.Done():
+				stopped = true
+			}
+		}
+	}()
+	return nil
+}
+
+// Listing a directory prefix returns the directory object before its child
+// keys. A durable cursor for that directory must preserve unfinished children.
+func TestBatchReviewDirectoryCursorKeepsChildrenPending(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	for range 12 {
+		batchPutVersion(t, f.obj, f.bucket, "a/", nil)
+	}
+	var pending string
+	for i := 0; i < 13; i++ {
+		older := f.putVersions(t, fmt.Sprintf("a/key-%02d", i))
+		if i == 11 {
+			pending = older
+		}
+	}
+	rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: ["a/"]
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	_ = job.Expire.Start(f.ctx, batchReviewWalkCutoff{ObjectLayer: f.obj, cutoff: "a/key-11"}, *job)
+	ri := f.jobInfo(t, job)
+	if ri.Object != "a/" {
+		t.Fatalf("control: the completed directory batch must be the cursor, got %q", ri.Object)
+	}
+	mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+	if f.versionExists(t, "a/key-11", pending) {
+		t.Error("a directory cursor must not suppress the unvisited child key on resume")
+	}
+}
+
+// Later prefixes must not advance the completion cursor past an earlier
+// prefix whose listing failed. This state is produced by Start itself.
+func TestBatchReviewExpireFailedPrefixKeepsWorkPending(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "a/todo")
+	for i := 0; i < 13; i++ {
+		f.putVersions(t, fmt.Sprintf("z/key-%02d", i))
+	}
+	rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: ["z/", "a/"]
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	_ = job.Expire.Start(f.ctx, batchReviewPrefixWalkFailure{f.obj}, *job)
+	ri := f.jobInfo(t, job)
+	if ri.Object >= "a/todo" {
+		t.Errorf("cursor must not cross an incomplete prefix, got %q", ri.Object)
+	}
+	mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+	if f.versionExists(t, "a/todo", older) {
+		t.Error("a fresh Start must revisit the key omitted by the failed listing")
+	}
+}
+
+// A repeated marker replication error must remain a failure on the second
+// attempt, including after the retry loop resets its counters.
+func TestBatchReviewMarkerRetryRemainsFailed(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchPutVersion(t, f.obj, f.bucket, "marker", nil)
+	_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
+	mustIAM(t, err)
+	oldTransport := globalRemoteTargetTransport
+	globalRemoteTargetTransport = batchLockTransport(func(r *http.Request) (*http.Response, error) {
+		code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
+		if r.URL.Query().Has("location") {
+			code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
+		}
+		return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
+	})
+	t.Cleanup(func() { globalRemoteTargetTransport = oldTransport })
+	job := BatchJobRequest{
+		ID: "replicate-marker-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+		Replicate: &BatchJobReplicateV1{
+			Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+			Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
+		},
+	}
+	job.Replicate.Flags.Retry.Attempts = 2
+	job.Replicate.Flags.Retry.Delay = time.Millisecond
+	mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
+	ri := f.jobInfo(t, &job)
+	if ri.Complete || !ri.Failed || ri.DeleteMarkersFailed != 1 {
+		t.Errorf("both marker attempts failed: complete=%v failed=%v markerFailures=%d attempts=%d", ri.Complete, ri.Failed, ri.DeleteMarkersFailed, ri.RetryAttempts)
+	}
+}
+
+// A current parent mapping suppresses the OpenID policy-name fallback.
+// Background-condition validation must inspect the same effective policies.
+func TestBatchReviewMappedPolicySuppressesClaimFallback(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
+	batchAuthzPolicy(f.ctx, t, f.sys, "unused-claim-policy", `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+	parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "mapped-full")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), iamPolicyClaimNameOpenID(): "unused-claim-policy",
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	p, err := sub.principal(f.ctx)
+	mustIAM(t, err)
+	req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "mapped/key", versionID: mustGetUUID()}
+	if !f.sys.isAllowedLocally(policy.Args{AccountName: p.accessKey, Claims: p.claims, Action: req.action, BucketName: req.bucket, ObjectName: req.object, ConditionValues: p.conditionValues(req)}) {
+		t.Fatal("control: the current mapped policy must allow this read")
+	}
+	if err := sub.authorize(f.ctx, req); err != nil {
+		t.Errorf("an unused fallback policy must not refuse a mapped-policy grant: %v", err)
+	}
+}
+
+func TestBatchReviewBindingAndExpiringServiceControls(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	root := batchRootSubmitter(t)
+	priorRoot := globalActiveCred
+	globalActiveCred.SecretKey = "different-test-root-secret"
+	_, err := root.principal(f.ctx)
+	globalActiveCred = priorRoot
+	if !isBatchJobAuthzError(err) {
+		t.Errorf("root-secret change must invalidate the binding: %v", err)
+	}
+	missing := *root
+	missing.Binding = nil
+	if _, err := missing.principal(f.ctx); !isBatchJobAuthzError(err) {
+		t.Errorf("missing fingerprint must refuse the principal: %v", err)
+	}
+	batchAuthzPolicy(f.ctx, t, f.sys, "service-parent-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "service-parent", "service-parent-full")
+	sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`))
+	mustIAM(t, err)
+	expires := UTCNow().Add(time.Hour)
+	cred, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp, expiration: &expires})
+	mustIAM(t, err)
+	stored, ok := f.sys.GetUser(f.ctx, cred.AccessKey)
+	if !ok {
+		t.Fatal("created service account must resolve")
+	}
+	cred = stored.Credentials
+	if !cred.IsTemp() || !cred.IsServiceAccount() {
+		t.Fatal("fixture must be an expiring service account")
+	}
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "allowed"}); err != nil {
+		t.Errorf("expiring service account must retain its allowed read: %v", err)
+	}
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectAction, bucket: f.bucket, object: "allowed"}); !isBatchJobAuthzError(err) {
+		t.Errorf("expiring service account must remain session-policy restricted: %v", err)
+	}
+}
+
+// An interrupted cross-pool copy can leave the same version in both pools.
+// A newer legal-hold field in either copy must prevent key rotation.
+func TestBatchReviewRotationUsesAuthoritativePoolHold(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const name = "rotation/split-hold"
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("batch-rotation-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	dek, err := GlobalKMS.GenerateKey(t.Context(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{bucket: bucket + "/" + name}})
+	mustIAM(t, err)
+	objectKey := crypto.GenerateKey(dek.Plaintext, nil)
+	sealed := objectKey.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), bucket, name)
+	first := crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed)
+	first[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "OFF"
+	first[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+	second := maps.Clone(first)
+	second[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
+	second[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
+	ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("protected payload"), objectKey))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if !enforceRetentionForDeletion(t.Context(), logical) {
+		t.Fatal("control: the authoritative logical version must be held")
+	}
+	r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+	err = r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
+	if !isErrObjectLocked(err) {
+		t.Errorf("rotation must refuse the authoritative legal hold, got %v", err)
+	}
+	after, gerr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, gerr)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Error("held logical version must not have its key rewritten")
+	}
+}
+
+// An encryption condition must describe what rotation actually commits,
+// even when the requested encryption type differs from the existing type.
+func TestBatchReviewRotationEncryptionConditionMatchesResult(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("required-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	const name = "rotation/encryption-condition"
+	meta := make(map[string]string)
+	key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, name, meta, nil)
+	mustIAM(t, err)
+	body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("encrypted payload"), key))
+	mustIAM(t, err)
+	before, err := f.obj.PutObject(f.ctx, f.bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, UserDefined: meta})
+	mustIAM(t, err)
+	batchAuthzPolicy(f.ctx, t, f.sys, "kms-write-only", `
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "kms-write-user", "kms-write-only")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	r := BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "required-key"}}
+	err = r.KeyRotate(f.ctx, f.obj, before, sub)
+	after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, gerr)
+	if err == nil && !crypto.S3KMS.IsEncrypted(after.UserDefined) {
+		t.Error("rotation allowed by an aws:kms-only write policy must produce SSE-KMS or refuse the operation")
+	}
+	if err != nil && before.UserDefined[crypto.MetaSealedKeyS3] != after.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Error("a refused rotation must leave the version unchanged")
+	}
+}
+
+// An explicit null marker collision must retain a held null data version.
+func TestBatchReviewPullMarkerNullLock(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	meta := batchLockMeta("LEGALHOLD")
+	before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, strings.NewReader("held null"), 9, "", ""), ObjectOptions{VersionID: nullVersionID, Versioned: true, UserDefined: meta})
+	mustIAM(t, err)
+	job := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+	}
+	err = job.ReplicateFromSource(f.ctx, f.obj, nil, ObjectInfo{
+		Name: before.Name, VersionID: nullVersionID, ModTime: UTCNow(), DeleteMarker: true,
+	}, false, batchRootSubmitter(t))
+	if !isErrObjectLocked(err) {
+		t.Errorf("marker collision with protected null data must return ObjectLocked, got %v", err)
+	}
+	if !batchVersionExists(t, f.obj, f.bucket, before.Name, nullVersionID) {
+		t.Fatal("protected null data must remain present")
+	}
+}

cmd/batch-authz-review3_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/auth"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A read policy must use the newest tag state of the logical version even
+// when an interrupted movement left an older copy in the preferred pool.
+func TestBatchReviewPushUsesAuthoritativePoolTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	const name = "push/tagged"
+	first := map[string]string{
+		xhttp.AmzObjectTagging:                         "classification=public",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
+	}
+	second := map[string]string{
+		xhttp.AmzObjectTagging:                         "classification=private",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+	}
+	before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if logical.UserTags != "classification=private" {
+		t.Fatal("control: logical version must have the newer private tag")
+	}
+	batchAuthzPolicy(f.ctx, t, f.sys, "pooled-push", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "pooled-push-user", "pooled-push")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: bucket, object: name, versionID: before.VersionID, tags: logical.UserTags}); !isBatchJobAuthzError(err) {
+		t.Fatal("control: authoritative tags must deny the read")
+	}
+	writes := 0
+	target, err := minio.NewCore("target.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+				_ = r.Body.Close()
+			}
+			if r.Method == http.MethodPut {
+				writes++
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
+	if !isBatchJobAuthzError(err) || writes != 0 {
+		t.Errorf("newer authoritative tags must deny push: err=%v writes=%d", err, writes)
+	}
+}
+
+// Verified token claims must not replace the actual version being authorized.
+func TestBatchReviewVersionConditionResistsClaimCollision(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	deniedVersion := mustGetUUID()
+	batchAuthzPolicy(f.ctx, t, f.sys, "version-claims", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, deniedVersion))
+	parent := iamAdminUser(f.ctx, t, f.sys, "version-claims-parent", "version-claims")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), "versionid": "an-unrelated-claim-value",
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	req := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: deniedVersion}
+	if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
+		t.Errorf("a token claim must not replace the actual denied version: %v", err)
+	}
+}
+
+// Retry totals describe a pass, including markers already successful before
+// another item caused that pass to retry.
+func TestBatchReviewMarkerSuccessCountResetsOnRetry(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchPutVersion(t, f.obj, f.bucket, "marker", nil)
+	_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
+	mustIAM(t, err)
+	oldTransport := remoteInstanceTransport
+	remoteInstanceTransport = atomic.Value{}
+	setRemoteInstanceTransport(batchLockTransport(func(r *http.Request) (*http.Response, error) {
+		code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
+		h := make(http.Header)
+		switch {
+		case r.URL.Query().Has("location"):
+			code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
+		case r.Method == http.MethodDelete:
+			code, body = http.StatusNoContent, ""
+		case r.Method == http.MethodHead && r.URL.Query().Get("versionId") != "":
+			code, body = http.StatusMethodNotAllowed, ""
+			h.Set("X-Amz-Delete-Marker", "true")
+		}
+		return &http.Response{StatusCode: code, Header: h, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
+	}))
+	t.Cleanup(func() { remoteInstanceTransport = oldTransport })
+	job := BatchJobRequest{
+		ID: "marker-success-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+		Replicate: &BatchJobReplicateV1{
+			Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+			Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
+		},
+	}
+	job.Replicate.Flags.Retry.Attempts = 2
+	job.Replicate.Flags.Retry.Delay = time.Millisecond
+	mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
+	ri := f.jobInfo(t, &job)
+	if !ri.Failed || ri.RetryAttempts != 2 {
+		t.Fatalf("control: data failure must cause two attempts: failed=%v attempts=%d", ri.Failed, ri.RetryAttempts)
+	}
+	if ri.DeleteMarkers != 1 {
+		t.Errorf("one successful marker must be counted once in the final attempt, got %d", ri.DeleteMarkers)
+	}
+}

cmd/batch-authz-review4_test.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/auth"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A tag removal in a newer copy must govern both the authorization and the
+// metadata sent by a push. The submitter can read data but cannot read tags.
+func TestBatchReviewPushSendsOnlyAuthorizedMetadata(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	const name = "push/removed-tags"
+	first := map[string]string{
+		xhttp.AmzObjectTagging:                         "classification=private",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
+	}
+	second := map[string]string{
+		xhttp.AmzObjectTagging:                         "",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+	}
+	before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if logical.UserTags != "" {
+		t.Fatal("control: the authoritative version must have no tags")
+	}
+	batchAuthzPolicy(f.ctx, t, f.sys, "data-without-tags", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "data-without-tags-user", "data-without-tags")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: name, versionID: before.VersionID}); !isBatchJobAuthzError(err) {
+		t.Fatal("control: reading tags must be denied")
+	}
+	writes := 0
+	var sentTags string
+	target, err := minio.NewCore("target.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+				_ = r.Body.Close()
+			}
+			if r.Method == http.MethodPut {
+				writes++
+				sentTags = r.Header.Get(xhttp.AmzObjectTagging)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
+	if err != nil && !isBatchJobAuthzError(err) {
+		t.Fatalf("unexpected replication failure: %v", err)
+	}
+	if writes > 0 && sentTags != logical.UserTags {
+		t.Errorf("push must send only metadata covered by authorization: writes=%d sentTags=%q authorizedTags=%q err=%v", writes, sentTags, logical.UserTags, err)
+	}
+}
+
+// LDAP's short username and its parent DN are distinct identities. The
+// background merge must not turn a deny for the verified short name into
+// a non-matching statement by replacing it with the parent DN.
+func TestBatchReviewLDAPUsernameDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "ldap-short-name-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"ldap:username":"alice"}}}`)
+	const dn = "uid=alice,ou=people,dc=example,dc=org"
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		"exp": UTCNow().Add(time.Hour).Unix(), ldapUser: dn, ldapActualUser: dn, ldapUserN: "alice",
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = dn
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "ldap-short-name-deny")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	p, err := sub.principal(f.ctx)
+	mustIAM(t, err)
+	cred.Claims = p.claims
+	req, err := http.NewRequest(http.MethodDelete, "https://localhost/batch-authz/data", nil)
+	mustIAM(t, err)
+	args := policy.Args{
+		AccountName: cred.AccessKey, Claims: p.claims, Action: policy.DeleteObjectVersionAction,
+		BucketName: f.bucket, ObjectName: "data", ConditionValues: getConditionValues(req, "", cred),
+	}
+	if args.ConditionValues["username"][0] != "alice" || f.sys.isAllowedLocally(args) {
+		t.Fatal("control: the request evaluator must deny the verified LDAP short username")
+	}
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: mustGetUUID()}); !isBatchJobAuthzError(err) {
+		t.Errorf("batch authorization must preserve the LDAP username deny or fail closed, got %v", err)
+	}
+}
+
+// Failure to read authoritative metadata is incomplete work, even if one
+// pool's copy can still be opened. In particular, skipErrs=false must not
+// turn an unreadable entry into a successful archive.
+func TestBatchReviewSnowballAuthoritativeReadFailure(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const name = "archive/unreadable-copy"
+	before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true})
+	set := z.serverPools[1].sets[0]
+	getDisks := set.getDisks
+	disks := append([]StorageAPI(nil), getDisks()...)
+	for i := range disks {
+		disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: name}
+	}
+	set.getDisks = func() []StorageAPI { return disks }
+	defer func() { set.getDisks = getDisks }()
+	gr, err := z.GetObjectNInfo(t.Context(), bucket, name, nil, nil, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	_, logicalErr := batchAuthoritativeRead(t.Context(), z, bucket, name, gr.ObjInfo)
+	gr.Close()
+	if logicalErr == nil {
+		t.Fatal("control: the authoritative read must fail")
+	}
+	remote, err := minio.New("archive.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	r.Source.Snowball.InMemory = ptr(true)
+	r.Source.Snowball.Compress = ptr(false)
+	r.Source.Snowball.SkipErrs = ptr(false)
+	if err := r.writeAsArchive(t.Context(), z, remote, []ObjectInfo{before}, "", batchRootSubmitter(t)); err == nil {
+		t.Error("archive with an unreadable authoritative version must not report success")
+	}
+}

cmd/batch-authz-review5_test.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+// A suspended destination does not keep a source version ID. Authorization
+// must therefore describe the null destination version that the write really
+// replaces, and must not let a source ID escape that model.
+func TestBatchReviewPullSuspendedVersionBinding(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
+	mustIAM(t, err)
+
+	sourceVersion := mustGetUUID()
+	batchAuthzPolicy(f.ctx, t, f.sys, "pull-source-version-deny", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, sourceVersion))
+	cred := iamAdminUser(f.ctx, t, f.sys, "pull-source-version-user", "pull-source-version-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	job := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+	}
+	err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+		Name: "pull/suspended-version", VersionID: sourceVersion, ModTime: UTCNow(), ETag: "batchetag",
+		UserDefined: map[string]string{},
+	}, false, sub)
+	if !isBatchJobAuthzError(err) {
+		t.Fatalf("a source version ID must not evade the destination deny: %v", err)
+	}
+	if _, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/suspended-version", ObjectOptions{VersionID: sourceVersion}); gerr == nil {
+		t.Fatal("a denied suspended-destination write created the source version")
+	}
+}
+
+// A failed source open in a non-skipping snowball must be reported to the
+// batch rather than silently omitted from a successful archive.
+type batchReviewOpenFailure struct {
+	ObjectLayer
+	name string
+}
+
+func (a batchReviewOpenFailure) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+	if object == a.name {
+		return nil, fmt.Errorf("test: source open failed")
+	}
+	return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
+}
+
+func TestBatchReviewSnowballSourceOpenFailure(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	name := "archive/open-failure"
+	entry := batchPutVersion(t, f.obj, f.bucket, name, nil)
+	remote, err := minio.New("archive.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	r.Source.Snowball.InMemory = ptr(true)
+	r.Source.Snowball.Compress = ptr(false)
+	r.Source.Snowball.SkipErrs = ptr(false)
+	api := batchReviewOpenFailure{ObjectLayer: f.obj, name: name}
+	if err := r.writeAsArchive(t.Context(), api, remote, []ObjectInfo{entry}, "", batchRootSubmitter(t)); err == nil {
+		t.Fatal("a source open failure must not be reported as a successful archive")
+	}
+}
+
+// Rotation must apply tag conditions to the logical version, including
+// a more recent tag update stored in a pool other than the selected copy.
+func TestBatchReviewRotationAuthoritativeTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("batch-round5-rotation-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	batchAuthzPolicy(f.ctx, t, f.sys, "round5-rotation-tags", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "round5-rotation-user", "round5-rotation-tags")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	for _, classification := range []string{"private", "public"} {
+		t.Run(classification, func(t *testing.T) {
+			name := "rotation/authoritative-" + classification
+			first := make(map[string]string)
+			key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
+			mustIAM(t, err)
+			first[xhttp.AmzObjectTagging] = "classification=public"
+			first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+			second := maps.Clone(first)
+			second[xhttp.AmzObjectTagging] = "classification=" + classification
+			second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+			ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("test encrypted payload"), key))
+			mustIAM(t, err)
+			before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+			putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+			logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if logical.UserTags != "classification="+classification {
+				t.Fatal("control: the second pool must govern the logical tags")
+			}
+			job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			err = job.KeyRotate(f.ctx, z, before, sub)
+			after, readErr := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, readErr)
+			changed := after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3]
+			if classification == "private" {
+				if !isBatchJobAuthzError(err) || changed {
+					t.Fatalf("authoritative tag deny must prevent rewrapping: err=%v changed=%v", err, changed)
+				}
+			} else if err != nil || !changed {
+				t.Fatalf("authorized unlocked version must rotate: err=%v changed=%v", err, changed)
+			}
+		})
+	}
+}

cmd/batch-authz-review6_test.go

@@ -0,0 +1,538 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+// Rotation and listing cases found in review.
+
+// Rotation rewrites the copy GetObjectInfo returns, which can carry another
+// tag revision than the merged version of all pools: both are authorized.
+func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("review6-rotation-copy-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	batchAuthzPolicy(f.ctx, t, f.sys, "review6-rotation-copy-deny", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "review6-rotation-copy-user", "review6-rotation-copy-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	const name = "rotation/written-copy"
+	first := make(map[string]string)
+	key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
+	mustIAM(t, err)
+	first[xhttp.AmzObjectTagging] = "classification=private"
+	first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+	second := maps.Clone(first)
+	second[xhttp.AmzObjectTagging] = "classification=public"
+	second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+	ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("review6 rotation payload"), key))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if logical.UserTags != "classification=public" {
+		t.Fatalf("control: merged tags=%q, want public", logical.UserTags)
+	}
+	job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+	if err = job.KeyRotate(f.ctx, z, before, sub); !isBatchJobAuthzError(err) {
+		t.Fatalf("rotation of a copy the submitter may not read returned %v", err)
+	}
+	after, err := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Fatal("a refused rotation rewrote the copy")
+	}
+}
+
+// batchWalkError fails every listing before it returns a result.
+type batchWalkError struct {
+	ObjectLayer
+	err error
+}
+
+func (w batchWalkError) Walk(context.Context, string, string, chan<- itemOrErr[ObjectInfo], WalkOptions) error {
+	return w.err
+}
+
+// A listing outage pauses the run, as a streamed listing error does.
+func TestBatchReviewExpireWalkErrorPausesRun(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	listErr := errors.New("review6: listing failed")
+	if err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job); !errors.Is(err, listErr) {
+		t.Fatalf("start returned %v, want %v", err, listErr)
+	}
+	if ri := f.jobInfo(t, job); ri.Failed || ri.Complete || ri.PausedReason == "" {
+		t.Fatalf("failed listing: complete=%v failed=%v", ri.Complete, ri.Failed)
+	}
+}
+
+// Two pools hold different writes of the null version, and a newer version
+// of the key is in the second one. Rotation rewrites the copy it read and
+// authorized, in the first pool, not the copy the latest version of the key
+// would route a write to; the other write keeps its key and stays readable.
+func TestBatchReviewRotationRewritesTheCopyItRead(t *testing.T) {
+	for _, otherLatest := range []bool{false, true} {
+		name := "addressed-version-is-latest"
+		if otherLatest {
+			name = "another-version-is-latest"
+		}
+		t.Run(name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			z, _ := consistencyPools(t)
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review7-rotation-copies"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, versionErr := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, versionErr)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review7-rotation-key")
+			t.Cleanup(func() { GlobalKMS = oldKMS })
+			const object = "rotation/null-copy"
+			const payload = "rotation must preserve this data"
+			readCopy := func(pool int) ([]byte, error) {
+				gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: nullVersionID})
+				if err != nil {
+					return nil, err
+				}
+				defer gr.Close()
+				return io.ReadAll(gr)
+			}
+			var before [2]ObjectInfo
+			var keys [2]crypto.ObjectKey
+			kinds := []crypto.Type{crypto.S3, crypto.S3KMS}
+			for pool, kind := range kinds {
+				meta := make(map[string]string)
+				key, err := newEncryptMetadata(t.Context(), kind, "review7-rotation-key", nil, bucket, object, meta, nil)
+				mustIAM(t, err)
+				keys[pool] = key
+				meta[xhttp.AmzObjectTagging] = "classification=public"
+				if pool == 1 {
+					meta[xhttp.AmzObjectTagging] = "classification=private"
+				}
+				meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Duration(pool+2) * time.Hour).Format(time.RFC3339Nano)
+				body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
+				mustIAM(t, err)
+				putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{
+					Versioned: true, VersionID: nullVersionID, UserDefined: meta,
+					MTime: UTCNow().Add(-time.Duration(pool+2) * time.Hour),
+				})
+				// As a read returns it, without the write's internal fields.
+				before[pool], err = z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
+				mustIAM(t, err)
+				data, err := readCopy(pool)
+				if err != nil || string(data) != payload {
+					t.Fatalf("control: pool %d was not readable before rotation: %v", pool, err)
+				}
+			}
+			if otherLatest {
+				putConsistencyObject(t, z, bucket, object, 1, "later version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+			}
+			selected, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID, Versioned: true})
+			mustIAM(t, err)
+			if selected.VersionID != nullVersionID || !selected.ModTime.Equal(before[0].ModTime) || !crypto.S3.IsEncrypted(selected.UserDefined) {
+				t.Fatal("control: the addressed null version must select the SSE-S3 write in pool zero")
+			}
+			logical, err := batchAuthoritativeVersion(t.Context(), z, bucket, object, selected)
+			mustIAM(t, err)
+			if logical.UserTags != "classification=public" || selected.UserTags != logical.UserTags {
+				t.Fatal("control: both current authorization inputs must have public tags")
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review7-rotation-key"}}
+			sub := batchRootSubmitter(t)
+			if otherLatest {
+				batchAuthzPolicy(t.Context(), t, globalIAMSys, "review7-rotate-public", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+				cred := iamAdminUser(t.Context(), t, globalIAMSys, "review7-rotate-user", "review7-rotate-public")
+				sub, err = newBatchJobSubmitter(cred)
+				mustIAM(t, err)
+				if err := sub.authorize(t.Context(), r.rotateAuthzRequests(before[1])...); !isBatchJobAuthzError(err) {
+					t.Fatal("control: the older write in pool one must be denied")
+				}
+			}
+			err = r.KeyRotate(t.Context(), z, selected, sub)
+			if err != nil {
+				t.Logf("rotation refused: %v", err)
+			}
+			for pool, want := range keys {
+				after, readErr := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
+				mustIAM(t, readErr)
+				if otherLatest && pool == 1 && (!maps.Equal(after.UserDefined, before[pool].UserDefined) || !after.ModTime.Equal(before[pool].ModTime) || after.UserTags != before[pool].UserTags) {
+					t.Error("rotation rewrote a copy denied by the submitter's tag-conditioned policy")
+				}
+				kind, _ := crypto.IsEncrypted(after.UserDefined)
+				var got crypto.ObjectKey
+				switch kind {
+				case crypto.S3:
+					got, readErr = crypto.S3.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
+				case crypto.S3KMS:
+					got, readErr = crypto.S3KMS.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
+				default:
+					t.Fatalf("pool %d: rotation lost encryption metadata", pool)
+				}
+				mustIAM(t, readErr)
+				if got != want {
+					t.Errorf("pool %d: rotation changed the data key without reencrypting its ciphertext (rotation error: %v)", pool, err)
+				}
+				data, readErr := readCopy(pool)
+				if readErr != nil || string(data) != payload {
+					t.Errorf("pool %d: rotation left unreadable ciphertext: %v", pool, readErr)
+				}
+			}
+		})
+	}
+}
+
+// A version in a pool that a decommission or a rebalance is moving is not
+// rewritten there: only the source holds it, so the rotation is left to a
+// later run.
+func TestBatchReviewRotationLeavesMovingSource(t *testing.T) {
+	resetTestGlobals()
+	t.Cleanup(resetTestGlobals)
+	z, _ := consistencyPools(t)
+	globalIAMSys.initStore(z, nil)
+	mustIAM(t, globalIAMSys.Load(t.Context(), true))
+	const bucket = "review8-rotation-draining"
+	mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+	_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+	mustIAM(t, err)
+	oldKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("review8-rotation-key")
+	t.Cleanup(func() { GlobalKMS = oldKMS })
+	const object = "rotation/draining"
+	meta := make(map[string]string)
+	key, err := newEncryptMetadata(t.Context(), crypto.S3, "review8-rotation-key", nil, bucket, object, meta, nil)
+	mustIAM(t, err)
+	body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("draining rotation"), key))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+	beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+
+	z.poolMetaMutex.Lock()
+	z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+	z.poolMetaMutex.Unlock()
+
+	r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
+	err = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+	if !errors.Is(err, errBatchRotateMoving) {
+		t.Errorf("rotation of a version only a draining pool holds returned %v", err)
+	}
+	after, readErr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, readErr)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Fatal("rotation rewrote a draining pool")
+	}
+	// The job pauses and resumes in-process once the selected pool stops moving.
+	job := BatchJobRequest{ID: "rotation-moving-resume", Started: UTCNow(), Submitter: batchRootSubmitter(t), KeyRotate: &r}
+	batchTestJobPool(t.Context(), t, z)
+	if err := r.Start(t.Context(), z, job); !isBatchJobUnavailable(err) {
+		t.Fatalf("moving job did not pause: %v", err)
+	}
+	ri := &batchJobInfo{}
+	mustIAM(t, ri.load(t.Context(), z, job))
+	if ri.PausedReason == "" || ri.Failed || ri.Complete || ri.Object != "" || ri.ObjectsFailed != 0 {
+		t.Fatalf("moving job report: %+v", ri)
+	}
+	z.poolMetaMutex.Lock()
+	z.poolMeta.Pools[0].Decommission = nil
+	z.poolMetaMutex.Unlock()
+	mustIAM(t, r.Start(t.Context(), z, job))
+	mustIAM(t, ri.load(t.Context(), z, job))
+	if !ri.Complete || ri.Failed || ri.PausedReason != "" || ri.Objects != 1 {
+		t.Fatalf("resumed rotation: %+v", ri)
+	}
+}
+
+// Once a move copied a version, the rotation leaves it to a later run, so the
+// source stays as the move's cleanup expects it.
+func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) {
+	for _, tc := range []struct {
+		kind   string
+		rotate bool
+	}{{"decommission", false}, {"decommission", true}, {"rebalance", false}, {"rebalance", true}} {
+		name := tc.kind + "/without-rotation"
+		if tc.rotate {
+			name = tc.kind + "/with-rotation"
+		}
+		t.Run(name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			z, _ := migrationTestPools(t)
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review8-rotation-migration"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, err)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review8-rotation-key")
+			t.Cleanup(func() { GlobalKMS = oldKMS })
+			const object = "rotation/migration"
+			meta := make(map[string]string)
+			key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+			mustIAM(t, err)
+			body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation and migration"), key))
+			mustIAM(t, err)
+			before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+			beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			move := migrationTestMover(t, z, 0, tc.kind)
+			mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, beforeInfo)))
+			if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); err != nil {
+				t.Fatalf("control: migration did not copy the version: %v", err)
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
+			var rotationErr error
+			if tc.rotate {
+				moved, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+				mustIAM(t, err)
+				rotationErr = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+				if !errors.Is(rotationErr, errBatchRotateMoving) {
+					t.Errorf("rotation during a move returned %v", rotationErr)
+				}
+				after, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+				mustIAM(t, err)
+				if !maps.Equal(after.UserDefined, moved.UserDefined) || after.UserTags != moved.UserTags {
+					t.Error("rotation during a move rewrote the moved copy")
+				}
+			}
+			after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if !after.ModTime.Equal(beforeInfo.ModTime) || !maps.Equal(after.UserDefined, beforeInfo.UserDefined) {
+				t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
+			}
+			snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
+			if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
+				t.Fatalf("rotation prevented migration cleanup: %v", err)
+			}
+			if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+				t.Fatalf("migration left a source version behind: %v", err)
+			}
+		})
+	}
+}
+
+// The rewrite keeps the tags and the expiry of the version, which a read
+// returns apart from its metadata.
+func TestBatchReviewRotationKeepsTagsAndExpiry(t *testing.T) {
+	for _, mode := range []string{"single-pool", "multiple-pools"} {
+		t.Run(mode, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			var z *erasureServerPools
+			if mode == "single-pool" {
+				z = purgeSinglePool(t, 4)
+			} else {
+				z, _ = consistencyPools(t)
+			}
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review8-rotation-metadata"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, err)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review8-rotation-metadata-key")
+			t.Cleanup(func() { GlobalKMS = oldKMS })
+			const object = "rotation/tagged"
+			meta := make(map[string]string)
+			key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+			mustIAM(t, err)
+			meta[xhttp.AmzObjectTagging] = "classification=private"
+			meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
+			meta["expires"] = UTCNow().Add(time.Hour).Format(http.TimeFormat)
+			body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("metadata preservation"), key))
+			mustIAM(t, err)
+			written := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta})
+			before, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: written.VersionID})
+			mustIAM(t, err)
+			if before.UserTags == "" || before.Expires.IsZero() {
+				t.Fatal("control: stored version is missing the metadata to preserve")
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			mustIAM(t, r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)))
+			after, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if after.UserDefined[crypto.MetaSealedKeyS3] == before.UserDefined[crypto.MetaSealedKeyS3] {
+				t.Error("control: rotation did not rewrap the key")
+			}
+			if after.UserTags != before.UserTags {
+				t.Errorf("rotation changed stored tags: got %q, want %q", after.UserTags, before.UserTags)
+			}
+			if !after.Expires.Equal(before.Expires) {
+				t.Errorf("rotation changed Expires: got %v, want %v", after.Expires, before.Expires)
+			}
+		})
+	}
+}
+
+// Two pools hold the same write with different tags, with or without
+// tagging timestamps. A rotation, also during a move, leaves the tags that
+// the merged version has.
+func TestBatchReviewRotationKeepsMergedTags(t *testing.T) {
+	for _, tc := range []struct {
+		ordered, moving bool
+	}{{false, false}, {false, true}, {true, false}, {true, true}} {
+		t.Run(fmt.Sprintf("timestamp=%t/moving=%t", tc.ordered, tc.moving), func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			z, _ := consistencyPools(t)
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review9-rotation-tag-tie"
+			const object = "rotation/tag-tie"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, err)
+			previousKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review9-tag-tie-key")
+			t.Cleanup(func() { GlobalKMS = previousKMS })
+			meta := make(map[string]string)
+			key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+			mustIAM(t, err)
+			meta[xhttp.AmzObjectTagging] = "classification=private"
+			if tc.ordered {
+				meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
+			}
+			other := maps.Clone(meta)
+			other[xhttp.AmzObjectTagging] = "classification=public"
+			body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("same write"), key))
+			mustIAM(t, err)
+			before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+			putConsistencyObject(t, z, bucket, object, 1, string(body), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: other})
+			prior, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if prior.UserTags != "classification=private" {
+				t.Fatalf("control: logical tags=%q", prior.UserTags)
+			}
+			if tc.moving {
+				migrationTestMover(t, z, 0, "decommission")
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			rotationErr := r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
+			after, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if after.UserTags != prior.UserTags {
+				t.Fatalf("rotation changed logical tags from %q to %q (rotation error: %v)", prior.UserTags, after.UserTags, rotationErr)
+			}
+		})
+	}
+}
+
+// batchRotateFlipDisk starts a decommission of pool 0 at the flipAt-th
+// version read, after the rotation's first read of the version.
+type batchRotateFlipDisk struct {
+	StorageAPI
+	z       *erasureServerPools
+	reads   *atomic.Int32
+	flipAt  int32
+	flipped *atomic.Bool
+}
+
+func (d batchRotateFlipDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
+	n := d.reads.Add(1)
+	if n == d.flipAt && d.flipped.CompareAndSwap(false, true) {
+		d.z.poolMetaMutex.Lock()
+		d.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+		d.z.poolMetaMutex.Unlock()
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
+}
+
+// A move that starts while the rotation authorizes and rewraps the version
+// defers it: the state is checked again right before the write, where the
+// pools layer checks it.
+func TestBatchReviewRotationRechecksPoolBeforeWrite(t *testing.T) {
+	resetTestGlobals()
+	t.Cleanup(resetTestGlobals)
+	z, _ := consistencyPools(t)
+	globalIAMSys.initStore(z, nil)
+	mustIAM(t, globalIAMSys.Load(t.Context(), true))
+	const bucket = "review11-rotation-window"
+	const object = "rotation/window"
+	mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+	_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+	mustIAM(t, err)
+	oldKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("review11-rotation-key")
+	t.Cleanup(func() { GlobalKMS = oldKMS })
+
+	meta := make(map[string]string)
+	key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+	mustIAM(t, err)
+	body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation window"), key))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{
+		Versioned:   true,
+		UserDefined: meta,
+		MTime:       UTCNow().Add(-time.Hour),
+	})
+	beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+
+	var reads atomic.Int32
+	var flipped atomic.Bool
+	firstPass := int32(0)
+	for _, pool := range z.serverPools {
+		firstPass += int32(pool.SetDriveCount())
+	}
+	var restore []func()
+	for _, pool := range z.serverPools {
+		restore = append(restore, conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI {
+			return batchRotateFlipDisk{
+				StorageAPI: d,
+				z:          z,
+				reads:      &reads,
+				flipAt:     firstPass + 1,
+				flipped:    &flipped,
+			}
+		}))
+	}
+	t.Cleanup(func() {
+		for _, fn := range restore {
+			fn()
+		}
+	})
+
+	r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+	gotErr := r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+	if !flipped.Load() {
+		t.Fatal("fault injection did not occur after the initial rotation read")
+	}
+	if !errors.Is(gotErr, errBatchRotateMoving) {
+		t.Errorf("rotation did not defer after the pool became moving during pre-work: %v", gotErr)
+	}
+	after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Error("rotation rewrote the pool after it became moving during pre-work")
+	}
+}

cmd/batch-authz-review_test.go

@@ -0,0 +1,287 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func batchPullJob(bucket string) BatchJobReplicateV1 {
+	return BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+	}
+}
+
+// A pull does not read the destination version it may replace, so a policy
+// that conditions the write on that version's tags refuses it; a version
+// condition sees the version the pull writes.
+func TestBatchPullDestinationConditions(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		action    policy.Action
+		marker    bool
+		condition func(versionID string) string
+	}{
+		{"tags-put", policy.ReplicateObjectAction, false, func(string) string {
+			return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
+		}},
+		{"tags-marker", policy.ReplicateDeleteAction, true, func(string) string {
+			return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
+		}},
+		{"version-put", policy.PutObjectAction, false, func(v string) string {
+			return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v)
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"})
+			batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID)))
+			cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard")
+			sub, err := newBatchJobSubmitter(cred)
+			mustIAM(t, err)
+			job := batchPullJob(f.bucket)
+			err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+				Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag",
+				DeleteMarker: tc.marker, UserDefined: map[string]string{},
+			}, false, sub)
+			if !isBatchJobAuthzError(err) {
+				t.Errorf("pull returned %v, want a refusal", err)
+			}
+			after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID})
+			if err != nil || after.ETag != before.ETag {
+				t.Errorf("refused pull changed the destination version: %v", err)
+			}
+		})
+	}
+}
+
+// A pull does not encrypt what it writes; encryption headers reported by the
+// source do not satisfy a destination policy that requires encryption.
+func TestBatchPullEncryptionCondition(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", `
+{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	job := batchPullJob(f.bucket)
+	version := mustGetUUID()
+	err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{
+		Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag",
+		UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"},
+	}, false, sub)
+	if err == nil {
+		oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version})
+		mustIAM(t, gerr)
+		if !crypto.S3KMS.IsEncrypted(oi.UserDefined) {
+			t.Fatal("a pull was allowed by an encryption condition it did not meet")
+		}
+	}
+}
+
+// A conditional Deny on an S3 Tables action covers the S3 data actions it
+// implies, under the tables resource form; the job refuses rather than skip it.
+func TestBatchJobTableConditionalDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context")
+	if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) {
+		t.Fatal("control: with the request context the Deny applies")
+	}
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) {
+		t.Fatalf("authorize returned %v, want a refusal", err)
+	}
+}
+
+type batchArchiveRevocation struct {
+	ObjectLayer
+	revoke func()
+}
+
+func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+	if object == "archive/second" {
+		a.revoke()
+	}
+	return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
+}
+
+// Each snowball entry is authorized for the submitter as it is then.
+func TestBatchPushSnowballEntryRevalidation(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/")
+	mustIAM(t, err)
+	previous := newGlobalAuthZPluginFn()
+	setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
+		URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() },
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil
+		}),
+	}))
+	t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
+	entries := []ObjectInfo{
+		batchPutVersion(t, f.obj, f.bucket, "archive/first", nil),
+		batchPutVersion(t, f.obj, f.bucket, "archive/second", nil),
+	}
+	remote, err := minio.New("archive.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}}
+	job.Source.Snowball.InMemory = ptr(true)
+	job.Source.Snowball.Compress = ptr(false)
+	job.Source.Snowball.SkipErrs = ptr(false)
+	api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }}
+	if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) {
+		t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err)
+	}
+}
+
+// A definition holds one job type; one with several is refused, so that what
+// is authorized is what runs.
+func TestBatchJobSeveralSections(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "data/a")
+	job := f.expireYAML("") + `keyrotate:
+  apiVersion: v1
+  bucket: other
+  encryption:
+    type: sse-s3
+`
+	rec, queued := f.submit(t, globalActiveCred, job)
+	if queued != nil {
+		f.run(t, queued)
+	}
+	if rec.Code != http.StatusBadRequest {
+		t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String())
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a definition with two job types ran")
+	}
+}
+
+// Prefixes are walked once each in one order: a cursor in a later prefix is
+// after every key of the earlier ones, and an overlapping prefix is not
+// walked twice.
+func TestBatchExpirePrefixes(t *testing.T) {
+	if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" {
+		t.Fatalf("canonical prefixes %q", got)
+	}
+	if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" {
+		t.Fatalf("canonical prefixes %q", got)
+	}
+	f := newBatchAuthzFixture(t)
+	done := f.putVersions(t, "a/done")
+	todo := f.putVersions(t, "z/todo")
+	batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD"))
+	batchPutVersion(t, f.obj, f.bucket, "x/data", nil)
+	rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: ["z/", "x/", "a/", "x/"]
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	// A run that completed a/ and stopped.
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"}
+	mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
+	f.run(t, job)
+	if !f.versionExists(t, "a/done", done) {
+		t.Error("resume did again a key before the cursor")
+	}
+	if f.versionExists(t, "z/todo", todo) {
+		t.Error("resume skipped a later prefix")
+	}
+	if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 {
+		t.Errorf("a locked version under two equal prefixes was counted %d times", got)
+	}
+}
+
+type batchPartialWalk struct {
+	ObjectLayer
+	version ObjectInfo
+	cancel  context.CancelFunc
+	read    chan struct{}
+}
+
+func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
+	go func() {
+		defer close(results)
+		results <- itemOrErr[ObjectInfo]{Item: a.version}
+		if a.cancel != nil {
+			<-a.read
+			a.cancel()
+			return
+		}
+		results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")}
+	}()
+	return nil
+}
+
+func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
+	if a.read != nil {
+		close(a.read)
+	}
+	return a.version, nil
+}
+
+// A rotation whose walk fails or is canceled within a key keeps that key
+// after its cursor and does not report completion.
+func TestBatchKeyRotationIncompleteWalk(t *testing.T) {
+	for _, canceled := range []bool{false, true} {
+		t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			ctx, cancel := context.WithCancel(f.ctx)
+			defer cancel()
+			meta := batchLockMeta("LEGALHOLD")
+			meta[crypto.MetaSealedKeyS3] = "test-sealed-key"
+			api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}}
+			if canceled {
+				api.cancel, api.read = cancel, make(chan struct{})
+			}
+			job := BatchJobRequest{
+				ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+				KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}},
+			}
+			_ = job.KeyRotate.Start(ctx, api, job)
+			ri := f.jobInfo(t, &job)
+			if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 {
+				t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked)
+			}
+		})
+	}
+}

cmd/batch-authz-snapshot_test.go

@@ -0,0 +1,205 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"github.com/puzpuzpuz/xsync/v3"
+)
+
+// batchBlockedIAMStore holds the lazy load of one user's policy mapping, or
+// of one policy document, until released or until the load's context ends.
+type batchBlockedIAMStore struct {
+	IAMStorageAPI
+	mapping string // access key whose mapping load blocks
+	doc     string // policy whose document load blocks
+	started chan struct{}
+	release chan struct{}
+	once    sync.Once
+}
+
+func (s *batchBlockedIAMStore) block(ctx context.Context) error {
+	s.once.Do(func() { close(s.started) })
+	select {
+	case <-s.release:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (s *batchBlockedIAMStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+	if name == s.mapping && !isGroup {
+		if err := s.block(ctx); err != nil {
+			return err
+		}
+	}
+	return s.IAMStorageAPI.loadMappedPolicy(ctx, name, userType, isGroup, m)
+}
+
+func (s *batchBlockedIAMStore) loadPolicyDoc(ctx context.Context, name string, out map[string]PolicyDoc) error {
+	if name == s.doc {
+		if err := s.block(ctx); err != nil {
+			return err
+		}
+	}
+	return s.IAMStorageAPI.loadPolicyDoc(ctx, name, out)
+}
+
+// The batch path and the request path share one IAM cache. A lazy load that
+// batch authorization needs, of a mapping that is never cached (a user whose
+// policies come from a group) or of an evicted policy document, must not
+// hold request authentication and cached authorization while storage is
+// slow, and must end at the deadline of the job's context.
+func TestBatchAuthorizeSlowLoadKeepsCachedAuthorizationAvailable(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		// submitter returns the submitter whose authorization lazily loads,
+		// and arranges for blocked to hold that load.
+		submitter func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter
+	}{
+		{"mapping", func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter {
+			// No mapping of the user itself is stored, so every lookup of
+			// its policies reads storage before it falls back to the group.
+			cred := iamAdminUser(f.ctx, t, f.sys, "snapshot-group-user", "")
+			_, err := f.sys.AddUsersToGroup(f.ctx, "snapshot-group", []string{cred.AccessKey})
+			mustIAM(t, err)
+			_, err = f.sys.PolicyDBSet(f.ctx, "snapshot-group", "snapshot-full", regUser, true)
+			mustIAM(t, err)
+			blocked.mapping = cred.AccessKey
+			sub, err := newBatchJobSubmitter(cred)
+			mustIAM(t, err)
+			return sub
+		}},
+		{"document", func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter {
+			batchAuthzPolicy(f.ctx, t, f.sys, "snapshot-evicted", batchAuthzFullAccess)
+			cred := iamAdminUser(f.ctx, t, f.sys, "snapshot-doc-user", "snapshot-evicted")
+			cache := f.sys.store.lock()
+			delete(cache.iamPolicyDocsMap, "snapshot-evicted")
+			f.sys.store.unlock()
+			blocked.doc = "snapshot-evicted"
+			sub, err := newBatchJobSubmitter(cred)
+			mustIAM(t, err)
+			return sub
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			batchAuthzPolicy(f.ctx, t, f.sys, "snapshot-full", batchAuthzFullAccess)
+			cached := iamAdminUser(f.ctx, t, f.sys, "snapshot-cached-user", "snapshot-full")
+			store := f.sys.store
+			original := store.IAMStorageAPI
+			blocked := &batchBlockedIAMStore{IAMStorageAPI: original, started: make(chan struct{}), release: make(chan struct{})}
+			sub := tc.submitter(t, f, blocked)
+			req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
+			if err := sub.authorize(f.ctx, req); err != nil {
+				t.Fatalf("control: %v", err)
+			}
+			store.IAMStorageAPI = blocked
+			var wg sync.WaitGroup
+			var once sync.Once
+			unblock := func() { once.Do(func() { close(blocked.release) }) }
+			t.Cleanup(func() { unblock(); wg.Wait(); store.IAMStorageAPI = original })
+
+			slowDone := make(chan error, 1)
+			wg.Go(func() { slowDone <- sub.authorize(f.ctx, req) })
+			select {
+			case <-blocked.started:
+			case <-time.After(5 * time.Second):
+				t.Fatal("the lazy load did not start")
+			}
+			cachedDone := make(chan bool, 2)
+			wg.Go(func() {
+				_, ok := f.sys.GetUser(f.ctx, cached.AccessKey)
+				cachedDone <- ok
+			})
+			wg.Go(func() {
+				cachedDone <- f.sys.IsAllowed(policy.Args{AccountName: cached.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "key"})
+			})
+			for range 2 {
+				select {
+				case ok := <-cachedDone:
+					if !ok {
+						t.Fatal("cached authentication or authorization was rejected")
+					}
+				case <-time.After(time.Second):
+					t.Fatal("a slow IAM load during batch authorization blocked cached authentication")
+				}
+			}
+			unblock()
+			if err := <-slowDone; err != nil {
+				t.Fatalf("authorization after the load completed: %v", err)
+			}
+
+			// A load that does not return ends with the job's context, as an
+			// unavailable dependency, not with the storage layer's patience.
+			blocked.release = make(chan struct{})
+			once = sync.Once{}
+			unblock = func() { once.Do(func() { close(blocked.release) }) }
+			ctx, cancel := context.WithTimeout(f.ctx, 200*time.Millisecond)
+			defer cancel()
+			deadlineDone := make(chan error, 1)
+			wg.Go(func() { deadlineDone <- sub.authorize(ctx, req) })
+			select {
+			case err := <-deadlineDone:
+				if !isBatchJobUnavailable(err) {
+					t.Fatalf("a load that ended with the job's context: %v", err)
+				}
+			case <-time.After(3 * time.Second):
+				t.Fatal("a lazy load outlived the job's context")
+			}
+		})
+	}
+}
+
+// A policy that a mapping still names after it was deleted takes no part in
+// the decision, for a batch job as for a request: the remaining mapped
+// policies decide. An STS parent user's mapping can name a deleted policy,
+// because deleting a policy checks only user and group mappings for use.
+func TestBatchAuthorizeSkipsDeletedMappedPolicy(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
+	batchAuthzPolicy(f.ctx, t, f.sys, "mapped-deleted", `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}`)
+	parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "")
+	_, err := f.sys.PolicyDBSet(f.ctx, parent.AccessKey, "mapped-full,mapped-deleted", stsUser, false)
+	mustIAM(t, err)
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
+	if err := sub.authorize(f.ctx, req); err != nil {
+		t.Fatalf("control: %v", err)
+	}
+	p, err := sub.principal(f.ctx)
+	mustIAM(t, err)
+	args := policy.Args{AccountName: cred.AccessKey, Action: req.action, BucketName: req.bucket, ObjectName: req.object, Claims: p.claims, ConditionValues: p.conditionValues(req)}
+
+	mustIAM(t, f.sys.DeletePolicy(f.ctx, "mapped-deleted", true))
+	if !f.sys.IsAllowed(args) {
+		t.Fatal("the request path denied the remaining mapped policy")
+	}
+	if err := sub.authorize(f.ctx, req); err != nil {
+		t.Fatalf("batch authorization diverged from the request after a mapped policy was deleted: %v", err)
+	}
+
+	// With every mapped policy deleted, both deny; the job is refused, not
+	// paused, as nothing is unavailable.
+	mustIAM(t, f.sys.DeletePolicy(f.ctx, "mapped-full", true))
+	if f.sys.IsAllowed(args) {
+		t.Fatal("the request path allowed without any mapped policy")
+	}
+	if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
+		t.Fatalf("batch authorization without any mapped policy: %v", err)
+	}
+}

cmd/batch-authz.go

@@ -0,0 +1,919 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/arn"
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"github.com/pgsty/silo-pkg/v3/policy/condition"
+	"github.com/puzpuzpuz/xsync/v3"
+)
+
+// A batch job acts on objects long after the request that submitted it has
+// gone. It therefore records who submitted it and acts only as that
+// principal:
+//
+//   - StartBatchJob binds the authenticated credential to the job
+//     (batchJobSubmitter). Nothing in the submitted definition names it.
+//   - Every start of the job, and every object it touches, resolves the
+//     binding against IAM again. A credential that is gone, disabled,
+//     expired, revoked or edited since submission fails the job; a reused
+//     access key does not match the binding.
+//   - Each action is evaluated with the submitter's policies and session
+//     policy but without a request. A policy statement that would need
+//     request context the job does not have (source address, transport,
+//     headers and so on) refuses the action instead of being skipped.
+//
+// A job without a binding was queued by an earlier release. It is never
+// authorized, whoever submitted it, and has to be submitted again.
+
+//go:generate msgp -file $GOFILE -unexported
+
+// batchJobSubmitter is the server-assigned identity of the credential that
+// submitted a batch job.
+type batchJobSubmitter struct {
+	Version   int    `msg:"v"`
+	Type      string `msg:"t"`
+	AccessKey string `msg:"ak"`
+	// ParentUser is the parent of a derived credential.
+	ParentUser string `msg:"pu"`
+	// UpdatedAt and RevokedBefore are the revision of the identity record;
+	// any edit of the identity changes them. They are unset for root.
+	UpdatedAt     time.Time `msg:"ua"`
+	RevokedBefore time.Time `msg:"rb"`
+	// Binding is a keyed fingerprint of the credential itself, see
+	// batchSubmitterBinding.
+	Binding []byte `msg:"bd"`
+}
+
+const batchJobSubmitterVersion = 1
+
+// Submitter types. A credential is classified in the order IsAllowed
+// evaluates it.
+const (
+	batchSubmitterRoot      = "root"
+	batchSubmitterUser      = "user"
+	batchSubmitterTemporary = "temporary"
+	batchSubmitterService   = "service-account"
+)
+
+//msgp:ignore batchJobAuthzError
+
+// batchJobAuthzError refuses a batch job, or one of its actions, for its
+// submitter. It ends the job: it is persisted as the job's failure and is
+// never retried.
+type batchJobAuthzError struct {
+	reason string
+}
+
+func (e batchJobAuthzError) Error() string {
+	return e.reason
+}
+
+func isBatchJobAuthzError(err error) bool {
+	var e batchJobAuthzError
+	return errors.As(err, &e)
+}
+
+var (
+	errBatchJobResubmit = batchJobAuthzError{reason: "the batch job does not record who submitted it; submit it again"}
+	errBatchJobChanged  = batchJobAuthzError{reason: "the credential that submitted the batch job has changed since submission; submit it again"}
+	errBatchJobInvalid  = batchJobAuthzError{reason: "the credential that submitted the batch job is no longer valid"}
+)
+
+func batchSubmitterType(cred auth.Credentials) string {
+	switch {
+	case cred.IsTemp():
+		return batchSubmitterTemporary
+	case cred.IsServiceAccount():
+		return batchSubmitterService
+	default:
+		return batchSubmitterUser
+	}
+}
+
+// batchSubmitterBinding is a fingerprint of one credential keyed with its own
+// secret. It changes when the secret, the session token or the parent
+// changes, and a new credential that reuses the access key does not have it.
+func batchSubmitterBinding(typ string, cred auth.Credentials) []byte {
+	mac := hmac.New(sha256.New, []byte(cred.SecretKey))
+	for _, s := range []string{"silo batch-job submitter v1", typ, cred.AccessKey, cred.ParentUser, cred.SessionToken} {
+		var n [8]byte
+		binary.BigEndian.PutUint64(n[:], uint64(len(s)))
+		mac.Write(n[:])
+		mac.Write([]byte(s))
+	}
+	return mac.Sum(nil)
+}
+
+// identityRecord returns the stored identity of a non-root access key.
+func (sys *IAMSys) identityRecord(accessKey string) (UserIdentity, bool, error) {
+	if !sys.Initialized() {
+		return UserIdentity{}, false, errServerNotInitialized
+	}
+	u, ok := sys.store.GetUser(accessKey)
+	return u, ok, nil
+}
+
+// newBatchJobSubmitter binds the credential that authenticated a
+// StartBatchJob request.
+func newBatchJobSubmitter(cred auth.Credentials) (*batchJobSubmitter, error) {
+	root := globalActiveCred
+	if cred.AccessKey == "" {
+		return nil, errBatchJobInvalid
+	}
+	if cred.AccessKey == root.AccessKey {
+		if subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(root.SecretKey)) != 1 {
+			return nil, errBatchJobChanged
+		}
+		return &batchJobSubmitter{
+			Version:   batchJobSubmitterVersion,
+			Type:      batchSubmitterRoot,
+			AccessKey: root.AccessKey,
+			Binding:   batchSubmitterBinding(batchSubmitterRoot, root),
+		}, nil
+	}
+	u, ok, err := globalIAMSys.identityRecord(cred.AccessKey)
+	if err != nil {
+		return nil, batchUnavailable("IAM identity lookup", err)
+	}
+	// The record must be the one the request was authenticated with.
+	if !ok || u.Credentials.AccessKey != cred.AccessKey || u.Credentials.ParentUser != cred.ParentUser ||
+		subtle.ConstantTimeCompare([]byte(u.Credentials.SecretKey), []byte(cred.SecretKey)) != 1 ||
+		subtle.ConstantTimeCompare([]byte(u.Credentials.SessionToken), []byte(cred.SessionToken)) != 1 {
+		return nil, errBatchJobChanged
+	}
+	typ := batchSubmitterType(u.Credentials)
+	return &batchJobSubmitter{
+		Version:       batchJobSubmitterVersion,
+		Type:          typ,
+		AccessKey:     u.Credentials.AccessKey,
+		ParentUser:    u.Credentials.ParentUser,
+		UpdatedAt:     u.UpdatedAt,
+		RevokedBefore: u.RevokedBefore,
+		Binding:       batchSubmitterBinding(typ, u.Credentials),
+	}, nil
+}
+
+//msgp:ignore batchJobPrincipal batchAuthzRequest
+
+// batchJobPrincipal is a submitter resolved against current IAM state.
+type batchJobPrincipal struct {
+	ctx        context.Context
+	typ        string
+	accessKey  string
+	parentUser string
+	groups     []string
+	claims     map[string]any
+	owner      bool
+	temp       bool // evaluated as a temporary credential
+	service    bool // evaluated as a service account
+}
+
+// principal resolves the binding against current IAM state. It fails if the
+// credential no longer exists, is no longer valid, or is no longer the one
+// that was bound.
+func (s *batchJobSubmitter) principal(ctx context.Context) (batchJobPrincipal, error) {
+	if s == nil || s.AccessKey == "" {
+		return batchJobPrincipal{}, errBatchJobResubmit
+	}
+	if s.Version != batchJobSubmitterVersion {
+		return batchJobPrincipal{}, batchJobAuthzError{reason: fmt.Sprintf("unsupported batch job submitter version %d; submit the job again", s.Version)}
+	}
+	if !batchIAMReady() {
+		return batchJobPrincipal{}, batchUnavailable("IAM is not initialized", errServerNotInitialized)
+	}
+	root := globalActiveCred
+	if s.Type == batchSubmitterRoot {
+		if s.AccessKey != root.AccessKey || !hmac.Equal(s.Binding, batchSubmitterBinding(batchSubmitterRoot, root)) {
+			return batchJobPrincipal{}, errBatchJobChanged
+		}
+		if !globalAPIConfig.permitRootAccess() {
+			return batchJobPrincipal{}, errBatchJobInvalid
+		}
+		return batchJobPrincipal{ctx: ctx, typ: batchSubmitterRoot, accessKey: root.AccessKey, owner: true}, nil
+	}
+	switch s.Type {
+	case batchSubmitterUser, batchSubmitterTemporary, batchSubmitterService:
+	default:
+		return batchJobPrincipal{}, errBatchJobResubmit
+	}
+	if s.AccessKey == root.AccessKey {
+		return batchJobPrincipal{}, errBatchJobChanged
+	}
+	u, ok, err := globalIAMSys.CheckKey(ctx, s.AccessKey)
+	if err != nil {
+		return batchJobPrincipal{}, batchUnavailable("IAM credential lookup", err)
+	}
+	if !ok {
+		// Deleted, disabled, expired or revoked.
+		return batchJobPrincipal{}, errBatchJobInvalid
+	}
+	cred := u.Credentials
+	typ := batchSubmitterType(cred)
+	if typ != s.Type || cred.AccessKey != s.AccessKey || cred.ParentUser != s.ParentUser ||
+		!u.UpdatedAt.Equal(s.UpdatedAt) || !u.RevokedBefore.Equal(s.RevokedBefore) ||
+		!hmac.Equal(s.Binding, batchSubmitterBinding(typ, cred)) {
+		return batchJobPrincipal{}, errBatchJobChanged
+	}
+	if cred.IsExpired() || cred.Status == auth.AccountOff {
+		return batchJobPrincipal{}, errBatchJobInvalid
+	}
+	p := batchJobPrincipal{
+		ctx:        ctx,
+		typ:        typ,
+		accessKey:  cred.AccessKey,
+		parentUser: cred.ParentUser,
+		groups:     cred.Groups,
+		claims:     map[string]any{},
+		temp:       cred.IsTemp(),
+		service:    !cred.IsTemp() && cred.IsServiceAccount(),
+	}
+	if typ != batchSubmitterUser {
+		// Verify the token as a request would, with the current signing keys.
+		jwtClaims, err := extractJWTClaims(u)
+		if err != nil {
+			if errors.Is(err, errTokenSigningKeyUnavailable) {
+				return batchJobPrincipal{}, batchUnavailable("token signing key", err)
+			}
+			return batchJobPrincipal{}, errBatchJobInvalid
+		}
+		p.claims = jwtClaims.Map()
+		if err := decodeBatchSessionPolicy(p.claims); err != nil {
+			return batchJobPrincipal{}, errBatchJobInvalid
+		}
+	}
+	p.owner = cred.ParentUser == root.AccessKey && cred.AccessKey != siteReplicatorSvcAcc
+	if p.owner && !globalAPIConfig.permitRootAccess() {
+		return batchJobPrincipal{}, errBatchJobInvalid
+	}
+	if _, ok := p.claims[policy.SessionPolicyName]; ok {
+		p.owner = false
+	}
+	return p, nil
+}
+
+// decodeBatchSessionPolicy decodes a session policy claim for evaluation.
+// A request decodes it while verifying its token, except when an
+// authorization plugin is configured; a batch job decodes it always, so the
+// session policy also limits what the plugin allows.
+func decodeBatchSessionPolicy(claims map[string]any) error {
+	sp, ok := claims[policy.SessionPolicyName]
+	if !ok {
+		return nil
+	}
+	s, ok := sp.(string)
+	if !ok {
+		return errAuthentication
+	}
+	b, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return errAuthentication
+	}
+	claims[sessionPolicyNameExtracted] = string(b)
+	return nil
+}
+
+// batchAuthzRequest is one action a batch job is about to take.
+type batchAuthzRequest struct {
+	action policy.Action
+	bucket string
+	object string // empty for a bucket action
+	// versionID is the version an object version action addresses.
+	versionID string
+	// tags are the current tags of the object version acted on.
+	tags string
+	// tagsUnknown is set when the job does not read the version it acts
+	// on, so s3:ExistingObjectTag is context it does not have.
+	tagsUnknown bool
+	// values are request values the job supplies itself, such as the
+	// listing prefix or the encryption and retention it writes, keyed by
+	// condition key name.
+	values map[string][]string
+}
+
+// batchAuthzMarker marks the condition values of a batch job for an
+// authorization plugin. Requests cannot set s3:authType.
+const batchAuthzMarker = "batch-job"
+
+// Condition keys a batch job always provides, as a request would: identity,
+// time, the addressed version and the stored tags. Keys a request takes from
+// itself are provided only when the job sets them (see batchAuthzRequest
+// values); every other key is request context the job does not have.
+var batchServerConditionKeys = map[string]bool{
+	condition.AWSCurrentTime.ToKey().String():    true,
+	condition.AWSEpochTime.ToKey().String():      true,
+	condition.AWSPrincipalType.ToKey().String():  true,
+	condition.AWSUserID.ToKey().String():         true,
+	condition.AWSUsername.ToKey().String():       true,
+	condition.AWSGroups.ToKey().String():         true,
+	condition.S3VersionID.ToKey().String():       true,
+	condition.ExistingObjectTag.ToKey().String(): true,
+}
+
+// Request keys a job may supply, by the name they have in condition values.
+var batchRequestConditionKeys = map[string]condition.KeyName{
+	condition.S3Prefix.Name():                              condition.S3Prefix,
+	condition.S3XAmzServerSideEncryption.Name():            condition.S3XAmzServerSideEncryption,
+	condition.S3XAmzServerSideEncryptionAwsKmsKeyID.Name(): condition.S3XAmzServerSideEncryptionAwsKmsKeyID,
+	condition.S3ObjectLockMode.Name():                      condition.S3ObjectLockMode,
+	condition.S3ObjectLockRetainUntilDate.Name():           condition.S3ObjectLockRetainUntilDate,
+	condition.S3ObjectLockLegalHold.Name():                 condition.S3ObjectLockLegalHold,
+	condition.RequestObjectTagKeys.Name():                  condition.RequestObjectTagKeys,
+}
+
+// suppliedConditionKeys returns the condition keys whose values req provides.
+func (req batchAuthzRequest) suppliedConditionKeys() map[string]bool {
+	supplied := make(map[string]bool, len(batchServerConditionKeys)+len(req.values))
+	for k := range batchServerConditionKeys {
+		supplied[k] = true
+	}
+	if req.tagsUnknown {
+		delete(supplied, condition.ExistingObjectTag.ToKey().String())
+	}
+	for name := range req.values {
+		if key, ok := batchRequestConditionKeys[name]; ok {
+			supplied[key.ToKey().String()] = true
+		}
+		if strings.HasPrefix(name, condition.RequestObjectTag.Name()+"/") {
+			supplied[condition.RequestObjectTag.ToKey().String()] = true
+		}
+	}
+	return supplied
+}
+
+func batchKeySupplied(supplied map[string]bool, name string) bool {
+	if i := strings.IndexByte(name, '/'); i >= 0 {
+		name = name[:i]
+	}
+	if strings.HasPrefix(name, "jwt:") || strings.HasPrefix(name, "ldap:") {
+		// Claims of the verified token, as a request has them.
+		return true
+	}
+	return supplied[name]
+}
+
+// conditionValues are the values a request with the same identity would
+// carry, without any value taken from the request itself.
+func (p batchJobPrincipal) conditionValues(req batchAuthzRequest) map[string][]string {
+	now := UTCNow()
+	username := p.accessKey
+	if p.temp || p.service {
+		username = p.parentUser
+	}
+	principalType := "User"
+	if len(p.claims) > 0 {
+		principalType = "AssumedRole"
+	}
+	if username == globalActiveCred.AccessKey {
+		principalType = "Account"
+	}
+	// Identity and time, then the claims over them, as a request has them
+	// (an LDAP session's short username replaces its parent DN).
+	values := map[string][]string{
+		"CurrentTime":   {now.Format(time.RFC3339)},
+		"EpochTime":     {strconv.FormatInt(now.Unix(), 10)},
+		"principaltype": {principalType},
+		"userid":        {username},
+		"username":      {username},
+	}
+	for k, v := range p.claims {
+		if s, ok := v.(string); ok {
+			values[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{s}
+		}
+	}
+	// No claim stands for a value of the object or of the job, which
+	// follow and always win.
+	for k := range values {
+		if k == "versionid" || k == "prefix" || k == "authtype" ||
+			strings.HasPrefix(k, "existingobjecttag/") || strings.HasPrefix(k, "requestobjecttag") ||
+			strings.HasPrefix(k, "x-amz-") || strings.HasPrefix(k, "object-lock-") {
+			delete(values, k)
+		}
+	}
+	// For an authorization plugin: this is a batch job, not a request.
+	values["authType"] = []string{batchAuthzMarker}
+	if req.versionID != "" {
+		values["versionid"] = []string{req.versionID}
+	}
+	if req.tags != "" {
+		if t, err := tags.ParseObjectTags(req.tags); err == nil {
+			for k, v := range t.ToMap() {
+				values[pathJoin("ExistingObjectTag", k)] = []string{v}
+			}
+		}
+	}
+	for k, v := range req.values {
+		values[k] = v
+	}
+	if grpsVal, ok := p.claims["groups"]; ok {
+		if grpsIs, ok := grpsVal.([]any); ok {
+			grps := []string{}
+			for _, gI := range grpsIs {
+				if g, ok := gI.(string); ok {
+					grps = append(grps, g)
+				}
+			}
+			if len(grps) > 0 {
+				values["groups"] = grps
+			}
+		}
+	}
+	if _, ok := values["groups"]; !ok && len(p.groups) > 0 {
+		values["groups"] = p.groups
+	}
+	return values
+}
+
+// authorize evaluates one action of the job as its submitter.
+func (p batchJobPrincipal) authorize(req batchAuthzRequest) error {
+	args := policy.Args{
+		AccountName:     p.accessKey,
+		Groups:          p.groups,
+		Action:          req.action,
+		BucketName:      req.bucket,
+		ObjectName:      req.object,
+		ConditionValues: p.conditionValues(req),
+		IsOwner:         p.owner,
+		Claims:          p.claims,
+	}
+	return p.authorizeArgs(args, &req)
+}
+
+// authorizeArgs uses the same policy snapshot for request-context checks and
+// the decision. No boolean IAM evaluator re-reads and silently drops policies.
+func (p batchJobPrincipal) authorizeArgs(args policy.Args, req *batchAuthzRequest) error {
+	denied := batchJobAuthzError{reason: "the batch job submitter is not allowed " + string(args.Action)}
+	if req != nil {
+		denied.reason = batchDeniedReason(*req)
+	}
+	ctx := p.ctx
+	if ctx == nil {
+		ctx = GlobalContext
+	}
+	plugin := newGlobalAuthZPluginFn()
+	var snap *batchPolicySnapshot
+	if plugin == nil && !args.IsOwner {
+		snap = globalIAMSys.store.batchSnapshot(ctx)
+		defer snap.release()
+	}
+	policies, err := globalIAMSys.batchLocalPolicies(p, args, plugin == nil, snap)
+	if err != nil {
+		if errors.Is(err, errNoSuchPolicy) || errors.Is(err, errAuthentication) || errors.Is(err, errNoSuchUser) || errors.Is(err, errNoSuchGroup) {
+			return denied
+		}
+		return batchUnavailable("IAM policy lookup", err)
+	}
+	if req != nil {
+		supplied := req.suppliedConditionKeys()
+		for _, pol := range policies {
+			for _, st := range pol.Statements {
+				if batchStatementNeedsRequest(st, args, supplied) {
+					return batchJobAuthzError{reason: denied.reason + ": a policy statement depends on request context that a batch job does not have"}
+				}
+			}
+		}
+	}
+	if plugin != nil {
+		ok, err := plugin.IsAllowedContext(ctx, args)
+		if err != nil {
+			return batchUnavailable("authorization plugin", err)
+		}
+		if !ok {
+			return denied
+		}
+		// The plugin does not see a session policy; it only limits further.
+		if has, allowed := p.sessionPolicyDecision(args); has && !allowed {
+			return denied
+		}
+		return nil
+	}
+	if args.IsOwner {
+		return nil
+	}
+	if p.service {
+		if p.claims[parentClaim] != p.parentUser {
+			return denied
+		}
+		if _, ok := p.claims[iamPolicyClaimNameSA()].(string); !ok {
+			return denied
+		}
+	}
+	if has, allowed := p.sessionPolicyDecision(args); has && !allowed {
+		return denied
+	}
+	ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey
+	if !ownerDerived {
+		if p.service {
+			args.AccountName = p.parentUser
+		}
+		if len(policies) == 0 || !policies[0].IsAllowed(args) {
+			return denied
+		}
+	}
+	return nil
+}
+
+func batchDeniedReason(req batchAuthzRequest) string {
+	resource := req.bucket
+	if req.object != "" {
+		resource = pathJoin(req.bucket, req.object)
+	}
+	if req.versionID != "" {
+		resource += " (versionId " + req.versionID + ")"
+	}
+	return fmt.Sprintf("the batch job submitter is not allowed %s on %s", req.action, resource)
+}
+
+// sessionPolicyDecision evaluates the session policy the way IsAllowed would
+// for this credential.
+func (p batchJobPrincipal) sessionPolicyDecision(args policy.Args) (bool, bool) {
+	switch {
+	case p.temp:
+		return isAllowedBySessionPolicy(args)
+	case p.service:
+		if p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
+			return false, false
+		}
+		return isAllowedBySessionPolicyForServiceAccount(args)
+	}
+	return false, false
+}
+
+// batchIAMLoadTimeout bounds one storage read of a lazy IAM load during
+// batch authorization, as MergePolicies bounds a policy document load.
+const batchIAMLoadTimeout = 5 * time.Second
+
+// batchPolicySnapshot reads the mappings and policy documents of a submitter
+// at one cache revision while cached authentication and authorization
+// proceed.
+//
+// It holds the store's writer mutex, so no writer changes the cache between
+// its reads, and the cache read lock, which it releases around each storage
+// read: a lazy mapping load or a policy document load. Request
+// authentication then neither waits for storage nor queues behind a writer
+// waiting for the cache lock, which taking the write lock back after each
+// read would cause. It writes nothing to the cache but the lazily loaded
+// mappings, which the request path stores under the read lock as well.
+//
+//msgp:ignore batchPolicySnapshot
+type batchPolicySnapshot struct {
+	*IAMStoreSys
+	ctx   context.Context
+	cache *iamCache
+	mu    sync.Mutex
+	reads int // storage reads in progress; the read lock is released while any
+}
+
+// batchSnapshot takes the snapshot; release it with release. rlock and
+// runlock are the cache's read lock (IAMStorageAPI), not IAMStoreSys.lock,
+// which also takes writeMu.
+func (store *IAMStoreSys) batchSnapshot(ctx context.Context) *batchPolicySnapshot {
+	store.writeMu.Lock()
+	return &batchPolicySnapshot{IAMStoreSys: store, ctx: ctx, cache: store.rlock()}
+}
+
+func (s *batchPolicySnapshot) release() {
+	s.runlock()
+	s.writeMu.Unlock()
+}
+
+// read runs one storage read without the cache lock, bounded by the job's
+// context and batchIAMLoadTimeout. Concurrent reads (the group fallback of
+// policyDBGetGroups) release the lock before the first and take it back
+// after the last.
+func (s *batchPolicySnapshot) read(fn func(ctx context.Context) error) error {
+	s.mu.Lock()
+	if s.reads == 0 {
+		s.runlock()
+	}
+	s.reads++
+	s.mu.Unlock()
+	defer func() {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		s.reads--
+		if s.reads == 0 {
+			s.rlock()
+		}
+	}()
+	ctx, cancel := context.WithTimeout(s.ctx, batchIAMLoadTimeout)
+	defer cancel()
+	return fn(ctx)
+}
+
+func (s *batchPolicySnapshot) loadMappedPolicy(_ context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+	return s.read(func(ctx context.Context) error {
+		return s.IAMStoreSys.loadMappedPolicy(ctx, name, userType, isGroup, m)
+	})
+}
+
+// loadGroup would write the cache under the read lock. The snapshot resolves
+// the policies of users, which never loads a group.
+func (s *batchPolicySnapshot) loadGroup(context.Context, string, map[string]GroupInfo) error {
+	return errors.New("batch authorization does not load groups")
+}
+
+func (s *batchPolicySnapshot) loadPolicyDoc(name string, out map[string]PolicyDoc) error {
+	return s.read(func(ctx context.Context) error { return s.IAMStoreSys.loadPolicyDoc(ctx, name, out) })
+}
+
+// batchLocalPolicies returns every local policy that can take part in the
+// decision on args: the mapped policies that IsAllowed would merge when
+// withMapped is set (no authorization plugin), and the session policy.
+func (sys *IAMSys) batchLocalPolicies(p batchJobPrincipal, args policy.Args, withMapped bool, snap *batchPolicySnapshot) ([]policy.Policy, error) {
+	var out []policy.Policy
+	if withMapped && args.IsOwner {
+		// IsAllowed does not evaluate any policy for the owner.
+		return nil, nil
+	}
+	if withMapped {
+		// The caller holds snap through evaluation. Lazy mapping loads and
+		// selected documents therefore belong to one cache revision.
+		cache := snap.cache
+		mappedPolicies := func(name string) ([]string, error) {
+			names, _, err := cache.policyDBGet(snap, name, false, false)
+			if err != nil {
+				return nil, err
+			}
+			groups, err := cache.policyDBGetGroups(snap, len(names) > 0, args.Groups...)
+			return append(names, groups...), err
+		}
+		var names []string
+		ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey
+		switch {
+		case p.typ == batchSubmitterRoot || ownerDerived:
+		case p.temp || p.service:
+			if roleArn := args.GetRoleArn(); roleArn != "" {
+				a, err := arn.Parse(roleArn)
+				if err != nil {
+					return nil, err
+				}
+				names = append(names, newMappedPolicy(sys.rolesMap[a]).toSlice()...)
+			} else {
+				mapped, err := mappedPolicies(p.parentUser)
+				if err != nil {
+					return nil, err
+				}
+				names = append(names, mapped...)
+				// The claimed policies apply only when nothing is mapped.
+				if len(mapped) == 0 {
+					if claimed, ok := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID()); ok {
+						names = append(names, claimed.ToSlice()...)
+					}
+				}
+			}
+		default:
+			mapped, err := mappedPolicies(p.accessKey)
+			if err != nil {
+				return nil, err
+			}
+			names = append(names, mapped...)
+		}
+		var selected []policy.Policy
+		for _, name := range names {
+			doc, ok := cache.iamPolicyDocsMap[name]
+			if !ok {
+				loaded := make(map[string]PolicyDoc)
+				err := snap.loadPolicyDoc(name, loaded)
+				if errors.Is(err, errNoSuchPolicy) {
+					// Confirmed absent, as when a mapping still names a
+					// deleted policy: it takes no part in the decision, as
+					// in MergePolicies. Any other failure could hide a Deny.
+					continue
+				}
+				if err != nil {
+					return nil, err
+				}
+				doc, ok = loaded[name]
+				if !ok {
+					return nil, fmt.Errorf("load IAM policy %q returned no document", name)
+				}
+			}
+			selected = append(selected, doc.Policy)
+		}
+		out = append(out, policy.MergePolicies(selected...))
+	}
+	if p.temp || p.service {
+		if p.service && p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
+			return out, nil
+		}
+		if sp, ok := p.claims[sessionPolicyNameExtracted]; ok {
+			s, ok := sp.(string)
+			if !ok {
+				return nil, errAuthentication
+			}
+			parsed, err := policy.ParseConfig(strings.NewReader(s))
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, *parsed)
+		}
+	}
+	return out, nil
+}
+
+// batchStatementNeedsRequest reports whether a statement that applies to
+// args depends on a condition key that the job does not supply. Applying
+// such a statement without its value could grant (Allow) or drop (Deny)
+// what a request with that context would not, so the job refuses the action.
+// Resources are matched without conditions; a statement is taken to apply
+// whenever it might.
+func batchStatementNeedsRequest(st policy.Statement, args policy.Args, supplied map[string]bool) bool {
+	if (!st.Actions.Match(args.Action) && !st.Actions.IsEmpty()) || st.NotActions.Match(args.Action) {
+		return false
+	}
+	// Policy variables in resources or condition values.
+	if data, err := json.Marshal(st); err == nil {
+		s := string(data)
+		for {
+			i := strings.Index(s, "${")
+			if i < 0 {
+				break
+			}
+			s = s[i+2:]
+			j := strings.IndexByte(s, '}')
+			if j < 0 {
+				break
+			}
+			if !batchKeySupplied(supplied, s[:j]) {
+				return true
+			}
+			s = s[j+1:]
+		}
+	}
+	if !batchStatementMayApply(st, args) {
+		return false
+	}
+	for key := range st.Conditions.Keys() {
+		if !batchKeySupplied(supplied, key.String()) {
+			return true
+		}
+	}
+	return false
+}
+
+// batchStatementMayApply matches the resources of a statement, in both forms
+// a bucket request can take. A statement whose resources the evaluator
+// translates (S3 Tables and vectors actions cover S3 data actions) is taken
+// to apply.
+func batchStatementMayApply(st policy.Statement, args policy.Args) bool {
+	for _, actions := range []policy.ActionSet{st.Actions, st.NotActions} {
+		for action := range actions {
+			if policy.TableAction(action).IsValid() || policy.VectorsAction(action).IsValid() {
+				return true
+			}
+		}
+	}
+	for _, resources := range []policy.ResourceSet{st.Resources, st.NotResources} {
+		for r := range resources {
+			if r.Type != policy.ResourceARNS3 && r.Type != policy.ResourceARNAll {
+				return true
+			}
+		}
+	}
+	resources := []string{args.BucketName + "/" + strings.TrimPrefix(args.ObjectName, "/")}
+	if args.ObjectName == "" {
+		resources = []string{args.BucketName, args.BucketName + "/"}
+	}
+	for _, resource := range resources {
+		if len(st.Resources) > 0 && !st.Resources.Match(resource, args.ConditionValues) {
+			continue
+		}
+		if len(st.NotResources) > 0 && st.NotResources.Match(resource, args.ConditionValues) {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+// authorize resolves the submitter and evaluates each request.
+func (s *batchJobSubmitter) authorize(ctx context.Context, reqs ...batchAuthzRequest) error {
+	p, err := s.principal(ctx)
+	if err != nil {
+		return err
+	}
+	for _, req := range reqs {
+		if err := p.authorize(req); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// startAuthzRequests are the bucket actions a job needs before it acts on any
+// object: the listings it walks. Object actions are authorized per object.
+func (j BatchJobRequest) startAuthzRequests() []batchAuthzRequest {
+	list := func(bucket string, prefixes []string) []batchAuthzRequest {
+		if len(prefixes) == 0 {
+			prefixes = []string{""}
+		}
+		var reqs []batchAuthzRequest
+		for _, prefix := range prefixes {
+			values := map[string][]string{condition.S3Prefix.Name(): {prefix}}
+			reqs = append(reqs,
+				batchAuthzRequest{action: policy.ListBucketAction, bucket: bucket, values: values},
+				batchAuthzRequest{action: policy.ListBucketVersionsAction, bucket: bucket, values: values},
+			)
+		}
+		return reqs
+	}
+	// The same order as Type() and the job pool.
+	switch {
+	case j.Replicate != nil:
+		if !j.Replicate.RemoteToLocal() {
+			return list(j.Replicate.Source.Bucket, j.Replicate.Source.Prefix.F())
+		}
+	case j.KeyRotate != nil:
+		return list(j.KeyRotate.Bucket, []string{j.KeyRotate.Prefix})
+	case j.Expire != nil:
+		return list(j.Expire.Bucket, batchCanonicalPrefixes(j.Expire.Prefix.F()))
+	}
+	return nil
+}
+
+// jobSections counts the job types a definition holds. A job is one of them.
+func (j BatchJobRequest) jobSections() int {
+	n := 0
+	for _, set := range []bool{j.Replicate != nil, j.KeyRotate != nil, j.Expire != nil} {
+		if set {
+			n++
+		}
+	}
+	return n
+}
+
+var errBatchJobSections = batchJobAuthzError{reason: "a batch job must define exactly one job type"}
+
+// refuse ends the job for its submitter.
+func (ri *batchJobInfo) refuse(err error) {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	if ri.FailureReason == "" {
+		ri.FailureReason = err.Error()
+	}
+	ri.Complete = false
+	ri.Failed = true
+	ri.PausedReason = ""
+}
+
+func (ri *batchJobInfo) refused() bool {
+	ri.mu.RLock()
+	defer ri.mu.RUnlock()
+	return ri.FailureReason != ""
+}
+
+// startAuthorized checks the submitter of a job that is about to run. It
+// returns done for a job that must not run now: one refused before, which
+// stays refused, or one refused now, which is persisted as failed first.
+// An error that is not a decision leaves the job to a later start.
+func (ri *batchJobInfo) startAuthorized(ctx context.Context, api ObjectLayer, job BatchJobRequest) (done bool, err error) {
+	if !ri.refused() {
+		err := job.Submitter.authorize(ctx, job.startAuthzRequests()...)
+		if err == nil && job.jobSections() != 1 {
+			err = errBatchJobSections
+		}
+		if err == nil {
+			ri.mu.Lock()
+			ri.PausedReason = ""
+			ri.mu.Unlock()
+			// Persist the report version before acting. Older binaries must not
+			// resume an authorization-bearing job with server authority.
+			if err := ri.updateAfter(ctx, api, 0, job); err != nil {
+				return true, batchUnavailable("saving job progress", err)
+			}
+			return false, nil
+		}
+		if isBatchJobUnavailable(err) {
+			return true, ri.savePaused(ctx, api, job, err)
+		}
+		if !isBatchJobAuthzError(err) {
+			return true, err
+		}
+		ri.refuse(err)
+		batchLogIf(ctx, fmt.Errorf("batch job %s refused: %w", job.ID, err))
+	}
+	globalBatchJobsMetrics.save(job.ID, ri)
+	if err := ri.updateAfter(ctx, api, 0, job); err != nil {
+		return true, batchUnavailable("saving job refusal", err)
+	}
+	return true, nil
+}

cmd/batch-authz_gen.go

@@ -0,0 +1,260 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package cmd
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *batchJobSubmitter) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "v":
+			z.Version, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		case "t":
+			z.Type, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "ak":
+			z.AccessKey, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "AccessKey")
+				return
+			}
+		case "pu":
+			z.ParentUser, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "ParentUser")
+				return
+			}
+		case "ua":
+			z.UpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "UpdatedAt")
+				return
+			}
+		case "rb":
+			z.RevokedBefore, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "RevokedBefore")
+				return
+			}
+		case "bd":
+			z.Binding, err = dc.ReadBytes(z.Binding)
+			if err != nil {
+				err = msgp.WrapError(err, "Binding")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *batchJobSubmitter) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 7
+	// write "v"
+	err = en.Append(0x87, 0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Version)
+	if err != nil {
+		err = msgp.WrapError(err, "Version")
+		return
+	}
+	// write "t"
+	err = en.Append(0xa1, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Type)
+	if err != nil {
+		err = msgp.WrapError(err, "Type")
+		return
+	}
+	// write "ak"
+	err = en.Append(0xa2, 0x61, 0x6b)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.AccessKey)
+	if err != nil {
+		err = msgp.WrapError(err, "AccessKey")
+		return
+	}
+	// write "pu"
+	err = en.Append(0xa2, 0x70, 0x75)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.ParentUser)
+	if err != nil {
+		err = msgp.WrapError(err, "ParentUser")
+		return
+	}
+	// write "ua"
+	err = en.Append(0xa2, 0x75, 0x61)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.UpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "UpdatedAt")
+		return
+	}
+	// write "rb"
+	err = en.Append(0xa2, 0x72, 0x62)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.RevokedBefore)
+	if err != nil {
+		err = msgp.WrapError(err, "RevokedBefore")
+		return
+	}
+	// write "bd"
+	err = en.Append(0xa2, 0x62, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.Binding)
+	if err != nil {
+		err = msgp.WrapError(err, "Binding")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *batchJobSubmitter) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 7
+	// string "v"
+	o = append(o, 0x87, 0xa1, 0x76)
+	o = msgp.AppendInt(o, z.Version)
+	// string "t"
+	o = append(o, 0xa1, 0x74)
+	o = msgp.AppendString(o, z.Type)
+	// string "ak"
+	o = append(o, 0xa2, 0x61, 0x6b)
+	o = msgp.AppendString(o, z.AccessKey)
+	// string "pu"
+	o = append(o, 0xa2, 0x70, 0x75)
+	o = msgp.AppendString(o, z.ParentUser)
+	// string "ua"
+	o = append(o, 0xa2, 0x75, 0x61)
+	o = msgp.AppendTime(o, z.UpdatedAt)
+	// string "rb"
+	o = append(o, 0xa2, 0x72, 0x62)
+	o = msgp.AppendTime(o, z.RevokedBefore)
+	// string "bd"
+	o = append(o, 0xa2, 0x62, 0x64)
+	o = msgp.AppendBytes(o, z.Binding)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *batchJobSubmitter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "v":
+			z.Version, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		case "t":
+			z.Type, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "ak":
+			z.AccessKey, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "AccessKey")
+				return
+			}
+		case "pu":
+			z.ParentUser, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ParentUser")
+				return
+			}
+		case "ua":
+			z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "UpdatedAt")
+				return
+			}
+		case "rb":
+			z.RevokedBefore, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RevokedBefore")
+				return
+			}
+		case "bd":
+			z.Binding, bts, err = msgp.ReadBytesBytes(bts, z.Binding)
+			if err != nil {
+				err = msgp.WrapError(err, "Binding")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *batchJobSubmitter) Msgsize() (s int) {
+	s = 1 + 2 + msgp.IntSize + 2 + msgp.StringPrefixSize + len(z.Type) + 3 + msgp.StringPrefixSize + len(z.AccessKey) + 3 + msgp.StringPrefixSize + len(z.ParentUser) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.BytesPrefixSize + len(z.Binding)
+	return
+}

cmd/batch-authz_gen_test.go

@@ -0,0 +1,123 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalbatchJobSubmitter(t *testing.T) {
+	v := batchJobSubmitter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgbatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgbatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalbatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodebatchJobSubmitter(t *testing.T) {
+	v := batchJobSubmitter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodebatchJobSubmitter Msgsize() is inaccurate")
+	}
+
+	vn := batchJobSubmitter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodebatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodebatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-discovery_test.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+)
+
+type batchDiscoveryLayer struct {
+	ObjectLayer
+	walk func(context.Context, chan<- itemOrErr[ObjectInfo]) error
+	read func(context.Context, string) ([]byte, error)
+}
+
+func (a batchDiscoveryLayer) Walk(ctx context.Context, _, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
+	return a.walk(ctx, results)
+}
+
+func (a batchDiscoveryLayer) GetObjectNInfo(ctx context.Context, _, object string, _ *HTTPRangeSpec, _ http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+	data, err := a.read(ctx, object)
+	if err != nil {
+		return nil, err
+	}
+	return NewGetObjectReaderFromReader(bytes.NewReader(data), ObjectInfo{Name: object}, opts)
+}
+
+// A partially successful startup scan must retry the missing work without
+// re-admitting work that finished while the scan was waiting to retry.
+func TestBatchDiscoveryRecoversInProcess(t *testing.T) {
+	for _, failure := range []string{"walk-start", "walk-stream", "manifest-read"} {
+		t.Run(failure, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				ctx, cancel := context.WithCancel(t.Context())
+				defer cancel()
+				pool := &BatchJobPool{ctx: ctx, jobCh: make(chan *BatchJobRequest, 10)}
+				attempts := 0
+				pool.objLayer = batchDiscoveryLayer{
+					walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
+						attempts++
+						if attempts == 1 && failure == "walk-start" {
+							return errErasureReadQuorum
+						}
+						defer close(results)
+						results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "finished"}}
+						if attempts == 1 && failure == "walk-stream" {
+							results <- itemOrErr[ObjectInfo]{Err: errErasureReadQuorum}
+						} else {
+							results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "pending"}}
+						}
+						return nil
+					},
+					read: func(_ context.Context, name string) ([]byte, error) {
+						if attempts == 1 && failure == "manifest-read" && name == "pending" {
+							return nil, errErasureReadQuorum
+						}
+						job := BatchJobRequest{ID: name}
+						return job.MarshalMsg(nil)
+					},
+				}
+				done := make(chan struct{})
+				go func() {
+					defer close(done)
+					pool.resume(func() time.Duration { return 0 })
+				}()
+				synctest.Wait()
+				if failure != "walk-start" {
+					select {
+					case job := <-pool.jobCh:
+						if job.ID != "finished" || !pool.beginJob(job) {
+							t.Fatal("first scan did not admit the healthy manifest")
+						}
+						pool.finishJob(job, nil)
+					default:
+						t.Fatal("first scan lost the healthy manifest")
+					}
+				}
+				select {
+				case <-done:
+				case <-time.After(20 * time.Minute):
+					t.Fatal("discovery failed to finish after storage recovered")
+				}
+				if attempts != 2 {
+					t.Fatalf("discovery attempts = %d, want 2", attempts)
+				}
+				want := 1
+				if failure == "walk-start" {
+					want = 2
+				}
+				if len(pool.jobCh) != want {
+					t.Fatalf("queued %d jobs, want %d; completed work was re-admitted or pending work was lost", len(pool.jobCh), want)
+				}
+				for len(pool.jobCh) > 0 {
+					job := <-pool.jobCh
+					if failure != "walk-start" && job.ID != "pending" {
+						t.Fatalf("re-admitted %q", job.ID)
+					}
+					if !pool.beginJob(job) {
+						t.Fatal("recovered job has no owner")
+					}
+					pool.finishJob(job, nil)
+				}
+			})
+		})
+	}
+}
+
+func TestBatchDiscoverySkipsUnusableManifests(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
+		attempts := 0
+		pool.objLayer = batchDiscoveryLayer{
+			walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
+				attempts++
+				defer close(results)
+				for _, name := range []string{"canceled", "deleted", "malformed", "future", "empty-id"} {
+					results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: name}}
+				}
+				return nil
+			},
+			read: func(_ context.Context, name string) ([]byte, error) {
+				switch name {
+				case "deleted":
+					return nil, errConfigNotFound
+				case "malformed":
+					return []byte("bad manifest"), nil
+				case "future":
+					return []byte("SILO-BATCH-V999\n"), nil
+				case "empty-id":
+					return (&BatchJobRequest{}).MarshalMsg(nil)
+				}
+				// The administrator cancels after Walk, during manifest load.
+				mustIAM(t, pool.canceler(name, true))
+				job := BatchJobRequest{ID: name}
+				return job.MarshalMsg(nil)
+			},
+		}
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pool.resume(func() time.Duration { return 0 })
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Hour):
+			t.Fatal("unusable manifests kept discovery retrying")
+		}
+		if len(pool.jobCh) != 0 || attempts != 1 {
+			t.Fatalf("canceled work was queued or retried: queue=%d scans=%d", len(pool.jobCh), attempts)
+		}
+	})
+}
+
+func TestBatchDiscoveryShutdownDuringBackoff(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+		attempts := 0
+		pool := &BatchJobPool{ctx: ctx, objLayer: batchDiscoveryLayer{
+			walk: func(context.Context, chan<- itemOrErr[ObjectInfo]) error {
+				attempts++
+				return errErasureReadQuorum
+			},
+		}}
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pool.resume(func() time.Duration { return 0 })
+		}()
+		synctest.Wait()
+		select {
+		case <-done:
+			t.Fatal("discovery abandoned a transient startup error")
+		default:
+		}
+		cancel()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatal("discovery did not stop on shutdown")
+		}
+		if attempts != 1 {
+			t.Fatalf("discovery scanned after shutdown: %d", attempts)
+		}
+	})
+}
+
+func TestBatchDiscoveryKeepsNewlyFinishedJobsTerminal(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
+		attempts := 0
+		pool.objLayer = batchDiscoveryLayer{
+			walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
+				attempts++
+				if attempts == 1 {
+					// This job was submitted and failed while startup discovery
+					// had not yet read any manifests. The retry must not rerun it.
+					job := &BatchJobRequest{ID: "new-job"}
+					mustIAM(t, pool.queueJob(job))
+					if !pool.beginJob(<-pool.jobCh) {
+						t.Fatal("submitted job has no owner")
+					}
+					pool.finishJob(job, errErasureWriteQuorum)
+					return errErasureReadQuorum
+				}
+				defer close(results)
+				if owner := pool.jobOwners["new-job"]; owner == nil || owner.state != "terminal" || owner.req != nil || owner.cancel != nil {
+					t.Fatalf("terminal marker lost or retained the job payload: %+v", owner)
+				}
+				results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "new-job"}}
+				return nil
+			},
+			read: func(_ context.Context, name string) ([]byte, error) {
+				return (&BatchJobRequest{ID: name}).MarshalMsg(nil)
+			},
+		}
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pool.resume(func() time.Duration { return 0 })
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Hour):
+			t.Fatal("discovery did not recover")
+		}
+		if attempts != 2 || len(pool.jobCh) != 0 {
+			t.Fatalf("discovery re-admitted a failed job: scans=%d queue=%d", attempts, len(pool.jobCh))
+		}
+		if !pool.discoveryDone || len(pool.jobOwners) != 0 {
+			t.Fatal("completed discovery retained terminal job markers")
+		}
+		job := &BatchJobRequest{ID: "after-discovery"}
+		mustIAM(t, pool.queueJob(job))
+		if !pool.beginJob(<-pool.jobCh) {
+			t.Fatal("new submission after discovery has no owner")
+		}
+		pool.finishJob(job, nil)
+		if len(pool.jobOwners) != 0 {
+			t.Fatal("normal completion after discovery leaked an owner")
+		}
+	})
+}

cmd/batch-expire-completed_test.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"go.yaml.in/yaml/v3"
+)
+
+// A completed expire job is not run again when the server restarts, as
+// replication and key rotation jobs are not: in particular it is not checked
+// against its submitter again, which could turn it into a failed job.
+
+// An expire job that completed before the upgrade records no submitter.
+func TestBatchExpireCompletedLegacyStaysComplete(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "data/a")
+	job := &BatchJobRequest{}
+	mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
+	job.ID = "expire-legacydone" + getKeySeparator() + "0"
+	job.User = globalActiveCred.AccessKey
+	job.Started = time.Now()
+	mustIAM(t, job.save(f.ctx, f.obj))
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Complete: true, Objects: 3}
+	mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
+
+	loaded := &BatchJobRequest{}
+	mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+	f.run(t, loaded) // resumed after a restart
+	got := f.jobInfo(t, job)
+	if !got.Complete || got.Failed || got.FailureReason != "" {
+		t.Fatalf("complete=%v failed=%v reason=%q, want the completed state kept", got.Complete, got.Failed, got.FailureReason)
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Error("a completed job ran again")
+	}
+}
+
+// A new expire job completed with a temporary credential that has expired
+// since.
+func TestBatchExpireCompletedAfterCredentialExpiry(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(3 * time.Second).Unix()}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	f.run(t, job)
+	if f.versionExists(t, "data/a", older) {
+		t.Fatal("the job did not expire the older version")
+	}
+	if ri := f.jobInfo(t, job); !ri.Complete {
+		t.Fatalf("complete=%v failed=%v", ri.Complete, ri.Failed)
+	}
+	time.Sleep(time.Until(cred.Expiration) + time.Second)
+
+	loaded := &BatchJobRequest{}
+	mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+	f.run(t, loaded) // resumed after a restart
+	if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed || ri.FailureReason != "" {
+		t.Errorf("complete=%v failed=%v reason=%q after a restart, want the completed state kept", ri.Complete, ri.Failed, ri.FailureReason)
+	}
+
+	// Control: a job that did not complete is still refused once its
+	// credential has expired.
+	f.putVersions(t, "data/b")
+	pending := &BatchJobRequest{}
+	mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), pending))
+	pending.ID = "expire-pending" + getKeySeparator() + "0"
+	pending.User = cred.AccessKey
+	pending.Started = time.Now()
+	pending.Submitter = loaded.Submitter
+	mustIAM(t, pending.save(f.ctx, f.obj))
+	f.run(t, pending)
+	if ri := f.jobInfo(t, pending); ri.Complete || !ri.Failed || ri.FailureReason == "" {
+		t.Errorf("pending job with an expired submitter: complete=%v failed=%v reason=%q, want refused", ri.Complete, ri.Failed, ri.FailureReason)
+	}
+}

cmd/batch-expire-purge-authz_test.go

@@ -0,0 +1,560 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/config/storageclass"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A batch expiration that keeps no versions purges each key it matched under
+// the purge's lock. What it authorizes for the submitter, and checks for
+// Object Lock, are the versions the purge deletes then, not the versions the
+// walk saw before.
+
+// batchPurgeHook runs change once, after the walk of a job and before its
+// first purge of all versions.
+type batchPurgeHook struct {
+	ObjectLayer
+	once   sync.Once
+	change func()
+}
+
+func (h *batchPurgeHook) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	if opts.DeleteAllVersions {
+		h.once.Do(h.change)
+	}
+	return h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+}
+
+func (f *batchAuthzFixture) purgeYAML(prefix string) string {
+	return fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: %q
+  rules:
+    - type: object
+      purge:
+        retainVersions: 0
+`, f.bucket, prefix)
+}
+
+// runChanged submits a purge of prefix as cred and runs it, with change made
+// between its walk and its purge, and returns the job's final state.
+func (f *batchAuthzFixture) runChanged(t *testing.T, cred auth.Credentials, prefix string, change func()) *batchJobInfo {
+	t.Helper()
+	rec, job := f.submit(t, cred, f.purgeYAML(prefix))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	hook := &batchPurgeHook{ObjectLayer: f.obj, change: change}
+	if err := job.Expire.Start(f.ctx, hook, *job); err != nil {
+		t.Logf("start: %v", err)
+	}
+	return f.jobInfo(t, job)
+}
+
+// putOlder writes a version of key older than every version it has, as a
+// replicated or pulled version can be, with metadata meta.
+func (f *batchAuthzFixture) putOlder(t *testing.T, key, versionID string, meta map[string]string) {
+	t.Helper()
+	data := []byte("older " + key)
+	_, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
+		ObjectOptions{Versioned: true, VersionID: versionID, MTime: UTCNow().Add(-time.Hour), UserDefined: meta})
+	mustIAM(t, err)
+}
+
+func batchPurgeVersionIDs(t *testing.T, f *batchAuthzFixture, key string) []string {
+	t.Helper()
+	var ids []string
+	marker, vmarker := "", ""
+	for {
+		res, err := f.obj.ListObjectVersions(f.ctx, f.bucket, key, marker, vmarker, "", 1000)
+		mustIAM(t, err)
+		for _, o := range res.Objects {
+			if o.Name == key {
+				ids = append(ids, o.VersionID)
+			}
+		}
+		if !res.IsTruncated {
+			return ids
+		}
+		marker, vmarker = res.NextMarker, res.NextVersionIDMarker
+	}
+}
+
+// A version that the submitter may not expire, because of its tags or its
+// version ID, is refused although the walk did not see it so: the job fails
+// and the key keeps all its versions.
+func TestBatchExpirePurgeAuthorizesCurrentVersions(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		action string // the action a conditional Deny denies
+		deny   string // its condition
+		// change returns the version IDs that change made.
+		change func(t *testing.T, f *batchAuthzFixture, key, older string) []string
+	}{
+		{
+			name:   "tagged-after-walk",
+			action: "s3:GetObjectVersion",
+			deny:   `{"StringEquals":{"s3:ExistingObjectTag/keep":["yes"]}}`,
+			change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
+				_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older})
+				mustIAM(t, err)
+				return nil
+			},
+		},
+		{
+			name:   "version-after-walk",
+			action: "s3:DeleteObjectVersion",
+			deny:   `{"StringEquals":{"s3:versionid":["%s"]}}`,
+			change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
+				f.putOlder(t, key, batchPurgeDeniedVersion, nil)
+				return []string{batchPurgeDeniedVersion}
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			deny := tc.deny
+			if strings.Contains(deny, "%s") {
+				deny = fmt.Sprintf(deny, batchPurgeDeniedVersion)
+			}
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-purge", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["`+tc.action+`"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":`+deny+`}`)
+			cred := iamAdminUser(f.ctx, t, f.sys, "batch-purge-user", "batch-purge")
+			const key = "purge/key"
+			older := f.putVersions(t, key)
+			want := batchPurgeVersionIDs(t, f, key)
+			ri := f.runChanged(t, cred, "purge/", func() {
+				want = append(want, tc.change(t, f, key, older)...)
+			})
+			for _, id := range want {
+				if !f.versionExists(t, key, id) {
+					t.Errorf("version %s was purged although the submitter may not delete it", id)
+				}
+			}
+			if !ri.Failed || ri.Complete || ri.FailureReason == "" {
+				t.Errorf("job complete=%v failed=%v reason=%q, want a refused job", ri.Complete, ri.Failed, ri.FailureReason)
+			}
+			if ri.Objects != 0 || ri.KeysRefused != 0 {
+				t.Errorf("job counts objects=%d refused=%d", ri.Objects, ri.KeysRefused)
+			}
+		})
+	}
+}
+
+const batchPurgeDeniedVersion = "6c4b2f8e-1d3a-4e5b-9c7d-0a1b2c3d4e5f"
+
+// A version that Object Lock protects refuses the whole key, also when it
+// was added after the walk. The job completes with the key counted as
+// refused, not failed.
+func TestBatchExpirePurgeRefusesLockedAfterWalk(t *testing.T) {
+	for _, kind := range batchLockKinds {
+		t.Run(kind, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			enableBucketObjectLock(t, f.bucket)
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+			cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			const key = "purge/locked"
+			f.putVersions(t, key)
+			want := batchPurgeVersionIDs(t, f, key)
+			locked := mustGetUUID()
+			ri := f.runChanged(t, cred, "purge/", func() {
+				f.putOlder(t, key, locked, batchLockMeta(kind))
+				want = append(want, locked)
+			})
+			for _, id := range want {
+				if !f.versionExists(t, key, id) {
+					t.Errorf("version %s of a key with a locked version was purged", id)
+				}
+			}
+			if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
+				t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
+					ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
+			}
+		})
+	}
+}
+
+// transition marks version versionID of key as transitioned to a remote
+// tier, as a lifecycle transition leaves it.
+func (f *batchAuthzFixture) transition(t *testing.T, key, versionID string) {
+	t.Helper()
+	set := f.obj.(*erasureServerPools).serverPools[0]
+	infos, errs := readAllFileInfo(f.ctx, set.getHashedSet(key).getDisks(), "", f.bucket, key, versionID, false, false)
+	mustIAM(t, errs[0])
+	fi := infos[0]
+	fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote/"+key+"/"+versionID
+	mustIAM(t, set.DecomTieredObject(f.ctx, f.bucket, key, fi, ObjectOptions{}))
+}
+
+// A key with a transitioned version is refused as a whole, as a key with a
+// locked version is: the job completes with the key counted as refused, not
+// failed, and the key keeps every version.
+func TestBatchExpirePurgeRefusesTieredKey(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+	const key = "purge/tiered"
+	older := f.putVersions(t, key)
+	f.transition(t, key, older)
+	want := batchPurgeVersionIDs(t, f, key)
+	ri := f.runChanged(t, cred, "purge/", func() {})
+	for _, id := range want {
+		if !f.versionExists(t, key, id) {
+			t.Errorf("version %s of a key with a transitioned version was purged", id)
+		}
+	}
+	if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
+		t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
+			ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
+	}
+}
+
+// Controls: a purge by root and by an authorized user removes every version,
+// including one added after the walk.
+func TestBatchExpirePurgeAuthorizedControls(t *testing.T) {
+	for _, root := range []bool{true, false} {
+		t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			cred := globalActiveCred
+			if !root {
+				batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+				cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			}
+			const key = "purge/key"
+			f.putVersions(t, key)
+			added := mustGetUUID()
+			ri := f.runChanged(t, cred, "purge/", func() {
+				f.putOlder(t, key, added, map[string]string{xhttp.AmzObjectTagging: "keep=yes"})
+			})
+			if ids := batchPurgeVersionIDs(t, f, key); len(ids) != 0 {
+				t.Errorf("an authorized purge kept %v", ids)
+			}
+			if !ri.Complete || ri.Failed || ri.Objects != 3 || ri.KeysRefused != 0 {
+				t.Errorf("job complete=%v failed=%v objects=%d refused=%d, want complete with 3 objects",
+					ri.Complete, ri.Failed, ri.Objects, ri.KeysRefused)
+			}
+		})
+	}
+}
+
+// Cases found in the sixth review round.
+
+// batchPurgeBudgetChange runs change once the first purge stopped at its
+// budget after removing versions.
+type batchPurgeBudgetChange struct {
+	ObjectLayer
+	change func()
+	calls  int
+}
+
+func (h *batchPurgeBudgetChange) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	oi, err := h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+	if opts.DeleteAllVersions {
+		h.calls++
+		if h.calls == 1 && errors.Is(err, errDeleteAllBudget) && opts.DeleteAllResult.Removed > 0 {
+			h.change()
+		}
+	}
+	return oi, err
+}
+
+// A purge that continues after its budget authorizes the versions again, as
+// they are then; the refusal and the versions already removed persist
+// across another start.
+func TestBatchExpirePurgeBudgetReauthorizes(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	const key = "purge/budget"
+	oldest := batchPutVersion(t, f.obj, f.bucket, key, nil)
+	middle := batchPutVersion(t, f.obj, f.bucket, key, nil)
+	latest := batchPutVersion(t, f.obj, f.bucket, key, nil)
+	batchAuthzPolicy(f.ctx, t, f.sys, "review6-budget-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "review6-budget-user", "review6-budget-deny")
+	rec, job := f.submit(t, cred, f.purgeYAML("purge/"))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	oldBudget := deleteAllBudget
+	deleteAllBudget.versions = 1
+	t.Cleanup(func() { deleteAllBudget = oldBudget })
+	hook := &batchPurgeBudgetChange{ObjectLayer: f.obj, change: func() {
+		_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: middle.VersionID})
+		mustIAM(t, err)
+	}}
+	mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
+	ri := f.jobInfo(t, job)
+	if hook.calls != 2 || !ri.Failed || ri.Complete || ri.FailureReason == "" || ri.Objects != 1 || ri.KeysRefused != 0 {
+		t.Fatalf("continuation: calls=%d complete=%v failed=%v reason=%q objects=%d refused=%d", hook.calls, ri.Complete, ri.Failed, ri.FailureReason, ri.Objects, ri.KeysRefused)
+	}
+	if f.versionExists(t, key, oldest.VersionID) || !f.versionExists(t, key, middle.VersionID) || !f.versionExists(t, key, latest.VersionID) {
+		t.Fatal("only the first authorized version may be removed")
+	}
+	_, err := f.obj.DeleteObjectTags(f.ctx, f.bucket, key, ObjectOptions{VersionID: middle.VersionID})
+	mustIAM(t, err)
+	mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
+	after := f.jobInfo(t, job)
+	if hook.calls != 2 || after.FailureReason != ri.FailureReason || after.Objects != 1 || after.Complete {
+		t.Fatal("persisted refusal must prevent a fresh purge after the tags change")
+	}
+}
+
+// The purge's lock excludes tag writers from its authorization through its
+// last delete, for a directory key in a lock domain of its own on one pool
+// and for a key in a second pool.
+func TestBatchExpirePurgeExcludesTagWriter(t *testing.T) {
+	for _, mode := range []string{ldDsync, "two-pools"} {
+		t.Run(mode, func(t *testing.T) {
+			var z *erasureServerPools
+			var bucket string
+			pool, key := 0, "purge/tags/"
+			if mode == "two-pools" {
+				z, bucket = consistencyPools(t)
+				pool = 1
+			} else {
+				z, bucket = ldPool(t, mode)
+				for i := 0; ; i++ {
+					key = fmt.Sprintf("purge/tags-%d/", i)
+					if z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) == 1 {
+						break
+					}
+				}
+			}
+			before := putConsistencyObject(t, z, bucket, encodeDirObject(key), pool, "", ObjectOptions{Versioned: true})
+			ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+			defer cancel()
+			entered, release := make(chan struct{}), make(chan struct{})
+			var released atomic.Bool
+			unblock := func() {
+				if released.CompareAndSwap(false, true) {
+					close(release)
+				}
+			}
+			defer unblock()
+			purged := make(chan error, 1)
+			go func() {
+				_, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{
+					DeleteAllVersions: true,
+					DeleteAllLatest:   &deleteAllMatch{VersionID: before.VersionID, ModTime: before.ModTime},
+					deleteAllAuthorize: func(versions []ObjectInfo) error {
+						close(entered)
+						select {
+						case <-release:
+							return nil
+						case <-ctx.Done():
+							return ctx.Err()
+						}
+					},
+				})
+				purged <- err
+			}()
+			select {
+			case <-entered:
+			case <-ctx.Done():
+				t.Fatal("purge never reached authorization")
+			}
+			written := make(chan error, 1)
+			go func() {
+				_, err := z.PutObjectTags(ctx, bucket, key, "keep=yes", ObjectOptions{VersionID: before.VersionID})
+				written <- err
+			}()
+			select {
+			case err := <-written:
+				t.Errorf("tag writer crossed the purge's authorization lock: %v", err)
+			case <-time.After(ldWait):
+				unblock()
+				err := <-written
+				if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+					t.Errorf("writer must observe the completed purge: %v", err)
+				}
+			}
+			unblock()
+			mustIAM(t, <-purged)
+		})
+	}
+}
+
+// batchTagWriteFault fails metadata updates of one key, so that a tag write
+// reaches only the drives it does not wrap.
+type batchTagWriteFault struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object string, fi FileInfo, opts UpdateMetadataOpts) error {
+	if bucket == d.bucket && object == d.object {
+		return errDiskNotFound
+	}
+	return d.StorageAPI.UpdateMetadata(ctx, bucket, object, fi, opts)
+}
+
+// A failed tag write can leave a revision that the submitter may not expire
+// on a minority of drives. The purge removes it with the version, so it is
+// authorized too.
+func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) {
+	resetTestGlobals()
+	t.Cleanup(resetTestGlobals)
+	// One drive of sixteen holds the failed tag write, so the read quorum of
+	// the metadata must stay below sixteen. Earlier tests can leave a
+	// storage class of parity 0 behind (newTestConfig on a single drive).
+	previous := globalStorageClass
+	globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
+	t.Cleanup(func() { globalStorageClass.Update(previous) })
+	z := purgeSinglePool(t, 16)
+	globalIAMSys.initStore(z, nil)
+	mustIAM(t, globalIAMSys.Load(t.Context(), true))
+	f := &batchAuthzFixture{ctx: t.Context(), sys: globalIAMSys, obj: z, bucket: "review6-batch-authz"}
+	mustIAM(t, z.MakeBucket(f.ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+	batchTestJobPool(f.ctx, t, z)
+	const key = "purge/minority-tags"
+	older := f.putVersions(t, key)
+	batchAuthzPolicy(f.ctx, t, f.sys, "review6-minority-deny", strings.ReplaceAll(batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`, "batch-authz", f.bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "review6-minority-user", "review6-minority-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectVersionAction, bucket: f.bucket, object: key, versionID: older, tags: "keep=yes"}); !isBatchJobAuthzError(err) {
+		t.Fatal("control: this tag revision must deny the read needed for expiration")
+	}
+	set := z.serverPools[0].getHashedSet(key)
+	original := set.getDisks
+	disks := append([]StorageAPI(nil), original()...)
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i := 1; i < len(wrapped); i++ {
+		wrapped[i] = batchTagWriteFault{StorageAPI: wrapped[i], bucket: f.bucket, object: key}
+	}
+	set.getDisks = func() []StorageAPI { return wrapped }
+	_, err = f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older, UserDefined: map[string]string{
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+	}})
+	set.getDisks = original
+	if err == nil {
+		t.Fatal("control: the tag write must fail its quorum")
+	}
+	fi, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{})
+	mustIAM(t, err)
+	if fi.Metadata[xhttp.AmzObjectTagging] != "keep=yes" {
+		t.Fatal("control: the successful drive must hold the denied tag revision")
+	}
+	history, err := z.deleteAllHistory(f.ctx, f.bucket, key)
+	mustIAM(t, err)
+	for _, v := range history {
+		for _, c := range v.copies {
+			if c.fi.Metadata[xhttp.AmzObjectTagging] != "" {
+				t.Fatal("control: the quorum history must still have the original tags")
+			}
+		}
+	}
+	ri := f.runChanged(t, cred, "purge/", func() {})
+	if !ri.Failed || ri.Complete || !strings.Contains(ri.FailureReason, "is not allowed") {
+		t.Errorf("denied physical revision must refuse the job: complete=%v failed=%v reason=%q", ri.Complete, ri.Failed, ri.FailureReason)
+	}
+	if _, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}); err != nil {
+		t.Errorf("denied physical revision was removed: %v", err)
+	}
+	if got := batchPurgeVersionIDs(t, f, key); len(got) != 2 {
+		t.Errorf("denial must keep the whole key; remaining versions=%v", got)
+	}
+}
+
+// Cases found in the seventh review round.
+
+// The time the authorization takes counts against the purge's budget: a
+// late allow and a deny both delete nothing.
+func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) {
+	z := purgeSinglePool(t, 16)
+	p := newPurgeTest(t, z, "review7-purge-auth-time", MakeBucketOptions{VersioningEnabled: true}, "")
+	const key = "purge/key"
+	p.version(0, key, "older", 2)
+	p.version(0, key, "latest", 1)
+	exp := p.matched(key)
+	oldBudget := deleteAllBudget
+	deleteAllBudget.duration = 10 * time.Millisecond
+	t.Cleanup(func() { deleteAllBudget = oldBudget })
+	denied := errors.New("review7: revision denied")
+	for _, decision := range []error{nil, denied} {
+		called := false
+		var res deleteAllResult
+		_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
+			DeleteAllVersions: true,
+			DeleteAllLatest:   &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+			DeleteAllResult:   &res,
+			deleteAllAuthorize: func(versions []ObjectInfo) error {
+				called = true
+				if len(versions) < 2 {
+					t.Error("authorization did not receive both approved versions")
+				}
+				time.Sleep(20 * time.Millisecond)
+				return decision
+			},
+		})
+		want := decision
+		if want == nil {
+			want = errDeleteAllBudget
+		}
+		if !called || !errors.Is(err, want) || res.Started || res.Complete || res.Removed != 0 {
+			t.Errorf("late authorization: called=%v err=%v result=%+v", called, err, res)
+		}
+		if got := p.held(0, key); len(got) != 2 {
+			t.Fatalf("purge removed versions after late authorization: %v", got)
+		}
+	}
+}
+
+// A listing outage pauses without moving the cursor or refusing the job;
+// a later run completes it in both modes.
+func TestBatchExpireWalkFailureResumes(t *testing.T) {
+	for _, purgeAll := range []bool{false, true} {
+		name := "retain-one"
+		if purgeAll {
+			name = "retain-zero"
+		}
+		t.Run(name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			const key = "data/key"
+			older := f.putVersions(t, key)
+			definition := f.expireYAML("data/")
+			if purgeAll {
+				definition = f.purgeYAML("data/")
+			}
+			rec, job := f.submit(t, globalActiveCred, definition)
+			if job == nil {
+				t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+			}
+			listErr := errors.New("review7: transient listing failure")
+			err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job)
+			if !errors.Is(err, listErr) {
+				t.Fatalf("listing error: %v", err)
+			}
+			ri := f.jobInfo(t, job)
+			if ri.Failed || ri.Complete || ri.PausedReason == "" || ri.FailureReason != "" || ri.Object != "" || !f.versionExists(t, key, older) {
+				t.Fatalf("listing failure changed progress or became terminal: failed=%v complete=%v reason=%q cursor=%q", ri.Failed, ri.Complete, ri.FailureReason, ri.Object)
+			}
+			mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+			ri = f.jobInfo(t, job)
+			wantRemoved := int64(1)
+			if purgeAll {
+				wantRemoved = 2
+			}
+			if ri.Failed || !ri.Complete || ri.FailureReason != "" || ri.Objects != wantRemoved || f.versionExists(t, key, older) {
+				t.Fatalf("resume did not complete: failed=%v complete=%v reason=%q removed=%d", ri.Failed, ri.Complete, ri.FailureReason, ri.Objects)
+			}
+		})
+	}
+}

cmd/batch-expire.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -34,6 +35,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/pgsty/silo-pkg/v3/env"
+	"github.com/pgsty/silo-pkg/v3/policy"
 	"github.com/pgsty/silo-pkg/v3/wildcard"
 	"github.com/pgsty/silo-pkg/v3/workers"
 	"github.com/pgsty/silo-pkg/v3/xtime"
@@ -289,16 +291,6 @@ type BatchJobExpire struct {
 
 var _ yaml.Unmarshaler = &BatchJobExpire{}
 
-// RedactSensitive will redact any sensitive information in b.
-func (r *BatchJobExpire) RedactSensitive() {
-	if r == nil {
-		return
-	}
-	if r.NotificationCfg.Token != "" {
-		r.NotificationCfg.Token = redactedText
-	}
-}
-
 // UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
 func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
 	type expireJob BatchJobExpire
@@ -345,32 +337,6 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
 	return nil
 }
 
-// Expire expires object versions which have already matched supplied filter conditions
-func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
-	opts := ObjectOptions{
-		PrefixEnabledFn:  vc.PrefixEnabled,
-		VersionSuspended: vc.Suspended(),
-	}
-
-	allErrs := make([]error, 0, len(objsToDel))
-
-	for {
-		count := len(objsToDel)
-		if count == 0 {
-			break
-		}
-		if count > maxDeleteList {
-			count = maxDeleteList
-		}
-		_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
-		allErrs = append(allErrs, errs...)
-		// Next batch of deletion
-		objsToDel = objsToDel[count:]
-	}
-
-	return allErrs
-}
-
 const (
 	batchExpireName                 = "batch-expire.bin"
 	batchExpireFormat               = 1
@@ -381,22 +347,230 @@ const (
 	batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
 )
 
-type objInfoCache map[string]*ObjectInfo
-
-func newObjInfoCache() objInfoCache {
-	return objInfoCache(make(map[string]*ObjectInfo))
+type expireObjInfo struct {
+	ObjectInfo
+	ExpireAll         bool
+	DeleteMarkerCount int64
 }
 
-func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
-	oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
+// isErrObjectLocked reports a version that Object Lock protects.
+func isErrObjectLocked(err error) bool {
+	var locked ObjectLocked
+	return errors.As(err, &locked)
 }
 
-func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
-	oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
-	return oi, ok
+func (r *BatchJobExpire) tagsFiltered() bool {
+	for _, rule := range r.Rules {
+		if len(rule.Tags) > 0 {
+			return true
+		}
+	}
+	return false
 }
 
-func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
+// expireAuthzRequests are the actions the job takes on one version: it reads
+// the version, and its tags when a rule filters by them, and deletes it.
+func (r *BatchJobExpire) expireAuthzRequests(name, versionID, userTags string) []batchAuthzRequest {
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	reqs := []batchAuthzRequest{
+		{action: policy.GetObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		{action: policy.DeleteObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		{action: policy.DeleteObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+	}
+	if r.tagsFiltered() {
+		reqs = append(reqs,
+			batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+			batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		)
+	}
+	return reqs
+}
+
+// expireVersion deletes one version that is older than the versions the rule
+// retains. Under the object's write lock, with the version as it is then, it
+// leaves a version that Object Lock protects (ObjectLocked) and refuses one
+// that the submitter may not delete (batchJobAuthzError). An unreadable
+// version is not deleted.
+func (r *BatchJobExpire) expireVersion(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, exp expireObjInfo) error {
+	p, err := job.Submitter.principal(ctx)
+	if err != nil {
+		return err
+	}
+	versionID := exp.VersionID
+	if versionID == "" {
+		// An empty version ID would add a delete marker instead.
+		versionID = nullVersionID
+	}
+	opts := ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        vc.PrefixEnabled(exp.Name),
+		VersionSuspended: vc.Suspended(),
+	}
+	opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) error {
+		// A delete marker reads as MethodNotAllowed; any other error,
+		// including a missing version, stops the delete.
+		marker := isErrMethodNotAllowed(gerr) && (goi.DeleteMarker || !goi.VersionPurgeStatus.Empty())
+		if gerr != nil && !marker {
+			return gerr
+		}
+		if enforceRetentionForDeletion(ctx, goi) {
+			return ObjectLocked{Bucket: r.Bucket, Object: exp.Name, VersionID: versionID}
+		}
+		for _, req := range r.expireAuthzRequests(exp.Name, versionID, goi.UserTags) {
+			if err := p.authorize(req); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	_, err = api.DeleteObject(ctx, r.Bucket, exp.Name, opts)
+	return err
+}
+
+// expireVersions deletes the versions of one batch entry and records the
+// outcome in res. It returns an error that ends the job.
+func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, delay time.Duration, res *batchCounts) error {
+	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+		err := r.expireVersion(ctx, api, vc, job, exp)
+		stopFn(exp, err)
+		switch {
+		case err == nil:
+			if exp.DeleteMarker {
+				ri.countRemoved(r.Bucket, 0, 1)
+			} else {
+				ri.countRemoved(r.Bucket, 1, 0)
+			}
+			return nil
+		case isBatchJobStopped(err):
+			return err
+		case isErrObjectLocked(err):
+			// Left in place, not retried and not a failure.
+			res.versionsSkipped++
+			return nil
+		case isErrVersionNotFound(err) || isErrObjectNotFound(err):
+			// Removed concurrently.
+			return nil
+		}
+		batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", r.Bucket, exp.Name, exp.VersionID, err, attempts))
+		if attempts < retryAttempts {
+			time.Sleep(delay)
+		}
+	}
+	if exp.DeleteMarker {
+		res.deleteMarkersFailed++
+	} else {
+		res.objectsFailed++
+	}
+	return nil
+}
+
+// expireAllVersions purges every version of a key whose rule retains none,
+// with the purge of exactly that key (erasureServerPools.deleteAllVersions),
+// while the version the walk matched is still its latest. Under its lock,
+// before it deletes anything, the purge refuses the key when a version is
+// transitioned or Object Lock protects one, and has every write it would
+// delete, each copy and each revision a drive holds, authorized for the
+// submitter, with the version ID and tags it has then.
+//
+// What the purge removed is counted at once; a refused key is counted in res
+// as refused, and the versions a failed purge left in res as failures. A
+// failed purge is retried up to retryAttempts times, each time with the
+// version the walk matched; a purge that stopped at its budget after
+// removing versions continues without spending an attempt. A key whose
+// latest version changed since the walk is not expired. It returns an error
+// that ends the job: an authorization failure or the end of ctx.
+func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error {
+	var objects, markers int64
+	matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
+	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+		latest := matched
+		var purge deleteAllResult
+		p, err := job.Submitter.principal(ctx)
+		if err == nil {
+			_, err = api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
+				DeleteAllVersions:  true,
+				DeleteAllLatest:    &latest,
+				DeleteAllResult:    &purge,
+				deleteAllAuthorize: r.expireAllAuthorize(p, exp.Name),
+			})
+		}
+		// Removed versions are gone for any later run: count them now.
+		if purge.Removed > 0 {
+			ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers))
+			objects += int64(purge.Removed - purge.Markers)
+			markers += int64(purge.Markers)
+		}
+		stopFn(exp, err)
+		switch {
+		case err == nil:
+			return nil
+		case isBatchJobStopped(err):
+			return err
+		case errors.Is(err, errDeleteAllChanged):
+			// Changed since the walk matched it: not expired.
+			return nil
+		case isErrObjectLocked(err), errors.Is(err, errDeleteAllTiered):
+			// Refused as a whole: nothing removed, not retried and not a
+			// failure. A later run of the job checks the key again.
+			res.keysRefused++
+			return nil
+		case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
+			attempts-- // progress
+			continue
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
+	}
+	res.expireAllFailed(exp, objects, markers)
+	return nil
+}
+
+// expireAllAuthorize returns the authorization of a purge of all versions of
+// name for p: each version the purge would delete, as it holds it, needs the
+// actions of expiring it (expireAuthzRequests).
+func (r *BatchJobExpire) expireAllAuthorize(p batchJobPrincipal, name string) func([]ObjectInfo) error {
+	return func(versions []ObjectInfo) error {
+		for _, v := range versions {
+			for _, req := range r.expireAuthzRequests(name, v.VersionID, v.UserTags) {
+				if err := p.authorize(req); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+}
+
+// expireAllFailed counts the versions of info, as the walk saw them, that a
+// failed purge of all its versions did not remove: those beyond the objects
+// and markers it removed, at least one.
+func (c *batchCounts) expireAllFailed(info expireObjInfo, objects, markers int64) {
+	failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
+	failedMarkers := max(info.DeleteMarkerCount-markers, 0)
+	if failedObjects+failedMarkers == 0 {
+		failedObjects = 1
+	}
+	c.objectsFailed += failedObjects
+	c.deleteMarkersFailed += failedMarkers
+}
+
+// batchObjsForDelete expires the batches of expireCh. walkDone reports, once
+// expireCh is closed, whether the walk reached its end: only then is its last
+// key known to be complete.
+func batchObjsForDelete(ctx context.Context, refuse func(error), walkDone func() bool, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
 	vc, _ := globalBucketVersioningSys.Get(r.Bucket)
 	retryAttempts := job.Expire.Retry.Attempts
 	if retryAttempts <= 0 {
@@ -407,130 +581,53 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 		delay = batchExpireJobDefaultRetryDelay
 	}
 
-	var i int
+	progress := newBatchProgress(ri, r.Bucket)
+	defer func() {
+		wk.Wait()
+		if walkDone() {
+			progress.finish()
+		}
+	}()
 	for toExpire := range expireCh {
 		select {
 		case <-ctx.Done():
 			return
 		default:
 		}
-		if i > 0 {
+		seq := progress.begin()
+		if seq > 0 {
 			if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
 				time.Sleep(wait)
 			}
 		}
-		i++
 		wk.Take()
-		go func(toExpire []expireObjInfo) {
+		go func(seq uint64, toExpire []expireObjInfo) {
 			defer wk.Give()
-
-			toExpireAll := make([]expireObjInfo, 0, len(toExpire))
-			toDel := make([]ObjectToDelete, 0, len(toExpire))
-			oiCache := newObjInfoCache()
+			var res batchCounts
 			for _, exp := range toExpire {
+				var err error
 				if exp.ExpireAll {
-					toExpireAll = append(toExpireAll, exp)
-					continue
+					err = r.expireAllVersions(ctx, api, job, ri, exp, retryAttempts, &res)
+				} else {
+					err = r.expireVersions(ctx, api, vc, job, ri, exp, retryAttempts, delay, &res)
 				}
-				// Cache ObjectInfo value via pointers for
-				// subsequent use to track objects which
-				// couldn't be deleted.
-				od := ObjectToDelete{
-					ObjectV: ObjectV{
-						ObjectName: exp.Name,
-						VersionID:  exp.VersionID,
-					},
-				}
-				toDel = append(toDel, od)
-				oiCache.Add(od, &exp.ObjectInfo)
-			}
-
-			// DeleteObject(deletePrefix: true) to expire all versions of an object
-			for _, exp := range toExpireAll {
-				var success bool
-				for attempts := 1; attempts <= retryAttempts; attempts++ {
-					select {
-					case <-ctx.Done():
-						ri.trackMultipleObjectVersions(exp, success)
-						return
-					default:
+				if err != nil {
+					// The batch is not done; a later run does it again.
+					if isBatchJobStopped(err) {
+						refuse(err)
 					}
-					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
-						DeletePrefix:       true,
-						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-					})
-					if err != nil {
-						stopFn(exp, err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
-					} else {
-						stopFn(exp, err)
-						success = true
-						break
-					}
-				}
-				ri.trackMultipleObjectVersions(exp, success)
-			}
-
-			// DeleteMultiple objects
-			toDelCopy := make([]ObjectToDelete, len(toDel))
-			for attempts := 1; attempts <= retryAttempts; attempts++ {
-				select {
-				case <-ctx.Done():
+					globalBatchJobsMetrics.save(ri.JobID, ri)
 					return
-				default:
-				}
-
-				stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-				// Copying toDel to select from objects whose
-				// deletion failed
-				copy(toDelCopy, toDel)
-				var failed int
-				errs := r.Expire(ctx, api, vc, toDel)
-				// reslice toDel in preparation for next retry attempt
-				toDel = toDel[:0]
-				for i, err := range errs {
-					if err != nil {
-						stopFn(toDelCopy[i], err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
-							err, attempts))
-						failed++
-						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
-							ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)
-						}
-						if attempts != retryAttempts {
-							// retry
-							toDel = append(toDel, toDelCopy[i])
-						}
-					} else {
-						stopFn(toDelCopy[i], nil)
-						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
-							ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts)
-						}
-					}
-				}
-
-				globalBatchJobsMetrics.save(ri.JobID, ri)
-
-				if failed == 0 {
-					break
-				}
-
-				// Add a delay between retry attempts
-				if attempts < retryAttempts {
-					time.Sleep(delay)
 				}
 			}
-		}(toExpire)
+			if n := len(toExpire); n > 0 {
+				progress.done(seq, toExpire[n-1].Name, res)
+			}
+			globalBatchJobsMetrics.save(ri.JobID, ri)
+		}(seq, toExpire)
 	}
 }
 
-type expireObjInfo struct {
-	ObjectInfo
-	ExpireAll         bool
-	DeleteMarkerCount int64
-}
-
 // Start the batch expiration job, resumes if there was a pending job via "job.ID"
 func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
 	ri := &batchJobInfo{
@@ -539,6 +636,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		StartTime: job.Started,
 	}
 	if err := ri.loadOrInit(ctx, api, job); err != nil {
+		return batchUnavailable("loading job progress", err)
+	}
+	// A completed job is not run again, as replication and key rotation
+	// jobs are not: its submitter is not checked again either, so a
+	// credential that expired since cannot turn it into a failed job.
+	if ri.Complete {
+		return nil
+	}
+	if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
 		return err
 	}
 
@@ -560,27 +666,47 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	ctx, cancelCause := context.WithCancelCause(ctx)
 	defer cancelCause(nil)
+	// refuse ends the job for its submitter: record why before stopping.
+	refuse := func(err error) {
+		if isBatchJobUnavailable(err) {
+			ri.pause(err)
+		} else {
+			ri.refuse(err)
+		}
+		cancelCause(err)
+	}
 
+	// The walk stops at a listing failure: work after it is left to a later
+	// run, so the cursor cannot move past what was not listed.
+	walkCtx, stopWalk := context.WithCancel(ctx)
+	defer stopWalk()
 	results := make(chan itemOrErr[ObjectInfo], workerSize)
 	go func() {
-		prefixes := r.Prefix.F()
-		if len(prefixes) == 0 {
-			prefixes = []string{""}
-		}
-		for _, prefix := range prefixes {
+		// One ascending walk over disjoint prefixes, which the cursor
+		// describes.
+		for _, prefix := range batchCanonicalPrefixes(r.Prefix.F()) {
+			if walkCtx.Err() != nil {
+				break
+			}
 			prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
-			err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
+			err := api.Walk(walkCtx, r.Bucket, prefix, prefixResultCh, WalkOptions{
 				Marker:       lastObject,
 				LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
 				VersionsSort: WalkVersionsSortDesc,
 			})
 			if err != nil {
-				cancelCause(err)
+				if walkCtx.Err() == nil {
+					cancelCause(batchListingError(err))
+				}
 				xioutil.SafeClose(results)
 				return
 			}
 			for result := range prefixResultCh {
-				results <- result
+				select {
+				case results <- result:
+				case <-walkCtx.Done():
+					// Stopped: let the walk end without a reader.
+				}
 			}
 		}
 		xioutil.SafeClose(results)
@@ -588,7 +714,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	// Goroutine to periodically save batch-expire job's in-memory state
 	saverQuitCh := make(chan struct{})
+	saverDoneCh := make(chan struct{})
 	go func() {
+		defer close(saverDoneCh)
 		saveTicker := time.NewTicker(10 * time.Second)
 		defer saveTicker.Stop()
 		quit := false
@@ -615,9 +743,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	expireCh := make(chan []expireObjInfo, workerSize)
 	expireDoneCh := make(chan struct{})
+	var walkDone atomic.Bool
 	go func() {
 		defer close(expireDoneCh)
-		batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+		batchObjsForDelete(ctx, refuse, walkDone.Load, r, ri, job, api, wk, expireCh)
 	}()
 
 	var (
@@ -629,6 +758,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		done          bool
 	)
 	deleteMarkerCountMap := map[string]int64{}
+	// pushToExpire is called between keys only, so that a batch holds all
+	// the versions of its keys and its last key is done with it.
 	pushToExpire := func() {
 		// set preObject deleteMarkerCount
 		if len(toDel) > 0 {
@@ -653,6 +784,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			}
 		}
 	}
+	var listingErr error
 	for {
 		select {
 		case result, ok := <-results:
@@ -662,7 +794,14 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			}
 			if result.Err != nil {
 				failed = true
+				listingErr = batchListingError(result.Err)
 				batchLogIf(ctx, result.Err)
+				stopWalk()
+				done = true
+				break
+			}
+			if !batchResumeAfter(lastObject, result.Item.Name) {
+				// Completed by the run that set the cursor.
 				continue
 			}
 			if result.Item.DeleteMarker {
@@ -720,7 +859,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			toDel = append(toDel, expireObjInfo{
 				ObjectInfo: result.Item,
 			})
-			pushToExpire()
 		case <-ctx.Done():
 			done = true
 		}
@@ -729,16 +867,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		}
 	}
 
-	if context.Cause(ctx) != nil {
-		xioutil.SafeClose(expireCh)
-		return context.Cause(ctx)
-	}
-	pushToExpire()
-	// Send any remaining objects downstream
-	if len(toDel) > 0 {
-		select {
-		case <-ctx.Done():
-		case expireCh <- toDel:
+	if context.Cause(ctx) == nil {
+		walkDone.Store(!failed)
+		pushToExpire()
+		// Send any remaining objects downstream
+		if len(toDel) > 0 {
+			select {
+			case <-ctx.Done():
+			case expireCh <- toDel:
+			}
 		}
 	}
 	xioutil.SafeClose(expireCh)
@@ -746,13 +883,40 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	<-expireDoneCh // waits for the expire goroutine to complete
 	wk.Wait()      // waits for all expire workers to retire
 
-	ri.Complete = !failed && ri.ObjectsFailed == 0
-	ri.Failed = failed || ri.ObjectsFailed > 0
-	globalBatchJobsMetrics.save(job.ID, ri)
-
-	// Close the saverQuitCh - this also triggers saving in-memory state
-	// immediately one last time before we exit this method.
+	// Persist the final state once the saver has stopped, so that its last
+	// save cannot overwrite it.
 	xioutil.SafeClose(saverQuitCh)
+	<-saverDoneCh
+	refused := ri.refused()
+	cause := context.Cause(ctx)
+	if cause == nil {
+		cause = listingErr
+	}
+	if cause != nil && !refused {
+		if isBatchJobUnavailable(cause) {
+			return ri.savePaused(ctx, api, job, cause)
+		}
+		if isErrBucketNotFound(cause) {
+			ri.mu.Lock()
+			ri.Failed = true
+			ri.mu.Unlock()
+		}
+		saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
+		batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
+		cancel()
+		globalBatchJobsMetrics.save(job.ID, ri)
+		return cause
+	}
+
+	ri.mu.Lock()
+	itemsFailed := ri.expireFailures()
+	ri.Complete = !refused && !failed && !itemsFailed
+	ri.Failed = refused || failed || itemsFailed
+	ri.mu.Unlock()
+	globalBatchJobsMetrics.save(job.ID, ri)
+	saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
+	batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
+	cancel()
 
 	// Notify expire jobs final status to the configured endpoint
 	buf, _ := json.Marshal(ri)

cmd/batch-handlers_gen.go

@@ -179,6 +179,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "Submitter":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Submitter")
+					return
+				}
+				z.Submitter = nil
+			} else {
+				if z.Submitter == nil {
+					z.Submitter = new(batchJobSubmitter)
+				}
+				err = z.Submitter.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Submitter")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -192,9 +210,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 7
 	// write "ID"
-	err = en.Append(0x86, 0xa2, 0x49, 0x44)
+	err = en.Append(0x87, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -274,15 +292,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "Submitter"
+	err = en.Append(0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
+	if err != nil {
+		return
+	}
+	if z.Submitter == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.Submitter.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Submitter")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
 	// string "ID"
-	o = append(o, 0x86, 0xa2, 0x49, 0x44)
+	o = append(o, 0x87, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -323,6 +358,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "Submitter"
+	o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
+	if z.Submitter == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.Submitter.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Submitter")
+			return
+		}
+	}
 	return
 }
 
@@ -413,6 +459,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "Submitter":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Submitter = nil
+			} else {
+				if z.Submitter == nil {
+					z.Submitter = new(batchJobSubmitter)
+				}
+				bts, err = z.Submitter.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Submitter")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -445,6 +508,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.Expire.Msgsize()
 	}
+	s += 10
+	if z.Submitter == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.Submitter.Msgsize()
+	}
 	return
 }
 
@@ -568,6 +637,30 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BytesFailed")
 				return
 			}
+		case "vsl":
+			z.VersionsSkippedLocked, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "VersionsSkippedLocked")
+				return
+			}
+		case "kr":
+			z.KeysRefused, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "KeysRefused")
+				return
+			}
+		case "fr":
+			z.FailureReason, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "FailureReason")
+				return
+			}
+		case "pr":
+			z.PausedReason, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "PausedReason")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -581,9 +674,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 17
+	// map header, size 21
 	// write "v"
-	err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76)
+	err = en.Append(0xde, 0x0, 0x15, 0xa1, 0x76)
 	if err != nil {
 		return
 	}
@@ -752,15 +845,55 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BytesFailed")
 		return
 	}
+	// write "vsl"
+	err = en.Append(0xa3, 0x76, 0x73, 0x6c)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.VersionsSkippedLocked)
+	if err != nil {
+		err = msgp.WrapError(err, "VersionsSkippedLocked")
+		return
+	}
+	// write "kr"
+	err = en.Append(0xa2, 0x6b, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.KeysRefused)
+	if err != nil {
+		err = msgp.WrapError(err, "KeysRefused")
+		return
+	}
+	// write "fr"
+	err = en.Append(0xa2, 0x66, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.FailureReason)
+	if err != nil {
+		err = msgp.WrapError(err, "FailureReason")
+		return
+	}
+	// write "pr"
+	err = en.Append(0xa2, 0x70, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.PausedReason)
+	if err != nil {
+		err = msgp.WrapError(err, "PausedReason")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 17
+	// map header, size 21
 	// string "v"
-	o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76)
+	o = append(o, 0xde, 0x0, 0x15, 0xa1, 0x76)
 	o = msgp.AppendInt(o, z.Version)
 	// string "jid"
 	o = append(o, 0xa3, 0x6a, 0x69, 0x64)
@@ -810,6 +943,18 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "bf"
 	o = append(o, 0xa2, 0x62, 0x66)
 	o = msgp.AppendInt64(o, z.BytesFailed)
+	// string "vsl"
+	o = append(o, 0xa3, 0x76, 0x73, 0x6c)
+	o = msgp.AppendInt64(o, z.VersionsSkippedLocked)
+	// string "kr"
+	o = append(o, 0xa2, 0x6b, 0x72)
+	o = msgp.AppendInt64(o, z.KeysRefused)
+	// string "fr"
+	o = append(o, 0xa2, 0x66, 0x72)
+	o = msgp.AppendString(o, z.FailureReason)
+	// string "pr"
+	o = append(o, 0xa2, 0x70, 0x72)
+	o = msgp.AppendString(o, z.PausedReason)
 	return
 }
 
@@ -933,6 +1078,30 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BytesFailed")
 				return
 			}
+		case "vsl":
+			z.VersionsSkippedLocked, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersionsSkippedLocked")
+				return
+			}
+		case "kr":
+			z.KeysRefused, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "KeysRefused")
+				return
+			}
+		case "fr":
+			z.FailureReason, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "FailureReason")
+				return
+			}
+		case "pr":
+			z.PausedReason, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PausedReason")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -947,6 +1116,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *batchJobInfo) Msgsize() (s int) {
-	s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
+	s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.StringPrefixSize + len(z.FailureReason) + 3 + msgp.StringPrefixSize + len(z.PausedReason)
 	return
 }

cmd/batch-object-lock_test.go

@@ -0,0 +1,479 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/auth"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+var batchLockKinds = []string{"COMPLIANCE", "GOVERNANCE", "LEGALHOLD"}
+
+func batchLockMeta(kind string) map[string]string {
+	m := map[string]string{}
+	switch kind {
+	case "COMPLIANCE", "GOVERNANCE":
+		m[strings.ToLower(xhttp.AmzObjectLockMode)] = kind
+		m[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(24 * time.Hour))
+	case "LEGALHOLD":
+		m[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
+	}
+	return m
+}
+
+func batchPutVersion(t *testing.T, obj ObjectLayer, bucket, key string, meta map[string]string) ObjectInfo {
+	t.Helper()
+	data := []byte("batch lock " + key)
+	oi, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
+		ObjectOptions{Versioned: true, UserDefined: meta})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return oi
+}
+
+func batchVersionExists(t *testing.T, obj ObjectLayer, bucket, key, versionID string) bool {
+	t.Helper()
+	_, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: versionID})
+	switch {
+	case err == nil:
+		return true
+	case isErrVersionNotFound(err) || isErrObjectNotFound(err):
+		return false
+	}
+	t.Fatal(err)
+	return false
+}
+
+// batchJobStatusJSON returns the status response of a job as decoded JSON.
+func batchJobStatusJSON(t *testing.T, jobID string) map[string]any {
+	t.Helper()
+	cred := globalActiveCred
+	req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/status-job?jobId="+jobID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = req.ParseForm(); err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	req = req.WithContext(newContext(req, rec, "BatchJobStatus"))
+	adminAPIHandlers{}.BatchJobStatus(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+	}
+	var out struct{ LastMetric map[string]any }
+	if err = json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	return out.LastMetric
+}
+
+// Batch expiration keeps versions under retention or legal hold, on both
+// purge paths, and reports them; unlocked versions are still expired.
+func TestBatchExpireObjectLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			f := &batchAuthzFixture{ctx: t.Context(), obj: obj, bucket: bucket}
+			batchTestJobPool(t.Context(), t, obj)
+
+			type version struct{ key, id string }
+			var locked, unlocked []version
+			for _, retain := range []int{0, 1} {
+				for _, kind := range batchLockKinds {
+					key := fmt.Sprintf("r%d/%s", retain, kind)
+					older := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
+					batchPutVersion(t, obj, bucket, key, nil)
+					locked = append(locked, version{key, older.VersionID})
+				}
+				key := fmt.Sprintf("r%d/plain", retain)
+				older := batchPutVersion(t, obj, bucket, key, nil)
+				batchPutVersion(t, obj, bucket, key, nil)
+				unlocked = append(unlocked, version{key, older.VersionID})
+			}
+
+			var jobs []*BatchJobRequest
+			for _, retain := range []int{0, 1} {
+				job := fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: r%d/
+  rules:
+    - type: object
+      purge:
+        retainVersions: %d
+`, bucket, retain, retain)
+				rec, queued := f.submit(t, globalActiveCred, job)
+				if queued == nil {
+					t.Fatalf("%s: submission status %d: %s", instance, rec.Code, rec.Body.String())
+				}
+				// Running a job again, as a resumed job does, must not count
+				// the versions it left a second time.
+				f.run(t, queued)
+				f.run(t, queued)
+				jobs = append(jobs, queued)
+			}
+
+			for _, v := range locked {
+				if !batchVersionExists(t, obj, bucket, v.key, v.id) {
+					t.Errorf("%s: %s: locked version %s was expired", instance, v.key, v.id)
+				}
+			}
+			for _, v := range unlocked {
+				if batchVersionExists(t, obj, bucket, v.key, v.id) {
+					t.Errorf("%s: %s: control: unlocked version %s was not expired", instance, v.key, v.id)
+				}
+			}
+			for i, field := range []string{"keysRefused", "versionsSkippedLocked"} {
+				m := batchJobStatusJSON(t, jobs[i].ID)
+				if got, _ := m[field].(float64); got != float64(len(batchLockKinds)) {
+					t.Errorf("%s: retain %d: %s = %v, want %d", instance, i, field, m[field], len(batchLockKinds))
+				}
+				if m["complete"] != true || m["failed"] != false {
+					t.Errorf("%s: retain %d: complete=%v failed=%v", instance, i, m["complete"], m["failed"])
+				}
+			}
+		},
+	})
+}
+
+type batchLockTransport func(*http.Request) (*http.Response, error)
+
+func (f batchLockTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// batchLockRemote is a source that serves replacement for every read.
+func batchLockRemote(t *testing.T, replacement string) *minio.Core {
+	t.Helper()
+	core, err := minio.NewCore("source.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusOK, Header: http.Header{
+				"Content-Length": []string{strconv.Itoa(len(replacement))},
+				"Last-Modified":  []string{UTCNow().Format(http.TimeFormat)},
+				"Etag":           []string{"\"batchetag\""},
+			}, Body: io.NopCloser(strings.NewReader(replacement)), ContentLength: int64(len(replacement)), Request: r}, nil
+		}),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return core
+}
+
+func batchRootSubmitter(t *testing.T) *batchJobSubmitter {
+	t.Helper()
+	sub, err := newBatchJobSubmitter(globalActiveCred)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return sub
+}
+
+// A pull leaves a destination version under Object Lock as it is, whether a
+// single PUT, a multipart completion or a delete marker would replace it.
+func TestBatchPullObjectLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			const replacement = "replacement from remote"
+			core := batchLockRemote(t, replacement)
+			sub := batchRootSubmitter(t)
+			job := BatchJobReplicateV1{
+				Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+				Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+			}
+			for _, kind := range batchLockKinds {
+				for _, path := range []string{"put", "multipart", "marker", "directory"} {
+					key := "pull/" + kind + "/" + path
+					if path == "directory" {
+						key += "/"
+					}
+					before := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
+					src := ObjectInfo{Name: key, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}}
+					switch path {
+					case "multipart":
+						src.ETag = "batchetag-1"
+					case "marker":
+						src.DeleteMarker = true
+					}
+					err := job.ReplicateFromSource(t.Context(), obj, core, src, false, sub)
+					if !isErrObjectLocked(err) {
+						t.Errorf("%s: %s: pull returned %v, want ObjectLocked", instance, key, err)
+					}
+					after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
+					if gerr != nil {
+						t.Errorf("%s: %s: locked version lost: %v", instance, key, gerr)
+						continue
+					}
+					if !enforceRetentionForDeletion(t.Context(), after) || after.ETag != before.ETag {
+						t.Errorf("%s: %s: locked version was replaced", instance, key)
+					}
+				}
+			}
+
+			// Controls: a new version, and a new delete marker that hides a
+			// locked version without replacing it.
+			locked := batchPutVersion(t, obj, bucket, "pull/new", batchLockMeta("COMPLIANCE"))
+			newVersion := mustGetUUID()
+			if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
+				Name: "pull/new", VersionID: newVersion, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
+			}, false, sub); err != nil {
+				t.Fatalf("%s: new version: %v", instance, err)
+			}
+			if got := readCopyChecksumObject(t, obj, bucket, "pull/new", ObjectOptions{VersionID: newVersion}); string(got) != replacement {
+				t.Errorf("%s: new version content %q", instance, got)
+			}
+			marker := mustGetUUID()
+			if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
+				Name: "pull/new", VersionID: marker, ModTime: UTCNow(), DeleteMarker: true,
+			}, false, sub); err != nil {
+				t.Fatalf("%s: new marker: %v", instance, err)
+			}
+			if !batchVersionExists(t, obj, bucket, "pull/new", locked.VersionID) {
+				t.Errorf("%s: a new marker removed a locked version", instance)
+			}
+		},
+	})
+}
+
+// A pull applies retention as a PUT to the destination would: copied
+// retention is validated, and a version without one gets the bucket default.
+func TestBatchPullRetentionMetadata(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			meta, err := globalBucketMetadataSys.Get(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			meta.ObjectLockConfigXML = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>`)
+			meta.VersioningConfigXML = enabledBucketVersioningConfig
+			if err = meta.parseAllConfigs(t.Context(), obj); err != nil {
+				t.Fatal(err)
+			}
+			globalBucketMetadataSys.Set(bucket, meta)
+			core := batchLockRemote(t, "retention content")
+			sub := batchRootSubmitter(t)
+			job := BatchJobReplicateV1{
+				Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+				Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+			}
+			pull := func(key string, meta map[string]string) (ObjectInfo, error) {
+				vid := mustGetUUID()
+				err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{Name: key, VersionID: vid, ModTime: UTCNow().Add(-time.Hour), ETag: "batchetag", UserDefined: meta}, false, sub)
+				if err != nil {
+					return ObjectInfo{}, err
+				}
+				return obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: vid})
+			}
+			oi, err := pull("retention/default", map[string]string{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
+			if ret.Mode != objectlock.RetGovernance || !ret.RetainUntilDate.After(UTCNow().Add(23*time.Hour)) {
+				t.Errorf("%s: default retention not applied with the destination's time: %+v", instance, ret)
+			}
+			until := UTCNow().Add(48 * time.Hour)
+			oi, err = pull("retention/copied", map[string]string{
+				xhttp.AmzObjectLockMode: "COMPLIANCE", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ret = objectlock.GetObjectRetentionMeta(oi.UserDefined); ret.Mode != objectlock.RetCompliance || !ret.RetainUntilDate.Equal(until.Truncate(time.Millisecond)) {
+				t.Errorf("%s: copied retention: %+v", instance, ret)
+			}
+			if _, err = pull("retention/invalid", map[string]string{xhttp.AmzObjectLockMode: "FOREVER", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until)}); err == nil {
+				t.Errorf("%s: invalid copied retention was accepted", instance)
+			}
+		},
+	})
+}
+
+// Key rotation does not rewrite a version under Object Lock.
+func TestBatchKeyRotationObjectLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject", "GetObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("batch-lock-key")
+			defer func() { GlobalKMS = oldKMS }()
+			sub := batchRootSubmitter(t)
+			job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			for _, kind := range append([]string{"none"}, batchLockKinds...) {
+				key := "rotate/" + kind
+				headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
+				switch kind {
+				case "COMPLIANCE", "GOVERNANCE":
+					headers[xhttp.AmzObjectLockMode] = kind
+					headers[xhttp.AmzObjectLockRetainUntilDate] = UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
+				case "LEGALHOLD":
+					headers[xhttp.AmzObjectLockLegalHold] = "ON"
+				}
+				putCopyChecksumSource(t, router, cred, bucket, key, []byte("rotate "+kind), headers)
+				before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
+				if err != nil || !crypto.S3.IsEncrypted(before.UserDefined) {
+					t.Fatalf("%s: %s: encrypted control: %v", instance, key, err)
+				}
+				err = job.KeyRotate(t.Context(), obj, before, sub)
+				after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
+				if gerr != nil {
+					t.Fatal(gerr)
+				}
+				if kind == "none" {
+					if err != nil || after.ModTime.Equal(before.ModTime) {
+						t.Errorf("%s: control: unlocked version not rotated: %v", instance, err)
+					}
+					continue
+				}
+				var locked ObjectLocked
+				if !errors.As(err, &locked) || !after.ModTime.Equal(before.ModTime) {
+					t.Errorf("%s: %s: rotation of a locked version returned %v; rewritten=%v", instance, key, err, !after.ModTime.Equal(before.ModTime))
+				}
+			}
+		},
+	})
+}
+
+// A push reads locked versions as any authorized read does, and a submitter
+// that may not read a version does not send it.
+func TestBatchPushReads(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			var puts []string
+			target, err := minio.NewCore("target.invalid", &minio.Options{
+				Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+				Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+					if r.Body != nil {
+						io.Copy(io.Discard, r.Body)
+						r.Body.Close()
+					}
+					if r.Method == http.MethodPut {
+						puts = append(puts, r.URL.Path)
+					}
+					return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": []string{"\"pushetag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+				}),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			job := BatchJobReplicateV1{
+				Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+				Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+			}
+			src := batchPutVersion(t, obj, bucket, "push/locked", batchLockMeta("COMPLIANCE"))
+			if err := job.ReplicateToTarget(t.Context(), obj, target, src, false, batchRootSubmitter(t)); err != nil {
+				t.Fatalf("%s: authorized push of a locked version: %v", instance, err)
+			}
+			if len(puts) != 1 {
+				t.Fatalf("%s: authorized push sent %d writes", instance, len(puts))
+			}
+
+			// A user that may list but not read.
+			p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:StartBatchJob"]},{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::` + bucket + `"]}]}`))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err = globalIAMSys.SetPolicy(t.Context(), "batch-list-only", *p); err != nil {
+				t.Fatal(err)
+			}
+			user := auth.Credentials{AccessKey: "batch-list-user", SecretKey: "batch-list-user-secret"}
+			if _, err = globalIAMSys.CreateUser(t.Context(), user.AccessKey, madmin.AddOrUpdateUserReq{SecretKey: user.SecretKey, Status: madmin.AccountEnabled}); err != nil {
+				t.Fatal(err)
+			}
+			if _, err = globalIAMSys.PolicyDBSet(t.Context(), user.AccessKey, "batch-list-only", regUser, false); err != nil {
+				t.Fatal(err)
+			}
+			sub, err := newBatchJobSubmitter(globalIAMSysUser(t, user.AccessKey))
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = job.ReplicateToTarget(t.Context(), obj, target, src, false, sub)
+			if !isBatchJobAuthzError(err) || len(puts) != 1 {
+				t.Fatalf("%s: unauthorized push returned %v and sent %d writes", instance, err, len(puts)-1)
+			}
+		},
+	})
+}
+
+// globalIAMSysUser returns the stored credential of accessKey, as a request
+// authenticated with it has.
+func globalIAMSysUser(t *testing.T, accessKey string) auth.Credentials {
+	t.Helper()
+	u, ok := globalIAMSys.store.GetUser(accessKey)
+	if !ok {
+		t.Fatalf("no user %s", accessKey)
+	}
+	return u.Credentials
+}
+
+// The cursor of a walk done out of order moves only past keys whose items
+// are all done, and counts each item once.
+func TestBatchProgressCursor(t *testing.T) {
+	ri := &batchJobInfo{}
+	p := newBatchProgress(ri, "bucket")
+	items := []string{"a", "a", "b", "c", "c"}
+	seqs := make([]uint64, len(items))
+	for i := range items {
+		seqs[i] = p.begin()
+	}
+	skip := batchCounts{versionsSkipped: 1}
+	p.done(seqs[1], "a", skip)
+	p.done(seqs[3], "c", skip)
+	if ri.Object != "" || ri.VersionsSkippedLocked != 0 {
+		t.Fatalf("cursor moved before the first item: %q %d", ri.Object, ri.VersionsSkippedLocked)
+	}
+	p.done(seqs[0], "a", skip)
+	p.done(seqs[2], "b", skip)
+	// a is done; b is done but c may still have items.
+	if ri.Object != "b" && ri.Object != "a" {
+		t.Fatalf("cursor %q", ri.Object)
+	}
+	if ri.Object == "a" && ri.VersionsSkippedLocked != 2 {
+		t.Fatalf("counts %d at cursor a", ri.VersionsSkippedLocked)
+	}
+	// c's last item never finishes, as when a run stops: the cursor stays
+	// before c and c is not counted.
+	p.finish()
+	if ri.Object != "b" || ri.VersionsSkippedLocked != 3 {
+		t.Fatalf("interrupted: cursor %q counts %d, want b 3", ri.Object, ri.VersionsSkippedLocked)
+	}
+	p.done(seqs[4], "c", skip)
+	p.finish()
+	if ri.Object != "c" || ri.VersionsSkippedLocked != 5 {
+		t.Fatalf("finished: cursor %q counts %d, want c 5", ri.Object, ri.VersionsSkippedLocked)
+	}
+}

cmd/batch-progress.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+// batchCounts are outcomes of a batch job that a later run of the job would
+// see again, and so must be persisted together with the cursor of the walk.
+type batchCounts struct {
+	objects             int64
+	objectsFailed       int64
+	deleteMarkersFailed int64
+	versionsSkipped     int64
+	keysRefused         int64
+}
+
+func (c *batchCounts) add(o batchCounts) {
+	c.objects += o.objects
+	c.objectsFailed += o.objectsFailed
+	c.deleteMarkersFailed += o.deleteMarkersFailed
+	c.versionsSkipped += o.versionsSkipped
+	c.keysRefused += o.keysRefused
+}
+
+//msgp:ignore batchProgress batchProgressItem batchCounts
+
+// batchProgress is the cursor of a walk whose items are done concurrently.
+//
+// The items of a walk are numbered in walk order. A key is done when every
+// item of it, and every item before it, is done; only then does the cursor
+// move past it and its counts become part of the job's counts. The cursor and
+// the counts are persisted together, so a job resumed after the cursor
+// neither skips an item that was not done nor counts an item twice.
+type batchProgress struct {
+	mu      sync.Mutex
+	ri      *batchJobInfo
+	bucket  string
+	issued  uint64
+	next    uint64
+	pending map[uint64]batchProgressItem
+	// key is the key of the item last committed, and held are the counts of
+	// its items so far.
+	key  string
+	held batchCounts
+}
+
+type batchProgressItem struct {
+	key    string
+	counts batchCounts
+}
+
+func newBatchProgress(ri *batchJobInfo, bucket string) *batchProgress {
+	return &batchProgress{ri: ri, bucket: bucket, pending: make(map[uint64]batchProgressItem)}
+}
+
+// begin numbers the next item of the walk.
+func (p *batchProgress) begin() uint64 {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	seq := p.issued
+	p.issued++
+	return seq
+}
+
+// done records item seq, of key, as done with counts.
+func (p *batchProgress) done(seq uint64, key string, counts batchCounts) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.pending[seq] = batchProgressItem{key: key, counts: counts}
+	for {
+		item, ok := p.pending[p.next]
+		if !ok {
+			return
+		}
+		delete(p.pending, p.next)
+		p.next++
+		if item.key != p.key {
+			p.commitLocked()
+			p.key = item.key
+		}
+		p.held.add(item.counts)
+	}
+}
+
+// finish commits the last key if every item of the walk is done. Call it
+// once no item is in progress.
+func (p *batchProgress) finish() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.next == p.issued {
+		p.commitLocked()
+	}
+}
+
+func (p *batchProgress) commitLocked() {
+	if p.key == "" {
+		return
+	}
+	ri := p.ri
+	ri.mu.Lock()
+	ri.Bucket = p.bucket
+	ri.Object = p.key
+	ri.Objects += p.held.objects
+	ri.ObjectsFailed += p.held.objectsFailed
+	ri.DeleteMarkersFailed += p.held.deleteMarkersFailed
+	ri.VersionsSkippedLocked += p.held.versionsSkipped
+	ri.KeysRefused += p.held.keysRefused
+	ri.mu.Unlock()
+	p.held = batchCounts{}
+}
+
+// countRemoved counts versions a job removed. A removed version is gone for
+// any later run, so it is counted at once, not at the cursor.
+func (ri *batchJobInfo) countRemoved(bucket string, objects, markers int64) {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	ri.Bucket = bucket
+	ri.Objects += objects
+	ri.DeleteMarkers += markers
+}
+
+// countSkipped counts a version a job left in place because Object Lock
+// protects it, for a job that is counted per run rather than at a cursor.
+func (ri *batchJobInfo) countSkipped() {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	ri.VersionsSkippedLocked++
+}
+
+// batchCanonicalPrefixes orders the prefixes of a walk and drops a prefix
+// that another one covers. The keys of the walk are then visited once each,
+// in one ascending order, which a single cursor can describe.
+func batchCanonicalPrefixes(prefixes []string) []string {
+	if len(prefixes) == 0 {
+		return []string{""}
+	}
+	sorted := slices.Clone(prefixes)
+	slices.Sort(sorted)
+	out := sorted[:0]
+	for _, p := range sorted {
+		if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) {
+			continue
+		}
+		out = append(out, p)
+	}
+	return out
+}
+
+// batchResumeAfter reports whether a resumed walk returned a key after the
+// cursor. The cursor key is done; the walk forwards conservatively and
+// returns it again. Keys compare as the walk orders them, by name, so a
+// directory object comes before the keys under it.
+func batchResumeAfter(cursor, name string) bool {
+	return cursor == "" || name > cursor
+}
+
+// batchJobStatus is the status response: madmin.BatchJobStatus with the
+// fields this server adds. The added fields are additive; a client that does
+// not know them ignores them.
+type batchJobStatus struct {
+	LastMetric batchJobStatusMetric
+}
+
+type batchJobStatusMetric struct {
+	madmin.JobMetric
+	VersionsSkippedLocked int64  `json:"versionsSkippedLocked,omitempty"`
+	KeysRefused           int64  `json:"keysRefused,omitempty"`
+	FailureReason         string `json:"failureReason,omitempty"`
+	PausedReason          string `json:"pausedReason,omitempty"`
+}
+
+func (ri *batchJobInfo) statusMetric() batchJobStatusMetric {
+	ri.mu.RLock()
+	defer ri.mu.RUnlock()
+	return batchJobStatusMetric{
+		JobMetric:             ri.metric(),
+		VersionsSkippedLocked: ri.VersionsSkippedLocked,
+		KeysRefused:           ri.KeysRefused,
+		FailureReason:         ri.FailureReason,
+		PausedReason:          ri.PausedReason,
+	}
+}

cmd/batch-retry.go

@@ -0,0 +1,225 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// An unavailable dependency is not an authorization decision. It stops the
+// current attempt without turning an outage into either an allow or a denial.
+type batchJobUnavailableError struct {
+	reason string
+	err    error
+}
+
+func (e batchJobUnavailableError) Error() string { return fmt.Sprintf("%s: %v", e.reason, e.err) }
+func (e batchJobUnavailableError) Unwrap() error { return e.err }
+func batchUnavailable(reason string, err error) error {
+	if err == nil {
+		return nil
+	}
+	if isBatchJobUnavailable(err) {
+		return err
+	}
+	return batchJobUnavailableError{reason: reason, err: err}
+}
+
+func isBatchJobUnavailable(err error) bool { var e batchJobUnavailableError; return errors.As(err, &e) }
+
+func isBatchJobStopped(err error) bool {
+	return isBatchJobAuthzError(err) || isBatchJobUnavailable(err)
+}
+
+func batchListingError(err error) error {
+	if isErrBucketNotFound(err) {
+		return err
+	}
+	return batchUnavailable("object listing unavailable", err)
+}
+
+func (ri *batchJobInfo) pause(err error) {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	// A concurrent explicit denial wins over a transient worker error.
+	if ri.FailureReason != "" {
+		return
+	}
+	ri.Complete, ri.Failed = false, false
+	ri.PausedReason = err.Error()
+}
+
+func (ri *batchJobInfo) savePaused(ctx context.Context, api ObjectLayer, job BatchJobRequest, err error) error {
+	ri.pause(err)
+	globalBatchJobsMetrics.save(job.ID, ri)
+	saveCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+	defer cancel()
+	if saveErr := ri.updateAfter(saveCtx, api, 0, job); saveErr != nil {
+		return batchUnavailable("saving paused job", errors.Join(err, saveErr))
+	}
+	return err
+}
+
+// Authenticate the HTTP request normally, but keep dependency errors through
+// the authorization step so submission returns 503, not a permanent 403.
+func validateBatchJobAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) (ObjectLayer, auth.Credentials) {
+	api := newObjectLayerFn()
+	if api == nil || globalNotificationSys == nil || !batchIAMReady() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return nil, auth.Credentials{}
+	}
+	cred, owner, code := validateAdminSignature(ctx, r, "")
+	if code != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(code), r.URL)
+		return nil, cred
+	}
+	submitter, err := newBatchJobSubmitter(cred)
+	if err == nil {
+		var principal batchJobPrincipal
+		principal, err = submitter.principal(ctx)
+		if err == nil {
+			err = principal.authorizeArgs(policy.Args{
+				AccountName: cred.AccessKey, Groups: cred.Groups,
+				Action: policy.Action(policy.StartBatchJobAction), ConditionValues: getConditionValues(r, "", cred), IsOwner: owner, Claims: principal.claims,
+			}, nil)
+		}
+	}
+	if err != nil {
+		apiErr := errorCodes.ToAPIErrWithErr(ErrAccessDenied, err)
+		if isBatchJobUnavailable(err) {
+			apiErr = batchServiceUnavailable(err)
+		}
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
+		return nil, cred
+	}
+	return api, cred
+}
+
+// One owner record spans queued, running and paused attempts. The request
+// pointer is the generation token. Canceled records fence stale requests until
+// shutdown. Terminal records fence discovery until the startup scan finishes,
+// including jobs submitted and terminated while the scan was unavailable.
+type batchJobOwner struct {
+	req     *BatchJobRequest
+	cancel  context.CancelFunc
+	state   string
+	retries uint
+	due     time.Time
+}
+
+func (j *BatchJobPool) beginJob(job *BatchJobRequest) bool {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	o := j.jobOwners[job.ID]
+	if o == nil || o.req != job || o.state != "queued" || job.ctx.Err() != nil || j.ctx.Err() != nil {
+		return false
+	}
+	o.state = "running"
+	return true
+}
+
+func (j *BatchJobPool) finishJob(job *BatchJobRequest, err error) {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	o := j.jobOwners[job.ID]
+	if o == nil || o.req != job || o.state != "running" {
+		return
+	}
+	if isBatchJobUnavailable(err) && job.ctx.Err() == nil && j.ctx.Err() == nil {
+		o.due = time.Now().Add(batchJobRetryDelay(o.retries))
+		o.retries++
+		o.state = "paused"
+		return
+	}
+	o.cancel()
+	if j.discoveryDone {
+		delete(j.jobOwners, job.ID)
+	} else {
+		j.jobOwners[job.ID] = &batchJobOwner{state: "terminal"}
+	}
+}
+
+// After the only startup discovery has returned, no loaded manifest remains
+// to re-admit a terminal job. Release its marker, but keep queued, running,
+// paused and canceled ownership intact. New submissions use new job IDs.
+func (j *BatchJobPool) finishDiscovery() {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	j.discoveryDone = true
+	for id, owner := range j.jobOwners {
+		if owner.state == "terminal" {
+			delete(j.jobOwners, id)
+		}
+	}
+}
+
+func batchJobRetryDelay(retries uint) time.Duration {
+	delay := 30 * time.Second * time.Duration(1<<min(retries, 4))
+	return delay/2 + time.Duration(rand.Float64()*float64(delay))
+}
+
+// retryDue holds the ownership lock across the nonblocking handoff. A fast
+// worker cannot replace a pause while the scheduler removes the old pause.
+func (j *BatchJobPool) retryDue(now time.Time) {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	if j.ctx.Err() != nil {
+		return
+	}
+	for _, o := range j.jobOwners {
+		if o.state != "paused" || now.Before(o.due) || o.req.ctx.Err() != nil {
+			continue
+		}
+		select {
+		case j.jobCh <- o.req:
+			o.state = "queued"
+		default:
+		}
+	}
+}
+
+func (j *BatchJobPool) retryLoop() {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-j.ctx.Done():
+			return
+		case now := <-ticker.C:
+			j.retryDue(now)
+		}
+	}
+}
+
+func batchServiceUnavailable(err error) APIError {
+	return APIError{Code: "ServiceUnavailable", Description: err.Error(), HTTPStatusCode: http.StatusServiceUnavailable}
+}
+
+// initStore precedes the first successful cache load. During that interval a
+// missing identity or policy is not a revocation decision, including for root.
+func batchIAMReady() bool {
+	sys := globalIAMSys
+	if sys == nil {
+		return false
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	if sys.store == nil {
+		return false
+	}
+	select {
+	case <-sys.configLoaded:
+		return true
+	default:
+		return false
+	}
+}

cmd/batch-retry_test.go

@@ -0,0 +1,349 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"github.com/tinylib/msgp/msgp"
+)
+
+func waitBatchState(t *testing.T, pool *BatchJobPool, id, state string) {
+	t.Helper()
+	timeout := time.NewTimer(10 * time.Second)
+	defer timeout.Stop()
+	tick := time.NewTicker(time.Millisecond)
+	defer tick.Stop()
+	for {
+		pool.jmu.Lock()
+		owner := pool.jobOwners[id]
+		ready := owner == nil && state == ""
+		if owner != nil {
+			ready = owner.state == state
+		}
+		pool.jmu.Unlock()
+		if ready {
+			return
+		}
+		select {
+		case <-timeout.C:
+			t.Fatalf("job did not reach %q", state)
+		case <-tick.C:
+		}
+	}
+}
+
+func TestBatchPluginPauseRecoversInProcess(t *testing.T) {
+	for _, midrun := range []bool{false, true} {
+		t.Run(map[bool]string{false: "before-start", true: "during-delete"}[midrun], func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			older := f.putVersions(t, "data/key")
+			var unavailable atomic.Bool
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				var request struct {
+					Input policy.Args `json:"input"`
+				}
+				_ = json.NewDecoder(r.Body).Decode(&request)
+				if unavailable.Load() && (!midrun || request.Input.Action == policy.DeleteObjectVersionAction) {
+					w.WriteHeader(503)
+					io.WriteString(w, `{"result":true}`)
+					return
+				}
+				io.WriteString(w, `{"result":true}`)
+			}))
+			defer server.Close()
+			url, err := xnet.ParseHTTPURL(server.URL)
+			mustIAM(t, err)
+			previous := newGlobalAuthZPluginFn()
+			setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: url, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
+			defer setGlobalAuthZPlugin(previous)
+			rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+			if job == nil {
+				t.Fatalf("submit %d: %s", rec.Code, rec.Body.String())
+			}
+			pool := globalBatchJobPool
+			ctx, cancel := context.WithCancel(f.ctx)
+			pool.ctx = ctx
+			// Re-admit the fixture's consumed request with a context owned by this worker.
+			pool.jmu.Lock()
+			pool.jobOwners[job.ID].cancel()
+			delete(pool.jobOwners, job.ID)
+			pool.jmu.Unlock()
+			mustIAM(t, pool.queueJob(job))
+			unavailable.Store(true)
+			done := make(chan struct{})
+			go func() { defer close(done); pool.AddWorker() }()
+			defer func() { cancel(); <-done }()
+			waitBatchState(t, pool, job.ID, "paused")
+			ri := f.jobInfo(t, job)
+			if ri.Failed || ri.Complete || ri.FailureReason != "" || ri.PausedReason == "" || ri.Object != "" {
+				t.Fatalf("paused report: %+v", ri)
+			}
+			if !f.versionExists(t, "data/key", older) {
+				t.Fatal("outage authorized a deletion")
+			}
+			unavailable.Store(false)
+			pool.retryDue(time.Now().Add(time.Hour))
+			waitBatchState(t, pool, job.ID, "terminal")
+			ri = f.jobInfo(t, job)
+			if !ri.Complete || ri.Failed || ri.PausedReason != "" || ri.Objects != 1 || f.versionExists(t, "data/key", older) {
+				t.Fatalf("resumed report: %+v", ri)
+			}
+		})
+	}
+}
+
+func TestBatchSubmissionUnavailableIs503(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(503); io.WriteString(w, `{"result":true}`) }))
+	defer server.Close()
+	url, err := xnet.ParseHTTPURL(server.URL)
+	mustIAM(t, err)
+	previous := newGlobalAuthZPluginFn()
+	setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: url, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
+	defer setGlobalAuthZPlugin(previous)
+	rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+	if rec.Code != 503 || job != nil {
+		t.Fatalf("outage submission: %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestBatchPoolOwnershipAndCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	pool := &BatchJobPool{ctx: ctx, jobCh: make(chan *BatchJobRequest, 1)}
+	first, second := &BatchJobRequest{ID: "first"}, &BatchJobRequest{ID: "second"}
+	mustIAM(t, pool.queueJob(first))
+	mustIAM(t, pool.queueJob(second))
+	if pool.jobOwners[second.ID].state != "paused" {
+		t.Fatal("full queue lost pending owner")
+	}
+	original := second.ctx
+	var wg sync.WaitGroup
+	for range 8 {
+		wg.Go(func() {
+			if err := pool.queueJob(&BatchJobRequest{ID: "second"}); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+	wg.Wait()
+	if second.ctx != original || len(pool.jobCh) != 1 {
+		t.Fatal("duplicate changed ownership")
+	}
+	if !pool.beginJob(<-pool.jobCh) {
+		t.Fatal("first not owned")
+	}
+	pool.finishJob(first, nil)
+	mustIAM(t, pool.queueJob(&BatchJobRequest{ID: first.ID}))
+	if len(pool.jobCh) != 0 {
+		t.Fatal("late discovery re-admitted a terminal job")
+	}
+	pool.retryDue(time.Now())
+	next := <-pool.jobCh
+	if next != second || !pool.beginJob(next) {
+		t.Fatal("pending handoff lost ownership")
+	}
+	for range 2 {
+		pool.finishJob(second, batchUnavailable("test", errors.New("outage")))
+		pool.retryDue(time.Now().Add(time.Hour))
+		if !pool.beginJob(<-pool.jobCh) {
+			t.Fatal("retry lost owner")
+		}
+	}
+	if pool.jobOwners[second.ID].retries != 2 {
+		t.Fatal("retry count reset during handoff")
+	}
+	mustIAM(t, pool.canceler(second.ID, true))
+	mustIAM(t, pool.canceler(second.ID, true))
+	pool.finishJob(second, batchUnavailable("late", errors.New("outage")))
+	pool.retryDue(time.Now().Add(time.Hour))
+	if len(pool.jobCh) != 0 || second.ctx.Err() == nil {
+		t.Fatal("cancel lost to retry")
+	}
+	if err := pool.queueJob(&BatchJobRequest{ID: second.ID}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("stale admission: %v", err)
+	}
+	// A cancellation before this node admitted a manifest is a tombstone too.
+	mustIAM(t, pool.canceler("not-loaded", true))
+	mustIAM(t, pool.canceler("not-loaded", true))
+	if err := pool.queueJob(&BatchJobRequest{ID: "not-loaded"}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("stale resume: %v", err)
+	}
+	cancel()
+	if err := pool.queueJob(&BatchJobRequest{ID: "shutdown"}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("shutdown admission: %v", err)
+	}
+	pool.retryDue(time.Now().Add(time.Hour)) // channel remains open; no send/close race.
+}
+
+// A cancellation of a job this node does not own leaves a record only while
+// startup discovery may still admit its manifest. Once discovery has
+// finished, canceling unknown IDs must not grow the ownership table.
+func TestBatchCancelUnknownJobAfterDiscovery(t *testing.T) {
+	pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
+	// Before discovery finished, the record fences the manifest it may admit.
+	mustIAM(t, pool.canceler("expire-before-discovery", true))
+	if err := pool.queueJob(&BatchJobRequest{ID: "expire-before-discovery"}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("admission of a job canceled during discovery: %v", err)
+	}
+	pool.finishDiscovery()
+	for i := range 1000 {
+		mustIAM(t, pool.canceler(fmt.Sprintf("expire-unknown-%d", i), true))
+	}
+	pool.jmu.Lock()
+	owners := len(pool.jobOwners)
+	pool.jmu.Unlock()
+	if owners != 1 {
+		t.Fatalf("%d owner records after canceling unknown jobs, want only the record from discovery", owners)
+	}
+	// An owned job is still canceled and fenced.
+	job := &BatchJobRequest{ID: "expire-owned"}
+	mustIAM(t, pool.queueJob(job))
+	mustIAM(t, pool.canceler(job.ID, true))
+	if job.ctx.Err() == nil {
+		t.Fatal("cancellation did not stop the owned job")
+	}
+	if err := pool.queueJob(&BatchJobRequest{ID: job.ID}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("stale admission of a canceled job: %v", err)
+	}
+}
+
+type batchPolicyOutage struct {
+	IAMStorageAPI
+	failed string
+	err    error
+}
+
+func (s *batchPolicyOutage) loadPolicyDoc(ctx context.Context, name string, out map[string]PolicyDoc) error {
+	if name == s.failed {
+		return s.err
+	}
+	return s.IAMStorageAPI.loadPolicyDoc(ctx, name, out)
+}
+
+func TestBatchPolicySnapshotKeepsUnavailableDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "retry-allow", batchAuthzFullAccess)
+	batchAuthzPolicy(f.ctx, t, f.sys, "retry-deny", `{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "retry-user", "retry-allow,retry-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	store := f.sys.store
+	original := store.IAMStorageAPI
+	defer func() { store.IAMStorageAPI = original }()
+	cache := store.lock()
+	delete(cache.iamPolicyDocsMap, "retry-deny")
+	store.unlock()
+	store.IAMStorageAPI = &batchPolicyOutage{IAMStorageAPI: original, failed: "retry-deny", err: errErasureReadQuorum}
+	request := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
+	if err := sub.authorize(f.ctx, request); !isBatchJobUnavailable(err) {
+		t.Fatalf("missing Deny must pause, got %v", err)
+	}
+	store.IAMStorageAPI = original
+	if err := sub.authorize(f.ctx, request); !isBatchJobAuthzError(err) {
+		t.Fatalf("recovered Deny was not enforced: %v", err)
+	}
+	// Root also waits for IAM readiness; startup must not bypass the pause gate.
+	root, err := newBatchJobSubmitter(globalActiveCred)
+	mustIAM(t, err)
+	f.sys.Lock()
+	f.sys.store = nil
+	f.sys.Unlock()
+	_, err = root.principal(f.ctx)
+	f.sys.Lock()
+	f.sys.store = store
+	f.sys.Unlock()
+	if !isBatchJobUnavailable(err) {
+		t.Fatalf("root before IAM readiness: %v", err)
+	}
+}
+
+func TestBatchPausedReportCompatibility(t *testing.T) {
+	before := &batchJobInfo{JobID: "paused", Object: "cursor", PausedReason: "IAM unavailable"}
+	data, err := before.MarshalMsg(nil)
+	mustIAM(t, err)
+	var after batchJobInfo
+	_, err = after.UnmarshalMsg(data)
+	mustIAM(t, err)
+	if after.PausedReason != before.PausedReason || after.clone().PausedReason != before.PausedReason || after.statusMetric().PausedReason != before.PausedReason {
+		t.Fatal("paused reason lost")
+	}
+	old := msgp.AppendMapHeader(nil, 2)
+	old = msgp.AppendString(old, "jid")
+	old = msgp.AppendString(old, "legacy")
+	old = msgp.AppendString(old, "cmp")
+	old = msgp.AppendBool(old, true)
+	var legacy batchJobInfo
+	_, err = legacy.UnmarshalMsg(old)
+	mustIAM(t, err)
+	if legacy.PausedReason != "" || !legacy.Complete {
+		t.Fatal("old report decode changed")
+	}
+	after.refuse(errBatchJobInvalid)
+	if after.PausedReason != "" || !after.Failed {
+		t.Fatal("terminal denial retained pause")
+	}
+}
+
+func TestBatchPersistenceFencesOlderExecutors(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	raw, err := readConfig(f.ctx, f.obj, getJobPath(*job))
+	mustIAM(t, err)
+	if !bytes.HasPrefix(raw, []byte(batchJobEnvelope)) {
+		t.Fatal("new manifest has no executor version fence")
+	}
+	var oldDecoder BatchJobRequest
+	if _, err := oldDecoder.UnmarshalMsg(raw); err == nil {
+		t.Fatal("old bare-msgpack executor could decode new manifest")
+	}
+	var loaded BatchJobRequest
+	mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+	if loaded.Submitter == nil {
+		t.Fatal("new executor lost submitter")
+	}
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started}
+	outage := batchUnavailable("test", errErasureReadQuorum)
+	if err := ri.savePaused(f.ctx, f.obj, *job, outage); !errors.Is(err, outage) {
+		t.Fatal(err)
+	}
+	reportPath, err := job.getJobReportPath()
+	mustIAM(t, err)
+	report, err := readConfig(f.ctx, f.obj, reportPath)
+	mustIAM(t, err)
+	if binary.LittleEndian.Uint16(report[2:4]) == batchExpireVersionV1 {
+		t.Fatal("old executor would accept paused report")
+	}
+	mustIAM(t, ri.load(f.ctx, f.obj, *job))
+	if ri.PausedReason == "" || ri.Failed {
+		t.Fatal("restart lost pause")
+	}
+	// A stale resume loaded the manifest before cancellation.
+	mustIAM(t, globalBatchJobPool.canceler(job.ID, true))
+	mustIAM(t, job.delete(f.ctx, f.obj))
+	if err := globalBatchJobPool.queueJob(&loaded); !errors.Is(err, context.Canceled) {
+		t.Fatalf("canceled job re-admitted: %v", err)
+	}
+	if err := loaded.load(f.ctx, f.obj, getJobPath(*job)); !errors.Is(err, errNoSuchJob) {
+		t.Fatalf("canceled manifest recreated: %v", err)
+	}
+}

cmd/batch-rotate.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"maps"
 	"math/rand"
@@ -35,6 +36,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/pgsty/silo-pkg/v3/env"
+	"github.com/pgsty/silo-pkg/v3/policy"
 	"github.com/pgsty/silo-pkg/v3/workers"
 )
 
@@ -161,8 +163,68 @@ func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error
 	return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
 }
 
-// KeyRotate rotates encryption key of an object
-func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
+// batchAuthoritativeVersion returns the version obj as all its copies
+// describe it. With several pools a version can have copies in more than one
+// pool, and a read returns one of them; the merged copy carries the newest
+// Object Lock and tag state. The caller holds the object's write lock.
+func batchAuthoritativeVersion(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
+	z, ok := api.(*erasureServerPools)
+	if !ok || z.SinglePool() {
+		return obj, nil
+	}
+	versionID := obj.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	return z.replicaObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{VersionID: versionID, NoLock: true, NoAuditLog: true})
+}
+
+// batchAuthoritativeRead is batchAuthoritativeVersion for a version that has
+// been opened for reading. It takes the object's read lock itself, since an
+// opened reader may hold none.
+func batchAuthoritativeRead(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
+	z, ok := api.(*erasureServerPools)
+	if !ok || z.SinglePool() {
+		return obj, nil
+	}
+	lk := z.NewNSLock(bucket, encodeDirObject(object))
+	lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.RUnlock(lkctx)
+	return batchAuthoritativeVersion(lkctx.Context(), api, bucket, object, obj)
+}
+
+// rotateAuthzRequests are the actions of rotating one version: it is read and
+// written again in place with the new key.
+func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzRequest {
+	versionID := oi.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	// The encryption the rewrite leaves: rotation keeps the kind of the
+	// version (rotateKey), and SSE-KMS takes the requested key.
+	values := map[string][]string{}
+	switch {
+	case crypto.S3KMS.IsEncrypted(oi.UserDefined):
+		values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
+		if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
+			values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
+		}
+	case crypto.S3.IsEncrypted(oi.UserDefined):
+		values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionAES}
+	}
+	return []batchAuthzRequest{
+		{action: policy.GetObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
+		{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
+		{action: policy.PutObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags, values: values},
+	}
+}
+
+// KeyRotate rotates encryption key of an object. A version that Object Lock
+// protects is not rewritten: it returns ObjectLocked.
+func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo, sub *batchJobSubmitter) error {
 	srcBucket := r.Bucket
 	srcObject := objInfo.Name
 
@@ -180,7 +242,8 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
 
-	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	// Lock the name writers lock: the stored name of a directory object.
+	lock := api.NewObjectNSLock(r.Bucket, encodeDirObject(objInfo.Name))
 	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err
@@ -194,10 +257,32 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 		VersionSuspended: versionSuspended,
 		NoLock:           true,
 	}
-	obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
+	obj, pool, err := batchRotateRead(ctx, api, r.Bucket, objInfo.Name, opts)
 	if err != nil {
 		return err
 	}
+	// Decide on the version as it is under the lock that the rewrite holds,
+	// before any key is requested, with the metadata of all its copies.
+	logical, err := batchAuthoritativeVersion(ctx, api, r.Bucket, objInfo.Name, obj)
+	if err != nil {
+		return err
+	}
+	if enforceRetentionForDeletion(ctx, logical) {
+		return ObjectLocked{Bucket: r.Bucket, Object: obj.Name, VersionID: obj.VersionID}
+	}
+	p, err := sub.principal(ctx)
+	if err != nil {
+		return err
+	}
+	// Authorize the copy that is read and rewritten, and the version as all
+	// its copies describe it.
+	for _, v := range []ObjectInfo{obj, logical} {
+		for _, req := range r.rotateAuthzRequests(v) {
+			if err := p.authorize(req); err != nil {
+				return err
+			}
+		}
+	}
 	oi := obj.Clone()
 	var (
 		newKeyID      string
@@ -225,16 +310,61 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	oi.metadataOnly = true
 	oi.keyRotation = true
 	maps.Copy(oi.UserDefined, encMetadata)
-	if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
-		VersionID: oi.VersionID,
-	}, ObjectOptions{
-		VersionID: oi.VersionID,
-		NoLock:    true,
-	}); err != nil {
-		return err
+	// The rewrite replaces the stored metadata with UserDefined, which a
+	// read returns without the tags and the expiry: keep them.
+	if oi.UserTags != "" {
+		oi.UserDefined[xhttp.AmzObjectTagging] = oi.UserTags
 	}
+	if !oi.Expires.IsZero() {
+		oi.UserDefined["expires"] = oi.ExpiresStr()
+	}
+	srcOpts := ObjectOptions{VersionID: oi.VersionID}
+	dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true}
+	if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
+		// Rewrite the copy that was read and authorized. The pools layer
+		// would route the rewrite by the latest version of the key, which
+		// can be in a pool that holds another write of this version ID.
+		//
+		// Check the pool's state again where the pools layer checks it,
+		// right before the write: a move can have started since the read.
+		if z.IsSuspended(pool) || z.IsPoolRebalancing(pool) {
+			return errBatchRotateMoving
+		}
+		dstOpts.replicaObjectInfo = z.replicaObjectInfo
+		name := encodeDirObject(oi.Name)
+		_, err = z.serverPools[pool].CopyObject(ctx, r.Bucket, name, r.Bucket, name, oi, srcOpts, dstOpts)
+	} else {
+		_, err = api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, srcOpts, dstOpts)
+	}
+	return err
+}
 
-	return nil
+// errBatchRotateMoving: the version is read from a pool that a decommission
+// or a rebalance is moving.
+var errBatchRotateMoving = errors.New("the version is being moved to another pool; rotate it once the move is done")
+
+// batchRotateRead reads the version that a rotation rewrites, as
+// GetObjectInfo does. With several pools it also returns the pool of the
+// copy it read, otherwise -1. The caller holds the object lock.
+//
+// A copy in a pool that a decommission or a rebalance is moving is the
+// source of that move, which must not change under it, and rewriting a copy
+// in another pool would change which copy the version's metadata comes
+// from. Such a version is left to a later run (errBatchRotateMoving).
+func batchRotateRead(ctx context.Context, api ObjectLayer, bucket, object string, opts ObjectOptions) (ObjectInfo, int, error) {
+	z, ok := api.(*erasureServerPools)
+	if !ok || z.SinglePool() {
+		oi, err := api.GetObjectInfo(ctx, bucket, object, opts)
+		return oi, -1, err
+	}
+	if err := checkGetObjArgs(ctx, bucket, object); err != nil {
+		return ObjectInfo{}, -1, err
+	}
+	oi, idx, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
+	if err == nil && (z.IsSuspended(idx) || z.IsPoolRebalancing(idx)) {
+		return ObjectInfo{}, -1, errBatchRotateMoving
+	}
+	return oi, idx, err
 }
 
 const (
@@ -255,11 +385,14 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		StartTime: job.Started,
 	}
 	if err := ri.loadOrInit(ctx, api, job); err != nil {
-		return err
+		return batchUnavailable("loading job progress", err)
 	}
 	if ri.Complete {
 		return nil
 	}
+	if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
+		return err
+	}
 
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
@@ -273,8 +406,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
 
-	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
-
 	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
@@ -355,65 +486,103 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(ctx)
+	ctx, cancelCause := context.WithCancelCause(ctx)
+	defer cancelCause(nil)
+	// refuse ends the job for its submitter: record why before stopping.
+	refuse := func(err error) {
+		if isBatchJobUnavailable(err) {
+			ri.pause(err)
+		} else {
+			ri.refuse(err)
+		}
+		cancelCause(err)
+	}
+	progress := newBatchProgress(ri, r.Bucket)
 
 	results := make(chan itemOrErr[ObjectInfo], 100)
 	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
 		Marker: lastObject,
 		Filter: selectObj,
 	}); err != nil {
-		cancel()
-		// Do not need to retry if we can't list objects on source.
-		return err
+		cause := batchListingError(err)
+		if isBatchJobUnavailable(cause) {
+			return ri.savePaused(ctx, api, job, cause)
+		}
+		return cause
 	}
 	failed := false
 	for res := range results {
+		if context.Cause(ctx) != nil {
+			// Refused or canceled: schedule nothing more.
+			continue
+		}
 		if res.Err != nil {
 			failed = true
+			cancelCause(batchListingError(res.Err))
 			batchLogIf(ctx, res.Err)
-			break
+			continue
 		}
 		result := res.Item
+		if !batchResumeAfter(lastObject, result.Name) {
+			// Completed by the run that set the cursor.
+			continue
+		}
 		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
 		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
 		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
 			continue
 		}
+		seq := progress.begin()
 		wk.Take()
 		go func() {
 			defer wk.Give()
+			var counts batchCounts
 			for attempts := 1; attempts <= retryAttempts; attempts++ {
 				stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
-				success := true
-				if err := r.KeyRotate(ctx, api, result); err != nil {
-					stopFn(result, err)
-					batchLogIf(ctx, err)
-					success = false
-					if attempts >= retryAttempts {
-						auditOptions := AuditLogOptions{
-							Event:     "KeyRotate",
-							APIName:   "StartBatchJob",
-							Bucket:    result.Bucket,
-							Object:    result.Name,
-							VersionID: result.VersionID,
-							Error:     err.Error(),
-						}
-						auditLogInternal(ctx, auditOptions)
-					}
-				} else {
-					stopFn(result, nil)
+				err := r.KeyRotate(ctx, api, result, job.Submitter)
+				if errors.Is(err, errBatchRotateMoving) {
+					err = batchUnavailable("object is being moved", err)
 				}
-				ri.trackCurrentBucketObject(r.Bucket, result, success, attempts)
-				globalBatchJobsMetrics.save(job.ID, ri)
-				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-				if success {
+				stopFn(result, err)
+				if err == nil {
+					counts.objects = 1
+					break
+				}
+				if isBatchJobStopped(err) {
+					refuse(err)
+					return
+				}
+				if isErrObjectLocked(err) {
+					// Left as it is, not retried and not a failure.
+					counts.versionsSkipped = 1
+					break
+				}
+				if ctx.Err() != nil {
+					// Not done; a later run rotates it.
+					return
+				}
+				batchLogIf(ctx, err)
+				if attempts >= retryAttempts {
+					auditOptions := AuditLogOptions{
+						Event:     "KeyRotate",
+						APIName:   "StartBatchJob",
+						Bucket:    result.Bucket,
+						Object:    result.Name,
+						VersionID: result.VersionID,
+						Error:     err.Error(),
+					}
+					auditLogInternal(ctx, auditOptions)
+					counts.objectsFailed = 1
 					break
 				}
 				if delay > 0 {
-					time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+					time.Sleep(delay + time.Duration(rand.Float64()*float64(delay)))
 				}
 			}
+			progress.done(seq, result.Name, counts)
+			globalBatchJobsMetrics.save(job.ID, ri)
+			// persist in-memory state to disk after every 10secs.
+			batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 			if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
 				time.Sleep(wait)
@@ -421,18 +590,34 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		}()
 	}
 	wk.Wait()
-
-	ri.Complete = !failed && ri.ObjectsFailed == 0
-	ri.Failed = failed || ri.ObjectsFailed > 0
-	globalBatchJobsMetrics.save(job.ID, ri)
-	// persist in-memory state to disk.
-	batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-
-	if err := r.Notify(ctx, ri); err != nil {
-		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	cause := context.Cause(ctx)
+	if !failed && cause == nil {
+		// The walk ended: its last key is done.
+		progress.finish()
+	}
+	if cause != nil && !ri.refused() {
+		if isBatchJobUnavailable(cause) {
+			return ri.savePaused(ctx, api, job, cause)
+		}
+		// Stopped before the end: keep the work after the cursor pending.
+		globalBatchJobsMetrics.save(job.ID, ri)
+		batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+		return cause
 	}
 
-	cancel()
+	if !ri.refused() {
+		ri.mu.Lock()
+		ri.Complete = !failed && ri.ObjectsFailed == 0
+		ri.Failed = failed || ri.ObjectsFailed > 0
+		ri.mu.Unlock()
+	}
+	globalBatchJobsMetrics.save(job.ID, ri)
+	// persist in-memory state to disk.
+	batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+
+	if err := r.Notify(GlobalContext, ri); err != nil {
+		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	}
 	return nil
 }
 

cmd/batch-submitter-authz_test.go

@@ -0,0 +1,456 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"go.yaml.in/yaml/v3"
+)
+
+// batchAuthzFixture runs batch jobs submitted through the admin API, one job
+// at a time and in the test's goroutine.
+type batchAuthzFixture struct {
+	ctx    context.Context
+	sys    *IAMSys
+	obj    ObjectLayer
+	bucket string
+}
+
+func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture {
+	t.Helper()
+	ctx, sys := iamAdminFixture(t)
+	obj := newObjectLayerFn()
+	batchTestJobPool(ctx, t, obj)
+	f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
+	mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+	_, err := globalBucketMetadataSys.Update(ctx, f.bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+	mustIAM(t, err)
+	return f
+}
+
+// batchTestJobPool queues submitted jobs without running them.
+func batchTestJobPool(ctx context.Context, t *testing.T, obj ObjectLayer) {
+	t.Helper()
+	prevPool := globalBatchJobPool
+	globalBatchJobPool = &BatchJobPool{
+		ctx:       ctx,
+		objLayer:  obj,
+		jobCh:     make(chan *BatchJobRequest, 10),
+		jobOwners: make(map[string]*batchJobOwner),
+	}
+	globalBatchJobsMetrics.Lock()
+	globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo)
+	globalBatchJobsMetrics.Unlock()
+	t.Cleanup(func() { globalBatchJobPool = prevPool })
+}
+
+// putVersions writes two versions of key and returns the older one.
+func (f *batchAuthzFixture) putVersions(t *testing.T, key string) string {
+	t.Helper()
+	var older string
+	for i := range 2 {
+		data := []byte(fmt.Sprintf("version %d", i))
+		oi, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
+		mustIAM(t, err)
+		if i == 0 {
+			older = oi.VersionID
+		}
+	}
+	return older
+}
+
+func (f *batchAuthzFixture) versionExists(t *testing.T, key, versionID string) bool {
+	t.Helper()
+	_, err := f.obj.GetObjectInfo(f.ctx, f.bucket, key, ObjectOptions{VersionID: versionID})
+	if err == nil {
+		return true
+	}
+	if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+		return false
+	}
+	t.Fatal(err)
+	return false
+}
+
+func (f *batchAuthzFixture) expireYAML(prefix string) string {
+	return fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: %q
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket, prefix)
+}
+
+// submit sends a job definition to StartBatchJob as cred. An accepted job is
+// returned from the queue, not run.
+func (f *batchAuthzFixture) submit(t *testing.T, cred auth.Credentials, job string) (*httptest.ResponseRecorder, *BatchJobRequest) {
+	t.Helper()
+	headers := map[string]string{}
+	if cred.IsTemp() && !cred.IsServiceAccount() {
+		headers["X-Amz-Security-Token"] = cred.SessionToken
+	}
+	body := []byte(job)
+	req, err := newTestSignedRequestV4(http.MethodPost, "http://localhost/minio/admin/v3/start-job", int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
+	mustIAM(t, err)
+	mustIAM(t, req.ParseForm())
+	rec := httptest.NewRecorder()
+	req = req.WithContext(newContext(req, rec, "StartBatchJob"))
+	adminAPIHandlers{}.StartBatchJob(rec, req)
+	if rec.Code != http.StatusOK {
+		return rec, nil
+	}
+	select {
+	case queued := <-globalBatchJobPool.jobCh:
+		return rec, queued
+	default:
+		t.Fatal("accepted job was not queued")
+		return rec, nil
+	}
+}
+
+func (f *batchAuthzFixture) run(t *testing.T, job *BatchJobRequest) {
+	t.Helper()
+	ctx := job.ctx
+	if ctx == nil {
+		ctx = f.ctx
+	}
+	switch {
+	case job.Expire != nil:
+		if err := job.Expire.Start(ctx, f.obj, *job); err != nil {
+			t.Logf("start: %v", err)
+		}
+	default:
+		t.Fatalf("unexpected job type %s", job.Type())
+	}
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
+	if err := ri.load(f.ctx, f.obj, *job); err == nil {
+		t.Logf("job %s: %+v", job.ID, ri.clone())
+	}
+}
+
+func (f *batchAuthzFixture) jobInfo(t *testing.T, job *BatchJobRequest) *batchJobInfo {
+	t.Helper()
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
+	mustIAM(t, ri.load(f.ctx, f.obj, *job))
+	return ri
+}
+
+func batchAuthzPolicy(ctx context.Context, t *testing.T, sys *IAMSys, name, statements string) {
+	t.Helper()
+	p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[` + statements + `]}`))
+	mustIAM(t, err)
+	_, err = sys.SetPolicy(ctx, name, *p)
+	mustIAM(t, err)
+}
+
+const batchAuthzFullAccess = `{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}`
+
+// A credential that may start batch jobs but has no access to the bucket
+// cannot delete through a job.
+func TestBatchJobSubmitterWithoutDataAccess(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-only", `{"Effect":"Allow","Action":["admin:StartBatchJob","admin:ListBatchJobs"]}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-only-user", "batch-only")
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job != nil {
+		f.run(t, job)
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job submitted without data access deleted a version")
+	}
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("submission status %d, want 403: %s", rec.Code, rec.Body.String())
+	}
+}
+
+// A job acts only while the credential that submitted it is the same valid
+// credential.
+func TestBatchJobSubmitterChangedBeforeStart(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		change func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials)
+	}{
+		{"unchanged", nil},
+		{"deleted", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+		}},
+		{"disabled", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			_, err := f.sys.SetUserStatus(f.ctx, cred.AccessKey, madmin.AccountDisabled)
+			mustIAM(t, err)
+		}},
+		{"secret-changed", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			mustIAM(t, f.sys.SetUserSecretKey(f.ctx, cred.AccessKey, "batch-authz-changed-secret"))
+		}},
+		{"access-key-reused", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+			time.Sleep(10 * time.Millisecond)
+			iamAdminUser(f.ctx, t, f.sys, cred.AccessKey, "batch-full")
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+			cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			older := f.putVersions(t, "data/a")
+			rec, job := f.submit(t, cred, f.expireYAML(""))
+			if job == nil {
+				t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+			}
+			if tc.change != nil {
+				tc.change(t, f, cred)
+			}
+			f.run(t, job)
+			exists := f.versionExists(t, "data/a", older)
+			if tc.change == nil && exists {
+				t.Fatal("control: an authorized job did not expire the older version")
+			}
+			if tc.change != nil && !exists {
+				t.Fatal("a job ran as a credential that changed after submission")
+			}
+		})
+	}
+}
+
+// An expired temporary credential does not keep its job running.
+func TestBatchJobSubmitterExpired(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(2 * time.Second).Unix()}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	time.Sleep(time.Until(cred.Expiration) + time.Second)
+	f.run(t, job)
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job ran as an expired credential")
+	}
+}
+
+// A session policy limits the job as it limits requests, also when an
+// authorization plugin decides the rest.
+func TestBatchJobSessionPolicyDeny(t *testing.T) {
+	session := `{"Version":"2012-10-17","Statement":[
+{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`
+	for _, tc := range []struct {
+		name   string
+		plugin bool
+	}{{"local", false}, {"plugin-allows", true}} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+			parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+			cred := iamAdminServiceAccount(f.ctx, t, f.sys, parent.AccessKey, "batch-restricted", session)
+			if tc.plugin {
+				iamAdminPlugin(t, true)
+			}
+			older := f.putVersions(t, "data/a")
+			rec, job := f.submit(t, cred, f.expireYAML(""))
+			if job != nil {
+				f.run(t, job)
+			} else {
+				t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+			}
+			if !f.versionExists(t, "data/a", older) {
+				t.Fatal("a job deleted a version its session policy denies")
+			}
+		})
+	}
+}
+
+// A temporary credential's session policy limits its job too.
+func TestBatchJobTemporarySessionPolicyDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+	session := `{"Version":"2012-10-17","Statement":[
+{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions","s3:GetObject","s3:GetObjectVersion","s3:DeleteObject"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}]}`
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
+		policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(session)),
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job != nil {
+		f.run(t, job)
+	} else {
+		t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job deleted a version its session policy does not allow")
+	}
+}
+
+// A Deny that depends on request context the job does not have refuses the
+// action instead of being skipped.
+func TestBatchJobAbsentContextDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-context", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-context")
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job != nil {
+		f.run(t, job)
+	} else {
+		t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job skipped a Deny that depends on request context")
+	}
+}
+
+// Each object is authorized: a Deny under an allowed prefix keeps its objects.
+func TestBatchJobPerObjectDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-keep", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/data/keep*"]}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-keep")
+	kept := f.putVersions(t, "data/keep")
+	rec, job := f.submit(t, cred, f.expireYAML("data/"))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	f.run(t, job)
+	if !f.versionExists(t, "data/keep", kept) {
+		t.Fatal("a job deleted a version that a per-object Deny protects")
+	}
+	if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+		t.Fatalf("refused job: complete=%v failed=%v", ri.Complete, ri.Failed)
+	}
+}
+
+// A job queued without its submitter, as an earlier release stored it, fails
+// and stays failed across another start.
+func TestBatchJobWithoutSubmitter(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "data/a")
+	job := &BatchJobRequest{}
+	mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
+	job.ID = "expire-legacy" + getKeySeparator() + "0"
+	job.User = globalActiveCred.AccessKey
+	job.Started = time.Now()
+	mustIAM(t, job.save(f.ctx, f.obj))
+	for range 2 {
+		loaded := &BatchJobRequest{}
+		mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+		f.run(t, loaded)
+		if !f.versionExists(t, "data/a", older) {
+			t.Fatal("a job without a recorded submitter deleted a version")
+		}
+		if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+			t.Fatalf("job without submitter: complete=%v failed=%v", ri.Complete, ri.Failed)
+		}
+	}
+}
+
+// Root and an authorized user can still run jobs.
+func TestBatchJobAuthorizedControls(t *testing.T) {
+	for _, root := range []bool{true, false} {
+		t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			cred := globalActiveCred
+			if !root {
+				batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+				cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			}
+			older := f.putVersions(t, "data/a")
+			rec, job := f.submit(t, cred, f.expireYAML(""))
+			if job == nil {
+				t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+			}
+			f.run(t, job)
+			if f.versionExists(t, "data/a", older) {
+				t.Fatal("an authorized job did not expire the older version")
+			}
+			if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed {
+				t.Fatalf("authorized job: complete=%v failed=%v", ri.Complete, ri.Failed)
+			}
+		})
+	}
+}
+
+// The describe output has no secrets of the job.
+func TestBatchJobDescribeRedactsSecrets(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	secrets := []string{"describe-source-secret", "describe-source-token", "describe-target-secret", "describe-target-token", "describe-notify-token"}
+	for _, job := range []BatchJobRequest{
+		{ID: "replicate-describe" + getKeySeparator() + "0", Replicate: &BatchJobReplicateV1{
+			APIVersion: batchReplJobAPIVersion,
+			Source: BatchJobReplicateSource{Bucket: "remote", Endpoint: "https://remote.invalid", Creds: BatchJobReplicateCredentials{
+				AccessKey: "describe-source-access", SecretKey: secrets[0], SessionToken: secrets[1],
+			}},
+			Target: BatchJobReplicateTarget{Bucket: f.bucket, Creds: BatchJobReplicateCredentials{
+				AccessKey: "describe-target-access", SecretKey: secrets[2], SessionToken: secrets[3],
+			}},
+			Flags: BatchJobReplicateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
+		}},
+		{ID: "expire-describe" + getKeySeparator() + "0", Expire: &BatchJobExpire{
+			APIVersion: batchExpireAPIVersion, Bucket: f.bucket,
+			NotificationCfg: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]},
+		}},
+		{ID: "keyrotate-describe" + getKeySeparator() + "0", KeyRotate: &BatchJobKeyRotateV1{
+			APIVersion: batchKeyRotateAPIVersion, Bucket: f.bucket,
+			Flags: BatchJobKeyRotateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
+		}},
+	} {
+		data, err := job.MarshalMsg(nil)
+		mustIAM(t, err)
+		mustIAM(t, saveConfig(f.ctx, f.obj, getJobPath(job), data))
+		cred := globalActiveCred
+		req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/describe-job?jobId="+job.ID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
+		mustIAM(t, err)
+		mustIAM(t, req.ParseForm())
+		rec := httptest.NewRecorder()
+		req = req.WithContext(newContext(req, rec, "DescribeBatchJob"))
+		adminAPIHandlers{}.DescribeBatchJob(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("%s: describe status %d: %s", job.ID, rec.Code, rec.Body.String())
+		}
+		for _, s := range secrets {
+			if strings.Contains(rec.Body.String(), s) {
+				t.Errorf("%s: describe output contains %s", job.ID, s)
+			}
+		}
+		if job.Replicate != nil && !strings.Contains(rec.Body.String(), "describe-source-access") {
+			t.Errorf("%s: describe output lost the access key", job.ID)
+		}
+		// The stored job keeps what a run needs.
+		stored := &BatchJobRequest{}
+		mustIAM(t, stored.load(f.ctx, f.obj, getJobPath(job)))
+		if stored.Replicate != nil && stored.Replicate.Source.Creds.SecretKey != secrets[0] {
+			t.Errorf("%s: describe changed the stored job", job.ID)
+		}
+	}
+}

cmd/bucket-cors-site-replication_test.go

@@ -511,14 +511,13 @@ func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ st
 	}))
 	defer remote.Close()
 
-	serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key")
+	serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "cors-status-service",
+		secretKey: "cors-status-service-secret-key",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	serviceCred.ParentUser = credentials.AccessKey
-	if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
-		t.Fatal(err)
-	}
 	defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
 
 	globalSiteReplicationSys.Lock()
@@ -585,14 +584,13 @@ func testSiteReplicationStatusCountsCorsPerSite(obj ObjectLayer, _ string, bucke
 	}))
 	defer remote.Close()
 
-	serviceCred, err := auth.CreateCredentials("cors-count-service", "cors-count-service-secret-key")
+	serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "cors-count-service",
+		secretKey: "cors-count-service-secret-key",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	serviceCred.ParentUser = credentials.AccessKey
-	if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
-		t.Fatal(err)
-	}
 	defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
 
 	globalSiteReplicationSys.Lock()
@@ -1190,14 +1188,13 @@ func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, buck
 	}))
 	defer remote.Close()
 
-	serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret")
+	serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "cors-integration-svc",
+		secretKey: "cors-integration-service-secret",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	serviceCred.ParentUser = credentials.AccessKey
-	if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
-		t.Fatal(err)
-	}
 	defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
 
 	globalSiteReplicationSys.Lock()

cmd/bucket-handlers.go

@@ -277,14 +277,6 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if keyMarker != "" {
-		// Marker not common with prefix is not implemented.
-		if !HasPrefix(keyMarker, prefix) {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-			return
-		}
-	}
-
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1281,6 +1273,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
+	if isDirObject(object) {
+		if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, ErrNone, ErrNone, false); s3Err != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+			return
+		}
+	}
 	opts.WantChecksum = checksum
 
 	fanOutOpts := fanOutOptions{Checksum: checksum}

cmd/bucket-handlers_test.go

@@ -425,7 +425,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			shouldPass:         true,
 		},
 		// Test case - 4.
-		// Setting Invalid prefix and marker combination.
+		// A key marker outside the prefix is valid and produces an empty page.
 		{
 			bucket:             bucketName,
 			prefix:             "asia",
@@ -435,8 +435,8 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotImplemented,
-			shouldPass:         false,
+			expectedRespStatus: http.StatusOK,
+			shouldPass:         true,
 		},
 		// Test case - 5.
 		// Invalid upload id and marker combination.

cmd/bucket-lifecycle-audit.go

@@ -39,7 +39,6 @@ const (
 	lcEventSrc_s3PutObject
 	lcEventSrc_s3CopyObject
 	lcEventSrc_s3CompleteMultipartUpload
-	lcEventSrc_AccessTier
 )
 
 //revive:enable:var-naming

cmd/bucket-lifecycle-incomplete.go

@@ -0,0 +1,174 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/event"
+)
+
+// An ILM expiry attempt that ends without removing what it addressed is
+// incomplete: the rule no longer applied when it was checked again under the
+// object lock, Object Lock protects the version, the history needed to decide
+// could not be read, or another error. It is counted by reason, traced with
+// the reason, and, unless it was refused, audited and logged. It never sends
+// a removal notification: what it addressed is still there.
+//
+// An expiry of all versions of a key can also be refused because a version
+// is in a remote tier, or because the latest version changed since a batch
+// matched it, and it can stop at its budget after it removed some versions.
+const (
+	ilmIncompleteNotApplicable      = "not_applicable"
+	ilmIncompleteLocked             = "locked"
+	ilmIncompleteHistoryUnavailable = "history_unavailable"
+	ilmIncompleteError              = "error"
+	ilmIncompleteTierUnsupported    = "tier_unsupported"
+	ilmIncompleteBudget             = "budget"
+	ilmIncompleteChanged            = "changed"
+	// ilmIncompleteTierSkipped counts the scanner's observations of a key
+	// whose expiry of all versions it did not queue, because a version it
+	// holds is in a remote tier and the purge would refuse it. Each drive's
+	// scanner of the key observes it, once per cycle.
+	ilmIncompleteTierSkipped = "tier_unsupported_skipped"
+)
+
+// ilmIncompleteReasons are the reasons expiry_incomplete_total counts, a
+// bounded set of label values.
+var ilmIncompleteReasons = []string{
+	ilmIncompleteNotApplicable,
+	ilmIncompleteLocked,
+	ilmIncompleteHistoryUnavailable,
+	ilmIncompleteError,
+	ilmIncompleteTierUnsupported,
+	ilmIncompleteBudget,
+	ilmIncompleteChanged,
+	ilmIncompleteTierSkipped,
+}
+
+// ilmIncompleteStats counts incomplete ILM expiries by reason.
+type ilmIncompleteStats struct {
+	counts map[string]*atomic.Int64
+}
+
+func newILMIncompleteStats(reasons ...string) *ilmIncompleteStats {
+	s := &ilmIncompleteStats{counts: make(map[string]*atomic.Int64, len(reasons))}
+	for _, r := range reasons {
+		s.counts[r] = &atomic.Int64{}
+	}
+	return s
+}
+
+func (s *ilmIncompleteStats) add(reason string) {
+	if c, ok := s.counts[reason]; ok {
+		c.Add(1)
+	}
+}
+
+// snapshot returns the counts by reason.
+func (s *ilmIncompleteStats) snapshot() map[string]int64 {
+	out := make(map[string]int64, len(s.counts))
+	for r, c := range s.counts {
+		out[r] = c.Load()
+	}
+	return out
+}
+
+var globalILMIncomplete = newILMIncompleteStats(ilmIncompleteReasons...)
+
+// ilmExpiryNotDone classifies the error of an ILM expiry that did not remove
+// what it addressed. A version that is already gone is not counted.
+func ilmExpiryNotDone(err error) (reason string, counted bool) {
+	switch {
+	case err == nil || isErrObjectNotFound(err) || isErrVersionNotFound(err):
+		return "", false
+	case errors.Is(err, errDeleteAllTiered):
+		return ilmIncompleteTierUnsupported, true
+	case errors.Is(err, errDeleteAllChanged):
+		return ilmIncompleteChanged, true
+	case errors.Is(err, errDeleteAllBudget):
+		return ilmIncompleteBudget, true
+	case errors.Is(err, errLifecycleNotApplicable):
+		return ilmIncompleteNotApplicable, true
+	}
+	var locked ObjectLocked
+	if errors.As(err, &locked) {
+		return ilmIncompleteLocked, true
+	}
+	if isErrReadQuorum(err) || isErrWriteQuorum(err) {
+		return ilmIncompleteHistoryUnavailable, true
+	}
+	return ilmIncompleteError, true
+}
+
+// ilmExpiryRefused reports whether reason is a refusal, which changes
+// nothing, rather than a failure.
+func ilmExpiryRefused(reason string) bool {
+	switch reason {
+	case ilmIncompleteNotApplicable, ilmIncompleteLocked, ilmIncompleteTierUnsupported, ilmIncompleteChanged:
+		return true
+	}
+	return false
+}
+
+// reportILMExpiryNotDone reports an ILM expiry of oi that ended with err. It
+// reports nothing and returns false when err says the version is already
+// gone; the caller reports that as it did before.
+func reportILMExpiryNotDone(ctx context.Context, oi ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, err error, logID string, traceFn func(event string, metadata map[string]string, err error)) bool {
+	reason, counted := ilmExpiryNotDone(err)
+	if !counted {
+		return false
+	}
+	globalILMIncomplete.add(reason)
+	if ilmExpiryRefused(reason) {
+		traceFn(ILMExpiry, map[string]string{"result": "refused", "reason": reason, "version-id": oi.VersionID}, err)
+		return true
+	}
+	traceFn(ILMExpiry, map[string]string{"result": "failed", "reason": reason, "version-id": oi.VersionID}, err)
+	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+	tags["version-id"] = oi.VersionID
+	tags["ilm-incomplete-reason"] = reason
+	auditLogInternal(ctx, AuditLogOptions{
+		Event:     ILMExpiry,
+		APIName:   "ILMExpiry",
+		Bucket:    oi.Bucket,
+		Object:    oi.Name,
+		VersionID: oi.VersionID,
+		Error:     err.Error(),
+		Tags:      tags,
+	})
+	ilmLogOnceIf(ctx, fmt.Errorf("ILM expiry of %s/%s (%s) not done: %w", oi.Bucket, oi.Name, oi.VersionID, err), logID)
+	return true
+}
+
+// ilmExpiryEventName is the notification of an expiry that completed. An
+// expiry that addressed no version ID (Expiration of a latest version in a
+// versioned or suspended bucket) and whose result is a delete marker created
+// that marker, also when the marker takes the null version ID of the version
+// it replaces. Any other expiry removed the version it addressed, also when
+// that version was itself a delete marker.
+func ilmExpiryEventName(requestedVersionID string, result ObjectInfo) event.Name {
+	if requestedVersionID == "" && result.DeleteMarker {
+		return event.ObjectRemovedDeleteMarkerCreated
+	}
+	return event.ObjectRemovedDelete
+}

cmd/bucket-lifecycle-incomplete_test.go

@@ -0,0 +1,364 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/logger"
+	loghttp "github.com/minio/minio/internal/logger/target/http"
+	"github.com/minio/minio/internal/pubsub"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestILMExpiryNotDoneClassification(t *testing.T) {
+	for _, tc := range []struct {
+		err     error
+		reason  string
+		counted bool
+	}{
+		{nil, "", false},
+		{ObjectNotFound{}, "", false},
+		{VersionNotFound{}, "", false},
+		{errLifecycleNotApplicable, ilmIncompleteNotApplicable, true},
+		{fmt.Errorf("wrapped: %w", errLifecycleNotApplicable), ilmIncompleteNotApplicable, true},
+		{ObjectLocked{}, ilmIncompleteLocked, true},
+		{InsufficientWriteQuorum{}, ilmIncompleteHistoryUnavailable, true},
+		{InsufficientReadQuorum{}, ilmIncompleteHistoryUnavailable, true},
+		{errFaultyDisk, ilmIncompleteError, true},
+	} {
+		reason, counted := ilmExpiryNotDone(tc.err)
+		if reason != tc.reason || counted != tc.counted {
+			t.Errorf("%v: %q %v, want %q %v", tc.err, reason, counted, tc.reason, tc.counted)
+		}
+	}
+}
+
+// fixedDeleteResults is an object layer whose bulk deletes return errs.
+type fixedDeleteResults struct {
+	ObjectLayer
+	errs []error
+}
+
+func (o fixedDeleteResults) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	deleted := make([]DeletedObject, len(objects))
+	for i, obj := range objects {
+		deleted[i] = DeletedObject{ObjectName: obj.ObjectName, VersionID: obj.VersionID}
+	}
+	return deleted, slices.Clone(o.errs)
+}
+
+// ilmTestAudit captures the ILMExpiry audit entries of bucket.
+func ilmTestAudit(t *testing.T, bucket string) <-chan map[string]any {
+	t.Helper()
+	if len(logger.AuditTargets()) != 0 {
+		t.Fatal("unexpected pre-existing test audit targets")
+	}
+	entries := make(chan map[string]any, 16)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var entry struct {
+			Error string         `json:"error"`
+			Tags  map[string]any `json:"tags"`
+			API   struct {
+				Name, Bucket string
+				Objects      []struct {
+					ObjectName string `json:"objectName"`
+				} `json:"objects"`
+			}
+		}
+		if err := json.NewDecoder(r.Body).Decode(&entry); err == nil && entry.API.Name == "ILMExpiry" && entry.API.Bucket == bucket {
+			tags := entry.Tags
+			if tags == nil {
+				tags = map[string]any{}
+			}
+			tags["error"] = entry.Error
+			if len(entry.API.Objects) > 0 {
+				tags["object"] = entry.API.Objects[0].ObjectName
+			}
+			entries <- tags
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	endpoint, err := xnet.ParseHTTPURL(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if errs := logger.UpdateAuditWebhooks(ctx, map[string]loghttp.Config{"ilm": {Enabled: true, Name: "ilm", Endpoint: endpoint, BatchSize: 1, QueueSize: 128, MaxRetry: 1, RetryIntvl: time.Millisecond, HTTPTimeout: time.Second}}); len(errs) > 0 {
+		t.Fatal(errs)
+	}
+	targets := logger.AuditTargets()
+	t.Cleanup(func() {
+		logger.UpdateAuditWebhooks(ctx, nil)
+		for _, target := range targets {
+			target.Cancel()
+		}
+		server.Close()
+	})
+	return entries
+}
+
+// A refused or failed noncurrent purge is counted and traced with its reason,
+// a failure is audited, and only the version that was removed is announced.
+func TestDeleteObjectVersionsReportsRefusalAndFailure(t *testing.T) {
+	s := newSplitILM(t, versioning.Enabled, splitILMRules)
+	ctx := t.Context()
+	audits := ilmTestAudit(t, s.bucket)
+	done := make(chan struct{})
+	defer close(done)
+	events := make(chan event.Event, 8)
+	if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedAll), events, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	traces := make(chan madmin.TraceInfo, 8)
+	if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	before := globalILMIncomplete.snapshot()
+
+	layer := fixedDeleteResults{ObjectLayer: s.z, errs: []error{errLifecycleNotApplicable, InsufficientWriteQuorum{}, nil}}
+	toDel := []ObjectToDelete{
+		{ObjectV: ObjectV{ObjectName: "ncv-a", VersionID: "11111111-1111-1111-1111-111111111111"}},
+		{ObjectV: ObjectV{ObjectName: "ncv-b", VersionID: "22222222-2222-2222-2222-222222222222"}},
+		{ObjectV: ObjectV{ObjectName: "ncv-c", VersionID: "33333333-3333-3333-3333-333333333333"}},
+	}
+	evs := []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}}
+	deleteObjectVersions(ctx, layer, s.bucket, toDel, evs)
+
+	after := globalILMIncomplete.snapshot()
+	if d := after[ilmIncompleteNotApplicable] - before[ilmIncompleteNotApplicable]; d != 1 {
+		t.Errorf("not_applicable moved by %d, want 1", d)
+	}
+	if d := after[ilmIncompleteHistoryUnavailable] - before[ilmIncompleteHistoryUnavailable]; d != 1 {
+		t.Errorf("history_unavailable moved by %d, want 1", d)
+	}
+	var removed []string
+	results := map[string]string{}
+	for drained := false; !drained; {
+		select {
+		case ev := <-events:
+			removed = append(removed, ev.S3.Object.Key)
+		case tr := <-traces:
+			if r := tr.Custom["result"]; r != "" {
+				results[tr.Custom["version-id"]] = r
+				if tr.Error == "" {
+					t.Errorf("%s expiry of %s traced without its error", r, tr.Custom["version-id"])
+				}
+			}
+		default:
+			drained = true
+		}
+	}
+	if !slices.Equal(removed, []string{"ncv-c"}) {
+		t.Errorf("removal events for %v, want only ncv-c", removed)
+	}
+	if results[toDel[0].VersionID] != "refused" || results[toDel[1].VersionID] != "failed" {
+		t.Errorf("traces %v, want refused and failed", results)
+	}
+	// Drain every audit entry of the bucket: until the removal of ncv-c,
+	// audited after the refusal and the failure were reported, has arrived,
+	// and then while the target empties its queue.
+	var got []map[string]any
+	deadline := time.After(10 * time.Second)
+	for removal := false; !removal; {
+		select {
+		case tags := <-audits:
+			got = append(got, tags)
+			removal = removal || tags["object"] == "ncv-c"
+		case <-deadline:
+			t.Fatalf("audits %v: the removal of ncv-c was not audited", got)
+		}
+	}
+	for idle := time.After(time.Second); idle != nil; {
+		select {
+		case tags := <-audits:
+			got = append(got, tags)
+		case <-idle:
+			idle = nil
+		}
+	}
+	perObject := map[any][]map[string]any{}
+	for _, tags := range got {
+		perObject[tags["object"]] = append(perObject[tags["object"]], tags)
+	}
+	if a := perObject["ncv-a"]; len(a) != 0 {
+		t.Errorf("the refusal was audited: %v", a)
+	}
+	if b := perObject["ncv-b"]; len(b) != 1 || b[0]["ilm-incomplete-reason"] != ilmIncompleteHistoryUnavailable || b[0]["error"] == "" {
+		t.Errorf("audits of the failure: %v, want one with its reason and error", b)
+	}
+	if c := perObject["ncv-c"]; len(c) != 1 || c[0]["ilm-incomplete-reason"] != nil {
+		t.Errorf("audits of the removal: %v, want one", c)
+	}
+}
+
+// expiryEvent returns the removal events sent while run runs.
+func expiryEvents(t *testing.T, run func()) []event.Event {
+	t.Helper()
+	done := make(chan struct{})
+	defer close(done)
+	events := make(chan event.Event, 8)
+	if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedAll), events, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	run()
+	var got []event.Event
+	for {
+		select {
+		case ev := <-events:
+			got = append(got, ev)
+		default:
+			return got
+		}
+	}
+}
+
+// The notification of a completed expiry names what its delete did: the
+// delete marker it created, or the version it removed.
+func TestExpiryEventsNameTheResult(t *testing.T) {
+	type expiry struct {
+		name      string
+		status    versioning.State
+		key       string
+		versions  []splitVersion
+		expire    string // label of the version expired
+		action    lifecycle.Action
+		wantEvent event.Name
+		wantID    func(s *splitILM, key string) []string // acceptable version IDs
+	}
+	marker := func(s *splitILM, key string) []string {
+		oi, err := s.z.GetObjectInfo(s.t.Context(), s.bucket, key, ObjectOptions{})
+		if !oi.DeleteMarker {
+			s.t.Fatalf("latest of %s is not a marker: %+v %v", key, oi, err)
+		}
+		return []string{oi.VersionID}
+	}
+	label := func(l string) func(s *splitILM, key string) []string {
+		return func(s *splitILM, key string) []string { return []string{s.ids[key+" "+l]} }
+	}
+	null := func(*splitILM, string) []string { return []string{"", nullVersionID} }
+	cases := []expiry{
+		{"latest-versioned", versioning.Enabled, "exp-ev-latest", []splitVersion{splitAt("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, marker},
+		{"noncurrent", versioning.Enabled, "ncv-ev-noncurrent", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 3, 0)}, "v1", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("v1")},
+		{"expired-marker", versioning.Enabled, "edm-ev-marker", []splitVersion{splitMarker("dm", 5, 0)}, "dm", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("dm")},
+		{"latest-suspended-null", versioning.Suspended, "exp-ev-null", []splitVersion{splitNull("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, null},
+		{"tiered-latest-versioned", versioning.Enabled, "exp-ev-tiered", []splitVersion{splitTiered("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, marker},
+		{"tiered-noncurrent", versioning.Enabled, "ncv-ev-tiered", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 3, 0)}, "v1", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("v1")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			s := newSplitILM(t, tc.status, splitILMRules)
+			installExpiryTestTier(t, map[string][]byte{"remote-v1": []byte("v1")})
+			for _, v := range tc.versions {
+				s.write(tc.key, v)
+			}
+			vid := s.ids[tc.key+" "+tc.expire]
+			if vid == "" {
+				vid = nullVersionID
+			}
+			oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, tc.key, ObjectOptions{VersionID: vid})
+			if err != nil && !isErrMethodNotAllowed(err) {
+				t.Fatal(err)
+			}
+			oi.Bucket, oi.Name = s.bucket, tc.key
+			lcEvent := lifecycle.Event{Action: tc.action}
+			got := expiryEvents(t, func() {
+				if oi.TransitionedObject.Status != "" {
+					if !applyExpiryOnTransitionedObject(t.Context(), s.z, oi, lcEvent, lcEventSrc_Scanner) {
+						t.Fatal("transitioned expiry did not apply")
+					}
+					return
+				}
+				if !applyExpiryOnNonTransitionedObjects(t.Context(), s.z, oi, lcEvent, lcEventSrc_Scanner) {
+					t.Fatal("expiry did not apply")
+				}
+			})
+			if len(got) != 1 {
+				t.Fatalf("events %+v, want one", got)
+			}
+			if got[0].EventName != tc.wantEvent {
+				t.Errorf("event %s, want %s", got[0].EventName, tc.wantEvent)
+			}
+			if want := tc.wantID(s, tc.key); !slices.Contains(want, got[0].S3.Object.VersionID) {
+				t.Errorf("event names version %q, want one of %v", got[0].S3.Object.VersionID, want)
+			}
+		})
+	}
+}
+
+// expiry_incomplete_total is exported as a counter with a reason label by
+// both metrics versions.
+func TestILMExpiryIncompleteMetrics(t *testing.T) {
+	// Metrics v3 exports a counter once it is above zero.
+	for _, r := range ilmIncompleteReasons {
+		globalILMIncomplete.add(r)
+	}
+	// v3, through the collector the metrics handler registers.
+	families, err := newMetricGroups(prometheus.NewRegistry()).mgGatherers[ilmCollectorPath].Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := ilmCollectorPath.metricPrefix() + "_" + string(expiryIncompleteTotal)
+	reasons := map[string]bool{}
+	for _, family := range families {
+		if family.GetName() != want {
+			continue
+		}
+		for _, m := range family.Metric {
+			if m.Counter == nil || m.GetCounter().GetValue() <= 0 {
+				t.Errorf("v3 %s: %v, want a counter above zero", want, m)
+			}
+			for _, l := range m.Label {
+				if l.GetName() == "reason" {
+					reasons[l.GetValue()] = true
+				}
+			}
+		}
+	}
+	for _, r := range ilmIncompleteReasons {
+		if !reasons[r] {
+			t.Errorf("v3: %s{reason=%q} not exported", want, r)
+		}
+	}
+	reasons = map[string]bool{}
+	for _, mv := range getILMNodeMetrics().Get() {
+		if mv.Description.Name == expiryIncompleteTotal {
+			if mv.Description.Type != counterMetric {
+				t.Errorf("v2 type %v", mv.Description.Type)
+			}
+			reasons[mv.VariableLabels["reason"]] = true
+		}
+	}
+	for _, r := range ilmIncompleteReasons {
+		if !reasons[r] {
+			t.Errorf("v2: reason %q not exported", r)
+		}
+	}
+}

cmd/bucket-lifecycle-restore-boundary_test.go

@@ -0,0 +1,277 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+// Read the actual metadata from every disk without object-layer read repair.
+func restoreBoundarySnapshot(t *testing.T, r *restoreTest) [][]byte {
+	t.Helper()
+	var result [][]byte
+	for _, pool := range r.z.serverPools {
+		for _, disk := range pool.getHashedSet(r.object).getDisks() {
+			raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+			if err != nil && err != errFileNotFound {
+				t.Fatal(err)
+			}
+			result = append(result, bytes.Clone(raw.Buf))
+		}
+	}
+	return result
+}
+
+func TestExpireRestoredKeepsReplacementDeleteMarker(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, pools, versioning.Suspended)
+			r.put("old", false, false)
+			oi := r.put("null", true, true)
+			r.restore(oi)
+			stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts("")); err != nil {
+				t.Fatal(err)
+			}
+			if !r.fileInfo(nullVersionID).Deleted {
+				t.Fatal("ordinary DELETE did not replace null with a delete marker")
+			}
+			before := restoreBoundarySnapshot(t, r)
+			err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+			if !isErrVersionNotFound(err) {
+				t.Errorf("stale cleanup should reject the replacement marker: %v", err)
+			}
+			if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+				t.Error("stale cleanup changed stored versions or their data")
+			}
+			if _, _, err := r.read(""); !isErrObjectNotFound(err) {
+				t.Errorf("stale cleanup exposed the old version: %v", err)
+			}
+			// An explicit user purge must still remove that same marker.
+			if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts(nullVersionID)); err != nil {
+				t.Fatal(err)
+			}
+			if label, data, err := r.read(""); err != nil || label != "old" || !bytes.Equal(data, r.bodies["old"]) {
+				t.Fatalf("ordinary marker purge: label=%s err=%v", label, err)
+			}
+		})
+	}
+}
+
+func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, operation := range []string{"cleanup", "restore", "restore-cleanup", "lookup-cleanup", "lookup-restore", "get", "delete"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, operation), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				oi := r.put("target", false, true)
+				r.restore(oi)
+				fi := r.fileInfo(oi.VersionID)
+				// A readable latest version lets pools lookup finish before the
+				// set reads the specifically addressed, incomplete old version.
+				r.put("newer", false, false)
+				er := r.z.serverPools[0].getHashedSet(r.object)
+				disks := er.getDisks()
+				for _, disk := range disks[1:] {
+					raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					var meta xlMetaV2
+					if err := meta.Load(raw.Buf); err != nil {
+						t.Fatal(err)
+					}
+					del := FileInfo{VersionID: oi.VersionID}
+					del.SetSkipTierFreeVersion()
+					if _, err := meta.DeleteVersion(del); err != nil {
+						t.Fatal(err)
+					}
+					buf, err := meta.AppendTo(nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					// Remove only this metadata entry; retain every real shard.
+					if err := disk.WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), buf); err != nil {
+						t.Fatal(err)
+					}
+				}
+				before := restoreBoundarySnapshot(t, r)
+				ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+				defer cancel()
+				opts := r.opts(oi.VersionID)
+				var err error
+				switch operation {
+				case "cleanup":
+					err = expireTransitionedObject(ctx, r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredVersionAction}, lcEventSrc_Scanner)
+				case "restore":
+					opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+					err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, opts)
+				case "restore-cleanup":
+					// The initial read succeeded before remote download failed;
+					// cleanup must not purge a now-incomplete target on reread.
+					err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil)
+				case "lookup-cleanup", "lookup-restore":
+					// Version-sensitive lookup also covers native restore routing.
+					opts.MetadataChg = true
+					if operation == "lookup-cleanup" {
+						opts.Transition.ExpireRestored = true
+					} else {
+						opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+					}
+					_, _, err = r.z.getPoolInfoExistingWithOpts(ctx, r.bucket, r.object, opts)
+				case "get":
+					_, err = r.z.GetObjectInfo(ctx, r.bucket, r.object, opts)
+				case "delete":
+					_, err = r.z.DeleteObject(ctx, r.bucket, r.object, opts)
+				}
+				if ctx.Err() != nil {
+					t.Fatalf("%s timed out instead of returning its read error: %v", operation, err)
+				}
+				if err == nil {
+					t.Fatal("incomplete target unexpectedly readable")
+				}
+				if operation != "get" && operation != "delete" {
+					if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+						t.Error("failed restore read changed metadata, inline data or free versions")
+					}
+					if n := r.dataDirs(fi.DataDir); n != len(disks) {
+						t.Errorf("failed restore read removed shards: %d/%d directories remain", n, len(disks))
+					}
+					return
+				}
+				// Ordinary GET/DELETE retain their existing dangling cleanup.
+				raw, err := disks[0].ReadXL(t.Context(), r.bucket, r.object, false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				versions, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if len(versions.Versions) != 1 || len(versions.FreeVersions) != 1 || r.dataDirs(fi.DataDir) != len(disks)-1 {
+					t.Fatal("ordinary dangling cleanup no longer removed the minority version")
+				}
+			})
+		}
+	}
+}
+
+func TestNativeRestoreInitialMissingReadDoesNotWrite(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	r.put("live", false, false)
+	before := restoreBoundarySnapshot(t, r)
+	opts := r.opts(mustGetUUID())
+	opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+		t.Fatalf("missing restore target: %v", err)
+	}
+	if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+		t.Fatal("initial restore failure wrote metadata without a resolved version")
+	}
+}
+
+func TestExpireRestoredStorageRejectsOtherTypes(t *testing.T) {
+	for _, kind := range []string{"null-marker", "uuid-marker", "legacy", "legacy-json"} {
+		t.Run(kind, func(t *testing.T) {
+			disk, _, err := newXLStorageTestSetup(t)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx := t.Context()
+			const bucket, object = "restore-types", "obj"
+			if err := disk.MakeVol(ctx, bucket); err != nil {
+				t.Fatal(err)
+			}
+			var meta xlMetaV2
+			versionID, metaName, dataDir := nullVersionID, xlStorageFormatFile, ""
+			var buf []byte
+			switch kind {
+			case "null-marker", "uuid-marker":
+				if kind == "uuid-marker" {
+					versionID = mustGetUUID()
+				}
+				err = meta.AddVersion(FileInfo{VersionID: versionID, Deleted: true, ModTime: time.Now().UTC()})
+			case "legacy", "legacy-json":
+				legacy := getSampleXLMeta(1)
+				legacy.DataDir, dataDir = "legacy", "legacy"
+				if kind == "legacy-json" {
+					metaName = xlStorageFormatFileV1
+					buf, err = json.Marshal(legacy)
+					disk.storage.Lock()
+					disk.storage.formatLegacy = true
+					disk.storage.Unlock()
+				} else {
+					err = meta.AddLegacy(&legacy)
+				}
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if buf == nil {
+				buf, err = meta.AppendTo(nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := disk.WriteAll(ctx, bucket, pathJoin(object, metaName), buf); err != nil {
+				t.Fatal(err)
+			}
+			payload := []byte("sole legacy contents")
+			dataPath := filepath.Join(disk.Endpoint().Path, bucket, object, dataDir, "part.1")
+			if dataDir != "" {
+				if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(dataPath, payload, 0o600); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID, ExpireRestored: true, TransitionStatus: lifecycle.TransitionComplete}, false, DeleteOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			after, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName))
+			if err != nil || !bytes.Equal(buf, after) {
+				t.Errorf("restore cleanup changed %s metadata: %v", kind, err)
+			}
+			if dataDir != "" {
+				if data, err := os.ReadFile(dataPath); err != nil || !bytes.Equal(data, payload) {
+					t.Errorf("restore cleanup removed legacy contents: %v", err)
+				}
+			}
+			if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID}, false, DeleteOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName)); err != errFileNotFound {
+				t.Errorf("ordinary purge left %s: %v", kind, err)
+			}
+		})
+	}
+}

cmd/bucket-lifecycle-restore-safety_test.go

@@ -0,0 +1,284 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+const restoreSafetyRules = `<LifecycleConfiguration><Rule><ID>tier</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Transition><Days>1000</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`
+
+// Reuse the original restore-copy fixture with a small, isolated real cluster.
+func newRestoreSafetyTest(t *testing.T, pools int, state versioning.State) *restoreTest {
+	t.Helper()
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	dirs, err := getRandomDisks(4 * pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { removeRoots(dirs) })
+	var endpoints EndpointServerPools
+	for i := range pools {
+		endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+	}
+	obj, _, err := initObjectLayer(t.Context(), endpoints)
+	if err != nil {
+		t.Fatal(err)
+	}
+	z := obj.(*erasureServerPools)
+	t.Cleanup(func() { z.Shutdown(context.Background()) })
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+	r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}}
+	if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: state != ""}); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := globalBucketMetadataSys.Get(r.bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state == versioning.Suspended {
+		meta.VersioningConfigXML = []byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)
+	}
+	meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+	if err := meta.Save(t.Context(), z); err != nil {
+		t.Fatal(err)
+	}
+	globalBucketMetadataSys.Set(r.bucket, meta)
+	return r
+}
+
+type restoreCleanupFaultDisk struct {
+	consistencyReadFaultDisk
+	deletes *atomic.Int64
+}
+
+func (d restoreCleanupFaultDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+	if bucket == d.bucket && object == d.object {
+		d.deletes.Add(1)
+	}
+	return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func TestExpireRestoredRequiresFreshMetadata(t *testing.T) {
+	for _, restored := range []bool{true, false} {
+		t.Run(fmt.Sprintf("expire-restored=%v", restored), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, "")
+			oi := r.put("v1", true, true)
+			r.restore(oi)
+			before := r.fileInfo(oi.VersionID)
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			original := er.getDisks
+			disks := original()
+			var deletes atomic.Int64
+			faulty := make([]StorageAPI, len(disks))
+			for i, d := range disks {
+				faulty[i] = restoreCleanupFaultDisk{consistencyReadFaultDisk{d, r.bucket, r.object}, &deletes}
+			}
+			er.getDisks = func() []StorageAPI { return faulty }
+			// Restore the callback before any fixture reads and on a fatal assertion.
+			defer func() { er.getDisks = original }()
+			opts := ObjectOptions{VersionID: nullVersionID, Transition: TransitionOptions{ExpireRestored: restored}, Expiration: ExpirationOptions{Expire: restored}}
+			_, err := er.DeleteObject(t.Context(), r.bucket, r.object, opts)
+			er.getDisks = original
+			if restored {
+				var quorum InsufficientWriteQuorum
+				if !errors.As(err, &quorum) {
+					t.Errorf("cleanup without fresh metadata: got %v, want write-quorum error", err)
+				}
+				if n := deletes.Load(); n != 0 {
+					t.Errorf("cleanup attempted %d blind disk deletes", n)
+				}
+				if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before, after) {
+					t.Error("failed metadata read changed the stored version")
+				}
+				if n := r.dataDirs(before.DataDir); n != len(disks) {
+					t.Errorf("cleanup removed data: %d/%d directories remain", n, len(disks))
+				}
+			} else {
+				// Ordinary unversioned DELETE retains the existing tryDel behavior.
+				if err != nil || deletes.Load() != int64(len(disks)) {
+					t.Fatalf("ordinary DELETE: err=%v, deletes=%d", err, deletes.Load())
+				}
+				if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); !isErrObjectNotFound(err) {
+					t.Fatalf("ordinary DELETE left the object: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestExpireRestoredRenewedOrMissing(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi := r.put("v1", false, true)
+	r.restore(oi)
+	stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil {
+		t.Fatal(err)
+	}
+	before := r.fileInfo(oi.VersionID)
+	renewed := completedRestoreObj(time.Now().Add(48 * time.Hour)).String()
+	opts := r.opts(oi.VersionID)
+	opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = renewed
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+		t.Fatal(err)
+	}
+	err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+	if !errors.Is(err, errLifecycleNotApplicable) {
+		t.Errorf("stale task should be rejected, got %v", err)
+	}
+	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir {
+		t.Error("stale task changed renewed restore")
+	}
+	if n := r.dataDirs(before.DataDir); n != 4 {
+		t.Errorf("renewed copy has %d directories, want 4", n)
+	}
+	if _, data, err := r.read(oi.VersionID); err != nil || !bytes.Equal(data, r.bodies["v1"]) {
+		t.Fatalf("renewed copy not readable: %v", err)
+	}
+
+	for _, id := range []string{mustGetUUID(), nullVersionID} {
+		opts := r.opts(id)
+		opts.Expiration.Expire = true
+		opts.Transition.ExpireRestored = true
+		if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+			t.Errorf("missing version %s: %v", id, err)
+		}
+	}
+	missing := r.opts(nullVersionID)
+	missing.Expiration.Expire = true
+	missing.Transition.ExpireRestored = true
+	if _, err := r.z.DeleteObject(t.Context(), r.bucket, "missing-object", missing); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+		t.Errorf("missing object: %v", err)
+	}
+	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed {
+		t.Error("missing-target cleanup changed existing restore")
+	}
+}
+
+func TestExpireRestoredStorageBoundary(t *testing.T) {
+	for _, status := range []string{"", lifecycle.TransitionPending, lifecycle.TransitionComplete} {
+		for _, inline := range []bool{false, true} {
+			for _, requestStatus := range []string{"", lifecycle.TransitionComplete} {
+				t.Run(fmt.Sprintf("stored=%s/inline=%v/request=%s", status, inline, requestStatus), func(t *testing.T) {
+					disk, _, err := newXLStorageTestSetup(t)
+					if err != nil {
+						t.Fatal(err)
+					}
+					const bucket, object = "restore-storage", "obj"
+					ctx := t.Context()
+					if err := disk.MakeVol(ctx, bucket); err != nil {
+						t.Fatal(err)
+					}
+					fi := newFileInfo(object, 2, 2)
+					fi.VersionID, fi.DataDir = mustGetUUID(), mustGetUUID()
+					fi.ModTime, fi.Size = time.Now().UTC(), 1024
+					fi.TransitionStatus = status
+					fi.AddObjectPart(1, "", fi.Size, fi.Size, fi.ModTime, nil, nil)
+					fi.Metadata = map[string]string{xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), "x-amz-meta-preserve": "keep"}
+					payload := []byte("sole local contents")
+					if inline {
+						fi.Data = payload
+						fi.SetInlineData()
+					}
+					if err := disk.WriteMetadata(ctx, "", bucket, object, fi); err != nil {
+						t.Fatal(err)
+					}
+					dir := filepath.Join(disk.Endpoint().Path, bucket, object, fi.DataDir)
+					if err := os.MkdirAll(dir, 0o700); err != nil {
+						t.Fatal(err)
+					}
+					if err := os.WriteFile(filepath.Join(dir, "part.1"), payload, 0o600); err != nil {
+						t.Fatal(err)
+					}
+					raw, err := disk.ReadXL(ctx, bucket, object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					var xl xlMetaV2
+					if err := xl.Load(raw.Buf); err != nil {
+						t.Fatal(err)
+					}
+					del := FileInfo{VersionID: fi.VersionID, ExpireRestored: true, TransitionStatus: requestStatus}
+					dataDir, err := xl.DeleteVersion(del)
+					if err != nil {
+						t.Fatal(err)
+					}
+					wantDir := ""
+					if status == lifecycle.TransitionComplete {
+						wantDir = fi.DataDir
+					}
+					if dataDir != wantDir {
+						t.Errorf("format authorized directory %q, want %q", dataDir, wantDir)
+					}
+					before, err := getFileInfo(raw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if err := disk.DeleteVersion(ctx, bucket, object, del, false, DeleteOptions{}); err != nil {
+						t.Fatal(err)
+					}
+					afterRaw, err := disk.ReadXL(ctx, bucket, object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					after, err := getFileInfo(afterRaw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					data, dirErr := os.ReadFile(filepath.Join(dir, "part.1"))
+					if status != lifecycle.TransitionComplete {
+						if !reflect.DeepEqual(before, after) {
+							t.Error("non-complete cleanup changed metadata or inline data")
+						}
+						if dirErr != nil || !bytes.Equal(data, payload) {
+							t.Errorf("sole local contents removed: %v", dirErr)
+						}
+					} else {
+						if !os.IsNotExist(dirErr) {
+							t.Errorf("completed restore directory still present: %v", dirErr)
+						}
+						if len(after.Data) != 0 || after.Metadata[xhttp.AmzRestore] != "" {
+							t.Error("completed restore kept cached data or restore headers")
+						}
+						if after.TransitionStatus != status || after.VersionID != before.VersionID || after.Metadata["x-amz-meta-preserve"] != "keep" {
+							t.Error("cleanup changed the transitioned version identity or user metadata")
+						}
+					}
+				})
+			}
+		}
+	}
+}

cmd/bucket-lifecycle-restore_test.go

@@ -0,0 +1,453 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreTestTier is a remote tier held in memory that counts its reads.
+type restoreTestTier struct {
+	*tierExpiryTestBackend
+	gets atomic.Int64
+}
+
+func (b *restoreTestTier) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	b.gets.Add(1)
+	return b.tierExpiryTestBackend.Get(ctx, object, rv, opts)
+}
+
+// restoreTest is a cluster with a remote tier WARM and a bucket whose
+// versions are transitioned to it and restored.
+type restoreTest struct {
+	t      *testing.T
+	z      *erasureServerPools
+	bucket string
+	object string
+	queue  chan expiryOp
+	labels map[string]string // version ID ("null" for the null version) -> label
+	bodies map[string][]byte // label -> data
+}
+
+func restoreTestVersionKey(versionID string) string {
+	if versionID == "" {
+		return nullVersionID
+	}
+	return versionID
+}
+
+// put writes a version of the object to the first pool. A restored version
+// is 1 MiB, so that its restored copy has a data directory of its own.
+func (r *restoreTest) put(label string, null, restored bool) ObjectInfo {
+	r.t.Helper()
+	body := []byte(r.object + " " + label)
+	if restored {
+		body = bytes.Repeat(body, (1<<20)/len(body)+1)
+	}
+	opts := ObjectOptions{Versioned: !null}
+	if null {
+		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+	}
+	oi, err := r.z.serverPools[0].PutObject(r.t.Context(), r.bucket, r.object,
+		mustGetPutObjReader(r.t, bytes.NewReader(body), int64(len(body)), "", ""), opts)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	r.labels[restoreTestVersionKey(oi.VersionID)] = label
+	r.bodies[label] = body
+	return oi
+}
+
+// opts returns the options of a request on a version of the object, as the
+// S3 handlers set them.
+func (r *restoreTest) opts(versionID string) ObjectOptions {
+	return ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object),
+	}
+}
+
+// restore transitions a version to the tier, restores it, and moves the
+// expiry date of the restore into the past.
+func (r *restoreTest) restore(oi ObjectInfo) {
+	r.t.Helper()
+	ctx := r.t.Context()
+	vopts := r.opts(oi.VersionID)
+	topts := vopts
+	topts.MTime = oi.ModTime
+	topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+	if err := r.z.TransitionObject(ctx, r.bucket, r.object, topts); err != nil {
+		r.t.Fatal(err)
+	}
+	ropts := vopts
+	ropts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+	// Through the pool, as with a single pool: with more than one pool the
+	// pools layer holds the object lock that the write of the restored copy
+	// waits for.
+	if err := r.z.serverPools[0].RestoreTransitionedObject(ctx, r.bucket, r.object, ropts); err != nil {
+		r.t.Fatal(err)
+	}
+	expired := completedRestoreObj(time.Now().Add(-24 * time.Hour).UTC()).String()
+	mopts := vopts
+	mopts.MTime = oi.ModTime
+	mopts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = expired
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, mopts); err != nil {
+		r.t.Fatal(err)
+	}
+}
+
+// fileInfo returns the version of the object from a drive of the first pool.
+func (r *restoreTest) fileInfo(versionID string) FileInfo {
+	r.t.Helper()
+	infos, errs := readAllFileInfo(r.t.Context(), r.z.serverPools[0].getHashedSet(r.object).getDisks(), "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false)
+	if errs[0] != nil {
+		r.t.Fatal(errs[0])
+	}
+	return infos[0]
+}
+
+// dataDirs returns how many drives of the first pool hold the data
+// directory of a version.
+func (r *restoreTest) dataDirs(dataDir string) int {
+	r.t.Helper()
+	n := 0
+	for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() {
+		_, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, dataDir))
+		if err == nil {
+			n++
+		} else if !os.IsNotExist(err) {
+			r.t.Fatal(err)
+		}
+	}
+	return n
+}
+
+// versions returns the versions of the object in each pool, newest first,
+// and its free versions in all pools.
+func (r *restoreTest) versions() (views []string, free int) {
+	r.t.Helper()
+	for _, pool := range r.z.serverPools {
+		rf, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+		if errors.Is(err, errFileNotFound) {
+			views = append(views, "")
+			continue
+		}
+		if err != nil {
+			r.t.Fatal(err)
+		}
+		fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+		if err != nil {
+			r.t.Fatal(err)
+		}
+		var labels []string
+		for _, fi := range fivs.Versions {
+			if fi.Deleted {
+				labels = append(labels, "DM")
+			} else {
+				labels = append(labels, r.labels[restoreTestVersionKey(fi.VersionID)])
+			}
+		}
+		views = append(views, strings.Join(labels, " "))
+		free += len(fivs.FreeVersions)
+	}
+	return views, free
+}
+
+// read returns the label and the data of a version, or of the latest version
+// when versionID is empty, or the error of the read.
+func (r *restoreTest) read(versionID string) (string, []byte, error) {
+	r.t.Helper()
+	gr, err := r.z.GetObjectNInfo(r.t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: versionID})
+	if err != nil {
+		return "", nil, err
+	}
+	defer gr.Close()
+	data, err := io.ReadAll(gr)
+	if err != nil {
+		return "", nil, err
+	}
+	return r.labels[restoreTestVersionKey(gr.ObjInfo.VersionID)], data, nil
+}
+
+// runQueue runs the queued expiries with an expiry worker.
+func (r *restoreTest) runQueue() {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		globalExpiryState.Worker(r.queue)
+	}()
+	r.queue <- nil // the worker returns once it ran every expiry queued before
+	<-done
+}
+
+// objInfos returns the versions of the object as a drive of the
+// first pool holds them, as the scanner and the walks of decommission and
+// the drive heal see them.
+func (r *restoreTest) objInfos() []ObjectInfo {
+	r.t.Helper()
+	rf, err := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	versioned := globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object) || globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+	objInfos := make([]ObjectInfo, len(fivs.Versions))
+	for i, fi := range fivs.Versions {
+		objInfos[i] = fi.ToObjectInfo(r.bucket, r.object, versioned)
+	}
+	return objInfos
+}
+
+// The restored copy of a transitioned version expires on the expiry date of
+// its restore. The expiry removes the copy and nothing else: the version
+// stays where it is in the version stack, transitioned, and is read from the
+// tier again. Every path that applies ILM runs the expiry: the scanner and a
+// GET or HEAD queue it for an expiry worker, decommission and the drive heal
+// run it at once. The expiry of the restored copy of the latest version used
+// to delete the object instead: a versioned bucket got a delete marker that
+// hid it. With versioning suspended it went to the null version, removed its
+// data if there was one and failed if not, and the restored copy stayed.
+func TestExpireRestoredCopy(t *testing.T) {
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	prevExpiry := globalExpiryState
+	t.Cleanup(func() { globalExpiryState = prevExpiry })
+
+	// Nothing here is old enough to transition; the rule makes the bucket
+	// one with a tier, as restored versions are in.
+	const rules = `<LifecycleConfiguration><Rule><ID>tier</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
+		`<Transition><Days>1000</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`
+
+	type version struct {
+		label    string
+		null     bool // the null version
+		restored bool // transitioned and restored, the restore has expired
+	}
+	testCases := []struct {
+		name       string
+		versioning versioning.State // "", Enabled or Suspended
+		versions   []version        // oldest first
+	}{
+		{name: "versioned", versioning: versioning.Enabled, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+		{name: "versioned-noncurrent", versioning: versioning.Enabled, versions: []version{{label: "v1", restored: true}, {label: "v2"}}},
+		{name: "versioned-null", versioning: versioning.Enabled, versions: []version{{label: "v1", null: true, restored: true}}},
+		{name: "suspended", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true}, {label: "v2", restored: true}}},
+		{name: "suspended-no-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+		{name: "suspended-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", null: true, restored: true}}},
+		{name: "suspended-null-noncurrent", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true, restored: true}, {label: "v2"}}},
+		{name: "unversioned", versions: []version{{label: "v1", null: true, restored: true}}},
+	}
+
+	sources := []struct {
+		name   string
+		expire func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event)
+	}{
+		{"scanner", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+			lc, err := globalLifecycleSys.Get(r.bucket)
+			if err != nil {
+				r.t.Fatal(err)
+			}
+			lr, err := globalBucketObjectLockSys.Get(r.bucket)
+			if err != nil {
+				r.t.Fatal(err)
+			}
+			item := scannerItem{bucket: r.bucket, objectName: r.object, lifeCycle: lc}
+			item.applyActions(r.t.Context(), r.z, r.objInfos(), lr, &sizeSummary{}, nil)
+			r.runQueue()
+		}},
+		{"head", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+			applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi)
+			r.runQueue()
+		}},
+		{"list", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+			// ListObjects of the key with max-keys=1 reads its latest
+			// version without the versioning of the bucket, a listing of
+			// versions reads them all.
+			if _, err := r.z.ListObjects(r.t.Context(), r.bucket, r.object, "", "", 1); err != nil {
+				r.t.Fatal(err)
+			}
+			if _, err := r.z.ListObjectVersions(r.t.Context(), r.bucket, r.object, "", "", "", 100); err != nil {
+				r.t.Fatal(err)
+			}
+			r.runQueue()
+		}},
+		{"decom", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+			for _, o := range r.objInfos() {
+				if o.VersionID == oi.VersionID {
+					applyExpiryOnTransitionedObject(r.t.Context(), r.z, o, evt, lcEventSrc_Decom)
+					return
+				}
+			}
+			r.t.Fatal("version not found on the drive")
+		}},
+	}
+
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			ctx := t.Context()
+			dirs, err := getRandomDisks(4 * pools)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { removeRoots(dirs) })
+			var endpoints EndpointServerPools
+			for i := range pools {
+				endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+			}
+			obj, _, err := initObjectLayer(ctx, endpoints)
+			if err != nil {
+				t.Fatal(err)
+			}
+			z := obj.(*erasureServerPools)
+			t.Cleanup(func() { z.Shutdown(context.Background()) })
+			// After initObjectLayer, which sets up a new tier config.
+			tier := &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+			globalTierConfigMgr = NewTierConfigMgr()
+			globalTierConfigMgr.drivercache["WARM"] = tier
+
+			for _, tc := range testCases {
+				for _, src := range sources {
+					t.Run(tc.name+"/"+src.name, func(t *testing.T) {
+						r := &restoreTest{
+							t: t, z: z,
+							bucket: "restore-" + tc.name + "-" + src.name, object: "obj",
+							queue:  make(chan expiryOp, 100),
+							labels: map[string]string{}, bodies: map[string][]byte{},
+						}
+						if err := z.MakeBucket(ctx, r.bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
+							t.Fatal(err)
+						}
+						meta, err := globalBucketMetadataSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						if tc.versioning == versioning.Suspended {
+							meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+						}
+						meta.LifecycleConfigXML = []byte(rules)
+						if err := meta.Save(ctx, z); err != nil {
+							t.Fatal(err)
+						}
+						globalBucketMetadataSys.Set(r.bucket, meta)
+						es := &expiryState{ctx: ctx, objAPI: z}
+						workers := []chan expiryOp{r.queue}
+						es.workers.Store(&workers)
+						globalExpiryState = es
+
+						var restored ObjectInfo
+						for _, v := range tc.versions {
+							oi := r.put(v.label, v.null, v.restored)
+							if v.restored {
+								r.restore(oi)
+								restored = oi
+							}
+						}
+						rlabel := r.labels[restoreTestVersionKey(restored.VersionID)]
+
+						// The fixture: the restored copy is on the drives,
+						// its restore has expired, ILM expires it.
+						fi := r.fileInfo(restored.VersionID)
+						if fi.TransitionStatus != lifecycle.TransitionComplete || fi.DataDir == "" || r.dataDirs(fi.DataDir) == 0 {
+							t.Fatalf("fixture: %s transition status %q, restored copy on %d drives", rlabel, fi.TransitionStatus, r.dataDirs(fi.DataDir))
+						}
+						oi, err := z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+						if err != nil {
+							t.Fatal(err)
+						}
+						lc, err := globalLifecycleSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						lr, err := globalBucketObjectLockSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
+						if !evt.Action.DeleteRestored() || oi.RestoreExpires.IsZero() {
+							t.Fatalf("fixture: %s restore expires %v, action %v", rlabel, oi.RestoreExpires, evt.Action)
+						}
+						before, _ := r.versions()
+						for key, label := range r.labels {
+							if _, data, err := r.read(key); err != nil || !bytes.Equal(data, r.bodies[label]) {
+								t.Fatalf("fixture: GET %s: %v", label, err)
+							}
+						}
+
+						src.expire(r, oi, evt)
+
+						after, free := r.versions()
+						if fmt.Sprint(after) != fmt.Sprint(before) || free != 0 {
+							t.Errorf("versions in each pool: before %q, after %q with %d free versions", before, after, free)
+						}
+						if label, _, err := r.read(""); err != nil || label != tc.versions[len(tc.versions)-1].label {
+							t.Errorf("GET reads %q, %v; want %q", label, err, tc.versions[len(tc.versions)-1].label)
+						}
+						if n := r.dataDirs(fi.DataDir); n != 0 {
+							t.Errorf("restored copy of %s still on %d drives", rlabel, n)
+						}
+						oi, err = z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+						if err != nil {
+							t.Fatalf("HEAD %s: %v", rlabel, err)
+						}
+						if oi.TransitionedObject.Status != lifecycle.TransitionComplete || oi.UserDefined[xhttp.AmzRestore] != "" || !oi.RestoreExpires.IsZero() {
+							t.Errorf("HEAD %s: transition status %q, x-amz-restore %q", rlabel, oi.TransitionedObject.Status, oi.UserDefined[xhttp.AmzRestore])
+						}
+						for _, v := range tc.versions {
+							for key, label := range r.labels {
+								if label != v.label {
+									continue
+								}
+								gets := tier.gets.Load()
+								_, data, err := r.read(key)
+								switch {
+								case err != nil:
+									t.Errorf("GET %s: %v", v.label, err)
+								case !bytes.Equal(data, r.bodies[v.label]):
+									t.Errorf("GET %s reads %d bytes that differ", v.label, len(data))
+								case v.restored && tier.gets.Load() == gets:
+									t.Errorf("GET %s does not read the tier", v.label)
+								}
+							}
+						}
+					})
+				}
+			}
+		})
+	}
+}

cmd/bucket-lifecycle.go

@@ -37,7 +37,8 @@ import (
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
@@ -161,7 +162,7 @@ func (n noncurrentVersionsTask) OpHash() uint64 {
 }
 
 func (j jentry) OpHash() uint64 {
-	return xxh3.HashString(j.TierName + j.ObjName)
+	return xxh3.HashString(j.Bucket + j.Object + j.TierName + j.RemoteObject + j.RemoteVersionID)
 }
 
 func (e expiryTask) OpHash() uint64 {
@@ -353,7 +354,16 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 			case noncurrentVersionsTask:
 				deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.events)
 			case jentry:
-				transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+				z, ok := es.objAPI.(*erasureServerPools)
+				if !ok {
+					transitionLogIf(es.ctx, errTierOwnershipUnavailable)
+					continue
+				}
+				transitionLogIf(es.ctx, z.deleteTierTuple(es.ctx, v.Bucket, v.Object, tierRemoteTuple{
+					Tier:      v.TierName,
+					Object:    v.RemoteObject,
+					VersionID: v.RemoteVersionID,
+				}))
 			case freeVersionTask:
 				oi := v.ObjectInfo
 				traceFn := globalLifecycleSys.trace(oi)
@@ -369,19 +379,26 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 					}
 					return err
 				}
-				// Remove the remote object
-				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
+				z, ok := es.objAPI.(*erasureServerPools)
+				if !ok {
+					transitionLogIf(es.ctx, errTierOwnershipUnavailable)
+					traceFn(ILMFreeVersionDelete, nil, errTierOwnershipUnavailable)
+					continue
+				}
+
+				// Check ownership, remove the remote object, and remove only the
+				// matching free record while the object lock is held.
+				err := z.deleteTierTuple(es.ctx, oi.Bucket, oi.Name, tierRemoteTuple{
+					Tier:      oi.TransitionedObject.Tier,
+					Object:    oi.TransitionedObject.Name,
+					VersionID: oi.TransitionedObject.VersionID,
+				})
 				if ignoreNotFoundErr(err) != nil {
 					transitionLogIf(es.ctx, err)
 					traceFn(ILMFreeVersionDelete, nil, err)
 					continue
 				}
 
-				// Remove this free version
-				_, err = es.objAPI.DeleteObject(es.ctx, oi.Bucket, oi.Name, ObjectOptions{
-					VersionID:        oi.VersionID,
-					InclFreeVersions: true,
-				})
 				if err == nil {
 					auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn)
 				}
@@ -610,8 +627,9 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error {
 	traceFn := globalLifecycleSys.trace(*oi)
 	opts := ObjectOptions{
-		Versioned:  globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
-		Expiration: ExpirationOptions{Expire: true},
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		Expiration:       ExpirationOptions{Expire: true},
 	}
 	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = oi.VersionID
@@ -621,6 +639,17 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
+		//
+		// The restored copy belongs to this version, also when it is the
+		// latest one (DeleteRestoredAction). Without its version ID the
+		// delete applies to the object: a versioned bucket gets a delete
+		// marker that hides it, a suspended one strips the null version.
+		// An empty version ID is the null version, as callers that read
+		// it without the versioning of the bucket leave it.
+		opts.VersionID = oi.VersionID
+		if opts.VersionID == "" {
+			opts.VersionID = nullVersionID
+		}
 		opts.Transition.ExpireRestored = true
 		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 		if err == nil {
@@ -631,32 +660,54 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		return err
 	}
 
-	// Delete remote object from warm-tier
-	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
-	if err == nil {
-		// Skip adding free version since we successfully deleted the
-		// remote object
-		opts.SkipFreeVersion = true
-	} else {
-		transitionLogIf(ctx, err)
+	// Recheck the lifecycle decision and create the hot-tier tombstone before
+	// touching the remote tier. If the lifecycle configuration changed after
+	// the scanner queued this task, DeleteObject rejects the stale action while
+	// the remote object is still protected.
+	keepRemote := opts.VersionID == "" &&
+		(opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID))
+
+	// Delete the hot-tier namespace first. For a transitioned version this
+	// leaves a durable free-version record until the remote delete succeeds;
+	// that record is the retry point if the process stops between the two
+	// operations.
+	dobj, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	if err != nil {
+		return err
 	}
 
-	// Now, delete object from hot-tier namespace
-	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-		return err
+	if !keepRemote {
+		z, ok := objectAPI.(*erasureServerPools)
+		if !ok {
+			transitionLogIf(ctx, errTierOwnershipUnavailable)
+			return nil // The completed local expiry owns a durable free record.
+		}
+		if err := z.deleteTierTuple(ctx, oi.Bucket, oi.Name, tierRemoteTuple{
+			Tier:      oi.TransitionedObject.Tier,
+			Object:    oi.TransitionedObject.Name,
+			VersionID: oi.TransitionedObject.VersionID,
+		}); err != nil {
+			// The hot delete has already created a durable free record. Keep
+			// the expiry successful while the scanner retries remote cleanup;
+			// ownership or tier errors must never cause a second remote delete
+			// without that record.
+			transitionLogIf(ctx, err)
+		}
 	}
 
 	// Send audit for the lifecycle delete operation
 	defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
 
-	eventName := event.ObjectRemovedDelete
-	if oi.DeleteMarker {
-		eventName = event.ObjectRemovedDeleteMarkerCreated
-	}
+	// Name what the delete did: a marker it created, or the version it
+	// removed.
+	eventName := ilmExpiryEventName(opts.VersionID, dobj)
 	objInfo := ObjectInfo{
 		Name:         oi.Name,
-		VersionID:    oi.VersionID,
-		DeleteMarker: oi.DeleteMarker,
+		VersionID:    dobj.VersionID,
+		DeleteMarker: dobj.DeleteMarker,
+	}
+	if objInfo.VersionID == "" {
+		objInfo.VersionID = opts.VersionID
 	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
@@ -931,8 +982,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
 	}, nil
 }
 
-// set ObjectOptions for PUT call to restore temporary copy of transitioned data
-func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data.
+// opts are the restore's options, with the versioning POST captured: the
+// bucket's versioning may have changed since, and must not mint a new version.
+func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) {
 	meta := make(map[string]string)
 	sc := rreq.OutputLocation.S3.StorageClass
 	if sc == "" {
@@ -955,8 +1008,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
-			Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+			Versioned:        opts.Versioned,
+			VersionSuspended: opts.VersionSuspended,
 			UserDefined:      meta,
 		}
 	}
@@ -964,19 +1017,37 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	if len(objInfo.UserTags) != 0 {
 		meta[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-	// Set restore object status
+	// Set restore object status; a completed restore has no attempt.
 	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
 	meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
+	delete(meta, restoreAttemptKey)
 	return ObjectOptions{
-		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
 		UserDefined:      meta,
-		VersionID:        objInfo.VersionID,
+		VersionID:        restoreVersionID(objInfo.VersionID),
 		MTime:            objInfo.ModTime,
 		Expires:          objInfo.Expires,
 	}
 }
 
+// restoredCopyExpires reports whether the restored copy of oi, as read now,
+// expires by its own reason: lifecycle expires the restored copy of this
+// version, its restore deadline is valid and has passed, its transition is
+// complete and no restore is running. Another rule falling due is no reason.
+func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool {
+	if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() {
+		return false
+	}
+	status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore])
+	if err != nil {
+		return false
+	}
+	expiry, completed := status.Expiry()
+	return completed && !expiry.IsZero() && UTCNow().After(expiry) &&
+		oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing
+}
+
 var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
 
 // IsRemote returns true if this object version's contents are in its remote

cmd/bucket-lifecycle_test.go

@@ -19,11 +19,17 @@ package cmd
 
 import (
 	"bytes"
+	"context"
+	"io"
 	"net/http"
+	"slices"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	xhttp "github.com/minio/minio/internal/http"
 )
 
@@ -245,3 +251,180 @@ func TestValidateTransitionTier(t *testing.T) {
 		}
 	}
 }
+
+// tierExpiryTestBackend is a remote tier held in memory.
+type tierExpiryTestBackend struct {
+	mu      sync.Mutex
+	objects map[string][]byte
+	removed []string
+}
+
+func (b *tierExpiryTestBackend) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+	return b.PutWithMeta(ctx, object, r, length, nil)
+}
+
+func (b *tierExpiryTestBackend) PutWithMeta(_ context.Context, object string, r io.Reader, _ int64, _ map[string]string) (remoteVersionID, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return "", err
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.objects[object] = data
+	return "", nil
+}
+
+func (b *tierExpiryTestBackend) Get(_ context.Context, object string, _ remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	data, ok := b.objects[object]
+	if !ok {
+		return nil, ObjectNotFound{Bucket: "tier", Object: object}
+	}
+	if opts.length > 0 {
+		data = data[opts.startOffset : opts.startOffset+opts.length]
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b *tierExpiryTestBackend) Remove(_ context.Context, object string, _ remoteVersionID) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	delete(b.objects, object)
+	b.removed = append(b.removed, object)
+	return nil
+}
+
+func (b *tierExpiryTestBackend) InUse(context.Context) (bool, error) { return false, nil }
+
+// The expiry of a transitioned latest version that only puts a delete marker
+// on top of it must keep its remote object, so the version stays readable as
+// a noncurrent version. The remote object goes when the version is deleted.
+func TestExpireTransitionedObjectKeepsNoncurrentData(t *testing.T) {
+	z, _ := migrationTestPools(t)
+	ctx := t.Context()
+	tier := &tierExpiryTestBackend{objects: map[string][]byte{}}
+	prevTier := globalTierConfigMgr
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.drivercache["WARM"] = tier
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+
+	const expire = `<LifecycleConfiguration><Rule><ID>exp</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
+		`<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration></Rule></LifecycleConfiguration>`
+
+	testCases := []struct {
+		name       string
+		versioning string   // "", Enabled or Suspended
+		nullLatest bool     // the latest version is the null version
+		keep       bool     // the latest version survives as noncurrent
+		want       []string // versions after the expiry, newest first
+	}{
+		{name: "versioned", versioning: "Enabled", keep: true, want: []string{"DM", "v2", "v1"}},
+		{name: "suspended", versioning: "Suspended", keep: true, want: []string{"DM", "v2", "v1"}},
+		{name: "suspended-null", versioning: "Suspended", nullLatest: true, want: []string{"DM", "v1"}},
+		{name: "unversioned", nullLatest: true, want: nil},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			bucket := "tier-expiry-" + tc.name
+			if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
+				t.Fatal(err)
+			}
+			meta, err := globalBucketMetadataSys.Get(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tc.versioning == "Suspended" {
+				meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+			}
+			meta.LifecycleConfigXML = []byte(expire)
+			if err := meta.Save(ctx, z); err != nil {
+				t.Fatal(err)
+			}
+			globalBucketMetadataSys.Set(bucket, meta)
+
+			// tiered writes a version whose data is in the remote tier.
+			labels := map[string]string{}
+			tiered := func(label string, opts ObjectOptions) ObjectInfo {
+				t.Helper()
+				oi := putConsistencyObject(t, z, bucket, "obj", 0, "obj "+label, opts)
+				vid := migrationTestVersion(oi)
+				infos, errs := readAllFileInfo(ctx, z.serverPools[0].getHashedSet("obj").getDisks(), "", bucket, "obj", vid, false, false)
+				if errs[0] != nil {
+					t.Fatal(errs[0])
+				}
+				fi := infos[0]
+				fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", bucket+"/"+label
+				if _, err := tier.Put(ctx, fi.TransitionedObjName, strings.NewReader("obj "+label), -1); err != nil {
+					t.Fatal(err)
+				}
+				if err := z.serverPools[0].DecomTieredObject(ctx, bucket, "obj", fi, ObjectOptions{}); err != nil {
+					t.Fatal(err)
+				}
+				labels[vid] = label
+				return oi
+			}
+			var latest ObjectInfo
+			if tc.versioning != "" {
+				tiered("v1", ObjectOptions{Versioned: true})
+			}
+			if tc.nullLatest {
+				latest = tiered("v2", ObjectOptions{})
+			} else {
+				latest = tiered("v2", ObjectOptions{Versioned: true})
+			}
+
+			// Expire the latest version as the expiry worker does for the scanner.
+			getOpts := ObjectOptions{Versioned: tc.versioning == "Enabled", VersionSuspended: tc.versioning == "Suspended"}
+			oi, err := z.GetObjectInfo(ctx, bucket, "obj", getOpts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			lc, err := globalLifecycleSys.Get(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			evt := evalActionFromLifecycle(ctx, *lc, objectlock.Retention{}, nil, oi)
+			if evt.Action != lifecycle.DeleteAction || oi.TransitionedObject.Status != lifecycle.TransitionComplete {
+				t.Fatalf("fixture: action %v, transition status %q", evt.Action, oi.TransitionedObject.Status)
+			}
+			if err := expireTransitionedObject(ctx, z, &oi, evt, lcEventSrc_Scanner); err != nil {
+				t.Errorf("expiry: %v", err)
+			}
+
+			var got []string
+			for _, set := range z.serverPools[0].sets {
+				err := set.listObjectsToDecommission(ctx, decomBucketInfo{Name: bucket}, func(entry metaCacheEntry) {
+					fivs, err := entry.fileInfoVersions(bucket)
+					if err != nil {
+						t.Error(err)
+						return
+					}
+					for _, fi := range fivs.Versions {
+						if fi.Deleted {
+							got = append(got, "DM")
+						} else {
+							got = append(got, labels[migrationTestVersion(fi.ToObjectInfo(bucket, "obj", true))])
+						}
+					}
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			if !slices.Equal(got, tc.want) {
+				t.Errorf("versions after expiry: got %v, want %v", got, tc.want)
+			}
+			remote := bucket + "/v2"
+			if removed := slices.Contains(tier.removed, remote); removed == tc.keep {
+				t.Errorf("remote object of the expired version removed: %v, want %v", removed, !tc.keep)
+			}
+			if tc.keep {
+				body := migrationTestBody(t, z.GetObjectNInfo, bucket, "obj", migrationTestVersion(latest))
+				if body != "obj v2" {
+					t.Errorf("v2 reads %q after the expiry, want %q", body, "obj v2")
+				}
+			}
+		})
+	}
+}

cmd/bucket-metadata_test.go

@@ -39,3 +39,28 @@ func TestBucketMetadataCorsRoundTrip(t *testing.T) {
 		t.Fatalf("CorsConfigUpdatedAt not preserved")
 	}
 }
+
+// A persisted retired extension must not prevent the whole bucket's metadata
+// from loading, including unrelated versioning and ordinary lifecycle rules.
+func TestBucketMetadataRetiredAccessTiering(t *testing.T) {
+	meta := newBucketMetadata("retired-access")
+	meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><AccessTierQuota>500GiB</AccessTierQuota><Rule><ID>access</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AccessTransition><Window>10m</Window><PromoteAfterAccesses>10</PromoteAfterAccesses></AccessTransition></Rule><Rule><ID>ordinary</ID><Status>Enabled</Status><Filter><Prefix>expired/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
+	meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
+	data, err := meta.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := newBucketMetadata(meta.Name)
+	if _, err := got.UnmarshalMsg(data); err != nil {
+		t.Fatal(err)
+	}
+	if err := got.parseAllConfigs(t.Context(), nil); err != nil {
+		t.Fatalf("bucket metadata failed to load: %v", err)
+	}
+	if got.lifecycleConfig == nil || got.lifecycleConfig.HasActiveRules("logs/") || !got.lifecycleConfig.HasActiveRules("expired/") {
+		t.Fatal("unexpected lifecycle behavior")
+	}
+	if got.versioningConfig == nil || !got.versioningConfig.Enabled() {
+		t.Fatal("unrelated versioning lost")
+	}
+}

cmd/bucket-object-lock.go

@@ -20,8 +20,8 @@ package cmd
 import (
 	"context"
 	"errors"
-	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -164,76 +164,176 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 	return nil
 }
 
-// enforceRetentionBypassForPut enforces whether an existing object under governance can be overwritten
-// with governance bypass headers set in the request.
-// Objects under site wide WORM cannot be overwritten.
-// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
-// governance bypass headers are set and user has governance bypass permissions.
-// Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
+// isRetentionActionAllowed evaluates action for a PutObjectRetention request
+// with the object-lock condition keys derived from the requested retention.
+// The keys come from the request, not from the stored version, so the
+// handler can decide s3:PutObjectRetention before reading any object state.
+func isRetentionActionAllowed(action policy.Action, bucket, object string, objRetention *objectlock.ObjectRetention, r *http.Request, cred auth.Credentials, owner bool) bool {
+	if cred.AccessKey == "" {
+		return false
+	}
+	t, err := objectlock.UTCNowNTP()
+	if err != nil {
+		return false
+	}
+	conditions := getConditionValues(r, "", cred)
+	conditions["object-lock-mode"] = []string{string(objRetention.Mode)}
+	conditions["object-lock-retain-until-date"] = []string{objRetention.RetainUntilDate.UTC().Format(time.RFC3339)}
+	// Relative days from now, for "object-lock-remaining-retention-days".
+	// Clearing retention (<Retention/>) has a zero RetainUntilDate, which must
+	// evaluate as 0 remaining days rather than a large positive math.Abs delta
+	// from year 0001.
+	days := 0
+	if !objRetention.RetainUntilDate.IsZero() && objRetention.RetainUntilDate.After(t) {
+		// Time.Sub saturates after about 292 years. Count seconds instead so
+		// maximum-retention policies also apply to distant RFC3339 dates.
+		seconds := objRetention.RetainUntilDate.Unix() - t.Unix()
+		days = int(seconds / 86400)
+		if seconds%86400 != 0 || objRetention.RetainUntilDate.Nanosecond() > t.Nanosecond() {
+			days++
+		}
+	}
+	conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(days)}
+	return globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          action,
+		BucketName:      bucket,
+		ConditionValues: conditions,
+		ObjectName:      object,
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+	})
+}
+
+// enforceRetentionBypassForPut enforces the stored retention of oi against a
+// PutObjectRetention request. The caller must already hold
+// s3:PutObjectRetention; this only decides changes that depend on the stored
+// state. Expired or absent retention may be changed freely. Unexpired
+// GOVERNANCE retention may be shortened, removed or have its mode changed only
+// with the bypass header and s3:BypassGovernanceRetention. Unexpired
+// COMPLIANCE retention may only be extended.
 func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi ObjectInfo, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) error {
-	byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header)
+	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
+	if !ret.Mode.Valid() {
+		return nil
+	}
 
 	t, err := objectlock.UTCNowNTP()
 	if err != nil {
 		internalLogIf(ctx, err, logger.WarningKind)
 		return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 	}
+	if ret.RetainUntilDate.Before(t) {
+		return nil
+	}
 
-	// Pass in relative days from current time, to additionally
-	// to verify "object-lock-remaining-retention-days" policy if any.
-	days := int(math.Ceil(math.Abs(objRetention.RetainUntilDate.Sub(t).Hours()) / 24))
-
-	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
-	if ret.Mode.Valid() {
-		// Retention has expired you may change whatever you like.
-		if ret.RetainUntilDate.Before(t) {
-			apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
-				days, objRetention.RetainUntilDate.Time,
-				objRetention.Mode, byPassSet, r, cred,
-				owner)
-			if apiErr == ErrAccessDenied {
-				return errAuthentication
-			}
-			return nil
+	weakens := objRetention.Mode != ret.Mode || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time)
+	if !weakens {
+		return nil
+	}
+	switch ret.Mode {
+	case objectlock.RetGovernance:
+		// https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html#object-lock-bypass-governance
+		if !objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
+			return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 		}
-
-		switch ret.Mode {
-		case objectlock.RetGovernance:
-			govPerm := isPutRetentionAllowed(oi.Bucket, oi.Name, days,
-				objRetention.RetainUntilDate.Time, objRetention.Mode,
-				byPassSet, r, cred, owner)
-			// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
-			if !byPassSet {
-				if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
-					return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
-				}
-			}
-			if govPerm == ErrAccessDenied {
-				return errAuthentication
-			}
-			return nil
-		case objectlock.RetCompliance:
-			// Compliance retention mode cannot be changed or shortened.
-			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
-			if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
-				return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
-			}
-			apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
-				days, objRetention.RetainUntilDate.Time, objRetention.Mode,
-				false, r, cred, owner)
-			if apiErr == ErrAccessDenied {
-				return errAuthentication
-			}
-			return nil
+		if !isRetentionActionAllowed(policy.BypassGovernanceRetentionAction, oi.Bucket, oi.Name, objRetention, r, cred, owner) {
+			return errAuthentication
 		}
 		return nil
-	} // No pre-existing retention metadata present.
+	default:
+		// Compliance retention mode cannot be changed or shortened.
+		// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
+		return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
+	}
+}
 
-	apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
-		days, objRetention.RetainUntilDate.Time,
-		objRetention.Mode, byPassSet, r, cred, owner)
-	if apiErr == ErrAccessDenied {
-		return errAuthentication
+// checkObjectLockedForOverwrite re-checks under the object write lock whether an
+// in-place write to the null version (such as a directory object) would
+// overwrite a locked version in an Object Lock bucket.
+func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn) error {
+	if isMinioMetaBucketName(bucket) {
+		return nil
+	}
+	retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
+	if err != nil {
+		return err
+	}
+	if !retentionCfg.LockEnabled {
+		return nil
+	}
+	objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
+		VersionID:  nullVersionID,
+		NoLock:     true,
+		NoAuditLog: true,
+	})
+	if err != nil {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
+			return nil
+		}
+		return err
+	}
+	if objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined).Status == objectlock.LegalHoldOn {
+		return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
+	}
+	r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
+	if r.Mode == objectlock.RetCompliance || r.Mode == objectlock.RetGovernance {
+		t, err := objectlock.UTCNowNTP()
+		if err != nil {
+			internalLogIf(ctx, err, logger.WarningKind)
+			return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
+		}
+		if r.RetainUntilDate.After(t) {
+			return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: nullVersionID}
+		}
+	}
+	return nil
+}
+
+// checkVersionLockedForOverwrite re-checks under the object write lock
+// whether a write would replace the stored version versionID (the null
+// version if empty) while Object Lock protects it. It reads the version's own
+// metadata, whatever the bucket configuration, and a failed read is an error.
+func checkVersionLockedForOverwrite(ctx context.Context, bucket, object, versionID string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error {
+	if isMinioMetaBucketName(bucket) {
+		return nil
+	}
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
+		VersionID:  versionID,
+		Transition: transition,
+		NoLock:     true,
+		NoAuditLog: true,
+	})
+	if err != nil {
+		// A delete marker is not protected.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
+			return nil
+		}
+		return err
+	}
+	if enforceRetentionForDeletion(ctx, objInfo) {
+		return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+	}
+	return nil
+}
+
+// refuseLockedMarkerOverwrite refuses a delete marker write that would replace
+// the stored version it names while Object Lock protects it. goi and gerr are
+// that version as read under the write lock. A new marker, or one replacing a
+// marker, is allowed.
+func refuseLockedMarkerOverwrite(ctx context.Context, bucket, object, versionID string, goi ObjectInfo, gerr error) error {
+	if gerr != nil {
+		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) || errors.As(gerr, &MethodNotAllowed{}) {
+			return nil
+		}
+		return gerr
+	}
+	if enforceRetentionForDeletion(ctx, goi) {
+		return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
 	}
 	return nil
 }
@@ -274,16 +374,24 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 	if err != nil {
 		return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 	}
+	if isDirObject(object) && opts.VersionID == "" {
+		opts.VersionID = nullVersionID
+	}
 
 	if opts.VersionID != "" && !replicaTrusted {
-		if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
+		objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
+		if err != nil {
+			if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !errors.As(err, &MethodNotAllowed{}) {
+				return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
+			}
+		} else {
 			r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
 				internalLogIf(ctx, err, logger.WarningKind)
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
-			if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
+			if (r.Mode == objectlock.RetCompliance || r.Mode == objectlock.RetGovernance) && r.RetainUntilDate.After(t) {
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
 			mode = r.Mode

cmd/bucket-replication-checksum_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"maps"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// checksumTestObject builds the xl.meta checksum of an object the way
+// PutObject and CompleteMultipartUpload store it.
+func checksumTestObject(t *testing.T, alg hash.ChecksumType, full bool, parts ...[]byte) FileInfo {
+	t.Helper()
+	fi := FileInfo{Volume: "bucket", Name: "object"}
+	if len(parts) == 1 {
+		fi.Size = int64(len(parts[0]))
+		fi.Checksum = hash.NewChecksumFromData(alg, parts[0]).AppendTo(nil, nil)
+		return fi
+	}
+	var combined, all []byte
+	for i, p := range parts {
+		combined = append(combined, hash.NewChecksumFromData(alg, p).Raw...)
+		all = append(all, p...)
+		fi.Parts = append(fi.Parts, ObjectPartInfo{Number: i + 1, Size: int64(len(p)), ActualSize: int64(len(p))})
+		fi.Size += int64(len(p))
+	}
+	typ := alg | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
+	if full {
+		typ |= hash.ChecksumFullObject
+	}
+	cs := hash.NewChecksumFromData(typ, combined)
+	if typ.FullObjectRequested() {
+		// Full object CRC, as merged from the parts on completion.
+		whole := hash.NewChecksumFromData(alg, all)
+		cs = &hash.Checksum{Type: typ, Raw: whole.Raw, Encoded: whole.Encoded}
+	}
+	fi.Checksum = cs.AppendTo(nil, combined)
+	return fi
+}
+
+// TestGetCRCMetaChecksumType checks the checksum headers that replication
+// sends for an object: a FULL_OBJECT checksum type only for multipart full
+// object CRCs, where the target needs it to create a matching upload.
+func TestGetCRCMetaChecksumType(t *testing.T) {
+	p1, p2 := []byte("first part"), []byte("second part")
+	enc := func(alg hash.ChecksumType, b []byte) string { return hash.NewChecksumFromData(alg, b).Encoded }
+	whole := append(append([]byte{}, p1...), p2...)
+	composite := func(alg hash.ChecksumType) string {
+		var combined []byte
+		combined = append(combined, hash.NewChecksumFromData(alg, p1).Raw...)
+		combined = append(combined, hash.NewChecksumFromData(alg, p2).Raw...)
+		return enc(alg, combined) + "-2"
+	}
+	full := map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject}
+	tests := []struct {
+		name  string
+		fi    FileInfo
+		alg   hash.ChecksumType
+		isMP  bool
+		value string
+		extra map[string]string
+	}{
+		{"single-part SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1), hash.ChecksumSHA256, false, enc(hash.ChecksumSHA256, p1), nil},
+		{"single-part CRC32", checksumTestObject(t, hash.ChecksumCRC32, false, p1), hash.ChecksumCRC32, false, enc(hash.ChecksumCRC32, p1), nil},
+		{"single-part CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1), hash.ChecksumCRC64NVME, false, enc(hash.ChecksumCRC64NVME, p1), nil},
+		{"multipart COMPOSITE CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, false, p1, p2), hash.ChecksumCRC32C, true, composite(hash.ChecksumCRC32C), nil},
+		{"multipart COMPOSITE SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1, p2), hash.ChecksumSHA256, true, composite(hash.ChecksumSHA256), nil},
+		{"multipart FULL_OBJECT CRC32", checksumTestObject(t, hash.ChecksumCRC32, true, p1, p2), hash.ChecksumCRC32, true, enc(hash.ChecksumCRC32, whole), full},
+		{"multipart FULL_OBJECT CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, true, p1, p2), hash.ChecksumCRC32C, true, enc(hash.ChecksumCRC32C, whole), full},
+		{"multipart CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1, p2), hash.ChecksumCRC64NVME, true, enc(hash.ChecksumCRC64NVME, whole), full},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			oi := tt.fi.ToObjectInfo("bucket", "object", true)
+			want := map[string]string{tt.alg.Key(): tt.value, xhttp.AmzChecksumAlgo: tt.alg.String()}
+			maps.Copy(want, tt.extra)
+			got, isMP := getCRCMeta(oi, 0, nil)
+			if isMP != tt.isMP {
+				t.Errorf("isMP: got %t, want %t", isMP, tt.isMP)
+			}
+			if !maps.Equal(got, want) {
+				t.Errorf("object headers: got %v, want %v", got, want)
+			}
+			if !tt.isMP {
+				return
+			}
+			// Part headers carry the part checksum and never a type.
+			want = map[string]string{tt.alg.Key(): enc(tt.alg, p2), xhttp.AmzChecksumAlgo: tt.alg.String()}
+			if got, _ := getCRCMeta(oi, 2, nil); !maps.Equal(got, want) {
+				t.Errorf("part 2 headers: got %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+// TestReplicateObjectChecksumTypes replicates objects with checksums through
+// the S3 API of a test server, and checks that the replica gets the same
+// checksum as the source.
+func TestReplicateObjectChecksumTypes(t *testing.T) {
+	ctx := context.Background()
+	// Leave no object layer behind: unit tests that run later in the package
+	// rely on newObjectLayerFn() returning nil.
+	defer resetTestGlobals()
+	ts := StartTestServer(t, ErasureSDStr)
+	defer ts.Stop()
+
+	clnt, err := minio.New(strings.TrimPrefix(ts.Server.URL, "http://"), &minio.Options{
+		Creds: credentials.NewStaticV4(ts.AccessKey, ts.SecretKey, ""),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	core := &minio.Core{Client: clnt}
+	for _, bucket := range []string{"src", "dst"} {
+		if err := clnt.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
+			t.Fatal(err)
+		}
+		if err := clnt.EnableVersioning(ctx, bucket); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	small := []byte("single part object")
+	p1 := bytes.Repeat([]byte("a"), 5<<20)
+	p2 := []byte("last part")
+	tests := []struct {
+		name      string
+		alg       hash.ChecksumType
+		multipart bool
+		full      bool
+		copyOf    string // server-side copy of an object created above
+	}{
+		{"put-SHA256", hash.ChecksumSHA256, false, false, ""},
+		{"put-CRC32", hash.ChecksumCRC32, false, false, ""},
+		{"mp-COMPOSITE-CRC32C", hash.ChecksumCRC32C, true, false, ""},
+		{"mp-COMPOSITE-SHA256", hash.ChecksumSHA256, true, false, ""},
+		{"mp-FULL_OBJECT-CRC32", hash.ChecksumCRC32, true, true, ""},
+		{"mp-FULL_OBJECT-CRC32C", hash.ChecksumCRC32C, true, true, ""},
+		{"mp-CRC64NVME", hash.ChecksumCRC64NVME, true, false, ""},
+		// The copy keeps the multipart FULL_OBJECT checksum of its source
+		// but is not multipart, so it replicates with a single PUT.
+		{"copy-FULL_OBJECT-CRC32", hash.ChecksumCRC32, false, true, "mp-FULL_OBJECT-CRC32"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			object := tt.name
+			// Upload the source object like an S3 client does.
+			switch {
+			case tt.copyOf != "":
+				_, err := core.CopyObject(ctx, "src", tt.copyOf, "src", object, nil, minio.CopySrcOptions{}, minio.PutObjectOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+			case !tt.multipart:
+				_, err := core.PutObject(ctx, "src", object, bytes.NewReader(small), int64(len(small)), "", "", minio.PutObjectOptions{
+					UserMetadata: map[string]string{tt.alg.Key(): hash.NewChecksumFromData(tt.alg, small).Encoded},
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+			default:
+				meta := map[string]string{xhttp.AmzChecksumAlgo: tt.alg.String()}
+				if tt.full {
+					meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
+				}
+				uploadID, err := core.NewMultipartUpload(ctx, "src", object, minio.PutObjectOptions{UserMetadata: meta})
+				if err != nil {
+					t.Fatal(err)
+				}
+				var parts []minio.CompletePart
+				for i, p := range [][]byte{p1, p2} {
+					h := http.Header{}
+					h.Set(tt.alg.Key(), hash.NewChecksumFromData(tt.alg, p).Encoded)
+					pi, err := core.PutObjectPart(ctx, "src", object, uploadID, i+1, bytes.NewReader(p), int64(len(p)), minio.PutObjectPartOptions{CustomHeader: h})
+					if err != nil {
+						t.Fatal(err)
+					}
+					parts = append(parts, minio.CompletePart{
+						PartNumber: pi.PartNumber, ETag: pi.ETag,
+						ChecksumCRC32: pi.ChecksumCRC32, ChecksumCRC32C: pi.ChecksumCRC32C,
+						ChecksumSHA1: pi.ChecksumSHA1, ChecksumSHA256: pi.ChecksumSHA256,
+						ChecksumCRC64NVME: pi.ChecksumCRC64NVME,
+					})
+				}
+				if _, err := core.CompleteMultipartUpload(ctx, "src", object, uploadID, parts, minio.PutObjectOptions{}); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			// Replicate it the way replicateObject does.
+			gr, err := ts.Obj.GetObjectNInfo(ctx, "src", object, nil, http.Header{}, ObjectOptions{Versioned: true, ReplicationRequest: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer gr.Close()
+			objInfo := gr.ObjInfo
+			putOpts, isMP, err := putReplicationOpts(ctx, "", objInfo)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if isMP != tt.multipart {
+				t.Fatalf("multipart replication: got %t, want %t", isMP, tt.multipart)
+			}
+			if isMP {
+				err = replicateObjectWithMultipart(ctx, core, "dst", object, gr, objInfo, putOpts)
+			} else {
+				size, serr := objInfo.GetActualSize()
+				if serr != nil {
+					t.Fatal(serr)
+				}
+				_, err = core.PutObject(ctx, "dst", object, gr, size, "", "", putOpts)
+			}
+			if err != nil {
+				t.Fatalf("replication failed: %v", err)
+			}
+
+			src, err := clnt.StatObject(ctx, "src", object, minio.StatObjectOptions{Checksum: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			dst, err := clnt.StatObject(ctx, "dst", object, minio.StatObjectOptions{Checksum: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			checksums := func(oi minio.ObjectInfo) string {
+				return strings.Join([]string{oi.ChecksumMode, oi.ChecksumCRC32, oi.ChecksumCRC32C, oi.ChecksumSHA1, oi.ChecksumSHA256, oi.ChecksumCRC64NVME}, "|")
+			}
+			if checksums(src) == "|||||" || checksums(dst) != checksums(src) {
+				t.Errorf("replica checksum %q, source %q", checksums(dst), checksums(src))
+			}
+		})
+	}
+}

cmd/bucket-replication-utils.go

@@ -418,6 +418,9 @@ func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID
 
 	for _, rinfo := range rinfos.Targets {
 		if rinfo.ResyncTimestamp != "" {
+			if rs.ResetStatusesMap == nil {
+				rs.ResetStatusesMap = make(map[string]string)
+			}
 			rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
 		}
 	}

cmd/bucket-replication.go

@@ -425,6 +425,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // the mere presence or absence of the target version.
 func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
 	var replicationStatus replication.StatusType
+	isPurge := dobj.isVersionPurge()
 	bucket := dobj.Bucket
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
@@ -484,6 +485,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
+		dobj.RetryCount++
 		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 		sendEvent(eventArgs{
 			BucketName: bucket,
@@ -502,6 +504,35 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	if !isPurge && dobj.DeleteMarkerVersionID != "" {
+		// A creation task carries the marker as it looked when it was queued
+		// by the DELETE handler, a GET/HEAD/LIST heal, the scanner or MRF.
+		// While it waited for this lock, another frontend may have purged the
+		// marker and replicated that purge. The targets no longer hold the
+		// marker, so the queued creation would recreate it from the stale
+		// snapshot. Confirm the source version under the lock first.
+		switch deleteMarkerCreationState(ctx, objectAPI, dobj) {
+		case creationStale:
+			return replicatedInfos{}
+		case creationUnverified:
+			dobj.RetryCount++
+			globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
+			sendEvent(eventArgs{
+				BucketName: bucket,
+				Object: ObjectInfo{
+					Bucket:       bucket,
+					Name:         dobj.ObjectName,
+					VersionID:    versionID,
+					DeleteMarker: dobj.DeleteMarker,
+				},
+				UserAgent: "Internal: [Replication]",
+				Host:      globalLocalNodeName,
+				EventName: event.ObjectReplicationNotTracked,
+			})
+			return replicatedInfos{}
+		}
+	}
+
 	rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(dsc.targetsMap))}
 	var wg sync.WaitGroup
 	var mu sync.Mutex
@@ -546,25 +577,38 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	replicationStatus = rinfos.ReplicationStatus()
 	prevStatus := dobj.DeleteMarkerReplicationStatus()
 
-	if dobj.VersionID != "" {
-		prevStatus = replication.StatusType(dobj.VersionPurgeStatus())
-		replicationStatus = replication.StatusType(rinfos.VersionPurgeStatus())
+	if isPurge {
+		prevStatus = purgeReplicationStatus(dobj.VersionPurgeStatus())
+		replicationStatus = purgeReplicationStatus(rinfos.VersionPurgeStatus())
 	}
 
 	// to decrement pending count later.
 	for _, rinfo := range rinfos.Targets {
-		if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
-			globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus,
-				prevStatus)
+		status, previous := rinfo.ReplicationStatus, rinfo.PrevReplicationStatus
+		if isPurge {
+			status = purgeReplicationStatus(rinfo.VersionPurgeStatus)
+			previous = purgeReplicationStatus(dobj.ReplicationState.PurgeTargets[rinfo.Arn])
+		}
+		if status != previous {
+			globalReplicationStats.Load().Update(dobj.Bucket, rinfo, status, previous)
 		}
 	}
 
 	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
+		dobj.RetryCount++
 		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
+	if isPurge {
+		// A purge must not rewrite the marker's creation/replica metadata.
+		// Multiple serialized empty target statuses can parse as nonempty,
+		// so explicitly send an empty creation update to the metadata writer.
+		drs.ReplicationStatusInternal = ""
+		drs.Targets = nil
+		drs.ReplicaStatus = ""
+	}
 	if replicationStatus != prevStatus {
 		drs.ReplicationTimeStamp = UTCNow()
 	}
@@ -606,6 +650,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 }
 
 func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+	isPurge := dobj.isVersionPurge()
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
@@ -615,42 +660,50 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	rinfo.OpType = dobj.OpType
 	rinfo.endpoint = tgt.EndpointURL().Host
 	rinfo.secure = tgt.EndpointURL().Scheme == "https"
+	// A purge leaves ReplicationStatus empty: the metadata writer interprets
+	// that as preserving the entire creation-status block, including targets
+	// outside this fan-out. Only VersionPurgeStatus records the purge outcome.
 	defer func() {
-		if rinfo.ReplicationStatus == replication.Completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
+		completed := rinfo.ReplicationStatus == replication.Completed
+		if isPurge {
+			completed = rinfo.VersionPurgeStatus == replication.VersionPurgeComplete
+		}
+		if completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
 			rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID)
 		}
 	}()
 
-	if dobj.VersionID == "" && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType {
+	if !isPurge && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType {
 		rinfo.ReplicationStatus = rinfo.PrevReplicationStatus
 		return rinfo
 	}
-	if dobj.VersionID != "" && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
+	if isPurge && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
 		return rinfo
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
+		rinfo.Err = fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN)
+		replLogOnceIf(ctx, rinfo.Err, "replication-target-offline-delete-"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
 				Bucket:       dobj.Bucket,
 				Name:         dobj.ObjectName,
-				VersionID:    dobj.VersionID,
+				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
 			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
-		if dobj.VersionID == "" {
-			rinfo.ReplicationStatus = replication.Failed
-		} else {
+		if isPurge {
 			rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
+		} else {
+			rinfo.ReplicationStatus = replication.Failed
 		}
 		return rinfo
 	}
 	// early return if already replicated delete marker for existing object replication/ healing delete markers
-	if dobj.DeleteMarkerVersionID != "" {
+	if !isPurge && dobj.DeleteMarkerVersionID != "" {
 		toi, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, minio.StatObjectOptions{
 			VersionID: versionID,
 			Internal: minio.AdvancedGetOptions{
@@ -662,16 +715,10 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		switch {
 		case isErrMethodNotAllowed(serr):
 			// delete marker already replicated
-			if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() {
-				rinfo.ReplicationStatus = replication.Completed
-				return rinfo
-			}
+			rinfo.ReplicationStatus = replication.Completed
+			return rinfo
 		case isErrObjectNotFound(serr), isErrVersionNotFound(serr):
-			// version being purged is already not found on target.
-			if !rinfo.VersionPurgeStatus.Empty() {
-				rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
-				return rinfo
-			}
+			// The marker still needs to be created on the target.
 		case isErrReadQuorum(serr), isErrWriteQuorum(serr):
 			// destination has some quorum issues, perform removeObject() anyways
 			// to complete the operation.
@@ -691,7 +738,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{
 		VersionID: versionID,
 		Internal: minio.AdvancedRemoveOptions{
-			ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "",
+			ReplicationDeleteMarker: !isPurge && dobj.DeleteMarkerVersionID != "",
 			ReplicationMTime:        dobj.DeleteMarkerMTime.Time,
 			ReplicationStatus:       minio.ReplicationStatusReplica,
 			ReplicationRequest:      true, // always set this to distinguish between `mc mirror` replication and serverside
@@ -699,20 +746,20 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	})
 	if rmErr != nil {
 		rinfo.Err = rmErr
-		if dobj.VersionID == "" {
-			rinfo.ReplicationStatus = replication.Failed
-		} else {
+		if isPurge {
 			rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
+		} else {
+			rinfo.ReplicationStatus = replication.Failed
 		}
 		replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
 		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
 		}
 	} else {
-		if dobj.VersionID == "" {
-			rinfo.ReplicationStatus = replication.Completed
-		} else {
+		if isPurge {
 			rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
+		} else {
+			rinfo.ReplicationStatus = replication.Completed
 		}
 	}
 	return rinfo
@@ -781,6 +828,18 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
+// replicationTaggingTimestamp carries a recorded removal even when tags are
+// empty. Only legacy nonempty tags use ModTime; absence is not a tombstone.
+func replicationTaggingTimestamp(objInfo ObjectInfo) (time.Time, error) {
+	if stamp, ok := caseInsensitiveMap(objInfo.UserDefined).Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
+		return time.Parse(time.RFC3339Nano, stamp)
+	}
+	if objInfo.UserTags != "" {
+		return objInfo.ModTime, nil
+	}
+	return time.Time{}, nil
+}
+
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) {
 	meta := make(map[string]string)
 	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
@@ -850,17 +909,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
 		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
 		if tag != nil {
 			putOpts.UserTags = tag.ToMap()
-			// set tag timestamp in opts
-			tagTimestamp := objInfo.ModTime
-			if tagTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
-				tagTimestamp, err = time.Parse(time.RFC3339Nano, tagTmstampStr)
-				if err != nil {
-					return putOpts, false, err
-				}
-			}
-			putOpts.Internal.TaggingTimestamp = tagTimestamp
 		}
 	}
+	putOpts.Internal.TaggingTimestamp, err = replicationTaggingTimestamp(objInfo)
+	if err != nil {
+		return putOpts, false, err
+	}
 
 	lkMap := caseInsensitiveMap(objInfo.UserDefined)
 	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
@@ -1003,6 +1057,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2Map, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
 		return replicateMetadata
 	}
+	// HEAD does not report the tag revision. Equal values can hide a newer
+	// deletion or re-addition, so scheduled metadata/heal work must deliver it.
+	// Completed objects are still excluded by the existing scanner gates.
+	if _, ok := caseInsensitiveMap(oi1.UserDefined).Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
+		return replicateMetadata
+	}
 
 	// Compare only necessary headers
 	compareKeys := []string{
@@ -1269,9 +1329,6 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 						oi.UserDefined[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
 					}
 				}
-				if ri.UserTags != "" {
-					oi.UserDefined[xhttp.AmzObjectTagging] = ri.UserTags
-				}
 				return dsc, nil
 			},
 		}
@@ -1689,14 +1746,11 @@ applyAction:
 		if _, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
 			dstOpts.Internal.RetentionTimestamp = objInfo.ModTime
 		}
-		if objInfo.UserTags != "" {
-			dstOpts.Internal.TaggingTimestamp = objInfo.ModTime
-		}
-		if tagTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
-			ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
-			if err == nil {
-				dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
-			}
+		dstOpts.Internal.TaggingTimestamp, rinfo.Err = replicationTaggingTimestamp(objInfo)
+		if rinfo.Err != nil {
+			rinfo.ReplicationStatus = replication.Failed
+			replLogIf(ctx, fmt.Errorf("invalid tagging timestamp for object %s/%s(%s): %w", bucket, object, objInfo.VersionID, rinfo.Err))
+			return rinfo
 		}
 		if retTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp); ok {
 			ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
@@ -1916,11 +1970,61 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 // DeletedObjectReplicationInfo has info on deleted object
 type DeletedObjectReplicationInfo struct {
 	DeletedObject
-	Bucket    string
-	EventType string
-	OpType    replication.Type
-	ResetID   string
-	TargetArn string
+	Bucket     string
+	EventType  string
+	OpType     replication.Type
+	ResetID    string
+	TargetArn  string
+	RetryCount int
+}
+
+// isVersionPurge also recognizes the old marker-shaped purge task. Use the
+// operation's state, rather than one target's possibly missing purge entry.
+func (di DeletedObjectReplicationInfo) isVersionPurge() bool {
+	return di.VersionID != "" || di.DeleteMarkerVersionID != "" && !di.VersionPurgeStatus().Empty()
+}
+
+// creationState is the outcome of re-reading a queued delete-marker creation
+// against the source.
+type creationState int
+
+const (
+	creationCurrent creationState = iota
+	creationStale
+	creationUnverified
+)
+
+// deleteMarkerCreationState re-reads the marker a queued creation task refers
+// to. The version being absent, no longer a marker, or under a version purge
+// makes the creation stale. A failed read is not absence: the caller retries
+// later instead of guessing.
+func deleteMarkerCreationState(ctx context.Context, objectAPI ObjectLayer, dobj DeletedObjectReplicationInfo) creationState {
+	oi, err := objectAPI.GetObjectInfo(ctx, dobj.Bucket, dobj.ObjectName, ObjectOptions{
+		VersionID:        dobj.DeleteMarkerVersionID,
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(dobj.Bucket, dobj.ObjectName),
+		VersionSuspended: globalBucketVersioningSys.Suspended(dobj.Bucket),
+	})
+	switch {
+	case isErrObjectNotFound(err), isErrVersionNotFound(err):
+		return creationStale
+	case err != nil && !isErrMethodNotAllowed(err):
+		return creationUnverified
+	}
+	if !oi.DeleteMarker || oi.VersionID != dobj.DeleteMarkerVersionID {
+		return creationStale
+	}
+	if !oi.VersionPurgeStatus.Empty() || oi.VersionPurgeStatusInternal != "" {
+		return creationStale
+	}
+	return creationCurrent
+}
+
+// Purge metadata uses COMPLETE; operation statistics and audit use COMPLETED.
+func purgeReplicationStatus(status VersionPurgeStatusType) replication.StatusType {
+	if replication.StatusType(status) == replication.CompletedLegacy {
+		return replication.Completed
+	}
+	return replication.StatusType(status)
 }
 
 // ToMRFEntry returns the relevant info needed by MRF
@@ -1930,9 +2034,10 @@ func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry {
 		versionID = di.VersionID
 	}
 	return MRFReplicateEntry{
-		Bucket:    di.Bucket,
-		Object:    di.ObjectName,
-		versionID: versionID,
+		Bucket:     di.Bucket,
+		Object:     di.ObjectName,
+		versionID:  versionID,
+		RetryCount: di.RetryCount,
 	}
 }
 
@@ -2424,6 +2529,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 	case <-p.ctx.Done():
 	case ch <- doi:
 	default:
+		doi.RetryCount++
 		p.queueMRFSave(doi.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
@@ -3787,9 +3893,10 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},
-			Bucket:    roi.Bucket,
-			OpType:    replication.HealReplicationType,
-			EventType: ReplicateHealDelete,
+			Bucket:     roi.Bucket,
+			OpType:     replication.HealReplicationType,
+			EventType:  ReplicateHealDelete,
+			RetryCount: retryCount,
 		}
 		// heal delete marker replication failure or versioned delete replication failure
 		if roi.ReplicationStatus == replication.Pending ||
@@ -4072,7 +4179,12 @@ func (p *ReplicationPool) queueMRFHeal() error {
 				VersionID: vID,
 			})
 			cancel()
-			if err != nil {
+			// A versioned marker lookup returns its metadata with a 405. Only
+			// accept that error with a real, matching marker identity.
+			validMarker := isErrMethodNotAllowed(err) && oi.DeleteMarker &&
+				vID != "" && oi.VersionID == vID && !oi.ModTime.IsZero() &&
+				oi.Bucket == e.Bucket && oi.Name != "" && oi.Name == decodeDirObject(e.Object)
+			if err != nil && !validMarker || oi.Name == "" {
 				continue
 			}
 
@@ -4138,6 +4250,11 @@ func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string
 		if cktype.IsSet() {
 			meta[cktype.Key()] = v
 			meta[xhttp.AmzChecksumAlgo] = cktype.String()
+			// Without the type the target creates a COMPOSITE multipart
+			// upload and rejects the full object CRC on completion.
+			if isMP && cktype.CanMerge() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject {
+				meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
+			}
 		}
 	}
 	return meta, isMP

cmd/common-main.go

@@ -56,6 +56,7 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/api"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -445,6 +446,7 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.SendBufSize = ctx.Int("send-buf-size")
 	ctxt.RecvBufSize = ctx.Int("recv-buf-size")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
+	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
 	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
 
 	if conf := ctx.String("config"); len(conf) > 0 {
@@ -880,6 +882,9 @@ func serverHandleEnvVars() {
 		logger.Warning("%s=%s is ignored: Silo in-place updates are permanently disabled", config.EnvUpdate, updateSetting)
 	}
 	globalInplaceUpdateDisabled = true
+	if env.IsSet(api.EnvAPIMultipartListing) {
+		logger.Warning("%s is ignored: multipart listing uses the legacy cache/exact-key behavior; persistent listing is deferred", api.EnvAPIMultipartListing)
+	}
 
 	// Check if the supported credential env vars,
 	// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided

cmd/config-current.go

@@ -227,7 +227,7 @@ func initHelp() {
 		},
 		config.HelpKV{
 			Key:         config.ILMSubSys,
-			Description: "manage ILM settings for expiration, transition, and access-tier workers",
+			Description: "manage ILM settings for expiration and transition workers",
 			Optional:    true,
 		},
 	}
@@ -609,7 +609,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 			loggerCfg.HTTP[n] = l
 		}
-		if errs := logger.UpdateHTTPWebhooks(ctx, loggerCfg.HTTP); len(errs) > 0 {
+		// Webhook workers outlive the bounded configuration reload.
+		if errs := logger.UpdateHTTPWebhooks(GlobalContext, loggerCfg.HTTP); len(errs) > 0 {
 			configLogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
@@ -627,7 +628,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			loggerCfg.AuditWebhook[n] = l
 		}
 
-		if errs := logger.UpdateAuditWebhooks(ctx, loggerCfg.AuditWebhook); len(errs) > 0 {
+		// Both the queue worker and disk replay follow the server lifetime.
+		if errs := logger.UpdateAuditWebhooks(GlobalContext, loggerCfg.AuditWebhook); len(errs) > 0 {
 			configLogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
@@ -644,7 +646,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 {
+		// The Kafka disk replay follows the server lifetime as well.
+		if errs := logger.UpdateAuditKafkaTargets(GlobalContext, loggerCfg); len(errs) > 0 {
 			configLogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:
@@ -704,9 +707,6 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			if globalExpiryState != nil {
 				globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
 			}
-			if globalAccessTierState != nil {
-				globalAccessTierState.UpdateWorkers(ilmCfg.AccessWorkers)
-			}
 			globalILMConfig.update(ilmCfg)
 		}
 	}

cmd/config-current_test.go

@@ -19,12 +19,246 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
+	"github.com/IBM/sarama"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/logger"
+	httpTarget "github.com/minio/minio/internal/logger/target/http"
+	types "github.com/minio/minio/internal/logger/target/loggertypes"
 )
 
+type webhookConfigObjectLayer struct {
+	ObjectLayer
+}
+
+func (webhookConfigObjectLayer) SetDriveCounts() []int { return []int{1} }
+
+// TestDynamicWebhookOutlivesConfigContext applies logger targets the way a
+// peer reload does: under a bounded context that ends when the handler returns.
+func TestDynamicWebhookOutlivesConfigContext(t *testing.T) {
+	for _, subSys := range []string{config.AuditWebhookSubSys, config.LoggerWebhookSubSys, config.AuditKafkaSubSys} {
+		for _, queue := range []struct {
+			name     string
+			disk     bool
+			failInit bool
+		}{
+			{name: "disk=false"},
+			{name: "disk=true", disk: true},
+			{name: "disk=unavailable", disk: true, failInit: true},
+		} {
+			if subSys == config.AuditKafkaSubSys && queue.failInit {
+				// An unopenable Kafka queue starts no goroutine.
+				continue
+			}
+			t.Run(subSys+"/"+queue.name, func(t *testing.T) {
+				// Isolate global target registries and wait for all workers to exit
+				// before the parent removes the disk queue.
+				if os.Getenv("_SILO_WEBHOOK_CONTEXT_TEST") != t.Name() {
+					executable, err := os.Executable()
+					if err != nil {
+						t.Fatal(err)
+					}
+					queueDir := ""
+					if queue.disk {
+						queueDir = t.TempDir()
+					}
+					ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+					defer cancel()
+					child := exec.CommandContext(ctx, executable,
+						"-test.run=^"+strings.ReplaceAll(t.Name(), "/", "$/^")+"$",
+						"-test.v", "-test.timeout=25s")
+					// Logger targets from the environment would join the registries.
+					for _, kv := range os.Environ() {
+						if !strings.HasPrefix(kv, "MINIO_AUDIT_") && !strings.HasPrefix(kv, "MINIO_LOGGER_") {
+							child.Env = append(child.Env, kv)
+						}
+					}
+					child.Env = append(child.Env,
+						"_SILO_WEBHOOK_CONTEXT_TEST="+t.Name(),
+						"_SILO_WEBHOOK_QUEUE_DIR="+queueDir)
+					output, err := child.CombinedOutput()
+					if err != nil {
+						t.Fatalf("isolated webhook test: %v\n%s", err, output)
+					}
+					t.Logf("%s", output)
+					return
+				}
+
+				if subSys == config.AuditKafkaSubSys {
+					testDynamicKafkaOutlivesConfigContext(t, os.Getenv("_SILO_WEBHOOK_QUEUE_DIR"))
+					return
+				}
+
+				targets := logger.AuditTargets
+				if subSys == config.LoggerWebhookSubSys {
+					targets = logger.SystemTargets
+				}
+				received := make(chan string, 8)
+				receiver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					var entry struct{ Marker string }
+					if err := json.NewDecoder(r.Body).Decode(&entry); err != nil {
+						t.Errorf("decode webhook: %v", err)
+					}
+					received <- entry.Marker
+					w.WriteHeader(http.StatusOK)
+				}))
+				t.Cleanup(receiver.Close)
+
+				queueDir := os.Getenv("_SILO_WEBHOOK_QUEUE_DIR")
+				if queue.failInit {
+					queueDir = filepath.Join(queueDir, "queue")
+					if err := os.WriteFile(queueDir, []byte("not a directory"), 0o600); err != nil {
+						t.Fatal(err)
+					}
+				}
+				cfg := newServerConfig()
+				_, err := cfg.SetKVS(fmt.Sprintf("%s:context-test enable=on endpoint=%s queue_size=16 batch_size=1 queue_dir=%s http_timeout=1s retry_interval=100ms",
+					subSys, receiver.URL, queueDir), config.DefaultKVS)
+				if err != nil {
+					t.Fatal(err)
+				}
+				// A reload cancels the target it replaces; clean up only the last one.
+				var current *httpTarget.Target
+				t.Cleanup(func() {
+					if current != nil {
+						current.Cancel()
+					}
+				})
+				// reload applies the subsystem the way a peer does and returns the
+				// registered target once the bounded reload context has ended.
+				reload := func() *httpTarget.Target {
+					t.Helper()
+					ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+					defer cancel()
+					if err := applyDynamicConfigForSubSys(ctx, webhookConfigObjectLayer{}, cfg, subSys); err != nil {
+						t.Fatal(err)
+					}
+					var target *httpTarget.Target
+					for _, candidate := range targets() {
+						if candidate.Endpoint() == receiver.URL {
+							if target != nil {
+								t.Fatal("configured webhook was registered twice")
+							}
+							target = candidate.(*httpTarget.Target)
+						}
+					}
+					if target == nil {
+						t.Fatal("configured webhook was not registered")
+					}
+					current = target
+					return target
+				}
+
+				target := reload()
+				if err := target.Send(t.Context(), struct{ Marker string }{"after-reload"}); err != nil {
+					t.Fatal(err)
+				}
+				if queue.failInit {
+					// A failed initialization starts no worker: the record stays queued.
+					select {
+					case got := <-received:
+						t.Fatalf("received %q before the queue was initialized", got)
+					case <-time.After(250 * time.Millisecond):
+					}
+					if got := target.Stats().QueueLength; got != 1 {
+						t.Fatalf("failed queue consumed an event: queue length %d", got)
+					}
+					// After the operator repairs queue_dir, applying the configuration
+					// again replaces the target, which must take over the record.
+					if err := os.Remove(queueDir); err != nil {
+						t.Fatal(err)
+					}
+					target = reload()
+				}
+				select {
+				case got := <-received:
+					if got != "after-reload" {
+						t.Fatalf("received marker %q, want after-reload", got)
+					}
+				case <-time.After(10 * time.Second):
+					t.Fatalf("webhook stopped after reload returned; stats: %+v", target.Stats())
+				}
+			})
+		}
+	}
+}
+
+// testDynamicKafkaOutlivesConfigContext runs in the isolated child process.
+// With a queue_dir, Kafka records reach the broker only through the disk
+// replay, which polls the queue every few seconds.
+func testDynamicKafkaOutlivesConfigContext(t *testing.T, queueDir string) {
+	const topic = "audit-context-test"
+	broker := sarama.NewMockBroker(t, 1)
+	t.Cleanup(broker.Close)
+	broker.SetHandlerByMap(map[string]sarama.MockResponse{
+		"ApiVersionsRequest": sarama.NewMockApiVersionsResponse(t).SetApiKeys([]sarama.ApiVersionsResponseKey{
+			{ApiKey: 0, MinVersion: 0, MaxVersion: 8},  // Produce
+			{ApiKey: 3, MinVersion: 0, MaxVersion: 11}, // Metadata
+			{ApiKey: 18, MinVersion: 0, MaxVersion: 3}, // ApiVersions
+		}),
+		"MetadataRequest": sarama.NewMockMetadataResponse(t).
+			SetBroker(broker.Addr(), broker.BrokerID()).
+			SetLeader(topic, 0, broker.BrokerID()),
+		"ProduceRequest": sarama.NewMockProduceResponse(t),
+	})
+	produced := func() (n int) {
+		for _, exchange := range broker.History() {
+			if _, ok := exchange.Request.(*sarama.ProduceRequest); ok {
+				n++
+			}
+		}
+		return n
+	}
+
+	cfg := newServerConfig()
+	_, err := cfg.SetKVS(fmt.Sprintf("%s:context-test enable=on brokers=%s topic=%s queue_size=16 queue_dir=%s",
+		config.AuditKafkaSubSys, broker.Addr(), topic, queueDir), config.DefaultKVS)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	if err := applyDynamicConfigForSubSys(ctx, webhookConfigObjectLayer{}, cfg, config.AuditKafkaSubSys); err != nil {
+		t.Fatal(err)
+	}
+	var target logger.Target
+	for _, candidate := range logger.AuditTargets() {
+		if candidate.Type() == types.TargetKafka {
+			if target != nil {
+				t.Fatal("more than one Kafka target was registered")
+			}
+			target = candidate
+		}
+	}
+	if target == nil {
+		t.Fatal("configured Kafka target was not registered")
+	}
+	t.Cleanup(target.Cancel)
+	// Returning from the peer reload cancels its bounded context.
+	cancel()
+
+	before := produced()
+	if err := target.Send(t.Context(), struct{ Marker string }{"after-reload"}); err != nil {
+		t.Fatal(err)
+	}
+	for deadline := time.Now().Add(10 * time.Second); produced() == before; time.Sleep(50 * time.Millisecond) {
+		if time.Now().After(deadline) {
+			t.Fatalf("Kafka target stopped after reload returned; stats: %+v", target.Stats())
+		}
+	}
+}
+
 func TestServerConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()

cmd/config-migrate.go

@@ -91,7 +91,7 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
 			return nil, err
 		}
 
-		data, err = decryptData(data, configFile)
+		data, err = decryptData(ctx, data, configFile)
 		if err != nil {
 			return nil, err
 		}

cmd/config.go

@@ -68,7 +68,7 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 					continue
 				}
 
-				data, err = decryptData(data, obj.Name)
+				data, err = decryptData(ctx, data, obj.Name)
 				if err != nil {
 					// ignore history file that cannot be loaded.
 					continue
@@ -110,7 +110,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
 		return nil, err
 	}
 
-	return decryptData(data, historyFile)
+	return decryptData(ctx, data, historyFile)
 }
 
 func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@@ -162,7 +162,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con
 			return nil, err
 		}
 
-		data, err = decryptData(data, configFile)
+		data, err = decryptData(ctx, data, configFile)
 		if err != nil {
 			lookupConfigs(srvCfg, objAPI)
 			return nil, err

cmd/data-scanner.go

@@ -24,10 +24,12 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"maps"
 	"math"
 	"math/rand"
 	"os"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -776,7 +778,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 
 					// We got an entry which we should be able to heal.
-					fiv, err := entry.fileInfoVersions(bucket)
+					fiv, err := entry.allFileInfoVersions(bucket)
 					if err != nil {
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
@@ -899,7 +901,6 @@ type scannerItem struct {
 	objectName  string // Only the object name without prefixes.
 	replication replicationConfig
 	lifeCycle   *lifecycle.Lifecycle
-	poolIdx     int
 	Typ         fs.FileMode
 	heal        struct {
 		enabled bool
@@ -910,7 +911,6 @@ type scannerItem struct {
 
 type sizeSummary struct {
 	totalSize       int64
-	hotTierSize     int64
 	versions        uint64
 	deleteMarkers   uint64
 	replicatedSize  int64
@@ -1120,8 +1120,31 @@ eventLoop:
 		size := actualSz
 		switch event.Action {
 		case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
-			remainingVersions = 0
-			applyExpiryRule(event, lcEventSrc_Scanner, oi)
+			// An expiry of a hot latest version purges the key, which refuses
+			// a key with a version in a remote tier: do not queue it when this
+			// pool holds one. A tiered version only another pool holds is
+			// not known here; the purge decides on it. A transitioned latest
+			// version takes the transitioned expiry, as before.
+			refused := oi.TransitionedObject.Status == "" && slices.ContainsFunc(objInfos, ObjectInfo.deleteAllTierRefuses)
+			if refused {
+				globalILMIncomplete.add(ilmIncompleteTierSkipped)
+			} else {
+				remainingVersions = 0
+				applyExpiryRule(event, lcEventSrc_Scanner, oi)
+			}
+			// The versions hold their space until the purge removes them,
+			// which can refuse the key or stop at its budget: account each
+			// at its size, with no other action. The alerts on many or
+			// large versions count those of a key that is not queued.
+			for _, v := range objInfos[idx:] {
+				sz, _ := v.GetActualSize()
+				if fn != nil {
+					fn(v, sz, sz, sizeS)
+				}
+				if refused {
+					cumulativeSize += sz
+				}
+			}
 			break eventLoop
 
 		case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1132,9 +1155,9 @@ eventLoop:
 			applyExpiryRule(event, lcEventSrc_Scanner, oi)
 
 		case lifecycle.DeleteVersionAction: // noncurrent versions expiration
-			opts := objOpts[idx]
 			remainingVersions--
 			size = 0
+			opts := objOpts[idx]
 			toDel = append(toDel, ObjectToDelete{
 				ObjectV: ObjectV{
 					ObjectName: opts.Name,
@@ -1161,14 +1184,6 @@ eventLoop:
 		globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents)
 	}
 	i.alertExcessiveVersions(remainingVersions, cumulativeSize)
-	if globalILMConfig.accessTieringEnabled() {
-		for idx, oi := range objInfos {
-			if oi.IsLatest && events[idx].Action == lifecycle.NoneAction {
-				applyAccessTransition(ctx, i, oi)
-				break
-			}
-		}
-	}
 }
 
 func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
@@ -1220,10 +1235,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) {
 	timeILM := globalScannerMetrics.timeILM(lcEvent.Action)
 	if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-			return false
-		}
-		ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err))
+		reportILMExpiryNotDone(ctx, obj, lcEvent, src, err, "transition-expiry"+obj.Name, globalLifecycleSys.trace(obj))
 		return false
 	}
 	timeILM(1)
@@ -1245,11 +1257,18 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
 	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 
+	// Purge every version of exactly this key, by version ID and under the
+	// object lock, after the pools approved it on the versions they hold;
+	// see erasureServerPools.deleteAllVersions.
+	var purge deleteAllResult
 	if lcEvent.Action.DeleteAll() {
-		opts.DeletePrefix = true
-		// use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-		opts.DeletePrefixObject = true
+		opts = ObjectOptions{
+			Expiration:        ExpirationOptions{Expire: true},
+			DeleteAllVersions: true,
+			DeleteAllResult:   &purge,
+		}
 	}
+
 	var (
 		dobj ObjectInfo
 		err  error
@@ -1264,22 +1283,21 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		if lcEvent.Action != lifecycle.NoneAction {
 			numVersions := uint64(1)
 			if lcEvent.Action.DeleteAll() {
-				numVersions = uint64(obj.NumVersions)
+				numVersions = uint64(purge.Removed)
 			}
 			timeILM(numVersions)
 		}
 	}()
 
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
+	if lcEvent.Action.DeleteAll() && (err != nil || purge.Removed == 0) {
+		return deleteAllNotCompleted(ctx, obj, lcEvent, src, purge, err, traceFn)
+	}
 	if err != nil {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+		// Assume it is still there, unless it is gone already.
+		if !reportILMExpiryNotDone(ctx, obj, lcEvent, src, err, "non-transition-expiry"+obj.Name, traceFn) {
 			traceFn(ILMExpiry, nil, nil)
-			return false
 		}
-		// Assume it is still there.
-		err := fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err)
-		ilmLogOnceIf(ctx, err, "non-transition-expiry"+obj.Name)
-		traceFn(ILMExpiry, nil, err)
 		return false
 	}
 	if dobj.Name == "" {
@@ -1288,14 +1306,14 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	tags["version-id"] = dobj.VersionID
+	if lcEvent.Action.DeleteAll() {
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+	}
 
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
 
-	eventName := event.ObjectRemovedDelete
-	if obj.DeleteMarker {
-		eventName = event.ObjectRemovedDeleteMarkerCreated
-	}
+	eventName := ilmExpiryEventName(opts.VersionID, dobj)
 	switch lcEvent.Action {
 	case lifecycle.DeleteAllVersionsAction:
 		eventName = event.ObjectRemovedDeleteAllVersions
@@ -1314,6 +1332,65 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	return true
 }
 
+// ilmVersionsRemoved is the audit tag with the number of versions that an
+// expiry of all versions removed; ilmPartial marks one that began to delete
+// and did not finish.
+const (
+	ilmVersionsRemoved = "ilm-versions-removed"
+	ilmPartial         = "ilm-partial"
+)
+
+// deleteAllNotCompleted reports an expiry of all versions of obj that did
+// not remove them all. A purge that had nothing to remove, or that failed
+// before it deleted anything, reports what a rejected expiry does. One that
+// failed or stopped after it began to delete is audited as partial, with the
+// versions it removed, but sends no removal event: the key is still there.
+//
+// Each is counted and traced with its reason, as reportILMExpiryNotDone
+// reports other expiries. A refusal because a version is in a remote tier is
+// also logged, since it lasts until such versions can be purged.
+func deleteAllNotCompleted(ctx context.Context, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, purge deleteAllResult, err error, traceFn func(event string, metadata map[string]string, err error)) bool {
+	logID := "non-transition-expiry" + obj.Name
+	switch {
+	case err == nil: // there was nothing to remove
+		traceFn(ILMExpiry, nil, nil)
+		return false
+	case purge.Started:
+		reason, _ := ilmExpiryNotDone(err)
+		globalILMIncomplete.add(reason)
+		tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+		tags["version-id"] = purge.Latest.VersionID
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+		tags[ilmPartial] = "true"
+		tags["ilm-incomplete-reason"] = reason
+		auditLogInternal(ctx, AuditLogOptions{
+			Event:     ILMExpiry,
+			APIName:   "ILMExpiry",
+			Bucket:    obj.Bucket,
+			Object:    obj.Name,
+			VersionID: purge.Latest.VersionID,
+			Error:     err.Error(),
+			Tags:      tags,
+		})
+		trace := maps.Clone(tags)
+		trace["result"], trace["reason"] = "partial", reason
+		traceFn(ILMExpiry, trace, err)
+		if errors.Is(err, errDeleteAllBudget) {
+			return false // a long history; the next expiry continues
+		}
+		// Assume it is still there.
+		ilmLogOnceIf(ctx, fmt.Errorf("ILM expiry of all versions of %s/%s stopped after it removed %d: %w", obj.Bucket, obj.Name, purge.Removed, err), logID)
+		return false
+	case errors.Is(err, errDeleteAllTiered):
+		// Kept until tiered versions can be purged; see deleteAllVersions.
+		ilmLogOnceIf(ctx, fmt.Errorf("ILM expiry of all versions of %s/%s refused: a version is in a remote tier, and a purge of all versions does not remove tiered versions yet", obj.Bucket, obj.Name), logID)
+	}
+	if !reportILMExpiryNotDone(ctx, obj, lcEvent, src, err, logID, traceFn) {
+		traceFn(ILMExpiry, nil, nil) // gone already
+	}
+	return false
+}
+
 // Apply object, object version, restored object or restored object version action on the given object
 func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
 	globalExpiryState.enqueueByDays(obj, event, src)
@@ -1484,8 +1561,6 @@ const (
 	ILMFreeVersionDelete = "ilm:free-version-delete"
 	// ILMTransition - audit trail for ILM transitioning.
 	ILMTransition = " ilm:transition"
-	// ILMAccessTier - audit trail for moving objects between server pools.
-	ILMAccessTier = "ilm:access-tier"
 )
 
 func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
@@ -1497,8 +1572,6 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
 		apiName = "ILMFreeVersionDelete"
 	case ILMTransition:
 		apiName = "ILMTransition"
-	case ILMAccessTier:
-		apiName = "ILMAccessTier"
 	}
 	auditLogInternal(ctx, AuditLogOptions{
 		Event:     event,

cmd/data-scanner_test.go

@@ -240,8 +240,10 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 			},
 		},
 		{
+			// Every version is accounted at its size until the purge
+			// removes it.
 			objInfos:    allVersExpObjInfos,
-			wants:       nil,
+			wants:       allVersExpObjInfos,
 			wantExpired: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: obj, VersionID: allVersExpObjInfos[0].VersionID}}},
 		},
 		{

cmd/data-usage-cache-v9_test.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"reflect"
+	"testing"
+)
+
+// Read a real pre-removal scanner cache with nonzero hot-tier bytes. A v8
+// payload with a relabeled header would not exercise the ignored hts field.
+func TestDataUsageCacheReadV9(t *testing.T) {
+	raw, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.bin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(raw) == 0 || raw[0] != 9 {
+		t.Fatal("fixture is not v9")
+	}
+	expected, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var want, got dataUsageCache
+	if err := json.Unmarshal(expected, &want); err != nil {
+		t.Fatal(err)
+	}
+	if err := got.deserialize(bytes.NewReader(raw)); err != nil {
+		t.Fatal(err)
+	}
+	if !got.Info.LastUpdate.Equal(want.Info.LastUpdate) {
+		t.Fatal("cache timestamp changed")
+	}
+	// msgp restores local time while JSON preserves the UTC representation.
+	want.Info.LastUpdate = got.Info.LastUpdate
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("v9 ordinary cache fields changed:\ngot: %#v\nwant: %#v", got, want)
+	}
+	flat := got.flatten(*got.root())
+	if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 {
+		t.Fatalf("statistics changed: %+v", flat)
+	}
+	if flat.AllTierStats == nil || flat.AllTierStats.Tiers["COLD"] != (tierStats{TotalSize: 65536, NumVersions: 1, NumObjects: 1}) {
+		t.Fatalf("remote tier statistics changed: %+v", flat.AllTierStats)
+	}
+	buckets := []BucketInfo{{Name: "v9-bucket"}}
+	if gotInfo, wantInfo := got.dui(dataUsageRoot, buckets), want.dui(dataUsageRoot, buckets); !reflect.DeepEqual(gotInfo, wantInfo) {
+		t.Fatalf("bucket usage aggregation changed: %+v != %+v", gotInfo, wantInfo)
+	}
+	var buf bytes.Buffer
+	if err := got.serializeTo(&buf); err != nil {
+		t.Fatal(err)
+	}
+	if buf.Bytes()[0] != 8 {
+		t.Fatalf("wrote cache version %d, want 8", buf.Bytes()[0])
+	}
+	var roundtrip dataUsageCache
+	if err := roundtrip.deserialize(&buf); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, roundtrip) {
+		t.Fatal("v8 round trip lost ordinary fields")
+	}
+	if err := roundtrip.deserialize(bytes.NewReader(raw[:len(raw)/2])); err == nil {
+		t.Fatal("truncated v9 cache accepted")
+	}
+	raw[0] = 10
+	if err := roundtrip.deserialize(bytes.NewReader(raw)); err == nil {
+		t.Fatal("unknown cache version accepted")
+	}
+}

cmd/data-usage-cache.go

@@ -61,7 +61,6 @@ type dataUsageEntry struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
 	Size          int64             `msg:"sz"`
-	HotTierSize   int64             `msg:"hts"`
 	Objects       uint64            `msg:"os"`
 	Versions      uint64            `msg:"vs"` // Versions that are not delete markers.
 	DeleteMarkers uint64            `msg:"dms"`
@@ -135,8 +134,8 @@ func (ts tierStats) add(u tierStats) tierStats {
 	}
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -200,30 +199,14 @@ type dataUsageEntryV7 struct {
 	Compacted     bool              `msg:"c"`
 }
 
-// dataUsageEntryV8 is the on-disk shape before access-tier accounting was
-// introduced. Keep it so caches written by the previous release decode
-// without being discarded.
-type dataUsageEntryV8 struct {
-	Children dataUsageHashMap `msg:"ch"`
-	// These fields do no include any children.
-	Size          int64             `msg:"sz"`
-	Objects       uint64            `msg:"os"`
-	Versions      uint64            `msg:"vs"`
-	DeleteMarkers uint64            `msg:"dms"`
-	ObjSizes      sizeHistogram     `msg:"szs"`
-	ObjVersions   versionsHistogram `msg:"vh"`
-	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
-	Compacted     bool              `msg:"c"`
-}
-
 // dataUsageCache contains a cache of data usage entries latest version.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Cache map[string]dataUsageEntry
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -261,12 +244,6 @@ type dataUsageCacheV7 struct {
 	Cache map[string]dataUsageEntryV7
 }
 
-// dataUsageCacheV8 contains a cache of data usage entries version 8.
-type dataUsageCacheV8 struct {
-	Info  dataUsageCacheInfo
-	Cache map[string]dataUsageEntryV8
-}
-
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -295,7 +272,6 @@ type dataUsageCacheInfo struct {
 
 func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Size += summary.totalSize
-	e.HotTierSize += summary.hotTierSize
 	e.Versions += summary.versions
 	e.DeleteMarkers += summary.deleteMarkers
 	e.ObjSizes.add(summary.totalSize)
@@ -315,7 +291,6 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Versions += other.Versions
 	e.DeleteMarkers += other.DeleteMarkers
 	e.Size += other.Size
-	e.HotTierSize += other.HotTierSize
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
@@ -456,7 +431,6 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	flat := d.flatten(*e)
 	dui := DataUsageInfo{
 		LastUpdate:              d.Info.LastUpdate,
-		ScannerCycle:            d.Info.NextCycle,
 		ObjectsTotalCount:       flat.Objects,
 		VersionsTotalCount:      flat.Versions,
 		DeleteMarkersTotalCount: flat.DeleteMarkers,
@@ -807,7 +781,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		flat := d.flatten(*e)
 		bui := BucketUsageInfo{
 			Size:                    uint64(flat.Size),
-			HotTierSize:             uint64(max(flat.HotTierSize, 0)),
 			VersionsCount:           flat.Versions,
 			ObjectsCount:            flat.Objects,
 			DeleteMarkersCount:      flat.DeleteMarkers,
@@ -1007,8 +980,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 9
-	dataUsageCacheVerV8      = 8
+	dataUsageCacheVerCurrent = 8
+	dataUsageCacheVerV9      = 9 // Retired access-tier cache; only adds the ignored "hts" entry key.
 	dataUsageCacheVerV7      = 7
 	dataUsageCacheVerV6      = 6
 	dataUsageCacheVerV5      = 5
@@ -1210,34 +1183,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		}
 
 		return nil
-	case dataUsageCacheVerV8:
-		// Zstd compressed.
-		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
-		if err != nil {
-			return err
-		}
-		defer dec.Close()
-		dold := &dataUsageCacheV8{}
-		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
-			return err
-		}
-		d.Info = dold.Info
-		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
-		for k, v := range dold.Cache {
-			d.Cache[k] = dataUsageEntry{
-				Children:      v.Children,
-				Size:          v.Size,
-				Objects:       v.Objects,
-				Versions:      v.Versions,
-				DeleteMarkers: v.DeleteMarkers,
-				ObjSizes:      v.ObjSizes,
-				ObjVersions:   v.ObjVersions,
-				AllTierStats:  v.AllTierStats,
-				Compacted:     v.Compacted,
-			}
-		}
-		return nil
-	case dataUsageCacheVerCurrent:
+	case dataUsageCacheVerCurrent, dataUsageCacheVerV9:
 		// Zstd compressed.
 		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
 		if err != nil {

cmd/data-usage-cache_gen.go

@@ -1591,145 +1591,6 @@ func (z *dataUsageCacheV7) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *dataUsageCacheV8) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Info":
-			err = z.Info.DecodeMsg(dc)
-			if err != nil {
-				err = msgp.WrapError(err, "Info")
-				return
-			}
-		case "Cache":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Cache")
-				return
-			}
-			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV8, zb0002)
-			} else if len(z.Cache) > 0 {
-				clear(z.Cache)
-			}
-			for zb0002 > 0 {
-				zb0002--
-				var za0001 string
-				za0001, err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Cache")
-					return
-				}
-				var za0002 dataUsageEntryV8
-				err = za0002.DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
-					return
-				}
-				z.Cache[za0001] = za0002
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageCacheV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Info":
-			bts, err = z.Info.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Info")
-				return
-			}
-		case "Cache":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Cache")
-				return
-			}
-			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV8, zb0002)
-			} else if len(z.Cache) > 0 {
-				clear(z.Cache)
-			}
-			for zb0002 > 0 {
-				var za0002 dataUsageEntryV8
-				zb0002--
-				var za0001 string
-				za0001, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache")
-					return
-				}
-				bts, err = za0002.UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
-					return
-				}
-				z.Cache[za0001] = za0002
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageCacheV8) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
-	if z.Cache != nil {
-		for za0001, za0002 := range z.Cache {
-			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
-		}
-	}
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -1762,12 +1623,6 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Size")
 				return
 			}
-		case "hts":
-			z.HotTierSize, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "HotTierSize")
-				return
-			}
 		case "os":
 			z.Objects, err = dc.ReadUint64()
 			if err != nil {
@@ -1946,12 +1801,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// check for omitted fields
-	zb0001Len := uint32(10)
-	var zb0001Mask uint16 /* 10 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x100
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -1981,16 +1836,6 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Size")
 			return
 		}
-		// write "hts"
-		err = en.Append(0xa3, 0x68, 0x74, 0x73)
-		if err != nil {
-			return
-		}
-		err = en.WriteInt64(z.HotTierSize)
-		if err != nil {
-			err = msgp.WrapError(err, "HotTierSize")
-			return
-		}
 		// write "os"
 		err = en.Append(0xa2, 0x6f, 0x73)
 		if err != nil {
@@ -2055,7 +1900,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 				return
 			}
 		}
-		if (zb0001Mask & 0x100) == 0 { // if not omitted
+		if (zb0001Mask & 0x80) == 0 { // if not omitted
 			// write "ats"
 			err = en.Append(0xa3, 0x61, 0x74, 0x73)
 			if err != nil {
@@ -2136,12 +1981,12 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// check for omitted fields
-	zb0001Len := uint32(10)
-	var zb0001Mask uint16 /* 10 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x100
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -2158,9 +2003,6 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 		// string "sz"
 		o = append(o, 0xa2, 0x73, 0x7a)
 		o = msgp.AppendInt64(o, z.Size)
-		// string "hts"
-		o = append(o, 0xa3, 0x68, 0x74, 0x73)
-		o = msgp.AppendInt64(o, z.HotTierSize)
 		// string "os"
 		o = append(o, 0xa2, 0x6f, 0x73)
 		o = msgp.AppendUint64(o, z.Objects)
@@ -2182,7 +2024,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 		for za0002 := range z.ObjVersions {
 			o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 		}
-		if (zb0001Mask & 0x100) == 0 { // if not omitted
+		if (zb0001Mask & 0x80) == 0 { // if not omitted
 			// string "ats"
 			o = append(o, 0xa3, 0x61, 0x74, 0x73)
 			if z.AllTierStats == nil {
@@ -2246,12 +2088,6 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Size")
 				return
 			}
-		case "hts":
-			z.HotTierSize, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "HotTierSize")
-				return
-			}
 		case "os":
 			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
 			if err != nil {
@@ -2429,7 +2265,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
 	if z.AllTierStats == nil {
 		s += msgp.NilSize
 	} else {
@@ -3422,438 +3258,6 @@ func (z *dataUsageEntryV7) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *dataUsageEntryV8) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var zb0001Mask uint8 /* 1 bits */
-	_ = zb0001Mask
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "ch":
-			err = z.Children.DecodeMsg(dc)
-			if err != nil {
-				err = msgp.WrapError(err, "Children")
-				return
-			}
-		case "sz":
-			z.Size, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "Size")
-				return
-			}
-		case "os":
-			z.Objects, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "Objects")
-				return
-			}
-		case "vs":
-			z.Versions, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "Versions")
-				return
-			}
-		case "dms":
-			z.DeleteMarkers, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "DeleteMarkers")
-				return
-			}
-		case "szs":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "ObjSizes")
-				return
-			}
-			if zb0002 != uint32(dataUsageBucketLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
-				return
-			}
-			for za0001 := range z.ObjSizes {
-				z.ObjSizes[za0001], err = dc.ReadUint64()
-				if err != nil {
-					err = msgp.WrapError(err, "ObjSizes", za0001)
-					return
-				}
-			}
-		case "vh":
-			var zb0003 uint32
-			zb0003, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "ObjVersions")
-				return
-			}
-			if zb0003 != uint32(dataUsageVersionLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
-				return
-			}
-			for za0002 := range z.ObjVersions {
-				z.ObjVersions[za0002], err = dc.ReadUint64()
-				if err != nil {
-					err = msgp.WrapError(err, "ObjVersions", za0002)
-					return
-				}
-			}
-		case "ats":
-			if dc.IsNil() {
-				err = dc.ReadNil()
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				z.AllTierStats = nil
-			} else {
-				if z.AllTierStats == nil {
-					z.AllTierStats = new(allTierStats)
-				}
-				var zb0004 uint32
-				zb0004, err = dc.ReadMapHeader()
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "AllTierStats")
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "ts":
-						var zb0005 uint32
-						zb0005, err = dc.ReadMapHeader()
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats", "Tiers")
-							return
-						}
-						if z.AllTierStats.Tiers == nil {
-							z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
-						} else if len(z.AllTierStats.Tiers) > 0 {
-							clear(z.AllTierStats.Tiers)
-						}
-						for zb0005 > 0 {
-							zb0005--
-							var za0003 string
-							za0003, err = dc.ReadString()
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers")
-								return
-							}
-							var za0004 tierStats
-							var zb0006 uint32
-							zb0006, err = dc.ReadMapHeader()
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-								return
-							}
-							for zb0006 > 0 {
-								zb0006--
-								field, err = dc.ReadMapKeyPtr()
-								if err != nil {
-									err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-									return
-								}
-								switch msgp.UnsafeString(field) {
-								case "ts":
-									za0004.TotalSize, err = dc.ReadUint64()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
-										return
-									}
-								case "nv":
-									za0004.NumVersions, err = dc.ReadInt()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
-										return
-									}
-								case "no":
-									za0004.NumObjects, err = dc.ReadInt()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
-										return
-									}
-								default:
-									err = dc.Skip()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-										return
-									}
-								}
-							}
-							z.AllTierStats.Tiers[za0003] = za0004
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats")
-							return
-						}
-					}
-				}
-			}
-			zb0001Mask |= 0x1
-		case "c":
-			z.Compacted, err = dc.ReadBool()
-			if err != nil {
-				err = msgp.WrapError(err, "Compacted")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	// Clear omitted fields.
-	if (zb0001Mask & 0x1) == 0 {
-		z.AllTierStats = nil
-	}
-
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageEntryV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var zb0001Mask uint8 /* 1 bits */
-	_ = zb0001Mask
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "ch":
-			bts, err = z.Children.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Children")
-				return
-			}
-		case "sz":
-			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Size")
-				return
-			}
-		case "os":
-			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Objects")
-				return
-			}
-		case "vs":
-			z.Versions, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Versions")
-				return
-			}
-		case "dms":
-			z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "DeleteMarkers")
-				return
-			}
-		case "szs":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ObjSizes")
-				return
-			}
-			if zb0002 != uint32(dataUsageBucketLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
-				return
-			}
-			for za0001 := range z.ObjSizes {
-				z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "ObjSizes", za0001)
-					return
-				}
-			}
-		case "vh":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ObjVersions")
-				return
-			}
-			if zb0003 != uint32(dataUsageVersionLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
-				return
-			}
-			for za0002 := range z.ObjVersions {
-				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "ObjVersions", za0002)
-					return
-				}
-			}
-		case "ats":
-			if msgp.IsNil(bts) {
-				bts, err = msgp.ReadNilBytes(bts)
-				if err != nil {
-					return
-				}
-				z.AllTierStats = nil
-			} else {
-				if z.AllTierStats == nil {
-					z.AllTierStats = new(allTierStats)
-				}
-				var zb0004 uint32
-				zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "AllTierStats")
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "ts":
-						var zb0005 uint32
-						zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats", "Tiers")
-							return
-						}
-						if z.AllTierStats.Tiers == nil {
-							z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
-						} else if len(z.AllTierStats.Tiers) > 0 {
-							clear(z.AllTierStats.Tiers)
-						}
-						for zb0005 > 0 {
-							var za0004 tierStats
-							zb0005--
-							var za0003 string
-							za0003, bts, err = msgp.ReadStringBytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers")
-								return
-							}
-							var zb0006 uint32
-							zb0006, bts, err = msgp.ReadMapHeaderBytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-								return
-							}
-							for zb0006 > 0 {
-								zb0006--
-								field, bts, err = msgp.ReadMapKeyZC(bts)
-								if err != nil {
-									err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-									return
-								}
-								switch msgp.UnsafeString(field) {
-								case "ts":
-									za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
-										return
-									}
-								case "nv":
-									za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
-										return
-									}
-								case "no":
-									za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
-										return
-									}
-								default:
-									bts, err = msgp.Skip(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-										return
-									}
-								}
-							}
-							z.AllTierStats.Tiers[za0003] = za0004
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats")
-							return
-						}
-					}
-				}
-			}
-			zb0001Mask |= 0x1
-		case "c":
-			z.Compacted, bts, err = msgp.ReadBoolBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Compacted")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	// Clear omitted fields.
-	if (zb0001Mask & 0x1) == 0 {
-		z.AllTierStats = nil
-	}
-
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageEntryV8) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
-	if z.AllTierStats == nil {
-		s += msgp.NilSize
-	} else {
-		s += 1 + 3 + msgp.MapHeaderSize
-		if z.AllTierStats.Tiers != nil {
-			for za0003, za0004 := range z.AllTierStats.Tiers {
-				_ = za0004
-				s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
-			}
-		}
-	}
-	s += 2 + msgp.BoolSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{

cmd/data-usage-utils.go

@@ -46,8 +46,7 @@ type BucketTargetUsageInfo struct {
 // - total objects in a bucket
 // - object size histogram per bucket
 type BucketUsageInfo struct {
-	Size        uint64 `json:"size"`
-	HotTierSize uint64 `json:"hotTierSize,omitempty"`
+	Size uint64 `json:"size"`
 	// Following five fields suffixed with V1 are here for backward compatibility
 	// Total Size for objects that have not yet been replicated
 	ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
@@ -79,10 +78,6 @@ type DataUsageInfo struct {
 	// LastUpdate is the timestamp of when the data usage info was last updated.
 	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
-	// ScannerCycle changes only after a complete scanner pass. Background
-	// consumers use it to distinguish a partial cache update from a baseline
-	// that has visited every bucket and server pool.
-	ScannerCycle uint32 `json:"scannerCycle,omitempty"`
 
 	// Objects total count across all buckets
 	ObjectsTotalCount uint64 `json:"objectsCount"`

cmd/endpoint-ellipses_test.go

@@ -27,6 +27,7 @@ import (
 
 // Tests create endpoints with ellipses and without.
 func TestCreateServerEndpoints(t *testing.T) {
+	useStaticTestEndpoints(t)
 	testCases := []struct {
 		serverAddr string
 		args       []string

cmd/endpoint_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -25,9 +26,42 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/minio/dnscache"
 )
 
+type endpointTestResolver struct {
+	dnscache.DNSResolver
+}
+
+func (r endpointTestResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
+	// These endpoint fixtures test locality and parsing, not public DNS.
+	for i, name := range []string{"example.org", "example.com", "example.net", "example.edu"} {
+		if host == name {
+			return []string{fmt.Sprintf("203.0.113.%d", i+1)}, nil
+		}
+	}
+	return r.DNSResolver.LookupHost(ctx, host)
+}
+
+func useTestEndpointDNS(t *testing.T) {
+	t.Helper()
+	previous := globalDNSCache
+	globalDNSCache = &dnscache.Resolver{Resolver: endpointTestResolver{net.DefaultResolver}}
+	t.Cleanup(func() { globalDNSCache = previous })
+}
+
+func useStaticTestEndpoints(t *testing.T) {
+	t.Helper()
+	useTestEndpointDNS(t)
+	// These fixtures deliberately use localhost, including inside containers.
+	previous := orchestrated
+	orchestrated = false
+	t.Cleanup(func() { orchestrated = previous })
+}
+
 func TestNewEndpoint(t *testing.T) {
+	useTestEndpointDNS(t)
 	u2, _ := url.Parse("https://example.org/path")
 	u4, _ := url.Parse("http://192.168.253.200/path")
 	rootSlashFoo, _ := filepath.Abs("/foo")
@@ -130,6 +164,7 @@ func TestNewEndpoints(t *testing.T) {
 }
 
 func TestCreateEndpoints(t *testing.T) {
+	useStaticTestEndpoints(t)
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {
 		globalMinioPort = tempGlobalMinioPort
@@ -353,6 +388,7 @@ func TestCreateEndpoints(t *testing.T) {
 // So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
 // is considered a remote service from localhost:9000 perspective.
 func TestGetLocalPeer(t *testing.T) {
+	useStaticTestEndpoints(t)
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {
 		globalMinioPort = tempGlobalMinioPort
@@ -393,6 +429,7 @@ func TestGetLocalPeer(t *testing.T) {
 }
 
 func TestGetRemotePeers(t *testing.T) {
+	useStaticTestEndpoints(t)
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {
 		globalMinioPort = tempGlobalMinioPort