fix(pools): keep rejected names out of a single-pool bulk delete

DeleteObjects checks every name with checkDelObjArgs, but a single pool
still sent all entries to the drives and cleared the error of an entry
that a drive reported as found. Local drives join the name to the bucket
directory, so a name with a ".." segment reached another bucket or the
meta bucket. An S3 DeleteObjects caller allowed to delete in one bucket
could remove a version elsewhere, including one under Object Lock
retention, and from a versioned bucket could write a delete marker on an
object of another bucket or on stored bucket metadata. The bulk lock was
taken on the normalized name as well, that is on the other object.

Entries that fail the check now keep their error, and the rest of the
request runs as a request of its own, before any lock or lookup. A
request whose names are all valid does not enter the new code. More than
one pool already skipped such entries.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  # Dockerfile.goreleaser and Dockerfile.distroless pin their bases by digest
+  # so a release tag rebuilds from the same bytes. Without this the pins would
+  # silently rot and stop receiving base-image CVE fixes.
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: build

.github/goreleaser.yml

@@ -27,6 +27,10 @@ archives:
     ids:
       - silo
     name_template: "silo_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
+    formats: ["tar.gz"]
+    format_overrides:
+      - goos: windows
+        formats: ["zip"]
     # Explicit so the license materials cannot silently drop out of the
     # binary archives: GoReleaser's default file globs would miss NOTICE.
     # CREDITS stays out deliberately -- at 1.8MB it would dominate the

.github/workflows/finalize-release.yml

@@ -148,7 +148,9 @@ jobs:
           for rpm_file in finalize/packages/*.rpm; do
             signature_output="$(sudo rpmkeys --checksig --verbose "${rpm_file}")"
             printf '%s\n' "${signature_output}"
-            if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q "key id ${key_id}: ok"; then
+            normalized_signature="$(printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]')"
+            full_fingerprint="$(printf '%s' "${expected_fingerprint}" | tr '[:upper:]' '[:lower:]')"
+            if ! printf '%s\n' "${normalized_signature}" | grep -Eq "key id ${key_id}: ok|key fingerprint: ${full_fingerprint}: ok"; then
               echo "Signature verification failed for ${rpm_file}" >&2
               exit 1
             fi

.github/workflows/go.yml

@@ -78,7 +78,7 @@ jobs:
   race-s3select:
     name: Race, S3 Select
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 25
     steps:
       - uses: actions/checkout@v7
 
@@ -90,14 +90,30 @@ jobs:
       - name: Run S3 Select tests under race detector
         run: go test -race ./internal/s3select/... -count=1
 
-      - name: Run conditional PUT tests under race detector
-        run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m
+      - name: Run conditional PUT and multipart tests under race detector
+        run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP|PoolsMultipartCondition)' -count=1 -timeout=5m
 
-      - name: Run multipart listing and cancellation tests under race detector
-        run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m
+      - name: Run multi-pool delete consistency tests under race detector
+        run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
+
+      # Build tags: kqueue selects the kqueue backend of rjeczalik/notify on
+      # BSD and macOS and changes nothing on Linux. dev came with upstream's
+      # 2024-06 Makefile change for development builds; no file in this
+      # repository or its dependencies carries a dev build constraint, so it
+      # selects nothing and is kept only so that the documented command
+      # lines (make test, make install-race) stay valid.
+      - name: Run lifecycle expiry and native restore tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(PoolsLifecycle|PoolsMigrationLifecycle|ExpireRestored|NativeRestore)' -count=1 -timeout=10m
+
+      - name: Run multipart listing, cancellation and cleanup tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race ./cmd -run '^Test(ObjectCompleteMultipartUpload|Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
 
       - name: Run listing metadata recovery, pagination and cancellation tests under race detector
-        run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity)' -count=1 -timeout=5m
+        run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m
 
       - name: Run lock cancellation and late-grant cleanup tests under race detector
         run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m
@@ -111,7 +127,18 @@ jobs:
       - name: Run Object Lock authorization and overwrite tests under race detector
         env:
           MINIO_API_REQUESTS_MAX: "10000"
-        run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|ReplicaDeleteObjectLock|PoolsReplicaDeleteObjectLock|BulkDeleteDirectoryObjectLock|BulkDeleteObjectLock|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
+
+      - name: Run maintenance storage timeout and compatibility tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(StorageTimeout|StorageRollback|StoragePrerelease|MaintenancePartialFailedOverwriteAvailability|SingleRequest|WithDeadline)' -count=1 -timeout=5m
+
+      - name: Run logger target lifecycle tests under race detector
+        run: go test -race ./cmd -run '^TestDynamicWebhookOutlivesConfigContext' -count=1 -timeout=5m
+
+      - name: Compile deferred consistency diagnostics
+        run: go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'
 
   crosscompile:
     name: Cross Compile

.github/workflows/release.yml

@@ -74,7 +74,7 @@ jobs:
       - name: Gate release on Object Lock authorization and overwrite tests under race detector
         env:
           MINIO_API_REQUESTS_MAX: "10000"
-        run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|ReplicaDeleteObjectLock|PoolsReplicaDeleteObjectLock|BulkDeleteDirectoryObjectLock|BulkDeleteObjectLock|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
 
       - name: Compute release variables
         env:
@@ -140,7 +140,7 @@ jobs:
       - name: Build Draft release with GoReleaser
         uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
-          version: v2.18.1
+          version: v2.18.3
           args: release --clean --skip=validate --config .github/goreleaser.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -156,7 +156,7 @@ jobs:
       - name: Install nFPM
         run: |
           set -euo pipefail
-          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
+          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
 
       - name: Build nFPM packages

.github/workflows/repository-cards.yml

@@ -34,10 +34,11 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
           persist-credentials: false
-      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: "3.13"
-      - run: python -m pip install -r buildscripts/repository-cards/requirements.txt
+      - name: Use the runner's Python in an isolated environment
+        run: |
+          python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
+          "$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
+          echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
       - run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v
 
   publish:
@@ -54,10 +55,11 @@ jobs:
       OUTPUT_BRANCH: codex/repository-cards
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: "3.13"
-      - run: python -m pip install -r buildscripts/repository-cards/requirements.txt
+      - name: Use the runner's Python in an isolated environment
+        run: |
+          python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
+          "$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
+          echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
 
       - name: Load the generated-assets branch
         shell: bash

.github/workflows/test-release.yml

@@ -60,7 +60,7 @@ jobs:
           docker build --build-arg TARGETARCH=${{ matrix.arch }} \
             --target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
           docker run --rm silo-curl-test --version | tee curl-version.txt
-          grep -F 'curl 8.22.0 ' curl-version.txt
+          grep -F 'curl 8.23.0 ' curl-version.txt
           grep -F 'HTTP2' curl-version.txt
           docker run --rm silo-curl-test --fail --silent --show-error \
             --connect-timeout 15 --max-time 60 https://curl.se/robots.txt
@@ -97,7 +97,7 @@ jobs:
       - name: GoReleaser config check
         uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
-          version: v2.18.1
+          version: v2.18.3
           args: check --config .github/goreleaser.yml
 
       - name: Validate Helm chart and legacy upgrade identity
@@ -111,7 +111,7 @@ jobs:
       - name: Build snapshot artifacts
         uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
         with:
-          version: v2.18.1
+          version: v2.18.3
           # A pull-request snapshot has no trusted release identity. Exercise
           # the SBOM/checksum pipeline here, and reserve keyless signing for
           # the tag-triggered release workflow with GitHub OIDC.
@@ -120,13 +120,34 @@ jobs:
           LDFLAGS: ${{ env.LDFLAGS }}
           PKG_VERSION: ${{ env.PKG_VERSION }}
 
-      - name: Verify archive SBOM and checksum coverage
+      - name: Verify archive formats, documentation, SBOMs, and checksums
         run: |
           set -euo pipefail
-          mapfile -t archives < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz' | sort)
-          mapfile -t sboms < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz.sbom.json' | sort)
+          mapfile -t archives < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz' -o -name 'silo_*.zip' \) | sort)
+          mapfile -t sboms < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz.sbom.json' -o -name 'silo_*.zip.sbom.json' \) | sort)
           test "${#archives[@]}" -eq 6
           test "${#sboms[@]}" -eq 6
+          for os in linux darwin windows; do
+            format=tar.gz
+            binary=silo
+            if [ "${os}" = windows ]; then
+              format=zip
+              binary=silo.exe
+            fi
+            for arch in amd64 arm64; do
+              archive="dist/silo_${PKG_VERSION}_${os}_${arch}.${format}"
+              test -s "${archive}"
+              jq -e '.spdxVersion | startswith("SPDX-")' "${archive}.sbom.json" >/dev/null
+              if [ "${format}" = zip ]; then
+                listing="$(unzip -Z1 "${archive}")"
+              else
+                listing="$(tar -tzf "${archive}")"
+              fi
+              for member in "${binary}" LICENSE NOTICE README.md; do
+                grep -Fx "${member}" <<< "${listing}"
+              done
+            done
+          done
           manifest="dist/silo_${PKG_VERSION}_checksums.txt"
           test -s "${manifest}"
           (
@@ -143,7 +164,7 @@ jobs:
       - name: Install package validation tools
         run: |
           set -euo pipefail
-          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
+          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
           sudo apt-get update
           sudo apt-get install --yes rpm binutils
@@ -461,6 +482,17 @@ jobs:
             fi
           done
 
+          # Both variants must trust the same roots: the UBI bundle with the
+          # RHSB-2026-012 blocklist applied, not Debian's bundle.
+          tar -xOf "${ctx}/rootfs.tar" etc/ssl/certs/ca-certificates.crt > "${ctx}/ca.pem"
+          openssl crl2pkcs7 -nocrl -certfile "${ctx}/ca.pem" \
+            | openssl pkcs7 -print_certs -noout > "${ctx}/ca-subjects.txt"
+          test "$(grep -c '^subject=' "${ctx}/ca-subjects.txt")" -gt 100
+          if grep -E '^subject=.*(AAA Certificate Services|emSign ECC Root CA - G3)' "${ctx}/ca-subjects.txt"; then
+            echo "distroless CA bundle still trusts a blocklisted root"
+            exit 1
+          fi
+
           # The baked-in healthcheck must drive Docker's health state on its
           # own, the probe binary must be directly exec-able without any
           # shell, and SIGTERM must still reach PID 1 (the server binary is

.github/workflows/vulncheck.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Install govulncheck
         run: |
-          go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
+          go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
 
       - name: Run govulncheck

.oss-scanner/Dockerfile

@@ -0,0 +1,34 @@
+# Copyright 2026 PGSTY contributors.
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+FROM golang:1.27.3-bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    GOTOOLCHAIN=local \
+    CGO_ENABLED=0 \
+    MINIO_API_REQUESTS_MAX=10000
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates curl git jq openssl python3 procps \
+    && rm -rf /var/lib/apt/lists/*
+
+# Cache dependencies in the final image: subsequent analysis runs offline.
+# Honor go.mod's PGSTY replacements and leave the module files unchanged.
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+COPY . /src
+
+# Keep normal Go debug information; the release Makefile strips it.
+RUN go build -tags kqueue -o /usr/local/bin/silo . \
+    && silo --version
+
+# Compile test packages while online, including the server and internal code.
+# Retain both the module and build caches so new reproducers work offline.
+RUN go test -tags kqueue,dev -run '^$' -timeout=10m ./cmd ./internal/...
+
+ENV GOPROXY=off \
+    GOSUMDB=off
+
+CMD ["bash"]

.oss-scanner/Dockerfile.dockerignore

@@ -0,0 +1,43 @@
+# Override the release .dockerignore: keep source, CI files and all test fixtures.
+# The scanner adds Git history separately after building the image.
+.git
+.bin
+.claude
+.codex
+.idea
+.release-sign
+.vscode
+_bmad
+_bmad-output
+dist
+vendor
+**/node_modules
+**/*.test
+**/.DS_Store
+default.etcd
+docs/investigations
+docs/security
+docs/rebranding.md
+
+# Local server and debugging binaries.
+minio
+silo
+hash-set
+healing-bin
+inspect
+pprofgoparser
+reorder-disks
+s3-check-md5
+s3-verify
+xattr
+xl-meta
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
+docs/debugging/reorder-disks/reorder-disks
+docs/debugging/populate-hard-links/populate-hardlinks
+docs/debugging/xattr/xattr

.oss-scanner/threat_model.md

@@ -0,0 +1,131 @@
+# PGSTY Silo security scope
+
+Silo is PGSTY's independently maintained S3-compatible object-storage server,
+derived from MinIO. It serves untrusted network requests and stores user data,
+including PostgreSQL backups. Scan the current `pgsty/silo` `main` branch.
+Read `/src/SECURITY.md`, `/src/VULNERABILITY_REPORT.md`, and `/src/CHANGELOG.md`.
+
+## Repository boundaries and lineage
+
+The Go module intentionally remains `github.com/minio/minio`. Historical
+`MINIO_*` environment variables, S3 headers, `/minio/*` routes, and `.minio.sys`
+storage names are compatibility interfaces, not evidence that upstream MinIO
+is the target. Use this checkout's code and dependency versions for reproducers.
+
+This enrollment covers the server code, including inherited code still present
+in the supported branch and its PGSTY changes. The maintained Console, client,
+and shared packages appear as dependencies in `go.mod`. Do not replace them
+with upstream releases. They are not additional independent scan targets under
+this enrollment. A server-side integration defect is in scope; clearly identify
+a dependency-only root cause rather than attributing it to this repository.
+
+## Attackers and important surfaces
+
+- Unauthenticated S3/HTTP clients, including hostile object names, headers,
+  query parameters, XML/JSON bodies, signatures, and multipart data.
+- Authenticated users, service accounts, and temporary STS credentials whose
+  authority must remain bounded by the effective IAM and bucket policies.
+- Bucket owners and delegated administrators with explicitly limited actions;
+  distinguish them from the unrestricted deployment/root administrator.
+- Inputs from configured identity providers, replication/tiering targets, and
+  webhooks where the configuration and permissions make an attack reachable.
+
+Prioritize authentication/signature verification, IAM/STS and policy
+composition, administrative API authorization, object/version isolation,
+Object Lock retention and legal holds, multipart operations, replication,
+lifecycle/tiering, and multi-pool deletion/migration. Include archive, XML,
+JSON and S3 Select parsing when remotely reachable. Investigate credential
+exposure, path traversal, unauthorized object access, SSRF across a meaningful
+boundary, and attacks that corrupt or irreversibly delete other users' data.
+
+The deployment administrator controls root credentials, storage directories,
+trusted peers, and identity/KMS/replication endpoint configuration. Host root
+access, stolen root credentials, or a malicious replacement of the server
+binary do not by themselves demonstrate a vulnerability. An unprivileged
+account gaining those capabilities does. An administrator's documented ability
+to configure an outbound endpoint is not sufficient evidence of SSRF; show
+an unauthorized actor, forbidden destination, or confused-deputy effect.
+
+Object Lock has explicit retention semantics. Do not discard a retention bypass
+just because the caller has some administrative permission: describe the mode,
+version, legal hold, and exact IAM actions, and compare with the intended rules.
+Likewise, do not assume bucket access implies access to other buckets or users.
+
+Known consistency limitations are described in `CONTRIBUTING.md` and the
+changelog. Tests tagged `consistency_strict` exercise stronger guarantees
+outside the current maintenance contract and can fail. A failing diagnostic
+alone is not a new vulnerability; demonstrate a reachable security impact.
+These limitations do not excuse an authorization or retention bypass.
+
+## Offline environment
+
+Source is in `/src`; the unstripped server is `/usr/local/bin/silo`. Go, the
+module cache, and the test build cache are retained. The scanner adds Git
+history to `/src/.git` after building this image; a plain Docker build omits it.
+The Dockerfile compiles the server and tests with the selected PGSTY dependency
+replacements. `GOTOOLCHAIN=local` and `GOPROXY=off` prevent surprise downloads.
+CGO is disabled as in the normal server/unit-test CI; enable it explicitly for
+race tests when needed (the Go base image includes a C compiler).
+
+Useful offline checks:
+
+```sh
+cd /src
+silo --version
+go test -tags kqueue,dev ./internal/auth ./internal/config/identity/... ./internal/s3select/... -count=1 -timeout=10m
+go test -tags kqueue,dev ./cmd -run '^(TestGetRequestAuthType|TestS3SupportedAuthType|TestIsReqAuthenticated|TestCheckAdminRequestAuthType|TestValidateAdminSignature|TestDoesPolicySignatureMatch|TestDoesPresignedSignatureMatch|TestDoesPresignedV2SignatureMatch|TestSTSLDAPLogin.*|TestGetSTSLDAPLoginSourceIP.*)$' -count=1 -timeout=10m
+```
+
+For HTTP reproducers, start a disposable loopback server with synthetic
+credentials and a fresh data directory:
+
+```sh
+MINIO_ROOT_USER=scanneradmin MINIO_ROOT_PASSWORD=scanner-local-test-only \
+  silo server /tmp/silo-scanner-data \
+  --address 127.0.0.1:9000 --console-address 127.0.0.1:9001
+```
+
+The broader suites are `go test -tags kqueue,dev ./internal/... -timeout=15m`
+and `go test -tags kqueue,dev ./cmd -timeout=30m`. Some integration suites need
+LDAP, OIDC, etcd, KMS, multiple servers, or IPv6; those services are not started
+by this Dockerfile. Do not treat a missing external service as a security
+finding. The focused commands above do not require those providers.
+
+## Reports and severity
+
+Include the commit, exact attacker privileges/policy, relevant deployment
+settings, API requests or a standalone test, observed impact, and affected
+versions where established. For data-loss or cross-pool reports, include the
+object/version state and a deterministic sequence. Deduplicate by root cause
+and distinguish a new flaw from an already documented limitation or fixed bug.
+
+Unauthenticated code execution or broad credential/authorization compromise
+can be critical. Cross-user object access, unauthorized destructive operations,
+and retention bypass can be high or critical according to their demonstrated
+reach. Rate availability issues by practical cost, scope and recovery; a
+single malformed request and an authorized full-disk upload are different
+threats. Do not claim exploitability solely from a static pattern.
+
+Prefer minimal patches and regression tests that preserve S3/storage format
+compatibility. Use only synthetic data and local test instances; do not test
+against public Silo/MinIO deployments or real backup repositories.
+
+## Maintaining the scanner image
+
+Build from the repository root, then run the checks above without networking:
+
+```sh
+docker build -f .oss-scanner/Dockerfile -t silo-scanner:local .
+docker run --rm -it --network=none silo-scanner:local bash
+```
+
+Keep the Go image version aligned with `go.mod`. The adjacent
+`Dockerfile.dockerignore` preserves compressed test fixtures that the release
+image's root `.dockerignore` excludes. Local build outputs, editor state, and
+private working directories are excluded from the scanner context.
+
+Enrollment lives in `anthropics/oss-scanner`, under `projects/silo/project.yaml`.
+Once these files are published on `main`, that configuration can use
+`dockerfile: .oss-scanner/Dockerfile` and
+`threat_model: .oss-scanner/threat_model.md`; remove its adjacent copies in the
+same update. Do not switch the enrollment paths before the files are public.

CHANGELOG.md

@@ -2,221 +2,363 @@
 
 ## Unreleased
 
-Preparation target: `RELEASE.2026-09-16T00-00-00Z` (package version
-`20260916000000.0.0`). The entries below describe the candidate changes since
-the latest published Server.
-**The latest published Server remains 20260903.** These changes are not in its
-binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
-and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
+- An edit of a service account or access key that names no policy keeps its
+  inline policy. `mcli admin user svcacct disable`, `enable` and `edit` without
+  `--policy`, the same `idp ldap accesskey` commands, and any other
+  `update-service-account` request without `newPolicy` removed the inline
+  policy at the site that took the request and, for an account of another user
+  than root, at its replication peers, and the account then had all its parent
+  is allowed. The inline policy is removed
+  only on request, by an empty policy document such as `{"Statement":[]}`
+  (`mcli admin user svcacct edit --policy`); a client that left the policy out
+  to remove it must send that document. A JSON `null` policy counts as none.
+  In the Console, clearing the policy editor of an access key no longer removes
+  the policy; enter `{}` to remove it. The fix does not bring back a policy
+  that was removed, and a site of a replicated deployment that does not run the
+  fix still removes the policy when such an edit reaches it, so sites can
+  differ. Upgrade every site, then review on each site the accounts that
+  `svcacct info` reports with `Policy: implied`, and set the policy again on
+  those that were meant to be restricted.
 
-### Authorization and security
+- S3 `DeleteObjects` on a single pool stops at an entry whose name fails the
+  object name check: the entry returns `XMinioInvalidObjectName` and is not
+  locked or sent to a drive, as with a single-object DELETE and with more than
+  one pool. Until now such an entry still reached the drives of the node, where
+  a name with a `..` segment was joined to the request bucket. A caller allowed
+  to delete in one bucket could remove a version in another bucket, including
+  one under Object Lock retention, and from a versioned bucket could put a
+  delete marker on an object of another bucket or on stored system metadata
+  such as a bucket's configuration. The other entries of a request are deleted
+  as before.
 
-- Synchronize CPU metrics reads with resource-metrics updates (#210), preventing
-  concurrent map access from terminating the server during Prometheus scraping.
-  Metric names, values and authentication requirements are unchanged.
-- Restrict embedded Console's anonymous sharing proxy to object-content GETs
-  at the configured S3 origin, and reject every redirect. Internal metrics,
-  system paths and non-download S3 operations cannot be reached through it.
-  Normal public, presigned and versioned downloads remain available without a
-  new setting; a full sharing-disable switch is not introduced. See
-  [Console #56](https://github.com/pgsty/silo-console/pull/56) and the
-  [design record](https://github.com/pgsty/silo-console/issues/52).
-  Thanks to Jiri Pejchal (@jiri-pejchal) for the report.
-- Persist IAM deletion revisions and parent revocation boundaries so stale site
-  events cannot restore deleted identities, policies or their older grants
-  (#191, #192). Peer deletion notifications reload committed storage; deliberate
-  recreation requires a newer revision, and credentials issued before the
-  parent's revocation remain invalid.
-  **Coordinated upgrade required:** upgrade every participating node and site.
-  Mixed old/new nodes sharing an IAM backend and rolling downgrade are
-  unsupported. Back up complete IAM storage and encryption material; an admin
-  export of live records omits deletion history. Reissue credentials for
-  recreated parents and explicitly reconcile pre-upgrade revocations whose
-  history is already lost. Restoring an older backup can lose later revocations;
-  keep affected sites isolated until reconciliation/rekeying is complete. See
-  [the operator runbook](https://silo.pgsty.com/operations/replication/iam-upgrade/).
-- Enforce an absolute HTTP/1 request-header deadline through the connection
-  wrapper (#196). Repeated small reads no longer extend that deadline, and
-  `--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
-  server. HTTP/1 request bodies retain the rolling idle timeout; this does not
-  impose a total upload/download duration. A shorter setting also constrains
-  TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
-- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
-  copy of another object accessible to the signer (SN-2026-011). The latest
-  public Server is affected; the fix is on main. See [the advisory ledger](https://silo.pgsty.com/about/security-advisories/).
-- Align signed request fields with policy conditions and enforce header-only
-  presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
-- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
-  from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
-  both denies if the previous combined restriction must survive upgrades or
-  rollback. Saved policies are not rewritten. Deploy with the matching Console
-  and pkg; see [the migration guide](docs/iam/password-permissions.md).
+## RELEASE.2026-11-09T00-00-00Z
 
-### Object storage and replication
+Package version: `20261109000000.0.0`. Publication status and artifacts are
+listed in [GitHub Releases](https://github.com/pgsty/silo/releases). This
+section covers the changes after the published 20261026 source `c3ad836aed0d`.
+Most fixes come from reports and patches by
+[@botdeniska](https://github.com/botdeniska); see [CONTRIBUTORS.md](CONTRIBUTORS.md).
 
-- Make durable `ListMultipartUploads` scanning the only listing path (#79):
-  apply prefix, delimiter, markers, ordering and the 1,000-entry page limit
-  globally across pools and sets, independently of node-local caches. Omit
-  readable historical uploads only when both bucket/key identity fields are
-  absent; malformed identities, uncertain coverage and resource limits remain
-  errors. Old uploads still support part upload, completion and cancellation
-  with a known bucket/key/upload ID. No backfill or new storage layout is added.
-  `MINIO_API_MULTIPART_LISTING` is retired and ignored with a startup warning;
-  historical shared `api multipart_listing` keys remain tolerated. The read-only
-  admin `multipart-preflight` reports `mode=persistent`; `ready` diagnoses the
-  absence of old records and no longer gates listing. Each page still scans
-  durable state within the existing 100,000-directory-entry, 30-second and
-  per-process concurrency budgets. See the
-  [design and limits](https://silo.pgsty.com/blog/design/list-multipart-uploads/#implementation).
-  Thanks to mr javad seydi (@mrjavadseydi) for the original metadata-and-scan
-  implementation (#198).
-- Keep the default read-quorum, best-effort multipart cancellation and
-  first-matching-pool behavior when retiring the listing switch. The optional
-  strict cancellation branch is removed. Wrong-target protection, upload-ID
-  path validation and peer notification with the request context remain.
-  HTTP 204 for an absent upload remains idempotent and does not prove physical
-  cleanup. **Known boundary:** delayed creation writes can still restore a
-  readable upload after cancellation; this change adds no creation fence.
-- Preserve object tags during multi-pool metadata reconciliation by reading the
-  resolved tag field together with its revision (#189). Previously, reconciliation
-  could replace existing tags with an empty value.
-- Preserve the tag revision on SSE-KMS metadata replication (#193), and advance
-  tag revisions monotonically on local PUT/DELETE tagging (#196). Empty tags
-  participate in reconciliation as an ordered deletion, preventing older
-  events from restoring removed tags. SSE-C key rotation also retains the tag
-  revision. Malformed historical revisions can fail and retry; their missing
-  history is not reconstructed by the upgrade.
-- Complete delete-marker version purges and preserve their identity and retry
-  state through MRF recovery (#196). Recovery accepts a 405 marker response only
-  when its version, bucket, object name and modification time match the task.
-  Purge audit status is normalized from `COMPLETE` to `COMPLETED`.
-  Thanks to Julien Laurenceau (@julienlau) for the investigation and proposed
-  fix in #184 that helped shape this follow-up.
-- Restore only the six replication-specific metadata fields after ordinary
-  request metadata extraction (#194). This prevents transport-only `aws-chunked`
-  from being stored as Content-Encoding while preserving the signed-header
-  protections. Trusted Snowball entries no longer inherit the outer archive's
-  ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
-  **Existing data:** these repairs prevent new errors; they do not scan or rewrite
-  historical object metadata, recover lost tags or prove that old purge work has
-  converged. Follow the [read-only audit procedure](https://silo.pgsty.com/operations/replication/replica-metadata-audit/)
-  before planning any repair of stored state.
+Lifecycle, multi-pool and tiering:
 
-- Make exact-version delete-marker purges converge in replicated buckets
-  (`eb4f5e5b3`, `254b19ac0`, `358ab38fb`). A purge no longer creates the
-  marker on drives that lacked it; a retried purge of a missing version is
-  acknowledged only when a write-quorum majority of drives report it absent;
-  purge results merge removed and reliably absent replies; healing a marker
-  preserves its stored replication and purge metadata; and a queued marker
-  creation is re-checked against the source under the replication lock before
-  it is sent, so a purge that already reached the targets is not undone by a
-  stale task from another frontend, a GET/LIST heal, the scanner or MRF.
-  Purging a data version whose earlier purge is still pending reports it as a
-  data version. **Known limitations:** creations already in flight or replayed
-  from another site, and minority marker copies left by a crash after a
-  majority-acknowledged purge, are tracked in #217. See
-  [the replication reliability record](https://silo.pgsty.com/blog/design/replication-reliability/).
-- Keep a null object version that has listing quorum when a newer minority of
-  drives sorts first (`8d06424b1`). The resolver recounts per header only when
-  the original selection lacks quorum, every non-empty drive stream holds
-  exactly one ordinary null version and all share the same erasure layout;
-  mixed histories keep their previous behavior. **Known limitation:** a
-  successful ListObjects can still omit readable keys during rolling restarts
-  with concurrent overwrites (#218). Do not run destination-deleting sync tools
-  against a listing taken during a rolling restart; list again once the
-  cluster is stable.
-- Carry object tags through rebalance and decommission for ordinary and
-  multipart writes (`fced86303`). Both migration entry points restore the tags
-  and their revision fields when rewriting the object in the destination pool.
-  Tags dropped by earlier migrations are not recovered; audit tag-dependent
-  lifecycle and policy rules for pools migrated with an older build.
+- Decide an ILM expiry under the object lock against the versions of the key
+  in all pools, and refuse one that does not hold for the whole object or whose
+  history or lock protection lacks an authoritative quorum. Decommission and
+  rebalance move such versions instead of dropping them, and an approved
+  version expiry deletes every copy ([#278](https://github.com/pgsty/silo/issues/278)).
+- Expire the restored copy of a current version instead of the object
+  ([#281](https://github.com/pgsty/silo/issues/281)). Keep lifecycle delete-all
+  (`ExpiredObjectAllVersions`, `DelMarkerExpiration`, batch expiration with
+  `retainVersions: 0`) on the expired key instead of the keys below it; a key
+  with a tiered or Object Lock protected version is refused
+  ([#297](https://github.com/pgsty/silo/issues/297)).
+- Stop taking the free version of a deleted tiered version for a version
+  ([#263](https://github.com/pgsty/silo/issues/263)), apply rules whose only
+  action is `DelMarkerExpiration` ([#268](https://github.com/pgsty/silo/issues/268)),
+  and report and account only the removals that happened
+  ([#305](https://github.com/pgsty/silo/issues/305)).
+- Keep remote tier data that another pool still references after a version
+  expiry or tier cleanup ([#306](https://github.com/pgsty/silo/issues/306),
+  [#308](https://github.com/pgsty/silo/issues/308)). Heal versions whose queued
+  expiry is refused ([#303](https://github.com/pgsty/silo/issues/303)), and read
+  a key's history once in migration prechecks ([#304](https://github.com/pgsty/silo/issues/304)).
+- Decommission and rebalance: move objects whose copies have two erasure
+  layouts ([#261](https://github.com/pgsty/silo/issues/261)) and ILM-expired
+  current versions ([#265](https://github.com/pgsty/silo/issues/265)), continue
+  with later buckets when one cannot finish and keep cancellation
+  ([#307](https://github.com/pgsty/silo/issues/307)), and do not panic when a
+  rebalance stop finds no metadata ([#270](https://github.com/pgsty/silo/issues/270)).
+- Do not replicate the delete markers and version purges that lifecycle
+  creates, as in S3. [PR #240](https://github.com/pgsty/silo/pull/240) remains
+  open as a proposal.
 
-- Evaluate conditional multipart completion against the logical current object
-  across all pools while holding the existing object lock. A stale `If-Match`
-  can no longer replace newer data in another pool, and the current ETag is no
-  longer rejected because the upload resides next to an older copy. Conditions
-  are evaluated once; a current delete marker counts as an absent object.
-  **Availability change:** if any pool's metadata cannot be read, conditional
-  completion fails even when another pool can still serve GET/HEAD. This also
-  applies when the unreadable pool may not hold the object: absence cannot be
-  verified. Retry after the pool recovers. Unconditional completion and the
-  single-pool path retain their existing behavior.
+Native restore:
 
-- Evaluate ordinary multi-pool conditional PUT against the logical current
-  object across all pools, including draining pools, under the existing object
-  lock (#207). A stale destination copy no longer accepts a stale ETag or rejects
-  the current one; a current delete marker is treated as absence.
-  **Availability change:** if any pool's object metadata cannot be verified,
-  the condition fails even when GET can use another pool; read-quorum failures
-  return 503. Restore readability or heal before retrying. Unconditional PUT,
-  single-pool conditions and internal replication retain their existing behavior.
-  A public condition with a destination `versionId` compares the current object
-  while preserving the requested write version. This change does not retire
-  stale copies in other pools, undo historical accepted overwrites or provide
-  a new global clock-ordering guarantee. The multipart-completion repair in #190
-  neither introduced nor repaired this separate PUT defect.
+- Restore in two phases, without holding the object write lock during the tier
+  download ([#284](https://github.com/pgsty/silo/issues/284)). Restores with more
+  than one pool no longer hang, stay in progress or change the restored version
+  ([#289](https://github.com/pgsty/silo/issues/289)), and a failed ordinary
+  multipart completion stays usable ([#327](https://github.com/pgsty/silo/issues/327)).
+  A failed or canceled restore returns at once and clears its marker in the
+  background, instead of waiting up to a minute for the object lock.
 
-- Reconcile ordinary single-object version DELETE across all pools, including
-  null versions, delete markers and unqualified directory-marker DELETE. This
-  applies the deletion to every resolved pool copy under existing quorum
-  rules. Pending outbound delete replication retains versions until the
-  existing replication worker completes their purge; a successful response
-  does not imply immediate physical removal from every drive. Unreadable
-  pools now consistently return 503 instead of depending on pool traversal
-  order; insufficient read quorum returns `SlowDownRead`. This extends the
-  existing failure surface. Retry after recovery.
-  Cleanup failures also return an error. Batch deletion already fans out across
-  pools; replication and scanner cleanup keep their existing contracts. See
-  [scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
+Locking, conditional writes and storage:
 
-- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
-  its tracker, mover, scanner hooks, configuration, XML actions and metrics.
-  Accept and ignore retired configuration/XML and preserve ordinary statistics
-  when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
-  The [decision record](https://silo.pgsty.com/compatibility/access-tiering-removal/) preserves
-  the feature's introduction, subsequent fixes, rollback scope and review history.
-- Preserve the independent multi-pool write, metadata, healing and conditional
-  deletion fixes from PR #178, including shared remote-tier reference protection.
-- Enforce `If-Match` on DELETE, preserve retention and independently ordered
-  Object Lock/tag updates, and correctly retransmit encrypted replicas.
-- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
-  compression, honor key-rotation checksums, and complete attributes pagination.
-- Repair federated CopyObject checksums, destination timestamps, reserved
-  metadata, encrypted-object forwarding, legal hold and KMS context.
-- Make resync counters, target selection, cancellation and worker lifetimes
-  reflect actual work, and report bounded MRF drops.
-- Converge bucket metadata with deterministic source state, deletion tombstones,
-  creation time recovery and diagnostics. The mixed-version export gate requires
-  coordinated upgrades before tombstones are exported. See [the #77 record](https://silo.pgsty.com/blog/design/bucket-metadata-convergence/).
-- Include per-bucket CORS in metadata export/import, close metadata publication
-  and logger races, and report effective bucket quotas in metrics.
+- Lock every write and delete of a single pool in the object's own erasure set
+  ([#298](https://github.com/pgsty/silo/issues/298)). Fix CopyObject waiting on
+  its own lock for normalized aliases ([#299](https://github.com/pgsty/silo/issues/299))
+  and starving its destination while waiting for the source
+  ([#300](https://github.com/pgsty/silo/issues/300)); delete the set groups of a
+  bulk delete concurrently ([#301](https://github.com/pgsty/silo/issues/301)).
+- Evaluate `If-Match` and `If-None-Match` of a multi-pool CreateMultipartUpload
+  against the current object ([#291](https://github.com/pgsty/silo/issues/291),
+  [PR #292](https://github.com/pgsty/silo/pull/292)).
+- Answer a retryable `503` instead of `NoSuchKey` when a node sees only some
+  drives of an existing object ([#302](https://github.com/pgsty/silo/issues/302)), remove the empty object
+  directory of a failed write ([#285](https://github.com/pgsty/silo/issues/285),
+  [PR #286](https://github.com/pgsty/silo/pull/286)), classify `xl.meta`
+  checksum failures for healing ([#309](https://github.com/pgsty/silo/issues/309)),
+  and keep a heal dry run from deleting data
+  ([PR #280](https://github.com/pgsty/silo/pull/280), thanks to
+  [@dihmandrake](https://github.com/dihmandrake)).
 
-### Console, dependencies and delivery
+Object Lock, IAM and batch jobs:
 
-- Restore embedded Console login over loopback TLS, trusted-proxy handling and
-  all four WebSocket connection limits. Preserve Go TLS defaults across transports.
-- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.1; select released Console
-  v2.4.1 (`v0.0.0-20260916075814-1360e26d976d`) and mcli 20260916
-  (`v0.0.0-20260916070421-e952aa78f10a`) with explicit PGSTY replacements.
-  The embedded frontend identifies itself as Console v2.4.1.
-- Pin upstream minio-go `v7.3.1-0.20260915093545-32e1f32cb176` to handle
-  CopyObject errors embedded in HTTP 200 responses. Update JWX to v3.3.0 for
-  JSON field-name escaping, strfmt to v0.27.2 for Go 1.27 hostname validation,
-  and LZ4 to v4.1.30 for frame-reader, partial-read and concurrency fixes.
-  Retain the earlier Go x/* and bounded AMQP frame updates, Go 1.27.1, and
-  go-systemd v22.6.0's NetBSD compatibility replacement.
-- Refresh container base digests and build static curl 8.22.0 from verified
-  source for both Linux architectures. Pin the published mcli 20260916 archives
-  and hashes in the container and update the client installer default.
-- Prepare Helm chart 7.0.3 with Server and client defaults for the September 16
-  batch. Publish the chart only after the corresponding Server image exists.
-- Pin GoReleaser v2.18.1 and its action commit identically in snapshot and release
-  workflows. Dependency-only PRs now run the Test Release Pipeline too.
+- Keep the target's legal hold and retention on a replicated delete; a source
+  purge that bypassed GOVERNANCE retention is refused by a target copy that
+  still carries it, and the source retries. Check a directory object's lock
+  before a bulk delete removes its null version. S3 `DeleteObjects` checks
+  physical version removals under the mutation lock and merges protection
+  across pools, including concurrent legal-hold and retention updates. These
+  removals in Object Lock buckets use the single-object path sequentially;
+  large version-purge batches can take longer. Marker creation and ordinary
+  buckets retain the bulk path.
+- Batch jobs enforce the submitter's data permissions throughout execution and
+  leave Object Lock protected versions alone ([#310](https://github.com/pgsty/silo/issues/310),
+  [#311](https://github.com/pgsty/silo/issues/311)). Key rotation keeps other
+  pool copies readable ([#312](https://github.com/pgsty/silo/issues/312)).
+  Transient failures pause work instead of ending it, saved jobs are found
+  after a failed startup scan, and running replication resumes
+  ([#313](https://github.com/pgsty/silo/issues/313), [#314](https://github.com/pgsty/silo/issues/314),
+  [#324](https://github.com/pgsty/silo/issues/324)). A completed expiration job
+  stays completed after a restart ([#315](https://github.com/pgsty/silo/issues/315)).
+  Authorization skips deleted policies a mapping still names, refuses a job
+  only for a Deny that needs request context, and survives metadata edits and
+  LDAP refreshes that keep the credential
+  ([#318](https://github.com/pgsty/silo/issues/318)–[#320](https://github.com/pgsty/silo/issues/320)).
+  Live metrics keep pause reasons and skipped or refused counts
+  ([#325](https://github.com/pgsty/silo/issues/325)), and `make check-gen`
+  leaves the runtime policy snapshot alone ([#326](https://github.com/pgsty/silo/issues/326)).
+- IAM refuses incomplete policy loads, keeps cached authentication available
+  during lazy loads, and applies edits, LDAP group refreshes and service-account
+  creation to the stored record, so a stale node cannot restore old credentials
+  or resurrect deleted identities ([#316](https://github.com/pgsty/silo/issues/316),
+  [#317](https://github.com/pgsty/silo/issues/317), [#321](https://github.com/pgsty/silo/issues/321)–[#323](https://github.com/pgsty/silo/issues/323)).
+- Log in LDAP users with long DNs on erasure sets, and root to the embedded
+  Console with LDAP enabled ([#271](https://github.com/pgsty/silo/issues/271),
+  [#272](https://github.com/pgsty/silo/issues/272)). Keep audit and logger
+  webhook targets running on every server after a configuration reload
+  ([#259](https://github.com/pgsty/silo/issues/259), reported by
+  [@haf](https://github.com/haf)).
 
-Validation of earlier source revisions does not establish acceptance of this
-candidate. Final source, package, image and multi-process checks are tracked
-separately in [#203](https://github.com/pgsty/silo/issues/203). No Server release
-or production rollout is implied by this preparation target.
+Components and packaging:
+
+- Embed the published Console v2.7.0 source `4e91ae506e70`, including its
+  ten-language UI, rebuilt frontend assets, Swagger array-validation fixes,
+  Redoc routing, plain-HTTP folder-size and key-copy actions, usage-worker TLS
+  roots and scan budgets, and malformed-multipart rejection. Select the
+  published mcli 20261109 source `e1d4cc5abfe0` and silo-pkg v3.16.0 directly
+  in the maintained dependency graph.
+- Adopt go-jose/v4 v4.1.9, cpuid/v2 v2.4.2, YAML v3.0.6 and gjson v1.20.1,
+  plus Console's runtime/server-middleware v0.33.4. Preserve Go 1.27.3, the
+  verified upstream minio-go commit, networking/crypto security floors and
+  terminal-width/NetBSD compatibility selections.
+- Update NATS credentials to jwt/v2 v2.8.3 and nkeys v0.4.17, including
+  rejection of identity-point and small-order public keys. Use nats-server
+  v2.15.2 for notification integration tests.
+- Update Google Cloud auth to v0.24.3 and oauth2adapt to v0.3.1 for REST
+  request validation and AWS FIPS endpoint signing fixes.
+- Bundle verified mcli 20261109 archives in source container builds and use
+  that release by default in the integration-test client installer.
+- Prevent replication worker metrics from panicking when the replication
+  pool is absent, while retaining historical averages and peak counts.
+- Give the distroless image the same CA roots as the standard image, and ship
+  mcli's license files in both ([#295](https://github.com/pgsty/silo/pull/295)).
+- Deploy SILO and mcli `RELEASE.2026-11-09T00-00-00Z` by default from Helm
+  chart 7.0.4; the server image tag was left at 20260916 before
+  ([#296](https://github.com/pgsty/silo/issues/296)).
+
+Upgrade notes:
+
+- A single pool whose erasure sets do not all use the same hosts (with the
+  automatic layout, 18 or more hosts) needs a drained restart, not a rolling
+  upgrade: old and new nodes lock deletes and copies in different sets and do
+  not exclude each other. Stop every old process before a new one serves.
+- During a rolling upgrade, upgraded nodes answer `POST ?restore` with `503`
+  while any peer is old or unreachable. Pause new restores until every node is
+  upgraded. Multi-pool lifecycle protection is complete only after a full
+  upgrade.
+- Stop batch submissions and drain or cancel unfinished jobs before changing
+  versions. Unfinished legacy jobs require resubmission after all nodes upgrade;
+  completed jobs stay completed. Submitter-bound manifests and progress reports
+  use a new format that 20261026 cannot execute or resume. Do not downgrade
+  with pending or paused new jobs. An old node can still execute old-format
+  jobs, so mixed-version execution is not protected by the new authorization
+  rules. Preserve job definitions and reconcile completed work before resubmitting.
+- `healthcheck ready` remains a process/dependency probe and can succeed before
+  storage or IAM load. `cluster` reports quorum, which can precede a restarted
+  node reconnecting its remote drives. Before resuming applications, run bounded
+  signed write/read checks through every node; neither HTTP probe alone proves
+  data-path readiness. Preserve peer discovery during bootstrap.
+- `helm upgrade --reuse-values` keeps the previously stored `image.tag`; set it
+  explicitly when upgrading such a release.
+
+Known issues: `ListMultipartUploads` prefix and paging
+([#79](https://github.com/pgsty/silo/issues/79)), multi-site delete-marker purge
+([#217](https://github.com/pgsty/silo/issues/217)), replayed bucket creation
+under site replication ([PR #241](https://github.com/pgsty/silo/pull/241)), and
+`403` instead of the server error for reads allowed only by object tags while
+the object is unreadable ([#293](https://github.com/pgsty/silo/issues/293)).
+
+## RELEASE.2026-10-26T00-00-00Z
+
+Package version: `20261026000000.0.0`; published from source `c3ad836aed0d`.
+Artifacts are listed in the [GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z).
+The coordinated [English release notes](https://silo.pgsty.com/blog/release/silo-20261026/)
+and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261026/)
+cover all components, upgrade guidance, and validation scope.
+
+- Fix LDAP login for long normalized DNs without hiding unreadable IAM
+  revocation records. Only length errors for valid keys with an ASCII path
+  component over 255 bytes are treated as absent; long-DN writes still fail.
+  Thanks to [@910patin](https://github.com/910patin) for the original fix and
+  reproduction in [#250](https://github.com/pgsty/silo/pull/250).
+- Build the server and standalone debugging tools with Go 1.27.3 and refresh
+  Go dependencies, including silo-pkg v3.15.2, mcli 20261026
+  and upstream minio-go `320af050c445` for multipart, read/seek,
+  batch-delete and bucket-replication removal fixes. Retain output-format and NetBSD compatibility selections.
+- Embed Console v2.5.1 source `fff1618d9abb`, including its GUI update,
+  diagnostic WebSocket closing handshake, complete health-report decoding,
+  group-membership preservation, OIDC group permissions, TLS-proxy session
+  cookies, credential exports and accessibility fixes.
+- Bundle mcli 20261026 in container builds, the verified client installer and
+  Helm client jobs. Include client identity isolation, resync status and rate
+  fixes, and bounded batch-status fallback. Scripts must handle the additive
+  batch `pending` state and nonzero exits for partial resync mutations.
+- Read current bucket inventory in account-info responses so Console refreshes
+  immediately reflect successful creation and deletion. This management endpoint
+  now performs a live list instead of returning a cached inventory.
+- Restore the native built-in policy immediately after resetting an override.
+  Peer and etcd notifications reload committed state, so delayed delete events
+  preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
+  custom-policy deletion behavior; storage failures retain the previous policy.
+- Reload policies without holding the authentication-cache lock, retain IAM
+  write ordering and publish replacements only after a complete successful load.
+  Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
+  the caller's policy-query deadline.
+- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
+  choose an available pool for a new delete marker, and keep tier data referenced
+  by surviving copies. This does not introduce strict cross-pool read recovery.
+- Preserve remote tier data when lifecycle expiry only adds a delete marker,
+  including non-null versions in suspended buckets. Forward `FULL_OBJECT`
+  checksum type for multipart replicas. Thanks to
+  [@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
+  and multi-pool deletion contributions.
+- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
+  fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
+- Forward storage RPC delete options, including the previous data directory
+  needed by overwrite rollback. Preserve existing drive/RPC timeouts, request
+  cancellation, object locking, read-quorum and healing behavior.
+- Update bundled curl to 8.23.0 and require patched static libraries. Apply
+  [Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
+  the root blocklist also rejects legitimate chains under the affected roots.
+- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
+
+Storage timeouts retain their existing meaning: a returned timeout does not
+prove that the underlying disk operation stopped or that a write did not
+commit. Late commits or delayed rollback can affect newer writes; stale reads
+remain possible after partial failures, lease loss or network partitions. This
+release does not provide distributed fencing or repair previously inconsistent
+objects.
+
+These source changes are not included in the published 20261018 artifacts.
+
+## RELEASE.2026-10-18T00-00-00Z
+
+Package version: `20261018000000.0.0`; published on 2026-10-18 from source
+`0cd388c666f8211f552604ae73448a066e9c25ce`. This section covers the changes
+after the public 20260916 source, `2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
+
+Full [English release notes](https://silo.pgsty.com/blog/release/silo-20261018/)
+and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261018/)
+cover the upgrade contract, evidence and remaining limits.
+
+- **Object Lock (#234, #235):** require `s3:PutObjectRetention` with request-body
+  conditions before object lookup; weakening active GOVERNANCE additionally
+  requires bypass intent and permission. Protect caller-addressed write versions,
+  directory null versions and persisted multipart targets under the write lock;
+  fail closed on unreadable lock state. See SN-2026-015/016 in the
+  [advisory ledger](https://silo.pgsty.com/about/security-advisories/).
+- **IAM Admin authorization (#242, `995439260`):** enforce explicit self denies,
+  session restrictions and plugin decisions; prevent restricted delegation and
+  non-root mutation of root-derived credentials under built-in authorization.
+  Preserve empty-policy inheritance and damaged-key cleanup. Normalize bulk
+  self-list aliases and allow ordinary policyless self inspection. See
+  [SN-2026-017](https://silo.pgsty.com/blog/security/20261018-release-hardening/#sn-2026-017).
+- **Scoped STS revocation (`0cd388c66`):** enumerate durable parent/type-selected
+  tokens, propagate each committed tombstone to sibling nodes and sites, and
+  recover missed notifications through existing refresh/healing. Tokens issued
+  only at an isolated site are outside the requesting cluster's snapshot;
+  propagation is eventual and newly issued keys remain valid.
+- **Pool migration (#248, `60c7179dd`, `1748f62c7`):** prevent stale moved
+  payloads from overwriting newer copies, preserve null/UUID delete markers,
+  recheck source versions and clean only confirmed moved versions under the
+  object lock. Added per-pool reads and per-version cleanup have maintenance
+  costs; earlier data loss is not reconstructed.
+
+- **LIST recovery (#239, `3dd53eb81`, `2080b3d03`):** confirm eligible uncertain
+  latest-object groups under the appropriate read locks, distinguish absence
+  from failure, preserve directory/common-prefix entries and terminal pagination
+  errors, and make lock backoff cancelable. Resolved entries keep their fast path;
+  version history, quorum and disk formats retain their contracts. The strict
+  sub-5% performance target remains unconfirmed in the recorded comparison.
+- **Multipart listing (`4733e1423`):** withdraw #243's default persistent scan.
+  Public listing uses node-local cache or exact-key lookup; restart omissions,
+  prefix and cache-pagination limits remain. The retired strict switch is ignored
+  with a startup warning. **Former strict users lose durable listing and strict
+  cancellation.** Keep identity metadata, path/wrong-target safety, default
+  best-effort Abort and peer notification. Diagnostic `multipart-preflight`
+  reports `mode=legacy`; its budgets do not gate public operations. #79 remains
+  open, including delayed creation after cancellation.
+- **Multipart cleanup (#236, `0f9199e0c`):** parse native timestamps before the
+  Base64 fallback and remove empty hash parents without a second expiry wait.
+  Keep active/unreadable uploads; default expiry/check intervals stay 24h/6h.
+- **Commit errors (#232):** missing staged `xl.meta` at final commit becomes
+  HTTP 503 `SlowDownWrite`; preserve this classification across storage RPC and
+  healing. Real incomplete bodies retain their existing errors.
+- **Reused metadata (#233):** clear absent legacy `EcM`/`Signature` and locally
+  reset mutually exclusive scanner response fields without changing disk/RPC formats.
+- **Checksum and Console errors (#225, #229, #238):** align Content-MD5 and
+  algorithm-specific messages, retaining the generic unknown-algorithm message;
+  embed Console's wrapped-error and `InvalidBucketState` handling.
+- **Lambda and erasure documentation (#220, `2fde3cf53`):** fix inverted
+  `TargetIDSet.IsEmpty`; document parity/quorum per object and erasure set.
+- **Coordinated dependencies:** Console v2.4.2 (`ef2dd2c41d21`), mcli 20261018
+  (`fa3b8bf72458`), silo-pkg v3.14.3, upstream minio-go v7.4.1 and Go 1.27.2.
+  Standalone Console/mcli still declare pkg v3.14.2. The Server package update
+  speeds admin-action classification without changing policy behavior. Refresh
+  Go x/*, go-jose v4.1.6, LDAP v3.4.15, compress v1.20.3, msgp v1.6.5, etcd v3.7.2,
+  fwd v1.2.1 and cpuid v2.4.1. OIDC RSA JWKs above 4096 bits are rejected.
+  Keep the NetBSD go-systemd v22.6.0 replacement; parallel SDK GET/Compose remain opt-in.
+- **Build and delivery:** gate releases on Object Lock regressions; use
+  Git for rebrand scanning without a ripgrep dependency (`7007c164a`). Refresh UBI 9 pins,
+  GoReleaser v2.18.3, nFPM v2.48.0, golangci-lint v2.15.0 and govulncheck v1.9.0.
+  Update bundled client hashes/helper and the Helm client image to mcli 20261018.
+  The chart's Server default stays at the public 20260916 image until publication.
+
+Upgrade every writer and replication site; follow the
+[IAM recovery guide](https://silo.pgsty.com/operations/replication/iam-upgrade/).
+Old strict multipart users must review the capability loss before upgrading.
+#217's multi-site delete-marker late creation and crash residue remain unresolved.
+Source tests, draft artifacts, public images and production rollout are separate
+acceptance boundaries; historical 20261015 test counts do not certify this source.
+
+## RELEASE.2026-09-16T00-00-00Z
+
+Package version: `20260916000000.0.0`; published source:
+`2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
+
+This release includes signed-header and payload-policy hardening (SN-2026-011/012),
+durable IAM revocations (SN-2026-013), the bounded Console sharing proxy
+(SN-2026-014), password-permission separation, multi-pool conditions/deletion/tag
+repairs, encrypted/federated copy repairs, delete-marker retry fixes, opt-in strict
+multipart discovery, Go-default TLS key exchange, absolute HTTP/1 header deadlines
+and synchronized CPU metrics. It embeds Console v2.4.1, mcli 20260916 and pkg v3.14.1.
+
+The complete release, contributor credits, historical artifact acceptance and
+upgrade requirements are preserved in the canonical
+[English](https://silo.pgsty.com/blog/release/silo-20260916/) and
+[Chinese](https://silo.pgsty.com/zh/blog/release/silo-20260916/) release records.
 
 ## RELEASE.2026-09-03T13-18-01Z
 

CONTRIBUTING.md

@@ -53,6 +53,20 @@ Before opening a pull request:
 - Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
   `.minio.sys`, ARN, module/import-path, or serialized compatibility name.
 
+The maintenance branch retains existing storage timeouts, cancellation,
+object locking, read-quorum and healing behavior. The normal suite checks local
+and RPC timeout/cancellation, remote rollback options, and readability after a
+failed partial overwrite. A timeout does not prove that a disk worker stopped
+or that a write did not commit.
+
+Stronger mutation lifetime, rollback ownership, generation ordering, strict-read
+and cross-pool recovery diagnostics are preserved behind `consistency_strict`.
+Compile them with `go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'`
+and run individual scenarios explicitly. They describe guarantees outside the
+maintenance contract and may fail; report their results separately from the
+normal suite. They must not be used to claim distributed fencing or repair of
+previously inconsistent objects.
+
 ### Commit changes
 
 After verification, commit your changes with a concise message and a DCO

Dockerfile.distroless

@@ -8,10 +8,30 @@
 # seeds fresh volumes from the image-layer mountpoint, no entrypoint
 # exists to repair ownership at runtime, and 0777 is what keeps every
 # privilege mode working, --user included.
-FROM busybox:1.37.0 AS prep
+FROM busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec AS prep
 RUN mkdir -p /prep/data && chmod 0777 /prep/data
 
-FROM gcr.io/distroless/static-debian12:latest
+# Debian's bundle in distroless still trusts the roots the classic image
+# blocklists. Build the same bundle as Dockerfile.goreleaser so both variants
+# trust the same roots; it is architecture-neutral, so build it natively.
+FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
+# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
+RUN dnf -y install ca-certificates && \
+    if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
+      blocklist=/etc/pki/ca-trust/source/blocklist; \
+    else \
+      blocklist=/etc/pki/ca-trust/source/blacklist; \
+    fi && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
+      "$blocklist/Comodo_AAA_Services_root.pem" && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
+      "$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
+    update-ca-trust && \
+    cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt
+
+FROM gcr.io/distroless/static-debian12:latest@sha256:d75cdd72874d4790092fcb1b058493ecf6bb5bf2b2b897045b00ff01d91843f2
 
 LABEL org.opencontainers.image.title="Silo" \
       org.opencontainers.image.description="S3-Interface Libre Object Storage (distroless)" \
@@ -28,6 +48,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     HOME=/tmp
 
+# The only file under /etc/ssl/certs, so replacing it is sufficient.
+COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --chmod=0755 silo /usr/bin/silo
 # COPY of a directory copies its contents, not the directory entry, so an
 # empty /prep/data would arrive as a default root:0755 /data and non-root

Dockerfile.goreleaser

@@ -1,4 +1,4 @@
-FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS curl-build
+FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build
 ARG TARGETARCH
 COPY dockerscripts/build-static-curl.sh /build/build-static-curl
 RUN /bin/sh /build/build-static-curl
@@ -9,7 +9,7 @@ COPY --from=curl-build /go/bin/curl /curl
 COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/curl"]
 
-FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
+FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build
 
 ARG TARGETARCH
 
@@ -17,9 +17,9 @@ ENV GOPATH=/go
 ENV CGO_ENABLED=0
 
 ARG MC_REPO=pgsty/mc
-ARG MC_VERSION=RELEASE.2026-09-16T00-00-00Z
-ARG MC_AMD64_SHA256=4ba2814fd5507fbe6b4d237c359750b9119d28d7217495fa5b48002fcbd397ef
-ARG MC_ARM64_SHA256=b7008ca2a1bc5735b6585981c59a3640a0daa152dc789df3d1a0d0438787de82
+ARG MC_VERSION=RELEASE.2026-11-09T00-00-00Z
+ARG MC_AMD64_SHA256=79e6cf4ad00a9e5eb2e7ceccc6e6496e4af4ec096e59695515ea8f223c287e4a
+ARG MC_ARM64_SHA256=5473c4b39a31dd66478f5c44686a09f3e9e37861a30f1faf18564fbe66118387
 
 RUN apk add -U --no-cache \
     ca-certificates \
@@ -65,16 +65,30 @@ RUN apk add -U --no-cache \
       echo "No mc or mcli binary found in archive:"; ls -la /tmp/mcli-extract/; exit 1; \
     fi && \
     chmod +x /go/bin/mcli && \
-    ln -sf mcli /go/bin/mc
+    ln -sf mcli /go/bin/mc && \
+    mkdir -p /go/share/mcli && \
+    cp /tmp/mcli-extract/LICENSE /tmp/mcli-extract/NOTICE /tmp/mcli-extract/CREDITS /go/share/mcli/
 
-FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
+FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
+# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
 RUN dnf -y install ca-certificates && \
+    if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
+      blocklist=/etc/pki/ca-trust/source/blocklist; \
+    else \
+      blocklist=/etc/pki/ca-trust/source/blacklist; \
+    fi && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
+      "$blocklist/Comodo_AAA_Services_root.pem" && \
+    trust extract --overwrite --format=pem-bundle \
+      --filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
+      "$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
     update-ca-trust && \
     cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
-FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
 
 LABEL org.opencontainers.image.title="Silo" \
       org.opencontainers.image.description="S3-Interface Libre Object Storage" \
@@ -95,6 +109,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY silo /usr/bin/silo
 COPY --from=build /go/bin/mcli /usr/bin/mcli
+# mcli's own legal files, from the same verified archive as the binary.
+COPY --from=build /go/share/mcli /licenses/mcli
 COPY --from=curl-build /go/bin/curl /usr/bin/curl
 COPY --from=curl-build /go/share/curl /licenses/curl
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

Makefile

@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
 GOOS ?= $(shell go env GOOS)
 GOARCH ?= $(shell go env GOARCH)
-GOLANGCI_VERSION ?= v2.14.0
+GOLANGCI_VERSION ?= v2.15.0
 
 VERSION ?= $(shell git describe --tags)
 REPO ?= docker.io/pgsty
@@ -73,6 +73,11 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
+# -tags kqueue,dev: kqueue selects the kqueue backend of rjeczalik/notify on
+# BSD and macOS (no-op on Linux); dev came with upstream's 2024-06 change for
+# development builds and no file in this repository or its dependencies
+# carries a dev build constraint, so it selects nothing. Kept for the
+# documented command lines; .github/workflows/go.yml carries the same note.
 test: verifiers build ## builds Silo, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue,dev ./...

README.md

@@ -24,7 +24,7 @@
 <p align="center">
   <a href="https://silo.pgsty.com/"><img alt="Website" src="https://img.shields.io/badge/Website-silo.pgsty.com-1d588c"></a>
   <a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
-  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
+  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
   <a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
   <a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
 </p>
@@ -32,29 +32,26 @@
 > [!IMPORTANT]
 > **PGSTY Silo** (hereinafter “Silo”) is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/silo`](https://github.com/pgsty/silo). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
 
-> [!NOTE]
-> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
-
-## Current release and main branch
-
-The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
-As of 2026-09-16, the main branch has newer security, storage, Console and
-shared-package changes that have not shipped in a Server release. See
-[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
-for the exact release/source boundary, including SN-2026-011 and password-policy migration.
-
 ## Overview
 
 PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
 
 It follows one rule — **the product and its delivery surfaces are renamed; the protocol and your data are not.** Everything else lives on [silo.pgsty.com](https://silo.pgsty.com/).
 
-**Related:** [`pgsty/mc`](https://github.com/pgsty/mc) client (shipped as `mcli`) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
+**Related:** [`pgsty/mc`](https://github.com/pgsty/mc) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/kes`](https://github.com/pgsty/kes) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
 
 <p align="center">
   <img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo Console">
 </p>
 
+#### SILO 20261109
+
+> [!WARNING]
+> [SILO 20261109](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z) fixes bugs and security vulnerabilities. We strongly recommend scheduling an upgrade to reduce security risks.
+
+> [!NOTE]
+> On 2026-08-06, `pgsty/minio` became `pgsty/silo`, and the default branch changed from `master` to `main`. The last MinIO-branded release is [`20260804`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) on the [`minio`](https://github.com/pgsty/silo/tree/minio) branch.
+
 ## Quick Start
 
 ```bash
@@ -62,7 +59,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
   -e MINIO_ROOT_USER=minioadmin \
   -e MINIO_ROOT_PASSWORD=change-me-long-password \
   -v "$PWD/data:/data" \
-  docker.io/pgsty/silo:latest server /data --console-address ":9001"
+  docker.io/pgsty/silo:RELEASE.2026-11-09T00-00-00Z server /data --console-address ":9001"
 ```
 
 <p align="center">
@@ -101,8 +98,8 @@ Every divergence from upstream is listed in the code-verified [compatibility aud
 
 ### TLS and Go upgrades
 
-The following TLS repair is on main and is not included in Server 20260903.
-With that repair, TLS key exchange follows Go's defaults across the S3 listener, node links,
+Server 20260916 restored Go's default key exchange; Server 20261018 retains it with Go 1.27.2.
+TLS key exchange follows Go's defaults across the S3 listener, node links,
 replication, identity providers, etcd, and external HTTP services. If an endpoint
 cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
 for the process; certificate verification remains enabled. This option does not
@@ -137,7 +134,7 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e
 
 <!-- Generated by silo.pgsty.com/bin/contributors.py. -->
 
-Every human issue or pull-request author is part of the SILO community, including open and unmerged work. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
+Every human issue or pull-request author is part of the SILO community, including open and unmerged work. We also recognize security reports, including reports sent by email, with the reporter's permission. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
 
 <a href="CONTRIBUTORS.md">
 <picture>

README_ZH.md

@@ -24,7 +24,7 @@
 <p align="center">
   <a href="https://silo.pgsty.com/zh/"><img alt="官网" src="https://img.shields.io/badge/%E5%AE%98%E7%BD%91-silo.pgsty.com%2Fzh-1d588c"></a>
   <a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
-  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
+  <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
   <a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
   <a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
 </p>
@@ -32,28 +32,26 @@
 > [!IMPORTANT]
 > **PGSTY Silo**（以下简称 Silo）是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/silo`](https://github.com/pgsty/silo) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系；文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
 
-> [!NOTE]
-> 2026-08-06，本仓库由 `pgsty/minio` 更名为 `pgsty/silo`，默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支，以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
-
-## 当前发行版与主分支
-
-最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
-截至 2026-09-16，主分支已合入更新的安全、存储、Console 与共享包改动，但尚未发布新 Server。
-准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/)，
-其中包括 SN-2026-011 修复状态与密码权限迁移要求。
-
 ## 概述
 
 上游停止社区发行后，Silo 为开源 MinIO 服务端维护一条持续可用的版本线：构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
 
 它只遵循一条原则：**改名的是产品与交付物，不是协议与你的数据。** 其余内容都在 [silo.pgsty.com](https://silo.pgsty.com/zh/)。
 
-**相关项目：**[`pgsty/mc`](https://github.com/pgsty/mc) 客户端（以 `mcli` 发行） · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
+**相关项目：**[`pgsty/mc`](https://github.com/pgsty/mc) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/kes`](https://github.com/pgsty/kes) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
 
 <p align="center">
   <img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo 控制台">
 </p>
 
+#### SILO 20261109
+
+> [!WARNING]
+> [SILO 20261109](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-11-09T00-00-00Z) 修复了缺陷与安全漏洞，强烈建议尽快安排升级，以降低安全风险。
+
+> [!NOTE]
+> 2026-08-06，`pgsty/minio` 更名为 `pgsty/silo`，默认分支由 `master` 改为 `main`。最后一个 MinIO 品牌发布是 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支上的 [`20260804`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z)。
+
 ## 快速上手
 
 ```bash
@@ -61,7 +59,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
   -e MINIO_ROOT_USER=minioadmin \
   -e MINIO_ROOT_PASSWORD=change-me-long-password \
   -v "$PWD/data:/data" \
-  docker.io/pgsty/silo:latest server /data --console-address ":9001"
+  docker.io/pgsty/silo:RELEASE.2026-11-09T00-00-00Z server /data --console-address ":9001"
 ```
 
 <p align="center">
@@ -100,7 +98,7 @@ Silo 保留 S3 与存储格式兼容性，包括既有 `MINIO_*` 环境变量、
 
 ### TLS 与 Go 升级
 
-Server 恢复 Go 默认密钥交换策略的修复已在 main，尚未包含在 Server 20260903。
+Server 20260916 已恢复 Go 默认密钥交换策略，20261018 在 Go 1.27.2 下保留该行为。
 `GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化，
 以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
 
@@ -119,7 +117,7 @@ Server 恢复 Go 默认密钥交换策略的修复已在 main，尚未包含在
 
 <!-- Generated by silo.pgsty.com/bin/contributors.py. -->
 
-每位 issue 或 PR 的作者都是 SILO 社区的一员，包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示，其余参考首次参与时间；金色圆环突出经过审核的显著贡献。
+每位 issue 或 PR 的作者，以及同意公开署名的安全报告者，都是 SILO 社区的一员，包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示，其余参考首次参与时间；金色圆环突出经过审核的显著贡献。
 
 <a href="CONTRIBUTORS.md">
 <picture>

SECURITY.md

@@ -31,15 +31,24 @@ separately even when the fork preserves the original commit object and SHA.
 
 ## Reporting a Vulnerability
 
+The security contact for Silo is [ron@pgsty.com](mailto:ron@pgsty.com).
+This address also receives automated security reports for this repository.
+
 For vulnerabilities in this fork:
 
 1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
 2. Prefer this repository's [private GitHub security advisory](https://github.com/pgsty/silo/security/advisories/new) workflow.
-3. If private reporting is unavailable, contact the maintainers through the
-   repository without publishing exploit details until a private channel is
-   established.
+3. If you cannot use GitHub private reporting, email [ron@pgsty.com](mailto:ron@pgsty.com) or rh@vonng.com
+   directly. Do not publish vulnerability details or exploit code in public
+   issues or pull requests.
 4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
 
+## Scanner Configuration
+
+The [OSS Scanner build environment](.oss-scanner/Dockerfile) and
+[scan scope and offline checks](.oss-scanner/threat_model.md) are maintained
+with the source. Update them when dependencies or security boundaries change.
+
 ## Disclosure Process
 
 Fork-specific fixes and user-visible upgrade notes are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).

buildscripts/checkdeps.sh

@@ -7,7 +7,7 @@ _init() {
 
 	## Minimum required versions for build dependencies
 	GIT_VERSION="1.0"
-	GO_VERSION="1.27.2"
+	GO_VERSION="1.27.3"
 	OSX_VERSION="10.8"
 	KNAME=$(uname -s)
 	ARCH=$(uname -m)

buildscripts/install-mcli.sh

@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
 	exit 0
 fi
 
-release=${MCLI_RELEASE:-RELEASE.2026-09-16T00-00-00Z}
+release=${MCLI_RELEASE:-RELEASE.2026-11-09T00-00-00Z}
 version_hyphen=${release#RELEASE.}
 package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
 if [ "${package_version}" = "${version_hyphen}" ]; then

buildscripts/rebrand-guard/compat-baseline.json

@@ -152,6 +152,7 @@
     "MINIO_API_TRANSITION_WORKERS",
     "MINIO_API_TRUSTED_PROXIES",
     "MINIO_ARGS",
+    "MINIO_AUDIT_",
     "MINIO_AUDIT_KAFKA_BROKERS",
     "MINIO_AUDIT_KAFKA_CLIENT_TLS_CERT",
     "MINIO_AUDIT_KAFKA_CLIENT_TLS_KEY",
@@ -313,6 +314,7 @@
     "MINIO_LAMBDA_WEBHOOK_ENABLE",
     "MINIO_LAMBDA_WEBHOOK_ENDPOINT",
     "MINIO_LISTENERS",
+    "MINIO_LOGGER_",
     "MINIO_LOGGER_HTTP_ENDPOINT",
     "MINIO_LOGGER_WEBHOOK_AUTH_TOKEN",
     "MINIO_LOGGER_WEBHOOK_BATCH_SIZE",
@@ -594,11 +596,13 @@
     "x-minio-from-peer",
     "x-minio-healing-drives",
     "x-minio-internal",
+    "x-minio-internal-a",
     "x-minio-internal-actual-object-size",
     "x-minio-internal-actual-size",
     "x-minio-internal-archive-info",
     "x-minio-internal-archive-type",
     "x-minio-internal-b",
+    "x-minio-internal-c",
     "x-minio-internal-compression",
     "x-minio-internal-encrypted-multipart",
     "x-minio-internal-encryptedmultipart",
@@ -611,6 +615,7 @@
     "x-minio-internal-replication-reset-arn",
     "x-minio-internal-replication-status",
     "x-minio-internal-replication-timestamp",
+    "x-minio-internal-restore-attempt",
     "x-minio-internal-server-side-encryption",
     "x-minio-internal-server-side-encryption-context",
     "x-minio-internal-server-side-encryption-iv",

buildscripts/verify-helm-migration.sh

@@ -112,8 +112,17 @@ helm_run template my-release "${new_chart}" \
 
 go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
 
+# The templates deploy values.image.tag, not appVersion; a release must move both.
+chart_version="$(sed -n 's/^version: *//p' helm/silo/Chart.yaml)"
+app_version="$(sed -n 's/^appVersion: *//p' helm/silo/Chart.yaml)"
+image_tag="$(sed -n '/^image:/,/^[^ ]/s/^  tag: *//p' helm/silo/values.yaml)"
+if [ -z "${app_version}" ] || [ "${image_tag}" != "${app_version}" ]; then
+	echo "helm/silo: image.tag '${image_tag}' must equal appVersion '${app_version}'" >&2
+	exit 1
+fi
+
 helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
-test -s "${work_dir}/silo-7.0.3.tgz"
+test -s "${work_dir}/silo-${chart_version}.tgz"
 if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
 	echo "Helm packaging emitted a legacy MinIO chart name" >&2
 	exit 1

buildscripts/verify-rebrand.sh

@@ -11,6 +11,33 @@ fail() {
 	exit 1
 }
 
+# Git is already a build prerequisite. --no-index also scans untracked files
+# and exported source trees. As before, honor .gitignore only in a worktree.
+search_ignore=--no-exclude-standard
+if git rev-parse --show-toplevel >/dev/null 2>&1; then
+	search_ignore=--exclude-standard
+fi
+search_errors="$(mktemp)"
+trap 'rm -f "${search_errors}"' EXIT
+
+search_text() {
+	local status=0
+	git --no-pager grep --no-index "${search_ignore}" --no-color -nIHE "$@" 2>"${search_errors}" || status=$?
+	# Git can return 1 for both no matches and a file read error. Check stderr
+	# as well, before callers filter matches through their allowlists.
+	if [ "${status}" -gt 1 ] || [ -s "${search_errors}" ]; then
+		cat "${search_errors}" >&2
+		fail "could not scan source files"
+	fi
+}
+
+# Keep hidden files and build output out of the repository-wide scans.
+source_excludes=(
+	':(exclude,glob)**/.*'
+	':(exclude,glob)**/.*/**'
+	':(exclude)dist/**'
+)
+
 require_file() {
 	[ -f "$1" ] || fail "missing required file: $1"
 }
@@ -152,7 +179,11 @@ if grep -Ev '^[[:space:]]*(#|$)' silo.env | grep -q '='; then
 	fail "silo.env must not contain active assignments that shadow /etc/default/minio"
 fi
 
-if rg -n 'pgsty/minio:' .github/workflows Dockerfile.goreleaser helm/silo; then
+delivery_hits="$(search_text 'pgsty/minio:' -- .github/workflows Dockerfile.goreleaser helm/silo \
+	':(exclude,glob).github/workflows/**/.*' ':(exclude,glob).github/workflows/**/.*/**' \
+	':(exclude,glob)helm/silo/**/.*' ':(exclude,glob)helm/silo/**/.*/**')"
+if [ -n "${delivery_hits}" ]; then
+	printf '%s\n' "${delivery_hits}" >&2
 	fail "an active delivery surface still publishes the frozen pgsty/minio image"
 fi
 
@@ -165,10 +196,10 @@ fi
 # So two rules. First, no live URL may resolve to the old repository anywhere,
 # READMEs and CONTRIBUTORS.md included.
 old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
-stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \
-	--glob '!.git/**' --glob '!dist/**' \
-	--glob '!SILO_REBRANDING_MIGRATION.md' \
-	--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
+stale_repo_url="$(search_text -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" -- . \
+	"${source_excludes[@]}" ':(exclude)SILO_REBRANDING_MIGRATION.md' \
+	':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
+stale_repo_url="$(printf '%s\n' "${stale_repo_url}" |
 	sed 's#^\./##' | grep -v '^buildscripts/verify-rebrand\.sh:' || true)"
 if [ -n "${stale_repo_url}" ]; then
 	printf '%s\n' "${stale_repo_url}" >&2
@@ -180,32 +211,32 @@ fi
 # legacy image, the two READMEs that document the rename and the archived
 # minio branch, and historical issue titles in CONTRIBUTORS.md.
 repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
-stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \
-	--glob '!SILO_REBRANDING_MIGRATION.md' \
-	--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
+stale_repo="$(search_text "${old_repo_pattern}" -- . "${source_excludes[@]}" \
+	':(exclude)SILO_REBRANDING_MIGRATION.md' ':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
+stale_repo="$(printf '%s\n' "${stale_repo}" |
 	sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
 if [ -n "${stale_repo}" ]; then
 	printf '%s\n' "${stale_repo}" >&2
 	fail "a source reference still names the pre-rename pgsty/minio repository"
 fi
 
-stale_branch="$(rg -n 'pgsty/silo/(blob/|tree/|raw/)?master' \
-	--glob '!.git/**' --glob '!dist/**' . || true)"
+stale_branch="$(search_text 'pgsty/silo/(blob/|tree/|raw/)?master' -- . "${source_excludes[@]}")"
 if [ -n "${stale_branch}" ]; then
 	printf '%s\n' "${stale_branch}" >&2
 	fail "a link still targets the retired master branch; raw and Actions URLs do not follow a branch rename"
 fi
 
 for workflow in .github/workflows/go.yml .github/workflows/vulncheck.yml; do
-	if rg -q '^\s+- master$' "${workflow}"; then
+	stale_filter="$(search_text '^[[:space:]]+- master$' -- "${workflow}")"
+	if [ -n "${stale_filter}" ]; then
 		fail "${workflow} still filters on master and would go silently dormant on main"
 	fi
 	require_text "${workflow}" "      - main"
 done
 
-network_hits="$(rg -n --glob '*.go' --glob '!**/*_test.go' \
-	'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' \
-	cmd internal || true)"
+network_hits="$(search_text \
+	'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' -- \
+	':(glob)cmd/**/*.go' ':(glob)internal/**/*.go' ':(exclude,glob)**/*_test.go' "${source_excludes[@]}")"
 network_hits="$(printf '%s\n' "${network_hits}" | grep -Ev '^[^:]+:[0-9]+:[[:space:]]*//' || true)"
 if [ -n "${network_hits}" ]; then
 	printf '%s\n' "${network_hits}" >&2

cmd/abandoned-mutation_test.go

@@ -0,0 +1,184 @@
+//go:build consistency_strict
+
+// Copyright (c) 2026 Feng Ruohang
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type abandonedCleanupCommitDisk struct {
+	StorageAPI
+	entered chan<- struct{}
+	release <-chan struct{}
+}
+
+func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
+	var success int64
+	gate := &scopeStagedContext{
+		Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
+		entered: d.entered, release: d.release,
+		tracker: healthDiskCtxValue{lastSuccess: &success},
+	}
+	return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
+}
+
+// A lost namespace lease can admit cleanup while a disk transaction is between
+// moving its external shards and replacing xl.meta. Those shards are live work,
+// even though the old metadata does not refer to them yet. Cleanup must wait.
+func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
+	z, bucket := mutationPools(t)
+	const object = "cleanup-during-commit"
+	body := strings.Repeat("new payload", 200000)
+	mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
+	set := z.serverPools[0].getHashedSet(object)
+	disks, swap := installListingDiskSwitch(set)
+	defer swap(disks)
+	entered, release := make(chan struct{}, len(disks)), make(chan struct{})
+	unblock := sync.OnceFunc(func() { close(release) })
+	defer unblock()
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i, d := range disks {
+		wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
+	}
+	swap(wrapped)
+	committed := make(chan error, 1)
+	go func() {
+		_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
+		committed <- err
+	}()
+	for range disks {
+		select {
+		case <-entered:
+		case err := <-committed:
+			t.Fatalf("PUT returned before shard barrier: %v", err)
+		case <-time.After(5 * time.Second):
+			t.Fatal("PUT did not move all external shards")
+		}
+	}
+	t.Log("All eight new external shard directories exist; old xl.meta is still active")
+	cleaned := make(chan error, len(disks))
+	for _, d := range disks {
+		go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
+	}
+	early := 0
+	timer := time.NewTimer(150 * time.Millisecond)
+wait:
+	for {
+		select {
+		case err := <-cleaned:
+			early++
+			t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
+			if early == len(disks) {
+				break wait
+			}
+		case <-timer.C:
+			break wait
+		}
+	}
+	timer.Stop()
+	unblock()
+	select {
+	case err := <-committed:
+		if err != nil {
+			t.Fatalf("PUT failed: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("PUT did not finish")
+	}
+	for i := early; i < len(disks); i++ {
+		select {
+		case err := <-cleaned:
+			if err != nil {
+				t.Fatal(err)
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatal("cleanup did not finish")
+		}
+	}
+	swap(disks)
+	r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
+	}
+	defer r.Close()
+	data, err := io.ReadAll(r)
+	if err != nil || string(data) != body {
+		t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
+	}
+}
+
+// Cleanup can also arrive after a disk has committed but before its delayed
+// reply lets the coordinator decide to undo a failed quorum. The previous
+// generation's backup is still owned by that unfinished transaction.
+func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
+	z, bucket := mutationPools(t)
+	const object = "cleanup-before-undo"
+	body := strings.Repeat("last acknowledged", 150000)
+	base := time.Now().Add(-time.Minute)
+	mutationPut(t, z, bucket, object, 0, body, base)
+	set := z.serverPools[0].getHashedSet(object)
+	disks, swap := installListingDiskSwitch(set)
+	defer swap(disks)
+	entered, release := make(chan struct{}, 4), make(chan struct{})
+	unblock := sync.OnceFunc(func() { close(release) })
+	defer unblock()
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i, d := range disks {
+		wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
+	}
+	swap(wrapped)
+	committed := make(chan error, 1)
+	go func() {
+		next := strings.Repeat("failed update", 180000)
+		_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
+		committed <- err
+	}()
+	for range 4 {
+		select {
+		case <-entered:
+		case err := <-committed:
+			t.Fatalf("PUT returned before reply barrier: %v", err)
+		case <-time.After(5 * time.Second):
+			t.Fatal("disk commit did not reach reply barrier")
+		}
+	}
+	for _, d := range disks[:4] {
+		if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
+			t.Fatal(err)
+		}
+	}
+	unblock()
+	select {
+	case err := <-committed:
+		if err == nil {
+			t.Fatal("four-of-eight commit unexpectedly reached write quorum")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("failed PUT did not finish undo")
+	}
+	swap(disks)
+	for i, d := range disks {
+		fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
+		if err != nil || !fi.ModTime.Equal(base) {
+			t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
+		}
+	}
+	r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
+	}
+	defer r.Close()
+	data, err := io.ReadAll(r)
+	if err != nil || string(data) != body {
+		t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
+	}
+}

cmd/admin-account-buckets_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/cachevalue"
+)
+
+func TestAccountInfoReflectsBucketMutations(t *testing.T) {
+	ctx := t.Context()
+	bed, err := prepareAdminErasureTestBed(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer bed.TearDown()
+	previousCache := listBucketsCache
+	defer func() { listBucketsCache = previousCache }()
+
+	// Hold a real pre-mutation snapshot past the assertions so the regression
+	// cannot disappear merely because a slow test outlasted the cache TTL.
+	freezeListing := func() {
+		t.Helper()
+		buckets, err := bed.objLayer.ListBuckets(ctx, BucketOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		listBucketsCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) ([]BucketInfo, error) {
+			return buckets, nil
+		})
+		if _, err := listBucketsCache.GetWithCtx(ctx); err != nil {
+			t.Fatal(err)
+		}
+	}
+	const bucket = "account-info-freshness"
+	assertPresent := func(want bool) {
+		t.Helper()
+		req, err := buildAdminRequest(nil, http.MethodGet, "/accountinfo", 0, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		rec := httptest.NewRecorder()
+		bed.router.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("account info: %d %s", rec.Code, rec.Body.String())
+		}
+		var info madmin.AccountInfo
+		if err := json.Unmarshal(rec.Body.Bytes(), &info); err != nil {
+			t.Fatal(err)
+		}
+		found := false
+		for _, item := range info.Buckets {
+			if item.Name == bucket {
+				found = true
+			}
+		}
+		if found != want {
+			t.Errorf("account info bucket present = %v, want %v", found, want)
+		}
+	}
+
+	freezeListing()
+	if err := bed.objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	assertPresent(true)
+	freezeListing()
+	if err := bed.objLayer.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	assertPresent(false)
+}

cmd/admin-handlers-pools.go

@@ -373,7 +373,9 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	// Cancel any ongoing rebalance operation
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
-	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	if err := pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt); !errors.Is(err, errRebalanceNotStarted) {
+		adminLogIf(ctx, err)
+	}
 	globalNotificationSys.LoadRebalanceMeta(ctx, false)
 }
 

cmd/admin-handlers-users-authz_test.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -248,8 +249,20 @@ func TestIAMAdminPolicylessSelf(t *testing.T) {
 	ctx, sys := iamAdminFixture(t)
 	caller := iamAdminUser(ctx, t, sys, "review-policyless", "")
 	svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "")
+	other := iamAdminUser(ctx, t, sys, "bulk-other", "readwrite")
+	_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "bulk-other-child", "")
 	api := adminAPIHandlers{}
 	assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200)
+	rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk)
+	assertIAMAdminStatus(t, "list bulk self", rec, 200)
+	data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
+	mustIAM(t, err)
+	var listed map[string]madmin.ListAccessKeysResp
+	mustIAM(t, json.Unmarshal(data, &listed))
+	entry, ok := listed[caller.AccessKey]
+	if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 1 || entry.ServiceAccounts[0].AccessKey != svc.AccessKey || len(entry.STSKeys) != 0 {
+		t.Fatalf("policyless bulk self returned unexpected identities: %+v", listed)
+	}
 	assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200)
 	assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200)
 	assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
@@ -384,6 +397,7 @@ func TestIAMAdminSelfDenials(t *testing.T) {
 			}
 			api := adminAPIHandlers{}
 			assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403)
+			assertIAMAdminStatus(t, "list bulk self", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk), 403)
 			assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403)
 			assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403)
 			assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403)
@@ -477,7 +491,10 @@ func TestIAMAdminListSelfAliases(t *testing.T) {
 			ctx, sys := iamAdminFixture(t)
 			parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite")
 			target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "")
-			caller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
+			serviceCaller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
+			caller := serviceCaller
+			other := iamAdminUser(ctx, t, sys, "alias-other", "readwrite")
+			_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "alias-other-child", "")
 			if kind == "sts" {
 				var err error
 				caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
@@ -501,6 +518,38 @@ func TestIAMAdminListSelfAliases(t *testing.T) {
 				if !found {
 					t.Fatalf("self alias %q did not list the parent's service accounts", alias)
 				}
+
+				query := url.Values{"listType": {madmin.AccessKeyListAll}}
+				if alias != "" {
+					query.Set("users", alias)
+				}
+				bulk := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?"+query.Encode(), nil, nil, api.ListAccessKeysBulk)
+				assertIAMAdminStatus(t, "list bulk self alias", bulk, 200)
+				data, err = madmin.DecryptData(caller.SecretKey, bulk.Body)
+				mustIAM(t, err)
+				var listed map[string]madmin.ListAccessKeysResp
+				mustIAM(t, json.Unmarshal(data, &listed))
+				entry, ok := listed[parent.AccessKey]
+				if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 2 {
+					t.Fatalf("bulk self alias %q returned unexpected identities: %+v", alias, listed)
+				}
+				want := map[string]bool{target.AccessKey: true, serviceCaller.AccessKey: true}
+				for _, account := range entry.ServiceAccounts {
+					if !want[account.AccessKey] {
+						t.Fatalf("bulk self alias %q returned unexpected or duplicate service account %s", alias, account.AccessKey)
+					}
+					delete(want, account.AccessKey)
+				}
+				if kind == "sts" {
+					if len(entry.STSKeys) != 1 || entry.STSKeys[0].AccessKey != caller.AccessKey {
+						t.Fatalf("bulk self alias %q returned unexpected STS accounts: %+v", alias, entry.STSKeys)
+					}
+				} else if len(entry.STSKeys) != 0 {
+					t.Fatalf("service caller returned unexpected STS accounts: %+v", entry.STSKeys)
+				}
+				if bytes.Contains(data, []byte(serviceCaller.SecretKey)) || bytes.Contains(data, []byte(`"secretKey"`)) || bytes.Contains(data, []byte(`"sessionToken"`)) {
+					t.Fatal("bulk listing returned credential secrets")
+				}
 			}
 		})
 	}

cmd/admin-handlers-users.go

@@ -622,7 +622,11 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 			policiesNames = policySet.ToSlice()
 		}
 
-		stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		stsAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	policyJSON, err := json.MarshalIndent(stsAccountPolicy, "", " ")
@@ -881,16 +885,16 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	// A request without a policy, or with a JSON null for one, keeps the
+	// policy the account has. An empty policy document is passed on as
+	// such: it is the request to remove the embedded policy.
 	var sp *policy.Policy
-	if len(updateReq.NewPolicy) > 0 {
+	if len(updateReq.NewPolicy) > 0 && !bytes.Equal(updateReq.NewPolicy, []byte("null")) {
 		sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		if sp.Version == "" && len(sp.Statements) == 0 {
-			sp = nil
-		}
 	}
 	opts := updateServiceAccountOpts{
 		secretKey:     updateReq.NewSecretKey,
@@ -899,6 +903,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		description:   updateReq.NewDescription,
 		expiration:    updateReq.NewExpiration,
 		sessionPolicy: sp,
+		// The edit applies to the stored account only while it still has
+		// the parent the checks above authorized.
+		parentUser: svcAccount.ParentUser,
 	}
 	updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
 	if err != nil {
@@ -987,7 +994,11 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		svcAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
@@ -1237,7 +1248,7 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
-	if !globalIAMSys.IsAllowed(policy.Args{
+	if !isAllowedServiceAccountAction(cred, policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.ListServiceAccountsAdminAction,
@@ -1250,12 +1261,12 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if selfOnly && len(users) == 0 {
+	if selfOnly {
 		selfUser := cred.AccessKey
 		if cred.ParentUser != "" {
 			selfUser = cred.ParentUser
 		}
-		users = append(users, selfUser)
+		users = []string{selfUser}
 	}
 
 	var checkedUserList []string
@@ -1447,7 +1458,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
+		// Console refreshes this inventory immediately after bucket mutations.
+		// A stale cached list can otherwise hide a creation or retain a deletion
+		// indefinitely in the client's snapshot.
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1463,6 +1477,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
 	policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
 	var effectivePolicy policy.Policy
+	var policyErr error
 
 	var buf []byte
 	switch {
@@ -1486,10 +1501,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return
 		}
 		policySlice := newMappedPolicy(policy).toSlice()
-		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
+		effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policySlice...)
 
 	case hasPolicyClaim:
-		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
+		effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
 
 	default:
 		policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
@@ -1498,7 +1513,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
+		effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policies...)
+	}
+	if policyErr != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, policyErr), r.URL)
+		return
 	}
 
 	buf, err = json.MarshalIndent(effectivePolicy, "", " ")
@@ -2186,7 +2205,11 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		svcAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
@@ -2946,7 +2969,15 @@ func canManageRootCredentials(cred auth.Credentials) bool {
 func isAllowedServiceAccountAction(cred auth.Credentials, args policy.Args) bool {
 	if args.DenyOnly && !args.IsOwner && !cred.IsTemp() && !cred.IsServiceAccount() && newGlobalAuthZPluginFn() == nil {
 		policies, err := globalIAMSys.PolicyDBGet(args.AccountName, args.Groups...)
-		return err == nil && globalIAMSys.GetCombinedPolicy(policies...).IsAllowed(args)
+		if err != nil {
+			return false
+		}
+		combined, err := globalIAMSys.GetCombinedPolicy(policies...)
+		if err != nil {
+			iamLogIf(GlobalContext, err)
+			return false
+		}
+		return combined.IsAllowed(args)
 	}
 	return globalIAMSys.IsAllowed(args)
 }

cmd/admin-handlers.go

@@ -791,7 +791,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 
 	enc := json.NewEncoder(w)
 	for n > 0 {
-		var m madmin.RealtimeMetrics
+		var m realtimeMetrics
 		mLocal := collectLocalMetrics(types, collectMetricsOpts{
 			hosts: hostMap,
 			disks: diskMap,
@@ -2675,7 +2675,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 	// collect all realtime metrics except disk
 	// disk metrics are already included under drive info of each server
 	getRealtimeMetrics := func() *madmin.RealtimeMetrics {
-		var m madmin.RealtimeMetrics
+		var m realtimeMetrics
 		types := madmin.MetricsAll &^ madmin.MetricsDisk
 		mLocal := collectLocalMetrics(types, collectMetricsOpts{})
 		m.Merge(&mLocal)
@@ -2690,7 +2690,8 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 			m.ByHost[anonAddr(host)] = metrics
 			delete(m.ByHost, host)
 		}
-		return &m
+		legacy := m.legacy()
+		return &legacy
 	}
 
 	anonymizeCmdLine := func(cmdLine string) string {

cmd/batch-authz-context_test.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// batchContextRead is the object read the tests below authorize.
+func batchContextRead(f *batchAuthzFixture) batchAuthzRequest {
+	return batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "data/a"}
+}
+
+// batchContextUser creates a user with statements as its only policy and
+// returns its submitter binding.
+func batchContextUser(t *testing.T, f *batchAuthzFixture, name, statements string) (auth.Credentials, *batchJobSubmitter) {
+	t.Helper()
+	batchAuthzPolicy(f.ctx, t, f.sys, name, statements)
+	cred := iamAdminUser(f.ctx, t, f.sys, name+"-user", name)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	return cred, sub
+}
+
+func isBatchContextRefusal(err error) bool {
+	return isBatchJobAuthzError(err) && strings.Contains(err.Error(), "request context")
+}
+
+// A statement whose resources rule the object out does not apply, whatever
+// its conditions need, policy variables in condition values included. One
+// whose resources match still refuses.
+func TestBatchAuthzContextResourceFirst(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	const unrelated = `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::other-bucket/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/owner":"${aws:UserAgent}"}}}`
+	_, sub := batchContextUser(t, f, "context-unrelated", batchAuthzFullAccess+",\n"+unrelated)
+	if err := sub.authorize(f.ctx, batchContextRead(f)); err != nil {
+		t.Errorf("a Deny on another bucket refused an allowed read: %v", err)
+	}
+
+	related := strings.Replace(unrelated, "other-bucket", f.bucket, 1)
+	_, sub = batchContextUser(t, f, "context-related", batchAuthzFullAccess+",\n"+related)
+	if err := sub.authorize(f.ctx, batchContextRead(f)); !isBatchContextRefusal(err) {
+		t.Errorf("a Deny on the object with an unsupplied variable must refuse, got %v", err)
+	}
+
+	// A variable in the resource leaves open whether it matches.
+	const resourceVar = `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/${aws:UserAgent}/*"]}`
+	_, sub = batchContextUser(t, f, "context-resource-var", batchAuthzFullAccess+",\n"+resourceVar)
+	if err := sub.authorize(f.ctx, batchContextRead(f)); !isBatchContextRefusal(err) {
+		t.Errorf("a Deny whose resource holds an unsupplied variable must refuse, got %v", err)
+	}
+}
+
+// An Allow that needs request context the job does not have grants nothing
+// to the job; it neither refuses the job nor grants by an absent key.
+func TestBatchAuthzContextAllowDropped(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	const ipAllow = `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::*"],"Condition":{"IpAddress":{"aws:SourceIp":["10.0.0.0/8"]}}}`
+	_, sub := batchContextUser(t, f, "context-ip-allow", batchAuthzFullAccess+",\n"+ipAllow)
+	if err := sub.authorize(f.ctx, batchContextRead(f)); err != nil {
+		t.Errorf("a conditional Allow refused a read another Allow grants: %v", err)
+	}
+
+	// A Null condition holds for an absent key: evaluated without the
+	// context, this Allow would grant what no request with an address gets.
+	const nullAllow = `{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Null":{"aws:SourceIp":"true"}}}`
+	cred, sub := batchContextUser(t, f, "context-null-allow", nullAllow)
+	if !f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "data/a", ConditionValues: map[string][]string{}}) {
+		t.Fatal("control: without the address the Null Allow grants")
+	}
+	err := sub.authorize(f.ctx, batchContextRead(f))
+	if !isBatchJobAuthzError(err) || isBatchContextRefusal(err) {
+		t.Errorf("an Allow that needs the address must grant nothing, and deny rather than refuse; got %v", err)
+	}
+}
+
+// The same holds for the Allows of a session policy, which only narrows.
+func TestBatchAuthzContextSessionAllow(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "context-session-parent", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "context-session-parent", "context-session-parent")
+	for _, tc := range []struct {
+		name, statements string
+		allowed          bool
+	}{
+		{"null-only", `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Null":{"aws:SourceIp":"true"}}}`, false},
+		{"with-plain", `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Null":{"aws:SourceIp":"true"}}},
+{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/data/*"]}`, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[` + tc.statements + `]}`))
+			mustIAM(t, err)
+			created, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp})
+			mustIAM(t, err)
+			stored, ok := f.sys.GetUser(f.ctx, created.AccessKey)
+			if !ok {
+				t.Fatal("service account missing")
+			}
+			sub, err := newBatchJobSubmitter(stored.Credentials)
+			mustIAM(t, err)
+			err = sub.authorize(f.ctx, batchContextRead(f))
+			switch {
+			case isBatchContextRefusal(err):
+				t.Errorf("a session-policy Allow refused the job: %v", err)
+			case tc.allowed && err != nil:
+				t.Errorf("the plain session Allow must still grant: %v", err)
+			case !tc.allowed && !isBatchJobAuthzError(err):
+				t.Errorf("a session policy whose only Allow needs the address must deny, got %v", err)
+			}
+		})
+	}
+}

cmd/batch-authz-review2_test.go

@@ -0,0 +1,356 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A pull into a suspended bucket replaces the null version. Its policy
+// decision must name that version even when the source is unversioned S3.
+func TestBatchReviewPullImplicitNullVersionAuthorization(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
+	mustIAM(t, err)
+	data := []byte("original null version")
+	before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: true})
+	mustIAM(t, err)
+	batchAuthzPolicy(f.ctx, t, f.sys, "null-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"null"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "null-deny-user", "null-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	job := batchPullJob(f.bucket)
+	job.Source.Type = BatchJobReplicateResourceS3
+	err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+		Name: before.Name, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
+	}, false, sub)
+	if !isBatchJobAuthzError(err) {
+		t.Errorf("null version policy must refuse the write, got %v", err)
+	}
+	after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: nullVersionID})
+	if gerr != nil || before.ETag != after.ETag {
+		t.Errorf("denied null version must remain unchanged: read=%v unchanged=%v", gerr, before.ETag == after.ETag)
+	}
+}
+
+type batchReviewPrefixWalkFailure struct{ ObjectLayer }
+
+func (a batchReviewPrefixWalkFailure) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
+	if prefix != "a/" {
+		return a.ObjectLayer.Walk(ctx, bucket, prefix, results, opts)
+	}
+	go func() {
+		defer close(results)
+		results <- itemOrErr[ObjectInfo]{Err: errors.New("test: incomplete first prefix")}
+	}()
+	return nil
+}
+
+type batchReviewWalkCutoff struct {
+	ObjectLayer
+	cutoff string
+}
+
+func (a batchReviewWalkCutoff) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
+	inner := make(chan itemOrErr[ObjectInfo], 1)
+	if err := a.ObjectLayer.Walk(ctx, bucket, prefix, inner, opts); err != nil {
+		close(results)
+		return err
+	}
+	go func() {
+		defer close(results)
+		stopped := false
+		for result := range inner {
+			if stopped {
+				continue
+			}
+			if result.Err == nil && result.Item.Name >= a.cutoff {
+				stopped = true
+				result = itemOrErr[ObjectInfo]{Err: errors.New("test: listing stopped before remaining child keys")}
+			}
+			select {
+			case results <- result:
+			case <-ctx.Done():
+				stopped = true
+			}
+		}
+	}()
+	return nil
+}
+
+// Listing a directory prefix returns the directory object before its child
+// keys. A durable cursor for that directory must preserve unfinished children.
+func TestBatchReviewDirectoryCursorKeepsChildrenPending(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	for range 12 {
+		batchPutVersion(t, f.obj, f.bucket, "a/", nil)
+	}
+	var pending string
+	for i := 0; i < 13; i++ {
+		older := f.putVersions(t, fmt.Sprintf("a/key-%02d", i))
+		if i == 11 {
+			pending = older
+		}
+	}
+	rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: ["a/"]
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	_ = job.Expire.Start(f.ctx, batchReviewWalkCutoff{ObjectLayer: f.obj, cutoff: "a/key-11"}, *job)
+	ri := f.jobInfo(t, job)
+	if ri.Object != "a/" {
+		t.Fatalf("control: the completed directory batch must be the cursor, got %q", ri.Object)
+	}
+	mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+	if f.versionExists(t, "a/key-11", pending) {
+		t.Error("a directory cursor must not suppress the unvisited child key on resume")
+	}
+}
+
+// Later prefixes must not advance the completion cursor past an earlier
+// prefix whose listing failed. This state is produced by Start itself.
+func TestBatchReviewExpireFailedPrefixKeepsWorkPending(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "a/todo")
+	for i := 0; i < 13; i++ {
+		f.putVersions(t, fmt.Sprintf("z/key-%02d", i))
+	}
+	rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: ["z/", "a/"]
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	_ = job.Expire.Start(f.ctx, batchReviewPrefixWalkFailure{f.obj}, *job)
+	ri := f.jobInfo(t, job)
+	if ri.Object >= "a/todo" {
+		t.Errorf("cursor must not cross an incomplete prefix, got %q", ri.Object)
+	}
+	mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+	if f.versionExists(t, "a/todo", older) {
+		t.Error("a fresh Start must revisit the key omitted by the failed listing")
+	}
+}
+
+// A repeated marker replication error must remain a failure on the second
+// attempt, including after the retry loop resets its counters.
+func TestBatchReviewMarkerRetryRemainsFailed(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchPutVersion(t, f.obj, f.bucket, "marker", nil)
+	_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
+	mustIAM(t, err)
+	oldTransport := globalRemoteTargetTransport
+	globalRemoteTargetTransport = batchLockTransport(func(r *http.Request) (*http.Response, error) {
+		code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
+		if r.URL.Query().Has("location") {
+			code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
+		}
+		return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
+	})
+	t.Cleanup(func() { globalRemoteTargetTransport = oldTransport })
+	job := BatchJobRequest{
+		ID: "replicate-marker-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+		Replicate: &BatchJobReplicateV1{
+			Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+			Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
+		},
+	}
+	job.Replicate.Flags.Retry.Attempts = 2
+	job.Replicate.Flags.Retry.Delay = time.Millisecond
+	mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
+	ri := f.jobInfo(t, &job)
+	if ri.Complete || !ri.Failed || ri.DeleteMarkersFailed != 1 {
+		t.Errorf("both marker attempts failed: complete=%v failed=%v markerFailures=%d attempts=%d", ri.Complete, ri.Failed, ri.DeleteMarkersFailed, ri.RetryAttempts)
+	}
+}
+
+// A current parent mapping suppresses the OpenID policy-name fallback.
+// Background-condition validation must inspect the same effective policies.
+func TestBatchReviewMappedPolicySuppressesClaimFallback(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
+	batchAuthzPolicy(f.ctx, t, f.sys, "unused-claim-policy", `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+	parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "mapped-full")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), iamPolicyClaimNameOpenID(): "unused-claim-policy",
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	p, err := sub.principal(f.ctx)
+	mustIAM(t, err)
+	req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "mapped/key", versionID: mustGetUUID()}
+	if !f.sys.isAllowedLocally(policy.Args{AccountName: p.accessKey, Claims: p.claims, Action: req.action, BucketName: req.bucket, ObjectName: req.object, ConditionValues: p.conditionValues(req)}) {
+		t.Fatal("control: the current mapped policy must allow this read")
+	}
+	if err := sub.authorize(f.ctx, req); err != nil {
+		t.Errorf("an unused fallback policy must not refuse a mapped-policy grant: %v", err)
+	}
+}
+
+func TestBatchReviewBindingAndExpiringServiceControls(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	root := batchRootSubmitter(t)
+	priorRoot := globalActiveCred
+	globalActiveCred.SecretKey = "different-test-root-secret"
+	_, err := root.principal(f.ctx)
+	globalActiveCred = priorRoot
+	if !isBatchJobAuthzError(err) {
+		t.Errorf("root-secret change must invalidate the binding: %v", err)
+	}
+	missing := *root
+	missing.Binding = nil
+	if _, err := missing.principal(f.ctx); !isBatchJobAuthzError(err) {
+		t.Errorf("missing fingerprint must refuse the principal: %v", err)
+	}
+	batchAuthzPolicy(f.ctx, t, f.sys, "service-parent-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "service-parent", "service-parent-full")
+	sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`))
+	mustIAM(t, err)
+	expires := UTCNow().Add(time.Hour)
+	cred, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp, expiration: &expires})
+	mustIAM(t, err)
+	stored, ok := f.sys.GetUser(f.ctx, cred.AccessKey)
+	if !ok {
+		t.Fatal("created service account must resolve")
+	}
+	cred = stored.Credentials
+	if !cred.IsTemp() || !cred.IsServiceAccount() {
+		t.Fatal("fixture must be an expiring service account")
+	}
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "allowed"}); err != nil {
+		t.Errorf("expiring service account must retain its allowed read: %v", err)
+	}
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectAction, bucket: f.bucket, object: "allowed"}); !isBatchJobAuthzError(err) {
+		t.Errorf("expiring service account must remain session-policy restricted: %v", err)
+	}
+}
+
+// An interrupted cross-pool copy can leave the same version in both pools.
+// A newer legal-hold field in either copy must prevent key rotation.
+func TestBatchReviewRotationUsesAuthoritativePoolHold(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const name = "rotation/split-hold"
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("batch-rotation-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	dek, err := GlobalKMS.GenerateKey(t.Context(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{bucket: bucket + "/" + name}})
+	mustIAM(t, err)
+	objectKey := crypto.GenerateKey(dek.Plaintext, nil)
+	sealed := objectKey.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), bucket, name)
+	first := crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed)
+	first[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "OFF"
+	first[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+	second := maps.Clone(first)
+	second[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
+	second[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
+	ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("protected payload"), objectKey))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if !enforceRetentionForDeletion(t.Context(), logical) {
+		t.Fatal("control: the authoritative logical version must be held")
+	}
+	r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+	err = r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
+	if !isErrObjectLocked(err) {
+		t.Errorf("rotation must refuse the authoritative legal hold, got %v", err)
+	}
+	after, gerr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, gerr)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Error("held logical version must not have its key rewritten")
+	}
+}
+
+// An encryption condition must describe what rotation actually commits,
+// even when the requested encryption type differs from the existing type.
+func TestBatchReviewRotationEncryptionConditionMatchesResult(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("required-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	const name = "rotation/encryption-condition"
+	meta := make(map[string]string)
+	key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, name, meta, nil)
+	mustIAM(t, err)
+	body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("encrypted payload"), key))
+	mustIAM(t, err)
+	before, err := f.obj.PutObject(f.ctx, f.bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, UserDefined: meta})
+	mustIAM(t, err)
+	batchAuthzPolicy(f.ctx, t, f.sys, "kms-write-only", `
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "kms-write-user", "kms-write-only")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	r := BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "required-key"}}
+	err = r.KeyRotate(f.ctx, f.obj, before, sub)
+	after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, gerr)
+	if err == nil && !crypto.S3KMS.IsEncrypted(after.UserDefined) {
+		t.Error("rotation allowed by an aws:kms-only write policy must produce SSE-KMS or refuse the operation")
+	}
+	if err != nil && before.UserDefined[crypto.MetaSealedKeyS3] != after.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Error("a refused rotation must leave the version unchanged")
+	}
+}
+
+// An explicit null marker collision must retain a held null data version.
+func TestBatchReviewPullMarkerNullLock(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	meta := batchLockMeta("LEGALHOLD")
+	before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, strings.NewReader("held null"), 9, "", ""), ObjectOptions{VersionID: nullVersionID, Versioned: true, UserDefined: meta})
+	mustIAM(t, err)
+	job := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+	}
+	err = job.ReplicateFromSource(f.ctx, f.obj, nil, ObjectInfo{
+		Name: before.Name, VersionID: nullVersionID, ModTime: UTCNow(), DeleteMarker: true,
+	}, false, batchRootSubmitter(t))
+	if !isErrObjectLocked(err) {
+		t.Errorf("marker collision with protected null data must return ObjectLocked, got %v", err)
+	}
+	if !batchVersionExists(t, f.obj, f.bucket, before.Name, nullVersionID) {
+		t.Fatal("protected null data must remain present")
+	}
+}

cmd/batch-authz-review3_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/auth"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A read policy must use the newest tag state of the logical version even
+// when an interrupted movement left an older copy in the preferred pool.
+func TestBatchReviewPushUsesAuthoritativePoolTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	const name = "push/tagged"
+	first := map[string]string{
+		xhttp.AmzObjectTagging:                         "classification=public",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
+	}
+	second := map[string]string{
+		xhttp.AmzObjectTagging:                         "classification=private",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+	}
+	before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if logical.UserTags != "classification=private" {
+		t.Fatal("control: logical version must have the newer private tag")
+	}
+	batchAuthzPolicy(f.ctx, t, f.sys, "pooled-push", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "pooled-push-user", "pooled-push")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: bucket, object: name, versionID: before.VersionID, tags: logical.UserTags}); !isBatchJobAuthzError(err) {
+		t.Fatal("control: authoritative tags must deny the read")
+	}
+	writes := 0
+	target, err := minio.NewCore("target.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+				_ = r.Body.Close()
+			}
+			if r.Method == http.MethodPut {
+				writes++
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
+	if !isBatchJobAuthzError(err) || writes != 0 {
+		t.Errorf("newer authoritative tags must deny push: err=%v writes=%d", err, writes)
+	}
+}
+
+// Verified token claims must not replace the actual version being authorized.
+func TestBatchReviewVersionConditionResistsClaimCollision(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	deniedVersion := mustGetUUID()
+	batchAuthzPolicy(f.ctx, t, f.sys, "version-claims", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, deniedVersion))
+	parent := iamAdminUser(f.ctx, t, f.sys, "version-claims-parent", "version-claims")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), "versionid": "an-unrelated-claim-value",
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	req := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: deniedVersion}
+	if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
+		t.Errorf("a token claim must not replace the actual denied version: %v", err)
+	}
+}
+
+// Retry totals describe a pass, including markers already successful before
+// another item caused that pass to retry.
+func TestBatchReviewMarkerSuccessCountResetsOnRetry(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchPutVersion(t, f.obj, f.bucket, "marker", nil)
+	_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
+	mustIAM(t, err)
+	oldTransport := remoteInstanceTransport
+	remoteInstanceTransport = atomic.Value{}
+	setRemoteInstanceTransport(batchLockTransport(func(r *http.Request) (*http.Response, error) {
+		code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
+		h := make(http.Header)
+		switch {
+		case r.URL.Query().Has("location"):
+			code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
+		case r.Method == http.MethodDelete:
+			code, body = http.StatusNoContent, ""
+		case r.Method == http.MethodHead && r.URL.Query().Get("versionId") != "":
+			code, body = http.StatusMethodNotAllowed, ""
+			h.Set("X-Amz-Delete-Marker", "true")
+		}
+		return &http.Response{StatusCode: code, Header: h, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
+	}))
+	t.Cleanup(func() { remoteInstanceTransport = oldTransport })
+	job := BatchJobRequest{
+		ID: "marker-success-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+		Replicate: &BatchJobReplicateV1{
+			Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+			Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
+		},
+	}
+	job.Replicate.Flags.Retry.Attempts = 2
+	job.Replicate.Flags.Retry.Delay = time.Millisecond
+	mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
+	ri := f.jobInfo(t, &job)
+	if !ri.Failed || ri.RetryAttempts != 2 {
+		t.Fatalf("control: data failure must cause two attempts: failed=%v attempts=%d", ri.Failed, ri.RetryAttempts)
+	}
+	if ri.DeleteMarkers != 1 {
+		t.Errorf("one successful marker must be counted once in the final attempt, got %d", ri.DeleteMarkers)
+	}
+}

cmd/batch-authz-review4_test.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/auth"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A tag removal in a newer copy must govern both the authorization and the
+// metadata sent by a push. The submitter can read data but cannot read tags.
+func TestBatchReviewPushSendsOnlyAuthorizedMetadata(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	const name = "push/removed-tags"
+	first := map[string]string{
+		xhttp.AmzObjectTagging:                         "classification=private",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
+	}
+	second := map[string]string{
+		xhttp.AmzObjectTagging:                         "",
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+	}
+	before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if logical.UserTags != "" {
+		t.Fatal("control: the authoritative version must have no tags")
+	}
+	batchAuthzPolicy(f.ctx, t, f.sys, "data-without-tags", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "data-without-tags-user", "data-without-tags")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: name, versionID: before.VersionID}); !isBatchJobAuthzError(err) {
+		t.Fatal("control: reading tags must be denied")
+	}
+	writes := 0
+	var sentTags string
+	target, err := minio.NewCore("target.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+				_ = r.Body.Close()
+			}
+			if r.Method == http.MethodPut {
+				writes++
+				sentTags = r.Header.Get(xhttp.AmzObjectTagging)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
+	if err != nil && !isBatchJobAuthzError(err) {
+		t.Fatalf("unexpected replication failure: %v", err)
+	}
+	if writes > 0 && sentTags != logical.UserTags {
+		t.Errorf("push must send only metadata covered by authorization: writes=%d sentTags=%q authorizedTags=%q err=%v", writes, sentTags, logical.UserTags, err)
+	}
+}
+
+// LDAP's short username and its parent DN are distinct identities. The
+// background merge must not turn a deny for the verified short name into
+// a non-matching statement by replacing it with the parent DN.
+func TestBatchReviewLDAPUsernameDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "ldap-short-name-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"ldap:username":"alice"}}}`)
+	const dn = "uid=alice,ou=people,dc=example,dc=org"
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		"exp": UTCNow().Add(time.Hour).Unix(), ldapUser: dn, ldapActualUser: dn, ldapUserN: "alice",
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = dn
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "ldap-short-name-deny")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	p, err := sub.principal(f.ctx)
+	mustIAM(t, err)
+	cred.Claims = p.claims
+	req, err := http.NewRequest(http.MethodDelete, "https://localhost/batch-authz/data", nil)
+	mustIAM(t, err)
+	args := policy.Args{
+		AccountName: cred.AccessKey, Claims: p.claims, Action: policy.DeleteObjectVersionAction,
+		BucketName: f.bucket, ObjectName: "data", ConditionValues: getConditionValues(req, "", cred),
+	}
+	if args.ConditionValues["username"][0] != "alice" || f.sys.isAllowedLocally(args) {
+		t.Fatal("control: the request evaluator must deny the verified LDAP short username")
+	}
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: mustGetUUID()}); !isBatchJobAuthzError(err) {
+		t.Errorf("batch authorization must preserve the LDAP username deny or fail closed, got %v", err)
+	}
+}
+
+// Failure to read authoritative metadata is incomplete work, even if one
+// pool's copy can still be opened. In particular, skipErrs=false must not
+// turn an unreadable entry into a successful archive.
+func TestBatchReviewSnowballAuthoritativeReadFailure(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const name = "archive/unreadable-copy"
+	before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true})
+	set := z.serverPools[1].sets[0]
+	getDisks := set.getDisks
+	disks := append([]StorageAPI(nil), getDisks()...)
+	for i := range disks {
+		disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: name}
+	}
+	set.getDisks = func() []StorageAPI { return disks }
+	defer func() { set.getDisks = getDisks }()
+	gr, err := z.GetObjectNInfo(t.Context(), bucket, name, nil, nil, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	_, logicalErr := batchAuthoritativeRead(t.Context(), z, bucket, name, gr.ObjInfo)
+	gr.Close()
+	if logicalErr == nil {
+		t.Fatal("control: the authoritative read must fail")
+	}
+	remote, err := minio.New("archive.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	r.Source.Snowball.InMemory = ptr(true)
+	r.Source.Snowball.Compress = ptr(false)
+	r.Source.Snowball.SkipErrs = ptr(false)
+	if err := r.writeAsArchive(t.Context(), z, remote, []ObjectInfo{before}, "", batchRootSubmitter(t)); err == nil {
+		t.Error("archive with an unreadable authoritative version must not report success")
+	}
+}

cmd/batch-authz-review5_test.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+// A suspended destination does not keep a source version ID. Authorization
+// must therefore describe the null destination version that the write really
+// replaces, and must not let a source ID escape that model.
+func TestBatchReviewPullSuspendedVersionBinding(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
+	mustIAM(t, err)
+
+	sourceVersion := mustGetUUID()
+	batchAuthzPolicy(f.ctx, t, f.sys, "pull-source-version-deny", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, sourceVersion))
+	cred := iamAdminUser(f.ctx, t, f.sys, "pull-source-version-user", "pull-source-version-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	job := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+	}
+	err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+		Name: "pull/suspended-version", VersionID: sourceVersion, ModTime: UTCNow(), ETag: "batchetag",
+		UserDefined: map[string]string{},
+	}, false, sub)
+	if !isBatchJobAuthzError(err) {
+		t.Fatalf("a source version ID must not evade the destination deny: %v", err)
+	}
+	if _, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/suspended-version", ObjectOptions{VersionID: sourceVersion}); gerr == nil {
+		t.Fatal("a denied suspended-destination write created the source version")
+	}
+}
+
+// A failed source open in a non-skipping snowball must be reported to the
+// batch rather than silently omitted from a successful archive.
+type batchReviewOpenFailure struct {
+	ObjectLayer
+	name string
+}
+
+func (a batchReviewOpenFailure) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+	if object == a.name {
+		return nil, fmt.Errorf("test: source open failed")
+	}
+	return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
+}
+
+func TestBatchReviewSnowballSourceOpenFailure(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	name := "archive/open-failure"
+	entry := batchPutVersion(t, f.obj, f.bucket, name, nil)
+	remote, err := minio.New("archive.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	r := BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+	}
+	r.Source.Snowball.InMemory = ptr(true)
+	r.Source.Snowball.Compress = ptr(false)
+	r.Source.Snowball.SkipErrs = ptr(false)
+	api := batchReviewOpenFailure{ObjectLayer: f.obj, name: name}
+	if err := r.writeAsArchive(t.Context(), api, remote, []ObjectInfo{entry}, "", batchRootSubmitter(t)); err == nil {
+		t.Fatal("a source open failure must not be reported as a successful archive")
+	}
+}
+
+// Rotation must apply tag conditions to the logical version, including
+// a more recent tag update stored in a pool other than the selected copy.
+func TestBatchReviewRotationAuthoritativeTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("batch-round5-rotation-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	batchAuthzPolicy(f.ctx, t, f.sys, "round5-rotation-tags", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "round5-rotation-user", "round5-rotation-tags")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	for _, classification := range []string{"private", "public"} {
+		t.Run(classification, func(t *testing.T) {
+			name := "rotation/authoritative-" + classification
+			first := make(map[string]string)
+			key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
+			mustIAM(t, err)
+			first[xhttp.AmzObjectTagging] = "classification=public"
+			first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+			second := maps.Clone(first)
+			second[xhttp.AmzObjectTagging] = "classification=" + classification
+			second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+			ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("test encrypted payload"), key))
+			mustIAM(t, err)
+			before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+			putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+			logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if logical.UserTags != "classification="+classification {
+				t.Fatal("control: the second pool must govern the logical tags")
+			}
+			job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			err = job.KeyRotate(f.ctx, z, before, sub)
+			after, readErr := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, readErr)
+			changed := after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3]
+			if classification == "private" {
+				if !isBatchJobAuthzError(err) || changed {
+					t.Fatalf("authoritative tag deny must prevent rewrapping: err=%v changed=%v", err, changed)
+				}
+			} else if err != nil || !changed {
+				t.Fatalf("authorized unlocked version must rotate: err=%v changed=%v", err, changed)
+			}
+		})
+	}
+}

cmd/batch-authz-review6_test.go

@@ -0,0 +1,538 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+// Rotation and listing cases found in review.
+
+// Rotation rewrites the copy GetObjectInfo returns, which can carry another
+// tag revision than the merged version of all pools: both are authorized.
+func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	f := newBatchAuthzFixture(t)
+	previousKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("review6-rotation-copy-key")
+	t.Cleanup(func() { GlobalKMS = previousKMS })
+	batchAuthzPolicy(f.ctx, t, f.sys, "review6-rotation-copy-deny", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "review6-rotation-copy-user", "review6-rotation-copy-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	const name = "rotation/written-copy"
+	first := make(map[string]string)
+	key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
+	mustIAM(t, err)
+	first[xhttp.AmzObjectTagging] = "classification=private"
+	first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+	second := maps.Clone(first)
+	second[xhttp.AmzObjectTagging] = "classification=public"
+	second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+	ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("review6 rotation payload"), key))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+	putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+	logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if logical.UserTags != "classification=public" {
+		t.Fatalf("control: merged tags=%q, want public", logical.UserTags)
+	}
+	job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+	if err = job.KeyRotate(f.ctx, z, before, sub); !isBatchJobAuthzError(err) {
+		t.Fatalf("rotation of a copy the submitter may not read returned %v", err)
+	}
+	after, err := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Fatal("a refused rotation rewrote the copy")
+	}
+}
+
+// batchWalkError fails every listing before it returns a result.
+type batchWalkError struct {
+	ObjectLayer
+	err error
+}
+
+func (w batchWalkError) Walk(context.Context, string, string, chan<- itemOrErr[ObjectInfo], WalkOptions) error {
+	return w.err
+}
+
+// A listing outage pauses the run, as a streamed listing error does.
+func TestBatchReviewExpireWalkErrorPausesRun(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	listErr := errors.New("review6: listing failed")
+	if err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job); !errors.Is(err, listErr) {
+		t.Fatalf("start returned %v, want %v", err, listErr)
+	}
+	if ri := f.jobInfo(t, job); ri.Failed || ri.Complete || ri.PausedReason == "" {
+		t.Fatalf("failed listing: complete=%v failed=%v", ri.Complete, ri.Failed)
+	}
+}
+
+// Two pools hold different writes of the null version, and a newer version
+// of the key is in the second one. Rotation rewrites the copy it read and
+// authorized, in the first pool, not the copy the latest version of the key
+// would route a write to; the other write keeps its key and stays readable.
+func TestBatchReviewRotationRewritesTheCopyItRead(t *testing.T) {
+	for _, otherLatest := range []bool{false, true} {
+		name := "addressed-version-is-latest"
+		if otherLatest {
+			name = "another-version-is-latest"
+		}
+		t.Run(name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			z, _ := consistencyPools(t)
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review7-rotation-copies"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, versionErr := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, versionErr)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review7-rotation-key")
+			t.Cleanup(func() { GlobalKMS = oldKMS })
+			const object = "rotation/null-copy"
+			const payload = "rotation must preserve this data"
+			readCopy := func(pool int) ([]byte, error) {
+				gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: nullVersionID})
+				if err != nil {
+					return nil, err
+				}
+				defer gr.Close()
+				return io.ReadAll(gr)
+			}
+			var before [2]ObjectInfo
+			var keys [2]crypto.ObjectKey
+			kinds := []crypto.Type{crypto.S3, crypto.S3KMS}
+			for pool, kind := range kinds {
+				meta := make(map[string]string)
+				key, err := newEncryptMetadata(t.Context(), kind, "review7-rotation-key", nil, bucket, object, meta, nil)
+				mustIAM(t, err)
+				keys[pool] = key
+				meta[xhttp.AmzObjectTagging] = "classification=public"
+				if pool == 1 {
+					meta[xhttp.AmzObjectTagging] = "classification=private"
+				}
+				meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Duration(pool+2) * time.Hour).Format(time.RFC3339Nano)
+				body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
+				mustIAM(t, err)
+				putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{
+					Versioned: true, VersionID: nullVersionID, UserDefined: meta,
+					MTime: UTCNow().Add(-time.Duration(pool+2) * time.Hour),
+				})
+				// As a read returns it, without the write's internal fields.
+				before[pool], err = z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
+				mustIAM(t, err)
+				data, err := readCopy(pool)
+				if err != nil || string(data) != payload {
+					t.Fatalf("control: pool %d was not readable before rotation: %v", pool, err)
+				}
+			}
+			if otherLatest {
+				putConsistencyObject(t, z, bucket, object, 1, "later version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+			}
+			selected, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID, Versioned: true})
+			mustIAM(t, err)
+			if selected.VersionID != nullVersionID || !selected.ModTime.Equal(before[0].ModTime) || !crypto.S3.IsEncrypted(selected.UserDefined) {
+				t.Fatal("control: the addressed null version must select the SSE-S3 write in pool zero")
+			}
+			logical, err := batchAuthoritativeVersion(t.Context(), z, bucket, object, selected)
+			mustIAM(t, err)
+			if logical.UserTags != "classification=public" || selected.UserTags != logical.UserTags {
+				t.Fatal("control: both current authorization inputs must have public tags")
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review7-rotation-key"}}
+			sub := batchRootSubmitter(t)
+			if otherLatest {
+				batchAuthzPolicy(t.Context(), t, globalIAMSys, "review7-rotate-public", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+				cred := iamAdminUser(t.Context(), t, globalIAMSys, "review7-rotate-user", "review7-rotate-public")
+				sub, err = newBatchJobSubmitter(cred)
+				mustIAM(t, err)
+				if err := sub.authorize(t.Context(), r.rotateAuthzRequests(before[1])...); !isBatchJobAuthzError(err) {
+					t.Fatal("control: the older write in pool one must be denied")
+				}
+			}
+			err = r.KeyRotate(t.Context(), z, selected, sub)
+			if err != nil {
+				t.Logf("rotation refused: %v", err)
+			}
+			for pool, want := range keys {
+				after, readErr := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
+				mustIAM(t, readErr)
+				if otherLatest && pool == 1 && (!maps.Equal(after.UserDefined, before[pool].UserDefined) || !after.ModTime.Equal(before[pool].ModTime) || after.UserTags != before[pool].UserTags) {
+					t.Error("rotation rewrote a copy denied by the submitter's tag-conditioned policy")
+				}
+				kind, _ := crypto.IsEncrypted(after.UserDefined)
+				var got crypto.ObjectKey
+				switch kind {
+				case crypto.S3:
+					got, readErr = crypto.S3.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
+				case crypto.S3KMS:
+					got, readErr = crypto.S3KMS.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
+				default:
+					t.Fatalf("pool %d: rotation lost encryption metadata", pool)
+				}
+				mustIAM(t, readErr)
+				if got != want {
+					t.Errorf("pool %d: rotation changed the data key without reencrypting its ciphertext (rotation error: %v)", pool, err)
+				}
+				data, readErr := readCopy(pool)
+				if readErr != nil || string(data) != payload {
+					t.Errorf("pool %d: rotation left unreadable ciphertext: %v", pool, readErr)
+				}
+			}
+		})
+	}
+}
+
+// A version in a pool that a decommission or a rebalance is moving is not
+// rewritten there: only the source holds it, so the rotation is left to a
+// later run.
+func TestBatchReviewRotationLeavesMovingSource(t *testing.T) {
+	resetTestGlobals()
+	t.Cleanup(resetTestGlobals)
+	z, _ := consistencyPools(t)
+	globalIAMSys.initStore(z, nil)
+	mustIAM(t, globalIAMSys.Load(t.Context(), true))
+	const bucket = "review8-rotation-draining"
+	mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+	_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+	mustIAM(t, err)
+	oldKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("review8-rotation-key")
+	t.Cleanup(func() { GlobalKMS = oldKMS })
+	const object = "rotation/draining"
+	meta := make(map[string]string)
+	key, err := newEncryptMetadata(t.Context(), crypto.S3, "review8-rotation-key", nil, bucket, object, meta, nil)
+	mustIAM(t, err)
+	body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("draining rotation"), key))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+	beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+
+	z.poolMetaMutex.Lock()
+	z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+	z.poolMetaMutex.Unlock()
+
+	r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
+	err = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+	if !errors.Is(err, errBatchRotateMoving) {
+		t.Errorf("rotation of a version only a draining pool holds returned %v", err)
+	}
+	after, readErr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, readErr)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Fatal("rotation rewrote a draining pool")
+	}
+	// The job pauses and resumes in-process once the selected pool stops moving.
+	job := BatchJobRequest{ID: "rotation-moving-resume", Started: UTCNow(), Submitter: batchRootSubmitter(t), KeyRotate: &r}
+	batchTestJobPool(t.Context(), t, z)
+	if err := r.Start(t.Context(), z, job); !isBatchJobUnavailable(err) {
+		t.Fatalf("moving job did not pause: %v", err)
+	}
+	ri := &batchJobInfo{}
+	mustIAM(t, ri.load(t.Context(), z, job))
+	if ri.PausedReason == "" || ri.Failed || ri.Complete || ri.Object != "" || ri.ObjectsFailed != 0 {
+		t.Fatalf("moving job report: %+v", ri)
+	}
+	z.poolMetaMutex.Lock()
+	z.poolMeta.Pools[0].Decommission = nil
+	z.poolMetaMutex.Unlock()
+	mustIAM(t, r.Start(t.Context(), z, job))
+	mustIAM(t, ri.load(t.Context(), z, job))
+	if !ri.Complete || ri.Failed || ri.PausedReason != "" || ri.Objects != 1 {
+		t.Fatalf("resumed rotation: %+v", ri)
+	}
+}
+
+// Once a move copied a version, the rotation leaves it to a later run, so the
+// source stays as the move's cleanup expects it.
+func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) {
+	for _, tc := range []struct {
+		kind   string
+		rotate bool
+	}{{"decommission", false}, {"decommission", true}, {"rebalance", false}, {"rebalance", true}} {
+		name := tc.kind + "/without-rotation"
+		if tc.rotate {
+			name = tc.kind + "/with-rotation"
+		}
+		t.Run(name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			z, _ := migrationTestPools(t)
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review8-rotation-migration"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, err)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review8-rotation-key")
+			t.Cleanup(func() { GlobalKMS = oldKMS })
+			const object = "rotation/migration"
+			meta := make(map[string]string)
+			key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+			mustIAM(t, err)
+			body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation and migration"), key))
+			mustIAM(t, err)
+			before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+			beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			move := migrationTestMover(t, z, 0, tc.kind)
+			mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, beforeInfo)))
+			if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); err != nil {
+				t.Fatalf("control: migration did not copy the version: %v", err)
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
+			var rotationErr error
+			if tc.rotate {
+				moved, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+				mustIAM(t, err)
+				rotationErr = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+				if !errors.Is(rotationErr, errBatchRotateMoving) {
+					t.Errorf("rotation during a move returned %v", rotationErr)
+				}
+				after, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+				mustIAM(t, err)
+				if !maps.Equal(after.UserDefined, moved.UserDefined) || after.UserTags != moved.UserTags {
+					t.Error("rotation during a move rewrote the moved copy")
+				}
+			}
+			after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if !after.ModTime.Equal(beforeInfo.ModTime) || !maps.Equal(after.UserDefined, beforeInfo.UserDefined) {
+				t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
+			}
+			snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
+			if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
+				t.Fatalf("rotation prevented migration cleanup: %v", err)
+			}
+			if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+				t.Fatalf("migration left a source version behind: %v", err)
+			}
+		})
+	}
+}
+
+// The rewrite keeps the tags and the expiry of the version, which a read
+// returns apart from its metadata.
+func TestBatchReviewRotationKeepsTagsAndExpiry(t *testing.T) {
+	for _, mode := range []string{"single-pool", "multiple-pools"} {
+		t.Run(mode, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			var z *erasureServerPools
+			if mode == "single-pool" {
+				z = purgeSinglePool(t, 4)
+			} else {
+				z, _ = consistencyPools(t)
+			}
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review8-rotation-metadata"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, err)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review8-rotation-metadata-key")
+			t.Cleanup(func() { GlobalKMS = oldKMS })
+			const object = "rotation/tagged"
+			meta := make(map[string]string)
+			key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+			mustIAM(t, err)
+			meta[xhttp.AmzObjectTagging] = "classification=private"
+			meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
+			meta["expires"] = UTCNow().Add(time.Hour).Format(http.TimeFormat)
+			body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("metadata preservation"), key))
+			mustIAM(t, err)
+			written := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta})
+			before, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: written.VersionID})
+			mustIAM(t, err)
+			if before.UserTags == "" || before.Expires.IsZero() {
+				t.Fatal("control: stored version is missing the metadata to preserve")
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			mustIAM(t, r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)))
+			after, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if after.UserDefined[crypto.MetaSealedKeyS3] == before.UserDefined[crypto.MetaSealedKeyS3] {
+				t.Error("control: rotation did not rewrap the key")
+			}
+			if after.UserTags != before.UserTags {
+				t.Errorf("rotation changed stored tags: got %q, want %q", after.UserTags, before.UserTags)
+			}
+			if !after.Expires.Equal(before.Expires) {
+				t.Errorf("rotation changed Expires: got %v, want %v", after.Expires, before.Expires)
+			}
+		})
+	}
+}
+
+// Two pools hold the same write with different tags, with or without
+// tagging timestamps. A rotation, also during a move, leaves the tags that
+// the merged version has.
+func TestBatchReviewRotationKeepsMergedTags(t *testing.T) {
+	for _, tc := range []struct {
+		ordered, moving bool
+	}{{false, false}, {false, true}, {true, false}, {true, true}} {
+		t.Run(fmt.Sprintf("timestamp=%t/moving=%t", tc.ordered, tc.moving), func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			z, _ := consistencyPools(t)
+			globalIAMSys.initStore(z, nil)
+			mustIAM(t, globalIAMSys.Load(t.Context(), true))
+			const bucket = "review9-rotation-tag-tie"
+			const object = "rotation/tag-tie"
+			mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+			_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+			mustIAM(t, err)
+			previousKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("review9-tag-tie-key")
+			t.Cleanup(func() { GlobalKMS = previousKMS })
+			meta := make(map[string]string)
+			key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+			mustIAM(t, err)
+			meta[xhttp.AmzObjectTagging] = "classification=private"
+			if tc.ordered {
+				meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
+			}
+			other := maps.Clone(meta)
+			other[xhttp.AmzObjectTagging] = "classification=public"
+			body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("same write"), key))
+			mustIAM(t, err)
+			before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+			putConsistencyObject(t, z, bucket, object, 1, string(body), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: other})
+			prior, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if prior.UserTags != "classification=private" {
+				t.Fatalf("control: logical tags=%q", prior.UserTags)
+			}
+			if tc.moving {
+				migrationTestMover(t, z, 0, "decommission")
+			}
+			r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			rotationErr := r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
+			after, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+			mustIAM(t, err)
+			if after.UserTags != prior.UserTags {
+				t.Fatalf("rotation changed logical tags from %q to %q (rotation error: %v)", prior.UserTags, after.UserTags, rotationErr)
+			}
+		})
+	}
+}
+
+// batchRotateFlipDisk starts a decommission of pool 0 at the flipAt-th
+// version read, after the rotation's first read of the version.
+type batchRotateFlipDisk struct {
+	StorageAPI
+	z       *erasureServerPools
+	reads   *atomic.Int32
+	flipAt  int32
+	flipped *atomic.Bool
+}
+
+func (d batchRotateFlipDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
+	n := d.reads.Add(1)
+	if n == d.flipAt && d.flipped.CompareAndSwap(false, true) {
+		d.z.poolMetaMutex.Lock()
+		d.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+		d.z.poolMetaMutex.Unlock()
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
+}
+
+// A move that starts while the rotation authorizes and rewraps the version
+// defers it: the state is checked again right before the write, where the
+// pools layer checks it.
+func TestBatchReviewRotationRechecksPoolBeforeWrite(t *testing.T) {
+	resetTestGlobals()
+	t.Cleanup(resetTestGlobals)
+	z, _ := consistencyPools(t)
+	globalIAMSys.initStore(z, nil)
+	mustIAM(t, globalIAMSys.Load(t.Context(), true))
+	const bucket = "review11-rotation-window"
+	const object = "rotation/window"
+	mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+	_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+	mustIAM(t, err)
+	oldKMS := GlobalKMS
+	GlobalKMS = kms.NewStub("review11-rotation-key")
+	t.Cleanup(func() { GlobalKMS = oldKMS })
+
+	meta := make(map[string]string)
+	key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+	mustIAM(t, err)
+	body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation window"), key))
+	mustIAM(t, err)
+	before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{
+		Versioned:   true,
+		UserDefined: meta,
+		MTime:       UTCNow().Add(-time.Hour),
+	})
+	beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+
+	var reads atomic.Int32
+	var flipped atomic.Bool
+	firstPass := int32(0)
+	for _, pool := range z.serverPools {
+		firstPass += int32(pool.SetDriveCount())
+	}
+	var restore []func()
+	for _, pool := range z.serverPools {
+		restore = append(restore, conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI {
+			return batchRotateFlipDisk{
+				StorageAPI: d,
+				z:          z,
+				reads:      &reads,
+				flipAt:     firstPass + 1,
+				flipped:    &flipped,
+			}
+		}))
+	}
+	t.Cleanup(func() {
+		for _, fn := range restore {
+			fn()
+		}
+	})
+
+	r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+	gotErr := r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+	if !flipped.Load() {
+		t.Fatal("fault injection did not occur after the initial rotation read")
+	}
+	if !errors.Is(gotErr, errBatchRotateMoving) {
+		t.Errorf("rotation did not defer after the pool became moving during pre-work: %v", gotErr)
+	}
+	after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+	mustIAM(t, err)
+	if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
+		t.Error("rotation rewrote the pool after it became moving during pre-work")
+	}
+}

cmd/batch-authz-review_test.go

@@ -0,0 +1,287 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func batchPullJob(bucket string) BatchJobReplicateV1 {
+	return BatchJobReplicateV1{
+		Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+		Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+	}
+}
+
+// A pull does not read the destination version it may replace, so a policy
+// that conditions the write on that version's tags refuses it; a version
+// condition sees the version the pull writes.
+func TestBatchPullDestinationConditions(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		action    policy.Action
+		marker    bool
+		condition func(versionID string) string
+	}{
+		{"tags-put", policy.ReplicateObjectAction, false, func(string) string {
+			return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
+		}},
+		{"tags-marker", policy.ReplicateDeleteAction, true, func(string) string {
+			return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
+		}},
+		{"version-put", policy.PutObjectAction, false, func(v string) string {
+			return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v)
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"})
+			batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID)))
+			cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard")
+			sub, err := newBatchJobSubmitter(cred)
+			mustIAM(t, err)
+			job := batchPullJob(f.bucket)
+			err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+				Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag",
+				DeleteMarker: tc.marker, UserDefined: map[string]string{},
+			}, false, sub)
+			if !isBatchJobAuthzError(err) {
+				t.Errorf("pull returned %v, want a refusal", err)
+			}
+			after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID})
+			if err != nil || after.ETag != before.ETag {
+				t.Errorf("refused pull changed the destination version: %v", err)
+			}
+		})
+	}
+}
+
+// A pull does not encrypt what it writes; encryption headers reported by the
+// source do not satisfy a destination policy that requires encryption.
+func TestBatchPullEncryptionCondition(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", `
+{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	job := batchPullJob(f.bucket)
+	version := mustGetUUID()
+	err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{
+		Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag",
+		UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"},
+	}, false, sub)
+	if err == nil {
+		oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version})
+		mustIAM(t, gerr)
+		if !crypto.S3KMS.IsEncrypted(oi.UserDefined) {
+			t.Fatal("a pull was allowed by an encryption condition it did not meet")
+		}
+	}
+}
+
+// A conditional Deny on an S3 Tables action covers the S3 data actions it
+// implies, under the tables resource form; the job refuses rather than skip it.
+func TestBatchJobTableConditionalDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context")
+	if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) {
+		t.Fatal("control: with the request context the Deny applies")
+	}
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) {
+		t.Fatalf("authorize returned %v, want a refusal", err)
+	}
+}
+
+type batchArchiveRevocation struct {
+	ObjectLayer
+	revoke func()
+}
+
+func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+	if object == "archive/second" {
+		a.revoke()
+	}
+	return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
+}
+
+// Each snowball entry is authorized for the submitter as it is then.
+func TestBatchPushSnowballEntryRevalidation(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/")
+	mustIAM(t, err)
+	previous := newGlobalAuthZPluginFn()
+	setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
+		URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() },
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil
+		}),
+	}))
+	t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
+	entries := []ObjectInfo{
+		batchPutVersion(t, f.obj, f.bucket, "archive/first", nil),
+		batchPutVersion(t, f.obj, f.bucket, "archive/second", nil),
+	}
+	remote, err := minio.New("archive.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			if r.Body != nil {
+				_, _ = io.Copy(io.Discard, r.Body)
+			}
+			return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+		}),
+	})
+	mustIAM(t, err)
+	job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}}
+	job.Source.Snowball.InMemory = ptr(true)
+	job.Source.Snowball.Compress = ptr(false)
+	job.Source.Snowball.SkipErrs = ptr(false)
+	api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }}
+	if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) {
+		t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err)
+	}
+}
+
+// A definition holds one job type; one with several is refused, so that what
+// is authorized is what runs.
+func TestBatchJobSeveralSections(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "data/a")
+	job := f.expireYAML("") + `keyrotate:
+  apiVersion: v1
+  bucket: other
+  encryption:
+    type: sse-s3
+`
+	rec, queued := f.submit(t, globalActiveCred, job)
+	if queued != nil {
+		f.run(t, queued)
+	}
+	if rec.Code != http.StatusBadRequest {
+		t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String())
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a definition with two job types ran")
+	}
+}
+
+// Prefixes are walked once each in one order: a cursor in a later prefix is
+// after every key of the earlier ones, and an overlapping prefix is not
+// walked twice.
+func TestBatchExpirePrefixes(t *testing.T) {
+	if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" {
+		t.Fatalf("canonical prefixes %q", got)
+	}
+	if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" {
+		t.Fatalf("canonical prefixes %q", got)
+	}
+	f := newBatchAuthzFixture(t)
+	done := f.putVersions(t, "a/done")
+	todo := f.putVersions(t, "z/todo")
+	batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD"))
+	batchPutVersion(t, f.obj, f.bucket, "x/data", nil)
+	rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: ["z/", "x/", "a/", "x/"]
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	// A run that completed a/ and stopped.
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"}
+	mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
+	f.run(t, job)
+	if !f.versionExists(t, "a/done", done) {
+		t.Error("resume did again a key before the cursor")
+	}
+	if f.versionExists(t, "z/todo", todo) {
+		t.Error("resume skipped a later prefix")
+	}
+	if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 {
+		t.Errorf("a locked version under two equal prefixes was counted %d times", got)
+	}
+}
+
+type batchPartialWalk struct {
+	ObjectLayer
+	version ObjectInfo
+	cancel  context.CancelFunc
+	read    chan struct{}
+}
+
+func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
+	go func() {
+		defer close(results)
+		results <- itemOrErr[ObjectInfo]{Item: a.version}
+		if a.cancel != nil {
+			<-a.read
+			a.cancel()
+			return
+		}
+		results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")}
+	}()
+	return nil
+}
+
+func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
+	if a.read != nil {
+		close(a.read)
+	}
+	return a.version, nil
+}
+
+// A rotation whose walk fails or is canceled within a key keeps that key
+// after its cursor and does not report completion.
+func TestBatchKeyRotationIncompleteWalk(t *testing.T) {
+	for _, canceled := range []bool{false, true} {
+		t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			ctx, cancel := context.WithCancel(f.ctx)
+			defer cancel()
+			meta := batchLockMeta("LEGALHOLD")
+			meta[crypto.MetaSealedKeyS3] = "test-sealed-key"
+			api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}}
+			if canceled {
+				api.cancel, api.read = cancel, make(chan struct{})
+			}
+			job := BatchJobRequest{
+				ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+				KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}},
+			}
+			_ = job.KeyRotate.Start(ctx, api, job)
+			ri := f.jobInfo(t, &job)
+			if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 {
+				t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked)
+			}
+		})
+	}
+}

cmd/batch-authz-snapshot_test.go

@@ -0,0 +1,205 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"github.com/puzpuzpuz/xsync/v3"
+)
+
+// batchBlockedIAMStore holds the lazy load of one user's policy mapping, or
+// of one policy document, until released or until the load's context ends.
+type batchBlockedIAMStore struct {
+	IAMStorageAPI
+	mapping string // access key whose mapping load blocks
+	doc     string // policy whose document load blocks
+	started chan struct{}
+	release chan struct{}
+	once    sync.Once
+}
+
+func (s *batchBlockedIAMStore) block(ctx context.Context) error {
+	s.once.Do(func() { close(s.started) })
+	select {
+	case <-s.release:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (s *batchBlockedIAMStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+	if name == s.mapping && !isGroup {
+		if err := s.block(ctx); err != nil {
+			return err
+		}
+	}
+	return s.IAMStorageAPI.loadMappedPolicy(ctx, name, userType, isGroup, m)
+}
+
+func (s *batchBlockedIAMStore) loadPolicyDoc(ctx context.Context, name string, out map[string]PolicyDoc) error {
+	if name == s.doc {
+		if err := s.block(ctx); err != nil {
+			return err
+		}
+	}
+	return s.IAMStorageAPI.loadPolicyDoc(ctx, name, out)
+}
+
+// The batch path and the request path share one IAM cache. A lazy load that
+// batch authorization needs, of a mapping that is never cached (a user whose
+// policies come from a group) or of an evicted policy document, must not
+// hold request authentication and cached authorization while storage is
+// slow, and must end at the deadline of the job's context.
+func TestBatchAuthorizeSlowLoadKeepsCachedAuthorizationAvailable(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		// submitter returns the submitter whose authorization lazily loads,
+		// and arranges for blocked to hold that load.
+		submitter func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter
+	}{
+		{"mapping", func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter {
+			// No mapping of the user itself is stored, so every lookup of
+			// its policies reads storage before it falls back to the group.
+			cred := iamAdminUser(f.ctx, t, f.sys, "snapshot-group-user", "")
+			_, err := f.sys.AddUsersToGroup(f.ctx, "snapshot-group", []string{cred.AccessKey})
+			mustIAM(t, err)
+			_, err = f.sys.PolicyDBSet(f.ctx, "snapshot-group", "snapshot-full", regUser, true)
+			mustIAM(t, err)
+			blocked.mapping = cred.AccessKey
+			sub, err := newBatchJobSubmitter(cred)
+			mustIAM(t, err)
+			return sub
+		}},
+		{"document", func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter {
+			batchAuthzPolicy(f.ctx, t, f.sys, "snapshot-evicted", batchAuthzFullAccess)
+			cred := iamAdminUser(f.ctx, t, f.sys, "snapshot-doc-user", "snapshot-evicted")
+			cache := f.sys.store.lock()
+			delete(cache.iamPolicyDocsMap, "snapshot-evicted")
+			f.sys.store.unlock()
+			blocked.doc = "snapshot-evicted"
+			sub, err := newBatchJobSubmitter(cred)
+			mustIAM(t, err)
+			return sub
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			batchAuthzPolicy(f.ctx, t, f.sys, "snapshot-full", batchAuthzFullAccess)
+			cached := iamAdminUser(f.ctx, t, f.sys, "snapshot-cached-user", "snapshot-full")
+			store := f.sys.store
+			original := store.IAMStorageAPI
+			blocked := &batchBlockedIAMStore{IAMStorageAPI: original, started: make(chan struct{}), release: make(chan struct{})}
+			sub := tc.submitter(t, f, blocked)
+			req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
+			if err := sub.authorize(f.ctx, req); err != nil {
+				t.Fatalf("control: %v", err)
+			}
+			store.IAMStorageAPI = blocked
+			var wg sync.WaitGroup
+			var once sync.Once
+			unblock := func() { once.Do(func() { close(blocked.release) }) }
+			t.Cleanup(func() { unblock(); wg.Wait(); store.IAMStorageAPI = original })
+
+			slowDone := make(chan error, 1)
+			wg.Go(func() { slowDone <- sub.authorize(f.ctx, req) })
+			select {
+			case <-blocked.started:
+			case <-time.After(5 * time.Second):
+				t.Fatal("the lazy load did not start")
+			}
+			cachedDone := make(chan bool, 2)
+			wg.Go(func() {
+				_, ok := f.sys.GetUser(f.ctx, cached.AccessKey)
+				cachedDone <- ok
+			})
+			wg.Go(func() {
+				cachedDone <- f.sys.IsAllowed(policy.Args{AccountName: cached.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "key"})
+			})
+			for range 2 {
+				select {
+				case ok := <-cachedDone:
+					if !ok {
+						t.Fatal("cached authentication or authorization was rejected")
+					}
+				case <-time.After(time.Second):
+					t.Fatal("a slow IAM load during batch authorization blocked cached authentication")
+				}
+			}
+			unblock()
+			if err := <-slowDone; err != nil {
+				t.Fatalf("authorization after the load completed: %v", err)
+			}
+
+			// A load that does not return ends with the job's context, as an
+			// unavailable dependency, not with the storage layer's patience.
+			blocked.release = make(chan struct{})
+			once = sync.Once{}
+			unblock = func() { once.Do(func() { close(blocked.release) }) }
+			ctx, cancel := context.WithTimeout(f.ctx, 200*time.Millisecond)
+			defer cancel()
+			deadlineDone := make(chan error, 1)
+			wg.Go(func() { deadlineDone <- sub.authorize(ctx, req) })
+			select {
+			case err := <-deadlineDone:
+				if !isBatchJobUnavailable(err) {
+					t.Fatalf("a load that ended with the job's context: %v", err)
+				}
+			case <-time.After(3 * time.Second):
+				t.Fatal("a lazy load outlived the job's context")
+			}
+		})
+	}
+}
+
+// A policy that a mapping still names after it was deleted takes no part in
+// the decision, for a batch job as for a request: the remaining mapped
+// policies decide. An STS parent user's mapping can name a deleted policy,
+// because deleting a policy checks only user and group mappings for use.
+func TestBatchAuthorizeSkipsDeletedMappedPolicy(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
+	batchAuthzPolicy(f.ctx, t, f.sys, "mapped-deleted", `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}`)
+	parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "")
+	_, err := f.sys.PolicyDBSet(f.ctx, parent.AccessKey, "mapped-full,mapped-deleted", stsUser, false)
+	mustIAM(t, err)
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
+	if err := sub.authorize(f.ctx, req); err != nil {
+		t.Fatalf("control: %v", err)
+	}
+	p, err := sub.principal(f.ctx)
+	mustIAM(t, err)
+	args := policy.Args{AccountName: cred.AccessKey, Action: req.action, BucketName: req.bucket, ObjectName: req.object, Claims: p.claims, ConditionValues: p.conditionValues(req)}
+
+	mustIAM(t, f.sys.DeletePolicy(f.ctx, "mapped-deleted", true))
+	if !f.sys.IsAllowed(args) {
+		t.Fatal("the request path denied the remaining mapped policy")
+	}
+	if err := sub.authorize(f.ctx, req); err != nil {
+		t.Fatalf("batch authorization diverged from the request after a mapped policy was deleted: %v", err)
+	}
+
+	// With every mapped policy deleted, both deny; the job is refused, not
+	// paused, as nothing is unavailable.
+	mustIAM(t, f.sys.DeletePolicy(f.ctx, "mapped-full", true))
+	if f.sys.IsAllowed(args) {
+		t.Fatal("the request path allowed without any mapped policy")
+	}
+	if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
+		t.Fatalf("batch authorization without any mapped policy: %v", err)
+	}
+}

cmd/batch-authz_gen.go

@@ -0,0 +1,260 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package cmd
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *batchJobSubmitter) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "v":
+			z.Version, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		case "t":
+			z.Type, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "ak":
+			z.AccessKey, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "AccessKey")
+				return
+			}
+		case "pu":
+			z.ParentUser, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "ParentUser")
+				return
+			}
+		case "ua":
+			z.UpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "UpdatedAt")
+				return
+			}
+		case "rb":
+			z.RevokedBefore, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "RevokedBefore")
+				return
+			}
+		case "bd":
+			z.Binding, err = dc.ReadBytes(z.Binding)
+			if err != nil {
+				err = msgp.WrapError(err, "Binding")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *batchJobSubmitter) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 7
+	// write "v"
+	err = en.Append(0x87, 0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Version)
+	if err != nil {
+		err = msgp.WrapError(err, "Version")
+		return
+	}
+	// write "t"
+	err = en.Append(0xa1, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Type)
+	if err != nil {
+		err = msgp.WrapError(err, "Type")
+		return
+	}
+	// write "ak"
+	err = en.Append(0xa2, 0x61, 0x6b)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.AccessKey)
+	if err != nil {
+		err = msgp.WrapError(err, "AccessKey")
+		return
+	}
+	// write "pu"
+	err = en.Append(0xa2, 0x70, 0x75)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.ParentUser)
+	if err != nil {
+		err = msgp.WrapError(err, "ParentUser")
+		return
+	}
+	// write "ua"
+	err = en.Append(0xa2, 0x75, 0x61)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.UpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "UpdatedAt")
+		return
+	}
+	// write "rb"
+	err = en.Append(0xa2, 0x72, 0x62)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.RevokedBefore)
+	if err != nil {
+		err = msgp.WrapError(err, "RevokedBefore")
+		return
+	}
+	// write "bd"
+	err = en.Append(0xa2, 0x62, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.Binding)
+	if err != nil {
+		err = msgp.WrapError(err, "Binding")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *batchJobSubmitter) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 7
+	// string "v"
+	o = append(o, 0x87, 0xa1, 0x76)
+	o = msgp.AppendInt(o, z.Version)
+	// string "t"
+	o = append(o, 0xa1, 0x74)
+	o = msgp.AppendString(o, z.Type)
+	// string "ak"
+	o = append(o, 0xa2, 0x61, 0x6b)
+	o = msgp.AppendString(o, z.AccessKey)
+	// string "pu"
+	o = append(o, 0xa2, 0x70, 0x75)
+	o = msgp.AppendString(o, z.ParentUser)
+	// string "ua"
+	o = append(o, 0xa2, 0x75, 0x61)
+	o = msgp.AppendTime(o, z.UpdatedAt)
+	// string "rb"
+	o = append(o, 0xa2, 0x72, 0x62)
+	o = msgp.AppendTime(o, z.RevokedBefore)
+	// string "bd"
+	o = append(o, 0xa2, 0x62, 0x64)
+	o = msgp.AppendBytes(o, z.Binding)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *batchJobSubmitter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "v":
+			z.Version, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		case "t":
+			z.Type, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "ak":
+			z.AccessKey, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "AccessKey")
+				return
+			}
+		case "pu":
+			z.ParentUser, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ParentUser")
+				return
+			}
+		case "ua":
+			z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "UpdatedAt")
+				return
+			}
+		case "rb":
+			z.RevokedBefore, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RevokedBefore")
+				return
+			}
+		case "bd":
+			z.Binding, bts, err = msgp.ReadBytesBytes(bts, z.Binding)
+			if err != nil {
+				err = msgp.WrapError(err, "Binding")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *batchJobSubmitter) Msgsize() (s int) {
+	s = 1 + 2 + msgp.IntSize + 2 + msgp.StringPrefixSize + len(z.Type) + 3 + msgp.StringPrefixSize + len(z.AccessKey) + 3 + msgp.StringPrefixSize + len(z.ParentUser) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.BytesPrefixSize + len(z.Binding)
+	return
+}

cmd/batch-authz_gen_test.go

@@ -0,0 +1,123 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalbatchJobSubmitter(t *testing.T) {
+	v := batchJobSubmitter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgbatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgbatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalbatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodebatchJobSubmitter(t *testing.T) {
+	v := batchJobSubmitter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodebatchJobSubmitter Msgsize() is inaccurate")
+	}
+
+	vn := batchJobSubmitter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodebatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodebatchJobSubmitter(b *testing.B) {
+	v := batchJobSubmitter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-discovery_test.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+	"testing/synctest"
+	"time"
+)
+
+type batchDiscoveryLayer struct {
+	ObjectLayer
+	walk func(context.Context, chan<- itemOrErr[ObjectInfo]) error
+	read func(context.Context, string) ([]byte, error)
+}
+
+func (a batchDiscoveryLayer) Walk(ctx context.Context, _, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
+	return a.walk(ctx, results)
+}
+
+func (a batchDiscoveryLayer) GetObjectNInfo(ctx context.Context, _, object string, _ *HTTPRangeSpec, _ http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+	data, err := a.read(ctx, object)
+	if err != nil {
+		return nil, err
+	}
+	return NewGetObjectReaderFromReader(bytes.NewReader(data), ObjectInfo{Name: object}, opts)
+}
+
+// A partially successful startup scan must retry the missing work without
+// re-admitting work that finished while the scan was waiting to retry.
+func TestBatchDiscoveryRecoversInProcess(t *testing.T) {
+	for _, failure := range []string{"walk-start", "walk-stream", "manifest-read"} {
+		t.Run(failure, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				ctx, cancel := context.WithCancel(t.Context())
+				defer cancel()
+				pool := &BatchJobPool{ctx: ctx, jobCh: make(chan *BatchJobRequest, 10)}
+				attempts := 0
+				pool.objLayer = batchDiscoveryLayer{
+					walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
+						attempts++
+						if attempts == 1 && failure == "walk-start" {
+							return errErasureReadQuorum
+						}
+						defer close(results)
+						results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "finished"}}
+						if attempts == 1 && failure == "walk-stream" {
+							results <- itemOrErr[ObjectInfo]{Err: errErasureReadQuorum}
+						} else {
+							results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "pending"}}
+						}
+						return nil
+					},
+					read: func(_ context.Context, name string) ([]byte, error) {
+						if attempts == 1 && failure == "manifest-read" && name == "pending" {
+							return nil, errErasureReadQuorum
+						}
+						job := BatchJobRequest{ID: name}
+						return job.MarshalMsg(nil)
+					},
+				}
+				done := make(chan struct{})
+				go func() {
+					defer close(done)
+					pool.resume(func() time.Duration { return 0 })
+				}()
+				synctest.Wait()
+				if failure != "walk-start" {
+					select {
+					case job := <-pool.jobCh:
+						if job.ID != "finished" || !pool.beginJob(job) {
+							t.Fatal("first scan did not admit the healthy manifest")
+						}
+						pool.finishJob(job, nil)
+					default:
+						t.Fatal("first scan lost the healthy manifest")
+					}
+				}
+				select {
+				case <-done:
+				case <-time.After(20 * time.Minute):
+					t.Fatal("discovery failed to finish after storage recovered")
+				}
+				if attempts != 2 {
+					t.Fatalf("discovery attempts = %d, want 2", attempts)
+				}
+				want := 1
+				if failure == "walk-start" {
+					want = 2
+				}
+				if len(pool.jobCh) != want {
+					t.Fatalf("queued %d jobs, want %d; completed work was re-admitted or pending work was lost", len(pool.jobCh), want)
+				}
+				for len(pool.jobCh) > 0 {
+					job := <-pool.jobCh
+					if failure != "walk-start" && job.ID != "pending" {
+						t.Fatalf("re-admitted %q", job.ID)
+					}
+					if !pool.beginJob(job) {
+						t.Fatal("recovered job has no owner")
+					}
+					pool.finishJob(job, nil)
+				}
+			})
+		})
+	}
+}
+
+func TestBatchDiscoverySkipsUnusableManifests(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
+		attempts := 0
+		pool.objLayer = batchDiscoveryLayer{
+			walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
+				attempts++
+				defer close(results)
+				for _, name := range []string{"canceled", "deleted", "malformed", "future", "empty-id"} {
+					results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: name}}
+				}
+				return nil
+			},
+			read: func(_ context.Context, name string) ([]byte, error) {
+				switch name {
+				case "deleted":
+					return nil, errConfigNotFound
+				case "malformed":
+					return []byte("bad manifest"), nil
+				case "future":
+					return []byte("SILO-BATCH-V999\n"), nil
+				case "empty-id":
+					return (&BatchJobRequest{}).MarshalMsg(nil)
+				}
+				// The administrator cancels after Walk, during manifest load.
+				mustIAM(t, pool.canceler(name, true))
+				job := BatchJobRequest{ID: name}
+				return job.MarshalMsg(nil)
+			},
+		}
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pool.resume(func() time.Duration { return 0 })
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Hour):
+			t.Fatal("unusable manifests kept discovery retrying")
+		}
+		if len(pool.jobCh) != 0 || attempts != 1 {
+			t.Fatalf("canceled work was queued or retried: queue=%d scans=%d", len(pool.jobCh), attempts)
+		}
+	})
+}
+
+func TestBatchDiscoveryShutdownDuringBackoff(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+		attempts := 0
+		pool := &BatchJobPool{ctx: ctx, objLayer: batchDiscoveryLayer{
+			walk: func(context.Context, chan<- itemOrErr[ObjectInfo]) error {
+				attempts++
+				return errErasureReadQuorum
+			},
+		}}
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pool.resume(func() time.Duration { return 0 })
+		}()
+		synctest.Wait()
+		select {
+		case <-done:
+			t.Fatal("discovery abandoned a transient startup error")
+		default:
+		}
+		cancel()
+		select {
+		case <-done:
+		case <-time.After(time.Second):
+			t.Fatal("discovery did not stop on shutdown")
+		}
+		if attempts != 1 {
+			t.Fatalf("discovery scanned after shutdown: %d", attempts)
+		}
+	})
+}
+
+func TestBatchDiscoveryKeepsNewlyFinishedJobsTerminal(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
+		attempts := 0
+		pool.objLayer = batchDiscoveryLayer{
+			walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
+				attempts++
+				if attempts == 1 {
+					// This job was submitted and failed while startup discovery
+					// had not yet read any manifests. The retry must not rerun it.
+					job := &BatchJobRequest{ID: "new-job"}
+					mustIAM(t, pool.queueJob(job))
+					if !pool.beginJob(<-pool.jobCh) {
+						t.Fatal("submitted job has no owner")
+					}
+					pool.finishJob(job, errErasureWriteQuorum)
+					return errErasureReadQuorum
+				}
+				defer close(results)
+				if owner := pool.jobOwners["new-job"]; owner == nil || owner.state != "terminal" || owner.req != nil || owner.cancel != nil {
+					t.Fatalf("terminal marker lost or retained the job payload: %+v", owner)
+				}
+				results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "new-job"}}
+				return nil
+			},
+			read: func(_ context.Context, name string) ([]byte, error) {
+				return (&BatchJobRequest{ID: name}).MarshalMsg(nil)
+			},
+		}
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			pool.resume(func() time.Duration { return 0 })
+		}()
+		select {
+		case <-done:
+		case <-time.After(time.Hour):
+			t.Fatal("discovery did not recover")
+		}
+		if attempts != 2 || len(pool.jobCh) != 0 {
+			t.Fatalf("discovery re-admitted a failed job: scans=%d queue=%d", attempts, len(pool.jobCh))
+		}
+		if !pool.discoveryDone || len(pool.jobOwners) != 0 {
+			t.Fatal("completed discovery retained terminal job markers")
+		}
+		job := &BatchJobRequest{ID: "after-discovery"}
+		mustIAM(t, pool.queueJob(job))
+		if !pool.beginJob(<-pool.jobCh) {
+			t.Fatal("new submission after discovery has no owner")
+		}
+		pool.finishJob(job, nil)
+		if len(pool.jobOwners) != 0 {
+			t.Fatal("normal completion after discovery leaked an owner")
+		}
+	})
+}

cmd/batch-expire-completed_test.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"go.yaml.in/yaml/v3"
+)
+
+// A completed expire job is not run again when the server restarts, as
+// replication and key rotation jobs are not: in particular it is not checked
+// against its submitter again, which could turn it into a failed job.
+
+// An expire job that completed before the upgrade records no submitter.
+func TestBatchExpireCompletedLegacyStaysComplete(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "data/a")
+	job := &BatchJobRequest{}
+	mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
+	job.ID = "expire-legacydone" + getKeySeparator() + "0"
+	job.User = globalActiveCred.AccessKey
+	job.Started = time.Now()
+	mustIAM(t, job.save(f.ctx, f.obj))
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Complete: true, Objects: 3}
+	mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
+
+	loaded := &BatchJobRequest{}
+	mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+	f.run(t, loaded) // resumed after a restart
+	got := f.jobInfo(t, job)
+	if !got.Complete || got.Failed || got.FailureReason != "" {
+		t.Fatalf("complete=%v failed=%v reason=%q, want the completed state kept", got.Complete, got.Failed, got.FailureReason)
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Error("a completed job ran again")
+	}
+}
+
+// A new expire job completed with a temporary credential that has expired
+// since.
+func TestBatchExpireCompletedAfterCredentialExpiry(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(3 * time.Second).Unix()}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	f.run(t, job)
+	if f.versionExists(t, "data/a", older) {
+		t.Fatal("the job did not expire the older version")
+	}
+	if ri := f.jobInfo(t, job); !ri.Complete {
+		t.Fatalf("complete=%v failed=%v", ri.Complete, ri.Failed)
+	}
+	time.Sleep(time.Until(cred.Expiration) + time.Second)
+
+	loaded := &BatchJobRequest{}
+	mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+	f.run(t, loaded) // resumed after a restart
+	if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed || ri.FailureReason != "" {
+		t.Errorf("complete=%v failed=%v reason=%q after a restart, want the completed state kept", ri.Complete, ri.Failed, ri.FailureReason)
+	}
+
+	// Control: a job that did not complete is still refused once its
+	// credential has expired.
+	f.putVersions(t, "data/b")
+	pending := &BatchJobRequest{}
+	mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), pending))
+	pending.ID = "expire-pending" + getKeySeparator() + "0"
+	pending.User = cred.AccessKey
+	pending.Started = time.Now()
+	pending.Submitter = loaded.Submitter
+	mustIAM(t, pending.save(f.ctx, f.obj))
+	f.run(t, pending)
+	if ri := f.jobInfo(t, pending); ri.Complete || !ri.Failed || ri.FailureReason == "" {
+		t.Errorf("pending job with an expired submitter: complete=%v failed=%v reason=%q, want refused", ri.Complete, ri.Failed, ri.FailureReason)
+	}
+}

cmd/batch-expire-purge-authz_test.go

@@ -0,0 +1,572 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/config/storageclass"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A batch expiration that keeps no versions purges each key it matched under
+// the purge's lock. What it authorizes for the submitter, and checks for
+// Object Lock, are the versions the purge deletes then, not the versions the
+// walk saw before.
+
+// batchPurgeHook runs change once, after the walk of a job and before its
+// first purge of all versions.
+type batchPurgeHook struct {
+	ObjectLayer
+	once   sync.Once
+	change func()
+}
+
+func (h *batchPurgeHook) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	if opts.DeleteAllVersions {
+		h.once.Do(h.change)
+	}
+	return h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+}
+
+func (f *batchAuthzFixture) purgeYAML(prefix string) string {
+	return fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: %q
+  rules:
+    - type: object
+      purge:
+        retainVersions: 0
+`, f.bucket, prefix)
+}
+
+// runChanged submits a purge of prefix as cred and runs it, with change made
+// between its walk and its purge, and returns the job's final state.
+func (f *batchAuthzFixture) runChanged(t *testing.T, cred auth.Credentials, prefix string, change func()) *batchJobInfo {
+	t.Helper()
+	rec, job := f.submit(t, cred, f.purgeYAML(prefix))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	hook := &batchPurgeHook{ObjectLayer: f.obj, change: change}
+	if err := job.Expire.Start(f.ctx, hook, *job); err != nil {
+		t.Logf("start: %v", err)
+	}
+	return f.jobInfo(t, job)
+}
+
+// putOlder writes a version of key older than every version it has, as a
+// replicated or pulled version can be, with metadata meta.
+func (f *batchAuthzFixture) putOlder(t *testing.T, key, versionID string, meta map[string]string) {
+	t.Helper()
+	data := []byte("older " + key)
+	_, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
+		ObjectOptions{Versioned: true, VersionID: versionID, MTime: UTCNow().Add(-time.Hour), UserDefined: meta})
+	mustIAM(t, err)
+}
+
+func batchPurgeVersionIDs(t *testing.T, f *batchAuthzFixture, key string) []string {
+	t.Helper()
+	var ids []string
+	marker, vmarker := "", ""
+	for {
+		res, err := f.obj.ListObjectVersions(f.ctx, f.bucket, key, marker, vmarker, "", 1000)
+		mustIAM(t, err)
+		for _, o := range res.Objects {
+			if o.Name == key {
+				ids = append(ids, o.VersionID)
+			}
+		}
+		if !res.IsTruncated {
+			return ids
+		}
+		marker, vmarker = res.NextMarker, res.NextVersionIDMarker
+	}
+}
+
+// A version that the submitter may not expire, because of its tags or its
+// version ID, is refused although the walk did not see it so: the job fails
+// and the key keeps all its versions.
+func TestBatchExpirePurgeAuthorizesCurrentVersions(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		action string // the action a conditional Deny denies
+		deny   string // its condition
+		// change returns the version IDs that change made.
+		change func(t *testing.T, f *batchAuthzFixture, key, older string) []string
+	}{
+		{
+			name:   "tagged-after-walk",
+			action: "s3:GetObjectVersion",
+			deny:   `{"StringEquals":{"s3:ExistingObjectTag/keep":["yes"]}}`,
+			change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
+				_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older})
+				mustIAM(t, err)
+				return nil
+			},
+		},
+		{
+			name:   "version-after-walk",
+			action: "s3:DeleteObjectVersion",
+			deny:   `{"StringEquals":{"s3:versionid":["%s"]}}`,
+			change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
+				f.putOlder(t, key, batchPurgeDeniedVersion, nil)
+				return []string{batchPurgeDeniedVersion}
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			deny := tc.deny
+			if strings.Contains(deny, "%s") {
+				deny = fmt.Sprintf(deny, batchPurgeDeniedVersion)
+			}
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-purge", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["`+tc.action+`"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":`+deny+`}`)
+			cred := iamAdminUser(f.ctx, t, f.sys, "batch-purge-user", "batch-purge")
+			const key = "purge/key"
+			older := f.putVersions(t, key)
+			want := batchPurgeVersionIDs(t, f, key)
+			ri := f.runChanged(t, cred, "purge/", func() {
+				want = append(want, tc.change(t, f, key, older)...)
+			})
+			for _, id := range want {
+				if !f.versionExists(t, key, id) {
+					t.Errorf("version %s was purged although the submitter may not delete it", id)
+				}
+			}
+			if !ri.Failed || ri.Complete || ri.FailureReason == "" {
+				t.Errorf("job complete=%v failed=%v reason=%q, want a refused job", ri.Complete, ri.Failed, ri.FailureReason)
+			}
+			if ri.Objects != 0 || ri.KeysRefused != 0 {
+				t.Errorf("job counts objects=%d refused=%d", ri.Objects, ri.KeysRefused)
+			}
+		})
+	}
+}
+
+const batchPurgeDeniedVersion = "6c4b2f8e-1d3a-4e5b-9c7d-0a1b2c3d4e5f"
+
+// A version that Object Lock protects refuses the whole key, also when it
+// was added after the walk. The job completes with the key counted as
+// refused, not failed.
+func TestBatchExpirePurgeRefusesLockedAfterWalk(t *testing.T) {
+	for _, kind := range batchLockKinds {
+		t.Run(kind, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			enableBucketObjectLock(t, f.bucket)
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+			cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			const key = "purge/locked"
+			f.putVersions(t, key)
+			want := batchPurgeVersionIDs(t, f, key)
+			locked := mustGetUUID()
+			ri := f.runChanged(t, cred, "purge/", func() {
+				f.putOlder(t, key, locked, batchLockMeta(kind))
+				want = append(want, locked)
+			})
+			for _, id := range want {
+				if !f.versionExists(t, key, id) {
+					t.Errorf("version %s of a key with a locked version was purged", id)
+				}
+			}
+			if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
+				t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
+					ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
+			}
+		})
+	}
+}
+
+// transition marks version versionID of key as transitioned to a remote
+// tier, as a lifecycle transition leaves it.
+func (f *batchAuthzFixture) transition(t *testing.T, key, versionID string) {
+	t.Helper()
+	set := f.obj.(*erasureServerPools).serverPools[0]
+	er := set.getHashedSet(key)
+	// DecomTieredObject takes its parity from the global storage class, which
+	// earlier tests can leave at a parity this set cannot have; it then
+	// writes on no drive and still succeeds. Use the set's own parity; the
+	// whole configuration is restored when the test ends.
+	setTestStandardParity(t, er.defaultParityCount)
+	infos, errs := readAllFileInfo(f.ctx, er.getDisks(), "", f.bucket, key, versionID, false, false)
+	mustIAM(t, errs[0])
+	fi := infos[0]
+	fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote/"+key+"/"+versionID
+	mustIAM(t, set.DecomTieredObject(f.ctx, f.bucket, key, fi, ObjectOptions{}))
+	infos, errs = readAllFileInfo(f.ctx, er.getDisks(), "", f.bucket, key, versionID, false, false)
+	for i := range infos {
+		if errs[i] != nil || infos[i].TransitionStatus != lifecycle.TransitionComplete {
+			t.Fatalf("drive %d does not hold the transition: %v", i, errs[i])
+		}
+	}
+}
+
+// A key with a transitioned version is refused as a whole, as a key with a
+// locked version is: the job completes with the key counted as refused, not
+// failed, and the key keeps every version.
+func TestBatchExpirePurgeRefusesTieredKey(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+	const key = "purge/tiered"
+	older := f.putVersions(t, key)
+	f.transition(t, key, older)
+	want := batchPurgeVersionIDs(t, f, key)
+	ri := f.runChanged(t, cred, "purge/", func() {})
+	for _, id := range want {
+		if !f.versionExists(t, key, id) {
+			t.Errorf("version %s of a key with a transitioned version was purged", id)
+		}
+	}
+	if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
+		t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
+			ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
+	}
+}
+
+// Controls: a purge by root and by an authorized user removes every version,
+// including one added after the walk.
+func TestBatchExpirePurgeAuthorizedControls(t *testing.T) {
+	for _, root := range []bool{true, false} {
+		t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			cred := globalActiveCred
+			if !root {
+				batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+				cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			}
+			const key = "purge/key"
+			f.putVersions(t, key)
+			added := mustGetUUID()
+			ri := f.runChanged(t, cred, "purge/", func() {
+				f.putOlder(t, key, added, map[string]string{xhttp.AmzObjectTagging: "keep=yes"})
+			})
+			if ids := batchPurgeVersionIDs(t, f, key); len(ids) != 0 {
+				t.Errorf("an authorized purge kept %v", ids)
+			}
+			if !ri.Complete || ri.Failed || ri.Objects != 3 || ri.KeysRefused != 0 {
+				t.Errorf("job complete=%v failed=%v objects=%d refused=%d, want complete with 3 objects",
+					ri.Complete, ri.Failed, ri.Objects, ri.KeysRefused)
+			}
+		})
+	}
+}
+
+// Cases found in the sixth review round.
+
+// batchPurgeBudgetChange runs change once the first purge stopped at its
+// budget after removing versions.
+type batchPurgeBudgetChange struct {
+	ObjectLayer
+	change func()
+	calls  int
+}
+
+func (h *batchPurgeBudgetChange) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	oi, err := h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+	if opts.DeleteAllVersions {
+		h.calls++
+		if h.calls == 1 && errors.Is(err, errDeleteAllBudget) && opts.DeleteAllResult.Removed > 0 {
+			h.change()
+		}
+	}
+	return oi, err
+}
+
+// A purge that continues after its budget authorizes the versions again, as
+// they are then; the refusal and the versions already removed persist
+// across another start.
+func TestBatchExpirePurgeBudgetReauthorizes(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	const key = "purge/budget"
+	oldest := batchPutVersion(t, f.obj, f.bucket, key, nil)
+	middle := batchPutVersion(t, f.obj, f.bucket, key, nil)
+	latest := batchPutVersion(t, f.obj, f.bucket, key, nil)
+	batchAuthzPolicy(f.ctx, t, f.sys, "review6-budget-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "review6-budget-user", "review6-budget-deny")
+	rec, job := f.submit(t, cred, f.purgeYAML("purge/"))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	oldBudget := deleteAllBudget
+	deleteAllBudget.versions = 1
+	t.Cleanup(func() { deleteAllBudget = oldBudget })
+	hook := &batchPurgeBudgetChange{ObjectLayer: f.obj, change: func() {
+		_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: middle.VersionID})
+		mustIAM(t, err)
+	}}
+	mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
+	ri := f.jobInfo(t, job)
+	if hook.calls != 2 || !ri.Failed || ri.Complete || ri.FailureReason == "" || ri.Objects != 1 || ri.KeysRefused != 0 {
+		t.Fatalf("continuation: calls=%d complete=%v failed=%v reason=%q objects=%d refused=%d", hook.calls, ri.Complete, ri.Failed, ri.FailureReason, ri.Objects, ri.KeysRefused)
+	}
+	if f.versionExists(t, key, oldest.VersionID) || !f.versionExists(t, key, middle.VersionID) || !f.versionExists(t, key, latest.VersionID) {
+		t.Fatal("only the first authorized version may be removed")
+	}
+	_, err := f.obj.DeleteObjectTags(f.ctx, f.bucket, key, ObjectOptions{VersionID: middle.VersionID})
+	mustIAM(t, err)
+	mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
+	after := f.jobInfo(t, job)
+	if hook.calls != 2 || after.FailureReason != ri.FailureReason || after.Objects != 1 || after.Complete {
+		t.Fatal("persisted refusal must prevent a fresh purge after the tags change")
+	}
+}
+
+// The purge's lock excludes tag writers from its authorization through its
+// last delete, for a directory key in a lock domain of its own on one pool
+// and for a key in a second pool.
+func TestBatchExpirePurgeExcludesTagWriter(t *testing.T) {
+	for _, mode := range []string{ldDsync, "two-pools"} {
+		t.Run(mode, func(t *testing.T) {
+			var z *erasureServerPools
+			var bucket string
+			pool, key := 0, "purge/tags/"
+			if mode == "two-pools" {
+				z, bucket = consistencyPools(t)
+				pool = 1
+			} else {
+				z, bucket = ldPool(t, mode)
+				for i := 0; ; i++ {
+					key = fmt.Sprintf("purge/tags-%d/", i)
+					if z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) == 1 {
+						break
+					}
+				}
+			}
+			before := putConsistencyObject(t, z, bucket, encodeDirObject(key), pool, "", ObjectOptions{Versioned: true})
+			ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+			defer cancel()
+			entered, release := make(chan struct{}), make(chan struct{})
+			var released atomic.Bool
+			unblock := func() {
+				if released.CompareAndSwap(false, true) {
+					close(release)
+				}
+			}
+			defer unblock()
+			purged := make(chan error, 1)
+			go func() {
+				_, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{
+					DeleteAllVersions: true,
+					DeleteAllLatest:   &deleteAllMatch{VersionID: before.VersionID, ModTime: before.ModTime},
+					deleteAllAuthorize: func(versions []ObjectInfo) error {
+						close(entered)
+						select {
+						case <-release:
+							return nil
+						case <-ctx.Done():
+							return ctx.Err()
+						}
+					},
+				})
+				purged <- err
+			}()
+			select {
+			case <-entered:
+			case <-ctx.Done():
+				t.Fatal("purge never reached authorization")
+			}
+			written := make(chan error, 1)
+			go func() {
+				_, err := z.PutObjectTags(ctx, bucket, key, "keep=yes", ObjectOptions{VersionID: before.VersionID})
+				written <- err
+			}()
+			select {
+			case err := <-written:
+				t.Errorf("tag writer crossed the purge's authorization lock: %v", err)
+			case <-time.After(ldWait):
+				unblock()
+				err := <-written
+				if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+					t.Errorf("writer must observe the completed purge: %v", err)
+				}
+			}
+			unblock()
+			mustIAM(t, <-purged)
+		})
+	}
+}
+
+// batchTagWriteFault fails metadata updates of one key, so that a tag write
+// reaches only the drives it does not wrap.
+type batchTagWriteFault struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object string, fi FileInfo, opts UpdateMetadataOpts) error {
+	if bucket == d.bucket && object == d.object {
+		return errDiskNotFound
+	}
+	return d.StorageAPI.UpdateMetadata(ctx, bucket, object, fi, opts)
+}
+
+// A failed tag write can leave a revision that the submitter may not expire
+// on a minority of drives. The purge removes it with the version, so it is
+// authorized too.
+func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) {
+	resetTestGlobals()
+	t.Cleanup(resetTestGlobals)
+	// One drive of sixteen holds the failed tag write, so the read quorum of
+	// the metadata must stay below sixteen. Earlier tests can leave a
+	// storage class of parity 0 behind (newTestConfig on a single drive).
+	previous := globalStorageClass
+	globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
+	t.Cleanup(func() { globalStorageClass.Update(previous) })
+	z := purgeSinglePool(t, 16)
+	globalIAMSys.initStore(z, nil)
+	mustIAM(t, globalIAMSys.Load(t.Context(), true))
+	f := &batchAuthzFixture{ctx: t.Context(), sys: globalIAMSys, obj: z, bucket: "review6-batch-authz"}
+	mustIAM(t, z.MakeBucket(f.ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+	batchTestJobPool(f.ctx, t, z)
+	const key = "purge/minority-tags"
+	older := f.putVersions(t, key)
+	batchAuthzPolicy(f.ctx, t, f.sys, "review6-minority-deny", strings.ReplaceAll(batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`, "batch-authz", f.bucket))
+	cred := iamAdminUser(f.ctx, t, f.sys, "review6-minority-user", "review6-minority-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectVersionAction, bucket: f.bucket, object: key, versionID: older, tags: "keep=yes"}); !isBatchJobAuthzError(err) {
+		t.Fatal("control: this tag revision must deny the read needed for expiration")
+	}
+	set := z.serverPools[0].getHashedSet(key)
+	original := set.getDisks
+	disks := append([]StorageAPI(nil), original()...)
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i := 1; i < len(wrapped); i++ {
+		wrapped[i] = batchTagWriteFault{StorageAPI: wrapped[i], bucket: f.bucket, object: key}
+	}
+	set.getDisks = func() []StorageAPI { return wrapped }
+	_, err = f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older, UserDefined: map[string]string{
+		ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+	}})
+	set.getDisks = original
+	if err == nil {
+		t.Fatal("control: the tag write must fail its quorum")
+	}
+	fi, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{})
+	mustIAM(t, err)
+	if fi.Metadata[xhttp.AmzObjectTagging] != "keep=yes" {
+		t.Fatal("control: the successful drive must hold the denied tag revision")
+	}
+	history, err := z.deleteAllHistory(f.ctx, f.bucket, key)
+	mustIAM(t, err)
+	for _, v := range history {
+		for _, c := range v.copies {
+			if c.fi.Metadata[xhttp.AmzObjectTagging] != "" {
+				t.Fatal("control: the quorum history must still have the original tags")
+			}
+		}
+	}
+	ri := f.runChanged(t, cred, "purge/", func() {})
+	if !ri.Failed || ri.Complete || !strings.Contains(ri.FailureReason, "is not allowed") {
+		t.Errorf("denied physical revision must refuse the job: complete=%v failed=%v reason=%q", ri.Complete, ri.Failed, ri.FailureReason)
+	}
+	if _, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}); err != nil {
+		t.Errorf("denied physical revision was removed: %v", err)
+	}
+	if got := batchPurgeVersionIDs(t, f, key); len(got) != 2 {
+		t.Errorf("denial must keep the whole key; remaining versions=%v", got)
+	}
+}
+
+// Cases found in the seventh review round.
+
+// The time the authorization takes counts against the purge's budget: a
+// late allow and a deny both delete nothing.
+func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) {
+	z := purgeSinglePool(t, 16)
+	p := newPurgeTest(t, z, "review7-purge-auth-time", MakeBucketOptions{VersioningEnabled: true}, "")
+	const key = "purge/key"
+	p.version(0, key, "older", 2)
+	p.version(0, key, "latest", 1)
+	exp := p.matched(key)
+	oldBudget := deleteAllBudget
+	deleteAllBudget.duration = 10 * time.Millisecond
+	t.Cleanup(func() { deleteAllBudget = oldBudget })
+	denied := errors.New("review7: revision denied")
+	for _, decision := range []error{nil, denied} {
+		called := false
+		var res deleteAllResult
+		_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
+			DeleteAllVersions: true,
+			DeleteAllLatest:   &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+			DeleteAllResult:   &res,
+			deleteAllAuthorize: func(versions []ObjectInfo) error {
+				called = true
+				if len(versions) < 2 {
+					t.Error("authorization did not receive both approved versions")
+				}
+				time.Sleep(20 * time.Millisecond)
+				return decision
+			},
+		})
+		want := decision
+		if want == nil {
+			want = errDeleteAllBudget
+		}
+		if !called || !errors.Is(err, want) || res.Started || res.Complete || res.Removed != 0 {
+			t.Errorf("late authorization: called=%v err=%v result=%+v", called, err, res)
+		}
+		if got := p.held(0, key); len(got) != 2 {
+			t.Fatalf("purge removed versions after late authorization: %v", got)
+		}
+	}
+}
+
+// A listing outage pauses without moving the cursor or refusing the job;
+// a later run completes it in both modes.
+func TestBatchExpireWalkFailureResumes(t *testing.T) {
+	for _, purgeAll := range []bool{false, true} {
+		name := "retain-one"
+		if purgeAll {
+			name = "retain-zero"
+		}
+		t.Run(name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			const key = "data/key"
+			older := f.putVersions(t, key)
+			definition := f.expireYAML("data/")
+			if purgeAll {
+				definition = f.purgeYAML("data/")
+			}
+			rec, job := f.submit(t, globalActiveCred, definition)
+			if job == nil {
+				t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+			}
+			listErr := errors.New("review7: transient listing failure")
+			err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job)
+			if !errors.Is(err, listErr) {
+				t.Fatalf("listing error: %v", err)
+			}
+			ri := f.jobInfo(t, job)
+			if ri.Failed || ri.Complete || ri.PausedReason == "" || ri.FailureReason != "" || ri.Object != "" || !f.versionExists(t, key, older) {
+				t.Fatalf("listing failure changed progress or became terminal: failed=%v complete=%v reason=%q cursor=%q", ri.Failed, ri.Complete, ri.FailureReason, ri.Object)
+			}
+			mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+			ri = f.jobInfo(t, job)
+			wantRemoved := int64(1)
+			if purgeAll {
+				wantRemoved = 2
+			}
+			if ri.Failed || !ri.Complete || ri.FailureReason != "" || ri.Objects != wantRemoved || f.versionExists(t, key, older) {
+				t.Fatalf("resume did not complete: failed=%v complete=%v reason=%q removed=%d", ri.Failed, ri.Complete, ri.FailureReason, ri.Objects)
+			}
+		})
+	}
+}

cmd/batch-expire.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -34,6 +35,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/pgsty/silo-pkg/v3/env"
+	"github.com/pgsty/silo-pkg/v3/policy"
 	"github.com/pgsty/silo-pkg/v3/wildcard"
 	"github.com/pgsty/silo-pkg/v3/workers"
 	"github.com/pgsty/silo-pkg/v3/xtime"
@@ -289,16 +291,6 @@ type BatchJobExpire struct {
 
 var _ yaml.Unmarshaler = &BatchJobExpire{}
 
-// RedactSensitive will redact any sensitive information in b.
-func (r *BatchJobExpire) RedactSensitive() {
-	if r == nil {
-		return
-	}
-	if r.NotificationCfg.Token != "" {
-		r.NotificationCfg.Token = redactedText
-	}
-}
-
 // UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
 func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
 	type expireJob BatchJobExpire
@@ -345,32 +337,6 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
 	return nil
 }
 
-// Expire expires object versions which have already matched supplied filter conditions
-func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
-	opts := ObjectOptions{
-		PrefixEnabledFn:  vc.PrefixEnabled,
-		VersionSuspended: vc.Suspended(),
-	}
-
-	allErrs := make([]error, 0, len(objsToDel))
-
-	for {
-		count := len(objsToDel)
-		if count == 0 {
-			break
-		}
-		if count > maxDeleteList {
-			count = maxDeleteList
-		}
-		_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
-		allErrs = append(allErrs, errs...)
-		// Next batch of deletion
-		objsToDel = objsToDel[count:]
-	}
-
-	return allErrs
-}
-
 const (
 	batchExpireName                 = "batch-expire.bin"
 	batchExpireFormat               = 1
@@ -381,22 +347,230 @@ const (
 	batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
 )
 
-type objInfoCache map[string]*ObjectInfo
-
-func newObjInfoCache() objInfoCache {
-	return objInfoCache(make(map[string]*ObjectInfo))
+type expireObjInfo struct {
+	ObjectInfo
+	ExpireAll         bool
+	DeleteMarkerCount int64
 }
 
-func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
-	oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
+// isErrObjectLocked reports a version that Object Lock protects.
+func isErrObjectLocked(err error) bool {
+	var locked ObjectLocked
+	return errors.As(err, &locked)
 }
 
-func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
-	oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
-	return oi, ok
+func (r *BatchJobExpire) tagsFiltered() bool {
+	for _, rule := range r.Rules {
+		if len(rule.Tags) > 0 {
+			return true
+		}
+	}
+	return false
 }
 
-func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
+// expireAuthzRequests are the actions the job takes on one version: it reads
+// the version, and its tags when a rule filters by them, and deletes it.
+func (r *BatchJobExpire) expireAuthzRequests(name, versionID, userTags string) []batchAuthzRequest {
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	reqs := []batchAuthzRequest{
+		{action: policy.GetObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		{action: policy.DeleteObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		{action: policy.DeleteObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+	}
+	if r.tagsFiltered() {
+		reqs = append(reqs,
+			batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+			batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+		)
+	}
+	return reqs
+}
+
+// expireVersion deletes one version that is older than the versions the rule
+// retains. Under the object's write lock, with the version as it is then, it
+// leaves a version that Object Lock protects (ObjectLocked) and refuses one
+// that the submitter may not delete (batchJobAuthzError). An unreadable
+// version is not deleted.
+func (r *BatchJobExpire) expireVersion(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, exp expireObjInfo) error {
+	p, err := job.Submitter.principal(ctx)
+	if err != nil {
+		return err
+	}
+	versionID := exp.VersionID
+	if versionID == "" {
+		// An empty version ID would add a delete marker instead.
+		versionID = nullVersionID
+	}
+	opts := ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        vc.PrefixEnabled(exp.Name),
+		VersionSuspended: vc.Suspended(),
+	}
+	opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) error {
+		// A delete marker reads as MethodNotAllowed; any other error,
+		// including a missing version, stops the delete.
+		marker := isErrMethodNotAllowed(gerr) && (goi.DeleteMarker || !goi.VersionPurgeStatus.Empty())
+		if gerr != nil && !marker {
+			return gerr
+		}
+		if enforceRetentionForDeletion(ctx, goi) {
+			return ObjectLocked{Bucket: r.Bucket, Object: exp.Name, VersionID: versionID}
+		}
+		for _, req := range r.expireAuthzRequests(exp.Name, versionID, goi.UserTags) {
+			if err := p.authorize(req); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	_, err = api.DeleteObject(ctx, r.Bucket, exp.Name, opts)
+	return err
+}
+
+// expireVersions deletes the versions of one batch entry and records the
+// outcome in res. It returns an error that ends the job.
+func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, delay time.Duration, res *batchCounts) error {
+	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+		err := r.expireVersion(ctx, api, vc, job, exp)
+		stopFn(exp, err)
+		switch {
+		case err == nil:
+			if exp.DeleteMarker {
+				ri.countRemoved(r.Bucket, 0, 1)
+			} else {
+				ri.countRemoved(r.Bucket, 1, 0)
+			}
+			return nil
+		case isBatchJobStopped(err):
+			return err
+		case isErrObjectLocked(err):
+			// Left in place, not retried and not a failure.
+			res.versionsSkipped++
+			return nil
+		case isErrVersionNotFound(err) || isErrObjectNotFound(err):
+			// Removed concurrently.
+			return nil
+		}
+		batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", r.Bucket, exp.Name, exp.VersionID, err, attempts))
+		if attempts < retryAttempts {
+			time.Sleep(delay)
+		}
+	}
+	if exp.DeleteMarker {
+		res.deleteMarkersFailed++
+	} else {
+		res.objectsFailed++
+	}
+	return nil
+}
+
+// expireAllVersions purges every version of a key whose rule retains none,
+// with the purge of exactly that key (erasureServerPools.deleteAllVersions),
+// while the version the walk matched is still its latest. Under its lock,
+// before it deletes anything, the purge refuses the key when a version is
+// transitioned or Object Lock protects one, and has every write it would
+// delete, each copy and each revision a drive holds, authorized for the
+// submitter, with the version ID and tags it has then.
+//
+// What the purge removed is counted at once; a refused key is counted in res
+// as refused, and the versions a failed purge left in res as failures. A
+// failed purge is retried up to retryAttempts times, each time with the
+// version the walk matched; a purge that stopped at its budget after
+// removing versions continues without spending an attempt. A key whose
+// latest version changed since the walk is not expired. It returns an error
+// that ends the job: an authorization failure or the end of ctx.
+func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error {
+	var objects, markers int64
+	matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
+	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+		latest := matched
+		var purge deleteAllResult
+		p, err := job.Submitter.principal(ctx)
+		if err == nil {
+			_, err = api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
+				DeleteAllVersions:  true,
+				DeleteAllLatest:    &latest,
+				DeleteAllResult:    &purge,
+				deleteAllAuthorize: r.expireAllAuthorize(p, exp.Name),
+			})
+		}
+		// Removed versions are gone for any later run: count them now.
+		if purge.Removed > 0 {
+			ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers))
+			objects += int64(purge.Removed - purge.Markers)
+			markers += int64(purge.Markers)
+		}
+		stopFn(exp, err)
+		switch {
+		case err == nil:
+			return nil
+		case isBatchJobStopped(err):
+			return err
+		case errors.Is(err, errDeleteAllChanged):
+			// Changed since the walk matched it: not expired.
+			return nil
+		case isErrObjectLocked(err), errors.Is(err, errDeleteAllTiered):
+			// Refused as a whole: nothing removed, not retried and not a
+			// failure. A later run of the job checks the key again.
+			res.keysRefused++
+			return nil
+		case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
+			attempts-- // progress
+			continue
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
+	}
+	res.expireAllFailed(exp, objects, markers)
+	return nil
+}
+
+// expireAllAuthorize returns the authorization of a purge of all versions of
+// name for p: each version the purge would delete, as it holds it, needs the
+// actions of expiring it (expireAuthzRequests).
+func (r *BatchJobExpire) expireAllAuthorize(p batchJobPrincipal, name string) func([]ObjectInfo) error {
+	return func(versions []ObjectInfo) error {
+		for _, v := range versions {
+			for _, req := range r.expireAuthzRequests(name, v.VersionID, v.UserTags) {
+				if err := p.authorize(req); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+}
+
+// expireAllFailed counts the versions of info, as the walk saw them, that a
+// failed purge of all its versions did not remove: those beyond the objects
+// and markers it removed, at least one.
+func (c *batchCounts) expireAllFailed(info expireObjInfo, objects, markers int64) {
+	failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
+	failedMarkers := max(info.DeleteMarkerCount-markers, 0)
+	if failedObjects+failedMarkers == 0 {
+		failedObjects = 1
+	}
+	c.objectsFailed += failedObjects
+	c.deleteMarkersFailed += failedMarkers
+}
+
+// batchObjsForDelete expires the batches of expireCh. walkDone reports, once
+// expireCh is closed, whether the walk reached its end: only then is its last
+// key known to be complete.
+func batchObjsForDelete(ctx context.Context, refuse func(error), walkDone func() bool, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
 	vc, _ := globalBucketVersioningSys.Get(r.Bucket)
 	retryAttempts := job.Expire.Retry.Attempts
 	if retryAttempts <= 0 {
@@ -407,130 +581,53 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 		delay = batchExpireJobDefaultRetryDelay
 	}
 
-	var i int
+	progress := newBatchProgress(ri, r.Bucket)
+	defer func() {
+		wk.Wait()
+		if walkDone() {
+			progress.finish()
+		}
+	}()
 	for toExpire := range expireCh {
 		select {
 		case <-ctx.Done():
 			return
 		default:
 		}
-		if i > 0 {
+		seq := progress.begin()
+		if seq > 0 {
 			if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
 				time.Sleep(wait)
 			}
 		}
-		i++
 		wk.Take()
-		go func(toExpire []expireObjInfo) {
+		go func(seq uint64, toExpire []expireObjInfo) {
 			defer wk.Give()
-
-			toExpireAll := make([]expireObjInfo, 0, len(toExpire))
-			toDel := make([]ObjectToDelete, 0, len(toExpire))
-			oiCache := newObjInfoCache()
+			var res batchCounts
 			for _, exp := range toExpire {
+				var err error
 				if exp.ExpireAll {
-					toExpireAll = append(toExpireAll, exp)
-					continue
+					err = r.expireAllVersions(ctx, api, job, ri, exp, retryAttempts, &res)
+				} else {
+					err = r.expireVersions(ctx, api, vc, job, ri, exp, retryAttempts, delay, &res)
 				}
-				// Cache ObjectInfo value via pointers for
-				// subsequent use to track objects which
-				// couldn't be deleted.
-				od := ObjectToDelete{
-					ObjectV: ObjectV{
-						ObjectName: exp.Name,
-						VersionID:  exp.VersionID,
-					},
-				}
-				toDel = append(toDel, od)
-				oiCache.Add(od, &exp.ObjectInfo)
-			}
-
-			// DeleteObject(deletePrefix: true) to expire all versions of an object
-			for _, exp := range toExpireAll {
-				var success bool
-				for attempts := 1; attempts <= retryAttempts; attempts++ {
-					select {
-					case <-ctx.Done():
-						ri.trackMultipleObjectVersions(exp, success)
-						return
-					default:
+				if err != nil {
+					// The batch is not done; a later run does it again.
+					if isBatchJobStopped(err) {
+						refuse(err)
 					}
-					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
-						DeletePrefix:       true,
-						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-					})
-					if err != nil {
-						stopFn(exp, err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
-					} else {
-						stopFn(exp, err)
-						success = true
-						break
-					}
-				}
-				ri.trackMultipleObjectVersions(exp, success)
-			}
-
-			// DeleteMultiple objects
-			toDelCopy := make([]ObjectToDelete, len(toDel))
-			for attempts := 1; attempts <= retryAttempts; attempts++ {
-				select {
-				case <-ctx.Done():
+					globalBatchJobsMetrics.save(ri.JobID, ri)
 					return
-				default:
-				}
-
-				stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-				// Copying toDel to select from objects whose
-				// deletion failed
-				copy(toDelCopy, toDel)
-				var failed int
-				errs := r.Expire(ctx, api, vc, toDel)
-				// reslice toDel in preparation for next retry attempt
-				toDel = toDel[:0]
-				for i, err := range errs {
-					if err != nil {
-						stopFn(toDelCopy[i], err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
-							err, attempts))
-						failed++
-						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
-							ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)
-						}
-						if attempts != retryAttempts {
-							// retry
-							toDel = append(toDel, toDelCopy[i])
-						}
-					} else {
-						stopFn(toDelCopy[i], nil)
-						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
-							ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts)
-						}
-					}
-				}
-
-				globalBatchJobsMetrics.save(ri.JobID, ri)
-
-				if failed == 0 {
-					break
-				}
-
-				// Add a delay between retry attempts
-				if attempts < retryAttempts {
-					time.Sleep(delay)
 				}
 			}
-		}(toExpire)
+			if n := len(toExpire); n > 0 {
+				progress.done(seq, toExpire[n-1].Name, res)
+			}
+			globalBatchJobsMetrics.save(ri.JobID, ri)
+		}(seq, toExpire)
 	}
 }
 
-type expireObjInfo struct {
-	ObjectInfo
-	ExpireAll         bool
-	DeleteMarkerCount int64
-}
-
 // Start the batch expiration job, resumes if there was a pending job via "job.ID"
 func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
 	ri := &batchJobInfo{
@@ -539,6 +636,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		StartTime: job.Started,
 	}
 	if err := ri.loadOrInit(ctx, api, job); err != nil {
+		return batchUnavailable("loading job progress", err)
+	}
+	// A completed job is not run again, as replication and key rotation
+	// jobs are not: its submitter is not checked again either, so a
+	// credential that expired since cannot turn it into a failed job.
+	if ri.Complete {
+		return nil
+	}
+	if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
 		return err
 	}
 
@@ -560,27 +666,47 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	ctx, cancelCause := context.WithCancelCause(ctx)
 	defer cancelCause(nil)
+	// refuse ends the job for its submitter: record why before stopping.
+	refuse := func(err error) {
+		if isBatchJobUnavailable(err) {
+			ri.pause(err)
+		} else {
+			ri.refuse(err)
+		}
+		cancelCause(err)
+	}
 
+	// The walk stops at a listing failure: work after it is left to a later
+	// run, so the cursor cannot move past what was not listed.
+	walkCtx, stopWalk := context.WithCancel(ctx)
+	defer stopWalk()
 	results := make(chan itemOrErr[ObjectInfo], workerSize)
 	go func() {
-		prefixes := r.Prefix.F()
-		if len(prefixes) == 0 {
-			prefixes = []string{""}
-		}
-		for _, prefix := range prefixes {
+		// One ascending walk over disjoint prefixes, which the cursor
+		// describes.
+		for _, prefix := range batchCanonicalPrefixes(r.Prefix.F()) {
+			if walkCtx.Err() != nil {
+				break
+			}
 			prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
-			err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
+			err := api.Walk(walkCtx, r.Bucket, prefix, prefixResultCh, WalkOptions{
 				Marker:       lastObject,
 				LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
 				VersionsSort: WalkVersionsSortDesc,
 			})
 			if err != nil {
-				cancelCause(err)
+				if walkCtx.Err() == nil {
+					cancelCause(batchListingError(err))
+				}
 				xioutil.SafeClose(results)
 				return
 			}
 			for result := range prefixResultCh {
-				results <- result
+				select {
+				case results <- result:
+				case <-walkCtx.Done():
+					// Stopped: let the walk end without a reader.
+				}
 			}
 		}
 		xioutil.SafeClose(results)
@@ -588,7 +714,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	// Goroutine to periodically save batch-expire job's in-memory state
 	saverQuitCh := make(chan struct{})
+	saverDoneCh := make(chan struct{})
 	go func() {
+		defer close(saverDoneCh)
 		saveTicker := time.NewTicker(10 * time.Second)
 		defer saveTicker.Stop()
 		quit := false
@@ -615,9 +743,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	expireCh := make(chan []expireObjInfo, workerSize)
 	expireDoneCh := make(chan struct{})
+	var walkDone atomic.Bool
 	go func() {
 		defer close(expireDoneCh)
-		batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+		batchObjsForDelete(ctx, refuse, walkDone.Load, r, ri, job, api, wk, expireCh)
 	}()
 
 	var (
@@ -629,6 +758,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		done          bool
 	)
 	deleteMarkerCountMap := map[string]int64{}
+	// pushToExpire is called between keys only, so that a batch holds all
+	// the versions of its keys and its last key is done with it.
 	pushToExpire := func() {
 		// set preObject deleteMarkerCount
 		if len(toDel) > 0 {
@@ -653,6 +784,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			}
 		}
 	}
+	var listingErr error
 	for {
 		select {
 		case result, ok := <-results:
@@ -662,7 +794,14 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			}
 			if result.Err != nil {
 				failed = true
+				listingErr = batchListingError(result.Err)
 				batchLogIf(ctx, result.Err)
+				stopWalk()
+				done = true
+				break
+			}
+			if !batchResumeAfter(lastObject, result.Item.Name) {
+				// Completed by the run that set the cursor.
 				continue
 			}
 			if result.Item.DeleteMarker {
@@ -720,7 +859,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			toDel = append(toDel, expireObjInfo{
 				ObjectInfo: result.Item,
 			})
-			pushToExpire()
 		case <-ctx.Done():
 			done = true
 		}
@@ -729,16 +867,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		}
 	}
 
-	if context.Cause(ctx) != nil {
-		xioutil.SafeClose(expireCh)
-		return context.Cause(ctx)
-	}
-	pushToExpire()
-	// Send any remaining objects downstream
-	if len(toDel) > 0 {
-		select {
-		case <-ctx.Done():
-		case expireCh <- toDel:
+	if context.Cause(ctx) == nil {
+		walkDone.Store(!failed)
+		pushToExpire()
+		// Send any remaining objects downstream
+		if len(toDel) > 0 {
+			select {
+			case <-ctx.Done():
+			case expireCh <- toDel:
+			}
 		}
 	}
 	xioutil.SafeClose(expireCh)
@@ -746,13 +883,40 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	<-expireDoneCh // waits for the expire goroutine to complete
 	wk.Wait()      // waits for all expire workers to retire
 
-	ri.Complete = !failed && ri.ObjectsFailed == 0
-	ri.Failed = failed || ri.ObjectsFailed > 0
-	globalBatchJobsMetrics.save(job.ID, ri)
-
-	// Close the saverQuitCh - this also triggers saving in-memory state
-	// immediately one last time before we exit this method.
+	// Persist the final state once the saver has stopped, so that its last
+	// save cannot overwrite it.
 	xioutil.SafeClose(saverQuitCh)
+	<-saverDoneCh
+	refused := ri.refused()
+	cause := context.Cause(ctx)
+	if cause == nil {
+		cause = listingErr
+	}
+	if cause != nil && !refused {
+		if isBatchJobUnavailable(cause) {
+			return ri.savePaused(ctx, api, job, cause)
+		}
+		if isErrBucketNotFound(cause) {
+			ri.mu.Lock()
+			ri.Failed = true
+			ri.mu.Unlock()
+		}
+		saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
+		batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
+		cancel()
+		globalBatchJobsMetrics.save(job.ID, ri)
+		return cause
+	}
+
+	ri.mu.Lock()
+	itemsFailed := ri.expireFailures()
+	ri.Complete = !refused && !failed && !itemsFailed
+	ri.Failed = refused || failed || itemsFailed
+	ri.mu.Unlock()
+	globalBatchJobsMetrics.save(job.ID, ri)
+	saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
+	batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
+	cancel()
 
 	// Notify expire jobs final status to the configured endpoint
 	buf, _ := json.Marshal(ri)

cmd/batch-handlers_gen.go

@@ -179,6 +179,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "Submitter":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Submitter")
+					return
+				}
+				z.Submitter = nil
+			} else {
+				if z.Submitter == nil {
+					z.Submitter = new(batchJobSubmitter)
+				}
+				err = z.Submitter.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Submitter")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -192,9 +210,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 7
 	// write "ID"
-	err = en.Append(0x86, 0xa2, 0x49, 0x44)
+	err = en.Append(0x87, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -274,15 +292,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "Submitter"
+	err = en.Append(0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
+	if err != nil {
+		return
+	}
+	if z.Submitter == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.Submitter.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Submitter")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
 	// string "ID"
-	o = append(o, 0x86, 0xa2, 0x49, 0x44)
+	o = append(o, 0x87, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -323,6 +358,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "Submitter"
+	o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
+	if z.Submitter == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.Submitter.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Submitter")
+			return
+		}
+	}
 	return
 }
 
@@ -413,6 +459,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "Submitter":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Submitter = nil
+			} else {
+				if z.Submitter == nil {
+					z.Submitter = new(batchJobSubmitter)
+				}
+				bts, err = z.Submitter.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Submitter")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -445,6 +508,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.Expire.Msgsize()
 	}
+	s += 10
+	if z.Submitter == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.Submitter.Msgsize()
+	}
 	return
 }
 
@@ -568,6 +637,30 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BytesFailed")
 				return
 			}
+		case "vsl":
+			z.VersionsSkippedLocked, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "VersionsSkippedLocked")
+				return
+			}
+		case "kr":
+			z.KeysRefused, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "KeysRefused")
+				return
+			}
+		case "fr":
+			z.FailureReason, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "FailureReason")
+				return
+			}
+		case "pr":
+			z.PausedReason, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "PausedReason")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -581,9 +674,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 17
+	// map header, size 21
 	// write "v"
-	err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76)
+	err = en.Append(0xde, 0x0, 0x15, 0xa1, 0x76)
 	if err != nil {
 		return
 	}
@@ -752,15 +845,55 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BytesFailed")
 		return
 	}
+	// write "vsl"
+	err = en.Append(0xa3, 0x76, 0x73, 0x6c)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.VersionsSkippedLocked)
+	if err != nil {
+		err = msgp.WrapError(err, "VersionsSkippedLocked")
+		return
+	}
+	// write "kr"
+	err = en.Append(0xa2, 0x6b, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.KeysRefused)
+	if err != nil {
+		err = msgp.WrapError(err, "KeysRefused")
+		return
+	}
+	// write "fr"
+	err = en.Append(0xa2, 0x66, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.FailureReason)
+	if err != nil {
+		err = msgp.WrapError(err, "FailureReason")
+		return
+	}
+	// write "pr"
+	err = en.Append(0xa2, 0x70, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.PausedReason)
+	if err != nil {
+		err = msgp.WrapError(err, "PausedReason")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 17
+	// map header, size 21
 	// string "v"
-	o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76)
+	o = append(o, 0xde, 0x0, 0x15, 0xa1, 0x76)
 	o = msgp.AppendInt(o, z.Version)
 	// string "jid"
 	o = append(o, 0xa3, 0x6a, 0x69, 0x64)
@@ -810,6 +943,18 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "bf"
 	o = append(o, 0xa2, 0x62, 0x66)
 	o = msgp.AppendInt64(o, z.BytesFailed)
+	// string "vsl"
+	o = append(o, 0xa3, 0x76, 0x73, 0x6c)
+	o = msgp.AppendInt64(o, z.VersionsSkippedLocked)
+	// string "kr"
+	o = append(o, 0xa2, 0x6b, 0x72)
+	o = msgp.AppendInt64(o, z.KeysRefused)
+	// string "fr"
+	o = append(o, 0xa2, 0x66, 0x72)
+	o = msgp.AppendString(o, z.FailureReason)
+	// string "pr"
+	o = append(o, 0xa2, 0x70, 0x72)
+	o = msgp.AppendString(o, z.PausedReason)
 	return
 }
 
@@ -933,6 +1078,30 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BytesFailed")
 				return
 			}
+		case "vsl":
+			z.VersionsSkippedLocked, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersionsSkippedLocked")
+				return
+			}
+		case "kr":
+			z.KeysRefused, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "KeysRefused")
+				return
+			}
+		case "fr":
+			z.FailureReason, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "FailureReason")
+				return
+			}
+		case "pr":
+			z.PausedReason, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PausedReason")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -947,6 +1116,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *batchJobInfo) Msgsize() (s int) {
-	s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
+	s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.StringPrefixSize + len(z.FailureReason) + 3 + msgp.StringPrefixSize + len(z.PausedReason)
 	return
 }

cmd/batch-object-lock_test.go

@@ -0,0 +1,479 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/auth"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+var batchLockKinds = []string{"COMPLIANCE", "GOVERNANCE", "LEGALHOLD"}
+
+func batchLockMeta(kind string) map[string]string {
+	m := map[string]string{}
+	switch kind {
+	case "COMPLIANCE", "GOVERNANCE":
+		m[strings.ToLower(xhttp.AmzObjectLockMode)] = kind
+		m[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(24 * time.Hour))
+	case "LEGALHOLD":
+		m[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
+	}
+	return m
+}
+
+func batchPutVersion(t *testing.T, obj ObjectLayer, bucket, key string, meta map[string]string) ObjectInfo {
+	t.Helper()
+	data := []byte("batch lock " + key)
+	oi, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
+		ObjectOptions{Versioned: true, UserDefined: meta})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return oi
+}
+
+func batchVersionExists(t *testing.T, obj ObjectLayer, bucket, key, versionID string) bool {
+	t.Helper()
+	_, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: versionID})
+	switch {
+	case err == nil:
+		return true
+	case isErrVersionNotFound(err) || isErrObjectNotFound(err):
+		return false
+	}
+	t.Fatal(err)
+	return false
+}
+
+// batchJobStatusJSON returns the status response of a job as decoded JSON.
+func batchJobStatusJSON(t *testing.T, jobID string) map[string]any {
+	t.Helper()
+	cred := globalActiveCred
+	req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/status-job?jobId="+jobID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = req.ParseForm(); err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	req = req.WithContext(newContext(req, rec, "BatchJobStatus"))
+	adminAPIHandlers{}.BatchJobStatus(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+	}
+	var out struct{ LastMetric map[string]any }
+	if err = json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	return out.LastMetric
+}
+
+// Batch expiration keeps versions under retention or legal hold, on both
+// purge paths, and reports them; unlocked versions are still expired.
+func TestBatchExpireObjectLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			f := &batchAuthzFixture{ctx: t.Context(), obj: obj, bucket: bucket}
+			batchTestJobPool(t.Context(), t, obj)
+
+			type version struct{ key, id string }
+			var locked, unlocked []version
+			for _, retain := range []int{0, 1} {
+				for _, kind := range batchLockKinds {
+					key := fmt.Sprintf("r%d/%s", retain, kind)
+					older := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
+					batchPutVersion(t, obj, bucket, key, nil)
+					locked = append(locked, version{key, older.VersionID})
+				}
+				key := fmt.Sprintf("r%d/plain", retain)
+				older := batchPutVersion(t, obj, bucket, key, nil)
+				batchPutVersion(t, obj, bucket, key, nil)
+				unlocked = append(unlocked, version{key, older.VersionID})
+			}
+
+			var jobs []*BatchJobRequest
+			for _, retain := range []int{0, 1} {
+				job := fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: r%d/
+  rules:
+    - type: object
+      purge:
+        retainVersions: %d
+`, bucket, retain, retain)
+				rec, queued := f.submit(t, globalActiveCred, job)
+				if queued == nil {
+					t.Fatalf("%s: submission status %d: %s", instance, rec.Code, rec.Body.String())
+				}
+				// Running a job again, as a resumed job does, must not count
+				// the versions it left a second time.
+				f.run(t, queued)
+				f.run(t, queued)
+				jobs = append(jobs, queued)
+			}
+
+			for _, v := range locked {
+				if !batchVersionExists(t, obj, bucket, v.key, v.id) {
+					t.Errorf("%s: %s: locked version %s was expired", instance, v.key, v.id)
+				}
+			}
+			for _, v := range unlocked {
+				if batchVersionExists(t, obj, bucket, v.key, v.id) {
+					t.Errorf("%s: %s: control: unlocked version %s was not expired", instance, v.key, v.id)
+				}
+			}
+			for i, field := range []string{"keysRefused", "versionsSkippedLocked"} {
+				m := batchJobStatusJSON(t, jobs[i].ID)
+				if got, _ := m[field].(float64); got != float64(len(batchLockKinds)) {
+					t.Errorf("%s: retain %d: %s = %v, want %d", instance, i, field, m[field], len(batchLockKinds))
+				}
+				if m["complete"] != true || m["failed"] != false {
+					t.Errorf("%s: retain %d: complete=%v failed=%v", instance, i, m["complete"], m["failed"])
+				}
+			}
+		},
+	})
+}
+
+type batchLockTransport func(*http.Request) (*http.Response, error)
+
+func (f batchLockTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// batchLockRemote is a source that serves replacement for every read.
+func batchLockRemote(t *testing.T, replacement string) *minio.Core {
+	t.Helper()
+	core, err := minio.NewCore("source.invalid", &minio.Options{
+		Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+		Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusOK, Header: http.Header{
+				"Content-Length": []string{strconv.Itoa(len(replacement))},
+				"Last-Modified":  []string{UTCNow().Format(http.TimeFormat)},
+				"Etag":           []string{"\"batchetag\""},
+			}, Body: io.NopCloser(strings.NewReader(replacement)), ContentLength: int64(len(replacement)), Request: r}, nil
+		}),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return core
+}
+
+func batchRootSubmitter(t *testing.T) *batchJobSubmitter {
+	t.Helper()
+	sub, err := newBatchJobSubmitter(globalActiveCred)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return sub
+}
+
+// A pull leaves a destination version under Object Lock as it is, whether a
+// single PUT, a multipart completion or a delete marker would replace it.
+func TestBatchPullObjectLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			const replacement = "replacement from remote"
+			core := batchLockRemote(t, replacement)
+			sub := batchRootSubmitter(t)
+			job := BatchJobReplicateV1{
+				Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+				Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+			}
+			for _, kind := range batchLockKinds {
+				for _, path := range []string{"put", "multipart", "marker", "directory"} {
+					key := "pull/" + kind + "/" + path
+					if path == "directory" {
+						key += "/"
+					}
+					before := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
+					src := ObjectInfo{Name: key, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}}
+					switch path {
+					case "multipart":
+						src.ETag = "batchetag-1"
+					case "marker":
+						src.DeleteMarker = true
+					}
+					err := job.ReplicateFromSource(t.Context(), obj, core, src, false, sub)
+					if !isErrObjectLocked(err) {
+						t.Errorf("%s: %s: pull returned %v, want ObjectLocked", instance, key, err)
+					}
+					after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
+					if gerr != nil {
+						t.Errorf("%s: %s: locked version lost: %v", instance, key, gerr)
+						continue
+					}
+					if !enforceRetentionForDeletion(t.Context(), after) || after.ETag != before.ETag {
+						t.Errorf("%s: %s: locked version was replaced", instance, key)
+					}
+				}
+			}
+
+			// Controls: a new version, and a new delete marker that hides a
+			// locked version without replacing it.
+			locked := batchPutVersion(t, obj, bucket, "pull/new", batchLockMeta("COMPLIANCE"))
+			newVersion := mustGetUUID()
+			if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
+				Name: "pull/new", VersionID: newVersion, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
+			}, false, sub); err != nil {
+				t.Fatalf("%s: new version: %v", instance, err)
+			}
+			if got := readCopyChecksumObject(t, obj, bucket, "pull/new", ObjectOptions{VersionID: newVersion}); string(got) != replacement {
+				t.Errorf("%s: new version content %q", instance, got)
+			}
+			marker := mustGetUUID()
+			if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
+				Name: "pull/new", VersionID: marker, ModTime: UTCNow(), DeleteMarker: true,
+			}, false, sub); err != nil {
+				t.Fatalf("%s: new marker: %v", instance, err)
+			}
+			if !batchVersionExists(t, obj, bucket, "pull/new", locked.VersionID) {
+				t.Errorf("%s: a new marker removed a locked version", instance)
+			}
+		},
+	})
+}
+
+// A pull applies retention as a PUT to the destination would: copied
+// retention is validated, and a version without one gets the bucket default.
+func TestBatchPullRetentionMetadata(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			meta, err := globalBucketMetadataSys.Get(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			meta.ObjectLockConfigXML = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>`)
+			meta.VersioningConfigXML = enabledBucketVersioningConfig
+			if err = meta.parseAllConfigs(t.Context(), obj); err != nil {
+				t.Fatal(err)
+			}
+			globalBucketMetadataSys.Set(bucket, meta)
+			core := batchLockRemote(t, "retention content")
+			sub := batchRootSubmitter(t)
+			job := BatchJobReplicateV1{
+				Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+				Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+			}
+			pull := func(key string, meta map[string]string) (ObjectInfo, error) {
+				vid := mustGetUUID()
+				err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{Name: key, VersionID: vid, ModTime: UTCNow().Add(-time.Hour), ETag: "batchetag", UserDefined: meta}, false, sub)
+				if err != nil {
+					return ObjectInfo{}, err
+				}
+				return obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: vid})
+			}
+			oi, err := pull("retention/default", map[string]string{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
+			if ret.Mode != objectlock.RetGovernance || !ret.RetainUntilDate.After(UTCNow().Add(23*time.Hour)) {
+				t.Errorf("%s: default retention not applied with the destination's time: %+v", instance, ret)
+			}
+			until := UTCNow().Add(48 * time.Hour)
+			oi, err = pull("retention/copied", map[string]string{
+				xhttp.AmzObjectLockMode: "COMPLIANCE", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if ret = objectlock.GetObjectRetentionMeta(oi.UserDefined); ret.Mode != objectlock.RetCompliance || !ret.RetainUntilDate.Equal(until.Truncate(time.Millisecond)) {
+				t.Errorf("%s: copied retention: %+v", instance, ret)
+			}
+			if _, err = pull("retention/invalid", map[string]string{xhttp.AmzObjectLockMode: "FOREVER", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until)}); err == nil {
+				t.Errorf("%s: invalid copied retention was accepted", instance)
+			}
+		},
+	})
+}
+
+// Key rotation does not rewrite a version under Object Lock.
+func TestBatchKeyRotationObjectLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject", "GetObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			oldKMS := GlobalKMS
+			GlobalKMS = kms.NewStub("batch-lock-key")
+			defer func() { GlobalKMS = oldKMS }()
+			sub := batchRootSubmitter(t)
+			job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+			for _, kind := range append([]string{"none"}, batchLockKinds...) {
+				key := "rotate/" + kind
+				headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
+				switch kind {
+				case "COMPLIANCE", "GOVERNANCE":
+					headers[xhttp.AmzObjectLockMode] = kind
+					headers[xhttp.AmzObjectLockRetainUntilDate] = UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
+				case "LEGALHOLD":
+					headers[xhttp.AmzObjectLockLegalHold] = "ON"
+				}
+				putCopyChecksumSource(t, router, cred, bucket, key, []byte("rotate "+kind), headers)
+				before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
+				if err != nil || !crypto.S3.IsEncrypted(before.UserDefined) {
+					t.Fatalf("%s: %s: encrypted control: %v", instance, key, err)
+				}
+				err = job.KeyRotate(t.Context(), obj, before, sub)
+				after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
+				if gerr != nil {
+					t.Fatal(gerr)
+				}
+				if kind == "none" {
+					if err != nil || after.ModTime.Equal(before.ModTime) {
+						t.Errorf("%s: control: unlocked version not rotated: %v", instance, err)
+					}
+					continue
+				}
+				var locked ObjectLocked
+				if !errors.As(err, &locked) || !after.ModTime.Equal(before.ModTime) {
+					t.Errorf("%s: %s: rotation of a locked version returned %v; rewritten=%v", instance, key, err, !after.ModTime.Equal(before.ModTime))
+				}
+			}
+		},
+	})
+}
+
+// A push reads locked versions as any authorized read does, and a submitter
+// that may not read a version does not send it.
+func TestBatchPushReads(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"PutObject"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			defer r5Capacity(obj.(*erasureServerPools))()
+			enableBucketObjectLock(t, bucket)
+			var puts []string
+			target, err := minio.NewCore("target.invalid", &minio.Options{
+				Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+				Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+					if r.Body != nil {
+						io.Copy(io.Discard, r.Body)
+						r.Body.Close()
+					}
+					if r.Method == http.MethodPut {
+						puts = append(puts, r.URL.Path)
+					}
+					return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": []string{"\"pushetag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+				}),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			job := BatchJobReplicateV1{
+				Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+				Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+			}
+			src := batchPutVersion(t, obj, bucket, "push/locked", batchLockMeta("COMPLIANCE"))
+			if err := job.ReplicateToTarget(t.Context(), obj, target, src, false, batchRootSubmitter(t)); err != nil {
+				t.Fatalf("%s: authorized push of a locked version: %v", instance, err)
+			}
+			if len(puts) != 1 {
+				t.Fatalf("%s: authorized push sent %d writes", instance, len(puts))
+			}
+
+			// A user that may list but not read.
+			p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:StartBatchJob"]},{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::` + bucket + `"]}]}`))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err = globalIAMSys.SetPolicy(t.Context(), "batch-list-only", *p); err != nil {
+				t.Fatal(err)
+			}
+			user := auth.Credentials{AccessKey: "batch-list-user", SecretKey: "batch-list-user-secret"}
+			if _, err = globalIAMSys.CreateUser(t.Context(), user.AccessKey, madmin.AddOrUpdateUserReq{SecretKey: user.SecretKey, Status: madmin.AccountEnabled}); err != nil {
+				t.Fatal(err)
+			}
+			if _, err = globalIAMSys.PolicyDBSet(t.Context(), user.AccessKey, "batch-list-only", regUser, false); err != nil {
+				t.Fatal(err)
+			}
+			sub, err := newBatchJobSubmitter(globalIAMSysUser(t, user.AccessKey))
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = job.ReplicateToTarget(t.Context(), obj, target, src, false, sub)
+			if !isBatchJobAuthzError(err) || len(puts) != 1 {
+				t.Fatalf("%s: unauthorized push returned %v and sent %d writes", instance, err, len(puts)-1)
+			}
+		},
+	})
+}
+
+// globalIAMSysUser returns the stored credential of accessKey, as a request
+// authenticated with it has.
+func globalIAMSysUser(t *testing.T, accessKey string) auth.Credentials {
+	t.Helper()
+	u, ok := globalIAMSys.store.GetUser(accessKey)
+	if !ok {
+		t.Fatalf("no user %s", accessKey)
+	}
+	return u.Credentials
+}
+
+// The cursor of a walk done out of order moves only past keys whose items
+// are all done, and counts each item once.
+func TestBatchProgressCursor(t *testing.T) {
+	ri := &batchJobInfo{}
+	p := newBatchProgress(ri, "bucket")
+	items := []string{"a", "a", "b", "c", "c"}
+	seqs := make([]uint64, len(items))
+	for i := range items {
+		seqs[i] = p.begin()
+	}
+	skip := batchCounts{versionsSkipped: 1}
+	p.done(seqs[1], "a", skip)
+	p.done(seqs[3], "c", skip)
+	if ri.Object != "" || ri.VersionsSkippedLocked != 0 {
+		t.Fatalf("cursor moved before the first item: %q %d", ri.Object, ri.VersionsSkippedLocked)
+	}
+	p.done(seqs[0], "a", skip)
+	p.done(seqs[2], "b", skip)
+	// a is done; b is done but c may still have items.
+	if ri.Object != "b" && ri.Object != "a" {
+		t.Fatalf("cursor %q", ri.Object)
+	}
+	if ri.Object == "a" && ri.VersionsSkippedLocked != 2 {
+		t.Fatalf("counts %d at cursor a", ri.VersionsSkippedLocked)
+	}
+	// c's last item never finishes, as when a run stops: the cursor stays
+	// before c and c is not counted.
+	p.finish()
+	if ri.Object != "b" || ri.VersionsSkippedLocked != 3 {
+		t.Fatalf("interrupted: cursor %q counts %d, want b 3", ri.Object, ri.VersionsSkippedLocked)
+	}
+	p.done(seqs[4], "c", skip)
+	p.finish()
+	if ri.Object != "c" || ri.VersionsSkippedLocked != 5 {
+		t.Fatalf("finished: cursor %q counts %d, want c 5", ri.Object, ri.VersionsSkippedLocked)
+	}
+}

cmd/batch-progress.go

@@ -0,0 +1,190 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+// batchCounts are outcomes of a batch job that a later run of the job would
+// see again, and so must be persisted together with the cursor of the walk.
+type batchCounts struct {
+	objects             int64
+	objectsFailed       int64
+	deleteMarkersFailed int64
+	versionsSkipped     int64
+	keysRefused         int64
+}
+
+func (c *batchCounts) add(o batchCounts) {
+	c.objects += o.objects
+	c.objectsFailed += o.objectsFailed
+	c.deleteMarkersFailed += o.deleteMarkersFailed
+	c.versionsSkipped += o.versionsSkipped
+	c.keysRefused += o.keysRefused
+}
+
+//msgp:ignore batchProgress batchProgressItem batchCounts
+
+// batchProgress is the cursor of a walk whose items are done concurrently.
+//
+// The items of a walk are numbered in walk order. A key is done when every
+// item of it, and every item before it, is done; only then does the cursor
+// move past it and its counts become part of the job's counts. The cursor and
+// the counts are persisted together, so a job resumed after the cursor
+// neither skips an item that was not done nor counts an item twice.
+type batchProgress struct {
+	mu      sync.Mutex
+	ri      *batchJobInfo
+	bucket  string
+	issued  uint64
+	next    uint64
+	pending map[uint64]batchProgressItem
+	// key is the key of the item last committed, and held are the counts of
+	// its items so far.
+	key  string
+	held batchCounts
+}
+
+type batchProgressItem struct {
+	key    string
+	counts batchCounts
+}
+
+func newBatchProgress(ri *batchJobInfo, bucket string) *batchProgress {
+	return &batchProgress{ri: ri, bucket: bucket, pending: make(map[uint64]batchProgressItem)}
+}
+
+// begin numbers the next item of the walk.
+func (p *batchProgress) begin() uint64 {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	seq := p.issued
+	p.issued++
+	return seq
+}
+
+// done records item seq, of key, as done with counts.
+func (p *batchProgress) done(seq uint64, key string, counts batchCounts) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.pending[seq] = batchProgressItem{key: key, counts: counts}
+	for {
+		item, ok := p.pending[p.next]
+		if !ok {
+			return
+		}
+		delete(p.pending, p.next)
+		p.next++
+		if item.key != p.key {
+			p.commitLocked()
+			p.key = item.key
+		}
+		p.held.add(item.counts)
+	}
+}
+
+// finish commits the last key if every item of the walk is done. Call it
+// once no item is in progress.
+func (p *batchProgress) finish() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.next == p.issued {
+		p.commitLocked()
+	}
+}
+
+func (p *batchProgress) commitLocked() {
+	if p.key == "" {
+		return
+	}
+	ri := p.ri
+	ri.mu.Lock()
+	ri.Bucket = p.bucket
+	ri.Object = p.key
+	ri.Objects += p.held.objects
+	ri.ObjectsFailed += p.held.objectsFailed
+	ri.DeleteMarkersFailed += p.held.deleteMarkersFailed
+	ri.VersionsSkippedLocked += p.held.versionsSkipped
+	ri.KeysRefused += p.held.keysRefused
+	ri.mu.Unlock()
+	p.held = batchCounts{}
+}
+
+// countRemoved counts versions a job removed. A removed version is gone for
+// any later run, so it is counted at once, not at the cursor.
+func (ri *batchJobInfo) countRemoved(bucket string, objects, markers int64) {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	ri.Bucket = bucket
+	ri.Objects += objects
+	ri.DeleteMarkers += markers
+}
+
+// countSkipped counts a version a job left in place because Object Lock
+// protects it, for a job that is counted per run rather than at a cursor.
+func (ri *batchJobInfo) countSkipped() {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	ri.VersionsSkippedLocked++
+}
+
+// batchCanonicalPrefixes orders the prefixes of a walk and drops a prefix
+// that another one covers. The keys of the walk are then visited once each,
+// in one ascending order, which a single cursor can describe.
+func batchCanonicalPrefixes(prefixes []string) []string {
+	if len(prefixes) == 0 {
+		return []string{""}
+	}
+	sorted := slices.Clone(prefixes)
+	slices.Sort(sorted)
+	out := sorted[:0]
+	for _, p := range sorted {
+		if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) {
+			continue
+		}
+		out = append(out, p)
+	}
+	return out
+}
+
+// batchResumeAfter reports whether a resumed walk returned a key after the
+// cursor. The cursor key is done; the walk forwards conservatively and
+// returns it again. Keys compare as the walk orders them, by name, so a
+// directory object comes before the keys under it.
+func batchResumeAfter(cursor, name string) bool {
+	return cursor == "" || name > cursor
+}
+
+// batchJobStatus is the status response: madmin.BatchJobStatus with the
+// fields this server adds. The added fields are additive; a client that does
+// not know them ignores them.
+type batchJobStatus struct {
+	LastMetric batchJobStatusMetric
+}
+
+type batchJobStatusMetric struct {
+	madmin.JobMetric
+	VersionsSkippedLocked *int64 `json:"versionsSkippedLocked,omitempty"`
+	KeysRefused           int64  `json:"keysRefused,omitempty"`
+	FailureReason         string `json:"failureReason,omitempty"`
+	PausedReason          string `json:"pausedReason,omitempty"`
+}
+
+func (ri *batchJobInfo) statusMetric() batchJobStatusMetric {
+	ri.mu.RLock()
+	defer ri.mu.RUnlock()
+	versionsSkipped := ri.VersionsSkippedLocked
+	return batchJobStatusMetric{
+		JobMetric:             ri.metric(),
+		VersionsSkippedLocked: &versionsSkipped,
+		KeysRefused:           ri.KeysRefused,
+		FailureReason:         ri.FailureReason,
+		PausedReason:          ri.PausedReason,
+	}
+}

cmd/batch-replicate-retry_test.go

@@ -0,0 +1,343 @@
+package cmd
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+type batchReplicationWriteObserver struct {
+	ObjectLayer
+	written func(string, string)
+}
+
+type batchReplicationWalkFailure struct {
+	ObjectLayer
+	streamed bool
+}
+
+func (failure batchReplicationWalkFailure) Walk(ctx context.Context, _, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
+	if !failure.streamed {
+		return errErasureReadQuorum
+	}
+	go func() {
+		defer close(results)
+		select {
+		case results <- itemOrErr[ObjectInfo]{Err: errErasureReadQuorum}:
+		case <-ctx.Done():
+		}
+	}()
+	return nil
+}
+
+func TestBatchReplicationListingUnavailable(t *testing.T) {
+	for _, snowball := range []bool{false, true} {
+		for _, streamed := range []bool{false, true} {
+			t.Run(fmt.Sprintf("snowball=%t/streamed=%t", snowball, streamed), func(t *testing.T) {
+				fixture := newBatchAuthzFixture(t)
+				job := BatchJobRequest{
+					ID: "replicate-listing" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+					Replicate: &BatchJobReplicateV1{
+						Source: BatchJobReplicateSource{
+							Type: BatchJobReplicateResourceMinIO, Bucket: fixture.bucket,
+							Snowball: BatchJobSnowball{
+								Disable: ptr(!snowball), InMemory: ptr(true), Compress: ptr(false), SkipErrs: ptr(false),
+								Batch: ptr(1), SmallerThan: ptr("1MiB"),
+							},
+						},
+						Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid"},
+					},
+				}
+				api := batchReplicationWalkFailure{ObjectLayer: fixture.obj, streamed: streamed}
+				if err := job.Replicate.Start(fixture.ctx, api, job); !isBatchJobUnavailable(err) {
+					t.Fatalf("listing failure did not pause: %v", err)
+				}
+				report := fixture.jobInfo(t, &job)
+				if report.PausedReason == "" || report.Failed || report.Complete || report.Object != "" || report.ObjectsFailed != 0 {
+					t.Fatalf("listing failure saved an invalid outcome: %+v", report)
+				}
+			})
+		}
+	}
+}
+
+func (observer batchReplicationWriteObserver) PutObject(ctx context.Context, bucket, object string, reader *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
+	info, err := observer.ObjectLayer.PutObject(ctx, bucket, object, reader, opts)
+	if err == nil {
+		observer.written(bucket, object)
+	}
+	return info, err
+}
+
+func TestBatchReplicationRemoteListingDenied(t *testing.T) {
+	fixture := newBatchAuthzFixture(t)
+	previousTransport := remoteInstanceTransport
+	remoteInstanceTransport = atomic.Value{}
+	setRemoteInstanceTransport(batchLockTransport(func(request *http.Request) (*http.Response, error) {
+		status := http.StatusForbidden
+		body := `<Error><Code>AccessDenied</Code><Message>source listing denied</Message></Error>`
+		if request.URL.Query().Has("location") {
+			status = http.StatusOK
+			body = `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
+		}
+		return &http.Response{
+			StatusCode: status, Header: make(http.Header),
+			Body: io.NopCloser(strings.NewReader(body)), ContentLength: int64(len(body)), Request: request,
+		}, nil
+	}))
+	defer func() { remoteInstanceTransport = previousTransport }()
+	job := BatchJobRequest{
+		ID: "replicate-listing-denied" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+		Replicate: &BatchJobReplicateV1{
+			Source: BatchJobReplicateSource{
+				Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://source.invalid",
+				Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"},
+			},
+			Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: fixture.bucket},
+		},
+	}
+	if err := job.Replicate.StartFromSource(fixture.ctx, fixture.obj, job); err != nil {
+		t.Fatalf("explicit remote denial was not terminal: %v", err)
+	}
+	report := fixture.jobInfo(t, &job)
+	if !report.Failed || report.Complete || report.FailureReason == "" || report.PausedReason != "" {
+		t.Fatalf("remote denial saved an invalid outcome: %+v", report)
+	}
+}
+
+func TestBatchReplicationDependencyRecovery(t *testing.T) {
+	for _, mode := range []string{"push", "pull", "snowball"} {
+		for _, denied := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/denied=%t", mode, denied), func(t *testing.T) {
+				t.Setenv("_MINIO_BATCH_REPLICATION_WORKERS", "2")
+				fixture := newBatchAuthzFixture(t)
+				keys := []string{"a-done", "b-pending", "c-later"}
+				versions := make(map[string]string)
+				for _, key := range keys {
+					versions[key] = mustGetUUID()
+					if mode != "pull" {
+						versions[key] = batchPutVersion(t, fixture.obj, fixture.bucket, key, nil).VersionID
+					}
+				}
+				laterDone := make(chan struct{})
+				signalLater := sync.OnceFunc(func() { close(laterDone) })
+				var writtenMu sync.Mutex
+				written := make(map[string]bool)
+				recordWrite := func(bucket, key string) {
+					if bucket != fixture.bucket && bucket != "remote" {
+						return
+					}
+					writtenMu.Lock()
+					written[key] = true
+					writtenMu.Unlock()
+					if key == "c-later" {
+						signalLater()
+					}
+				}
+				var outage atomic.Bool
+				outage.Store(true)
+				var decisions atomic.Int32
+				pluginServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+					var input struct {
+						Input policy.Args `json:"input"`
+					}
+					if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
+						t.Error(err)
+						writer.WriteHeader(http.StatusBadRequest)
+						return
+					}
+					action := policy.Action(policy.GetObjectAction)
+					if mode == "pull" {
+						action = policy.PutObjectAction
+					}
+					if outage.Load() && input.Input.Action == action && input.Input.ObjectName == "b-pending" {
+						decisions.Add(1)
+						if mode != "snowball" {
+							select {
+							case <-laterDone:
+							case <-request.Context().Done():
+								return
+							}
+						}
+						if denied {
+							io.WriteString(writer, `{"result":false}`)
+							return
+						}
+						writer.WriteHeader(http.StatusServiceUnavailable)
+					}
+					io.WriteString(writer, `{"result":true}`)
+				}))
+				defer pluginServer.Close()
+				pluginURL, err := xnet.ParseHTTPURL(pluginServer.URL)
+				mustIAM(t, err)
+				previousPlugin := newGlobalAuthZPluginFn()
+				setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
+					URL: pluginURL, CloseRespFn: func(reader io.ReadCloser) { reader.Close() },
+				}))
+				defer setGlobalAuthZPlugin(previousPlugin)
+				previousTransport := remoteInstanceTransport
+				remoteInstanceTransport = atomic.Value{}
+				setRemoteInstanceTransport(batchLockTransport(func(request *http.Request) (*http.Response, error) {
+					if err := request.Context().Err(); err != nil {
+						return nil, err
+					}
+					body := ""
+					header := make(http.Header)
+					query := request.URL.Query()
+					switch {
+					case query.Has("location"):
+						body = `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
+					case query.Has("versions"):
+						body = `<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>remote</Name><IsTruncated>false</IsTruncated>`
+						for _, key := range keys {
+							body += fmt.Sprintf(`<Version><Key>%s</Key><VersionId>%s</VersionId><IsLatest>true</IsLatest><LastModified>2026-11-01T00:00:00.000Z</LastModified><ETag>"etag"</ETag><Size>7</Size><StorageClass>STANDARD</StorageClass></Version>`, key, versions[key])
+						}
+						body += `</ListVersionsResult>`
+					case request.Method == http.MethodGet:
+						body = "payload"
+						header.Set("Etag", `"etag"`)
+						header.Set("Last-Modified", UTCNow().Format(http.TimeFormat))
+						header.Set("X-Amz-Version-Id", query.Get("versionId"))
+					case request.Method == http.MethodPut:
+						if mode == "snowball" && request.Body != nil {
+							archive := tar.NewReader(request.Body)
+							for {
+								entry, readErr := archive.Next()
+								if readErr == io.EOF {
+									break
+								}
+								if readErr != nil {
+									return nil, readErr
+								}
+								recordWrite("remote", entry.Name)
+							}
+						} else if mode != "snowball" {
+							if _, err := io.Copy(io.Discard, request.Body); err != nil {
+								return nil, err
+							}
+							recordWrite("remote", strings.TrimPrefix(request.URL.Path, "/remote/"))
+						}
+						header.Set("Etag", `"etag"`)
+					default:
+						return nil, fmt.Errorf("unexpected replication request: %s %s", request.Method, request.URL)
+					}
+					header.Set("Content-Length", strconv.Itoa(len(body)))
+					return &http.Response{
+						StatusCode: http.StatusOK, Header: header,
+						Body: io.NopCloser(strings.NewReader(body)), ContentLength: int64(len(body)), Request: request,
+					}, nil
+				}))
+				defer func() { remoteInstanceTransport = previousTransport }()
+				job := &BatchJobRequest{
+					ID: "replicate-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+					Replicate: &BatchJobReplicateV1{
+						Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: fixture.bucket},
+						Target: BatchJobReplicateTarget{
+							Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid",
+							Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"},
+						},
+					},
+				}
+				if mode == "pull" {
+					job.Replicate.Source.Endpoint = job.Replicate.Target.Endpoint
+					job.Replicate.Source.Creds = job.Replicate.Target.Creds
+					job.Replicate.Source.Bucket = "remote"
+					job.Replicate.Target.Endpoint = ""
+					job.Replicate.Target.Bucket = fixture.bucket
+				}
+				if mode == "snowball" {
+					job.Replicate.Source.Snowball = BatchJobSnowball{
+						Disable: ptr(false), InMemory: ptr(true), Compress: ptr(false), SkipErrs: ptr(false),
+						Batch: ptr(1), SmallerThan: ptr("1MiB"),
+					}
+				}
+				job.Replicate.Flags.Retry.Attempts = 1
+				job.Replicate.Flags.Retry.Delay = time.Millisecond
+				interrupted := &batchJobInfo{
+					JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started,
+					Object: "c-later", Objects: 2,
+				}
+				mustIAM(t, interrupted.updateAfter(fixture.ctx, fixture.obj, 0, *job))
+				pool := globalBatchJobPool
+				pool.objLayer = batchReplicationWriteObserver{ObjectLayer: fixture.obj, written: recordWrite}
+				ctx, cancel := context.WithCancel(fixture.ctx)
+				pool.ctx = ctx
+				mustIAM(t, pool.queueJob(job))
+				done := make(chan struct{})
+				go func() { defer close(done); pool.AddWorker() }()
+				defer func() {
+					cancel()
+					<-done
+					if t.Failed() {
+						t.Logf("last report: %+v", fixture.jobInfo(t, job))
+					}
+				}()
+				state := "paused"
+				if denied {
+					state = "terminal"
+				}
+				waitBatchState(t, pool, job.ID, state)
+				report := fixture.jobInfo(t, job)
+				if decisions.Load() == 0 {
+					t.Fatal("test did not reach per-object authorization")
+				}
+				if denied {
+					if !report.Failed || report.Complete || report.FailureReason == "" || report.PausedReason != "" {
+						t.Fatalf("denial was not terminal: %+v", report)
+					}
+					pool.retryDue(time.Now().Add(time.Hour))
+					if len(pool.jobCh) != 0 {
+						t.Fatal("denied job was retried")
+					}
+					return
+				}
+				if report.Failed || report.Complete || report.FailureReason != "" || report.PausedReason == "" ||
+					report.Object != "" || report.Objects != 0 || report.ObjectsFailed != 0 {
+					t.Fatalf("outage was not saved with a safe restart point: %+v", report)
+				}
+				pool.jmu.Lock()
+				owner := pool.jobOwners[job.ID]
+				sameOwner := owner.req == job && owner.retries == 1
+				pool.jmu.Unlock()
+				if !sameOwner {
+					t.Fatal("pause lost the job owner")
+				}
+				outage.Store(false)
+				pool.retryDue(time.Now().Add(time.Hour))
+				waitBatchState(t, pool, job.ID, "terminal")
+				report = fixture.jobInfo(t, job)
+				if !report.Complete || report.Failed || report.PausedReason != "" || report.Objects != int64(len(keys)) || report.ObjectsFailed != 0 {
+					t.Fatalf("retry did not finish with consistent counts: %+v", report)
+				}
+				writtenMu.Lock()
+				for _, key := range keys {
+					if !written[key] {
+						t.Errorf("retry skipped %s", key)
+					}
+				}
+				writtenMu.Unlock()
+				if mode == "pull" {
+					for _, key := range keys {
+						if !batchVersionExists(t, fixture.obj, fixture.bucket, key, versions[key]) {
+							t.Errorf("retry lost remote version of %s", key)
+						}
+					}
+				}
+			})
+		}
+	}
+}

cmd/batch-retry.go

@@ -0,0 +1,225 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// An unavailable dependency is not an authorization decision. It stops the
+// current attempt without turning an outage into either an allow or a denial.
+type batchJobUnavailableError struct {
+	reason string
+	err    error
+}
+
+func (e batchJobUnavailableError) Error() string { return fmt.Sprintf("%s: %v", e.reason, e.err) }
+func (e batchJobUnavailableError) Unwrap() error { return e.err }
+func batchUnavailable(reason string, err error) error {
+	if err == nil {
+		return nil
+	}
+	if isBatchJobUnavailable(err) {
+		return err
+	}
+	return batchJobUnavailableError{reason: reason, err: err}
+}
+
+func isBatchJobUnavailable(err error) bool { var e batchJobUnavailableError; return errors.As(err, &e) }
+
+func isBatchJobStopped(err error) bool {
+	return isBatchJobAuthzError(err) || isBatchJobUnavailable(err)
+}
+
+func batchListingError(err error) error {
+	if isErrBucketNotFound(err) {
+		return err
+	}
+	return batchUnavailable("object listing unavailable", err)
+}
+
+func (ri *batchJobInfo) pause(err error) {
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+	// A concurrent explicit denial wins over a transient worker error.
+	if ri.FailureReason != "" {
+		return
+	}
+	ri.Complete, ri.Failed = false, false
+	ri.PausedReason = err.Error()
+}
+
+func (ri *batchJobInfo) savePaused(ctx context.Context, api ObjectLayer, job BatchJobRequest, err error) error {
+	ri.pause(err)
+	globalBatchJobsMetrics.save(job.ID, ri)
+	saveCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+	defer cancel()
+	if saveErr := ri.updateAfter(saveCtx, api, 0, job); saveErr != nil {
+		return batchUnavailable("saving paused job", errors.Join(err, saveErr))
+	}
+	return err
+}
+
+// Authenticate the HTTP request normally, but keep dependency errors through
+// the authorization step so submission returns 503, not a permanent 403.
+func validateBatchJobAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) (ObjectLayer, auth.Credentials) {
+	api := newObjectLayerFn()
+	if api == nil || globalNotificationSys == nil || !batchIAMReady() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return nil, auth.Credentials{}
+	}
+	cred, owner, code := validateAdminSignature(ctx, r, "")
+	if code != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(code), r.URL)
+		return nil, cred
+	}
+	submitter, err := newBatchJobSubmitter(cred)
+	if err == nil {
+		var principal batchJobPrincipal
+		principal, err = submitter.principal(ctx)
+		if err == nil {
+			err = principal.authorizeArgs(policy.Args{
+				AccountName: cred.AccessKey, Groups: cred.Groups,
+				Action: policy.Action(policy.StartBatchJobAction), ConditionValues: getConditionValues(r, "", cred), IsOwner: owner, Claims: principal.claims,
+			}, nil)
+		}
+	}
+	if err != nil {
+		apiErr := errorCodes.ToAPIErrWithErr(ErrAccessDenied, err)
+		if isBatchJobUnavailable(err) {
+			apiErr = batchServiceUnavailable(err)
+		}
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
+		return nil, cred
+	}
+	return api, cred
+}
+
+// One owner record spans queued, running and paused attempts. The request
+// pointer is the generation token. Canceled records fence stale requests until
+// shutdown. Terminal records fence discovery until the startup scan finishes,
+// including jobs submitted and terminated while the scan was unavailable.
+type batchJobOwner struct {
+	req     *BatchJobRequest
+	cancel  context.CancelFunc
+	state   string
+	retries uint
+	due     time.Time
+}
+
+func (j *BatchJobPool) beginJob(job *BatchJobRequest) bool {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	o := j.jobOwners[job.ID]
+	if o == nil || o.req != job || o.state != "queued" || job.ctx.Err() != nil || j.ctx.Err() != nil {
+		return false
+	}
+	o.state = "running"
+	return true
+}
+
+func (j *BatchJobPool) finishJob(job *BatchJobRequest, err error) {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	o := j.jobOwners[job.ID]
+	if o == nil || o.req != job || o.state != "running" {
+		return
+	}
+	if isBatchJobUnavailable(err) && job.ctx.Err() == nil && j.ctx.Err() == nil {
+		o.due = time.Now().Add(batchJobRetryDelay(o.retries))
+		o.retries++
+		o.state = "paused"
+		return
+	}
+	o.cancel()
+	if j.discoveryDone {
+		delete(j.jobOwners, job.ID)
+	} else {
+		j.jobOwners[job.ID] = &batchJobOwner{state: "terminal"}
+	}
+}
+
+// After the only startup discovery has returned, no loaded manifest remains
+// to re-admit a terminal job. Release its marker, but keep queued, running,
+// paused and canceled ownership intact. New submissions use new job IDs.
+func (j *BatchJobPool) finishDiscovery() {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	j.discoveryDone = true
+	for id, owner := range j.jobOwners {
+		if owner.state == "terminal" {
+			delete(j.jobOwners, id)
+		}
+	}
+}
+
+func batchJobRetryDelay(retries uint) time.Duration {
+	delay := 30 * time.Second * time.Duration(1<<min(retries, 4))
+	return delay/2 + time.Duration(rand.Float64()*float64(delay))
+}
+
+// retryDue holds the ownership lock across the nonblocking handoff. A fast
+// worker cannot replace a pause while the scheduler removes the old pause.
+func (j *BatchJobPool) retryDue(now time.Time) {
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	if j.ctx.Err() != nil {
+		return
+	}
+	for _, o := range j.jobOwners {
+		if o.state != "paused" || now.Before(o.due) || o.req.ctx.Err() != nil {
+			continue
+		}
+		select {
+		case j.jobCh <- o.req:
+			o.state = "queued"
+		default:
+		}
+	}
+}
+
+func (j *BatchJobPool) retryLoop() {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-j.ctx.Done():
+			return
+		case now := <-ticker.C:
+			j.retryDue(now)
+		}
+	}
+}
+
+func batchServiceUnavailable(err error) APIError {
+	return APIError{Code: "ServiceUnavailable", Description: err.Error(), HTTPStatusCode: http.StatusServiceUnavailable}
+}
+
+// initStore precedes the first successful cache load. During that interval a
+// missing identity or policy is not a revocation decision, including for root.
+func batchIAMReady() bool {
+	sys := globalIAMSys
+	if sys == nil {
+		return false
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	if sys.store == nil {
+		return false
+	}
+	select {
+	case <-sys.configLoaded:
+		return true
+	default:
+		return false
+	}
+}

cmd/batch-retry_test.go

@@ -0,0 +1,349 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"github.com/tinylib/msgp/msgp"
+)
+
+func waitBatchState(t *testing.T, pool *BatchJobPool, id, state string) {
+	t.Helper()
+	timeout := time.NewTimer(10 * time.Second)
+	defer timeout.Stop()
+	tick := time.NewTicker(time.Millisecond)
+	defer tick.Stop()
+	for {
+		pool.jmu.Lock()
+		owner := pool.jobOwners[id]
+		ready := owner == nil && state == ""
+		if owner != nil {
+			ready = owner.state == state
+		}
+		pool.jmu.Unlock()
+		if ready {
+			return
+		}
+		select {
+		case <-timeout.C:
+			t.Fatalf("job did not reach %q", state)
+		case <-tick.C:
+		}
+	}
+}
+
+func TestBatchPluginPauseRecoversInProcess(t *testing.T) {
+	for _, midrun := range []bool{false, true} {
+		t.Run(map[bool]string{false: "before-start", true: "during-delete"}[midrun], func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			older := f.putVersions(t, "data/key")
+			var unavailable atomic.Bool
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				var request struct {
+					Input policy.Args `json:"input"`
+				}
+				_ = json.NewDecoder(r.Body).Decode(&request)
+				if unavailable.Load() && (!midrun || request.Input.Action == policy.DeleteObjectVersionAction) {
+					w.WriteHeader(503)
+					io.WriteString(w, `{"result":true}`)
+					return
+				}
+				io.WriteString(w, `{"result":true}`)
+			}))
+			defer server.Close()
+			url, err := xnet.ParseHTTPURL(server.URL)
+			mustIAM(t, err)
+			previous := newGlobalAuthZPluginFn()
+			setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: url, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
+			defer setGlobalAuthZPlugin(previous)
+			rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+			if job == nil {
+				t.Fatalf("submit %d: %s", rec.Code, rec.Body.String())
+			}
+			pool := globalBatchJobPool
+			ctx, cancel := context.WithCancel(f.ctx)
+			pool.ctx = ctx
+			// Re-admit the fixture's consumed request with a context owned by this worker.
+			pool.jmu.Lock()
+			pool.jobOwners[job.ID].cancel()
+			delete(pool.jobOwners, job.ID)
+			pool.jmu.Unlock()
+			mustIAM(t, pool.queueJob(job))
+			unavailable.Store(true)
+			done := make(chan struct{})
+			go func() { defer close(done); pool.AddWorker() }()
+			defer func() { cancel(); <-done }()
+			waitBatchState(t, pool, job.ID, "paused")
+			ri := f.jobInfo(t, job)
+			if ri.Failed || ri.Complete || ri.FailureReason != "" || ri.PausedReason == "" || ri.Object != "" {
+				t.Fatalf("paused report: %+v", ri)
+			}
+			if !f.versionExists(t, "data/key", older) {
+				t.Fatal("outage authorized a deletion")
+			}
+			unavailable.Store(false)
+			pool.retryDue(time.Now().Add(time.Hour))
+			waitBatchState(t, pool, job.ID, "terminal")
+			ri = f.jobInfo(t, job)
+			if !ri.Complete || ri.Failed || ri.PausedReason != "" || ri.Objects != 1 || f.versionExists(t, "data/key", older) {
+				t.Fatalf("resumed report: %+v", ri)
+			}
+		})
+	}
+}
+
+func TestBatchSubmissionUnavailableIs503(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(503); io.WriteString(w, `{"result":true}`) }))
+	defer server.Close()
+	url, err := xnet.ParseHTTPURL(server.URL)
+	mustIAM(t, err)
+	previous := newGlobalAuthZPluginFn()
+	setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: url, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
+	defer setGlobalAuthZPlugin(previous)
+	rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+	if rec.Code != 503 || job != nil {
+		t.Fatalf("outage submission: %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestBatchPoolOwnershipAndCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	pool := &BatchJobPool{ctx: ctx, jobCh: make(chan *BatchJobRequest, 1)}
+	first, second := &BatchJobRequest{ID: "first"}, &BatchJobRequest{ID: "second"}
+	mustIAM(t, pool.queueJob(first))
+	mustIAM(t, pool.queueJob(second))
+	if pool.jobOwners[second.ID].state != "paused" {
+		t.Fatal("full queue lost pending owner")
+	}
+	original := second.ctx
+	var wg sync.WaitGroup
+	for range 8 {
+		wg.Go(func() {
+			if err := pool.queueJob(&BatchJobRequest{ID: "second"}); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+	wg.Wait()
+	if second.ctx != original || len(pool.jobCh) != 1 {
+		t.Fatal("duplicate changed ownership")
+	}
+	if !pool.beginJob(<-pool.jobCh) {
+		t.Fatal("first not owned")
+	}
+	pool.finishJob(first, nil)
+	mustIAM(t, pool.queueJob(&BatchJobRequest{ID: first.ID}))
+	if len(pool.jobCh) != 0 {
+		t.Fatal("late discovery re-admitted a terminal job")
+	}
+	pool.retryDue(time.Now())
+	next := <-pool.jobCh
+	if next != second || !pool.beginJob(next) {
+		t.Fatal("pending handoff lost ownership")
+	}
+	for range 2 {
+		pool.finishJob(second, batchUnavailable("test", errors.New("outage")))
+		pool.retryDue(time.Now().Add(time.Hour))
+		if !pool.beginJob(<-pool.jobCh) {
+			t.Fatal("retry lost owner")
+		}
+	}
+	if pool.jobOwners[second.ID].retries != 2 {
+		t.Fatal("retry count reset during handoff")
+	}
+	mustIAM(t, pool.canceler(second.ID, true))
+	mustIAM(t, pool.canceler(second.ID, true))
+	pool.finishJob(second, batchUnavailable("late", errors.New("outage")))
+	pool.retryDue(time.Now().Add(time.Hour))
+	if len(pool.jobCh) != 0 || second.ctx.Err() == nil {
+		t.Fatal("cancel lost to retry")
+	}
+	if err := pool.queueJob(&BatchJobRequest{ID: second.ID}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("stale admission: %v", err)
+	}
+	// A cancellation before this node admitted a manifest is a tombstone too.
+	mustIAM(t, pool.canceler("not-loaded", true))
+	mustIAM(t, pool.canceler("not-loaded", true))
+	if err := pool.queueJob(&BatchJobRequest{ID: "not-loaded"}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("stale resume: %v", err)
+	}
+	cancel()
+	if err := pool.queueJob(&BatchJobRequest{ID: "shutdown"}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("shutdown admission: %v", err)
+	}
+	pool.retryDue(time.Now().Add(time.Hour)) // channel remains open; no send/close race.
+}
+
+// A cancellation of a job this node does not own leaves a record only while
+// startup discovery may still admit its manifest. Once discovery has
+// finished, canceling unknown IDs must not grow the ownership table.
+func TestBatchCancelUnknownJobAfterDiscovery(t *testing.T) {
+	pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
+	// Before discovery finished, the record fences the manifest it may admit.
+	mustIAM(t, pool.canceler("expire-before-discovery", true))
+	if err := pool.queueJob(&BatchJobRequest{ID: "expire-before-discovery"}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("admission of a job canceled during discovery: %v", err)
+	}
+	pool.finishDiscovery()
+	for i := range 1000 {
+		mustIAM(t, pool.canceler(fmt.Sprintf("expire-unknown-%d", i), true))
+	}
+	pool.jmu.Lock()
+	owners := len(pool.jobOwners)
+	pool.jmu.Unlock()
+	if owners != 1 {
+		t.Fatalf("%d owner records after canceling unknown jobs, want only the record from discovery", owners)
+	}
+	// An owned job is still canceled and fenced.
+	job := &BatchJobRequest{ID: "expire-owned"}
+	mustIAM(t, pool.queueJob(job))
+	mustIAM(t, pool.canceler(job.ID, true))
+	if job.ctx.Err() == nil {
+		t.Fatal("cancellation did not stop the owned job")
+	}
+	if err := pool.queueJob(&BatchJobRequest{ID: job.ID}); !errors.Is(err, context.Canceled) {
+		t.Fatalf("stale admission of a canceled job: %v", err)
+	}
+}
+
+type batchPolicyOutage struct {
+	IAMStorageAPI
+	failed string
+	err    error
+}
+
+func (s *batchPolicyOutage) loadPolicyDoc(ctx context.Context, name string, out map[string]PolicyDoc) error {
+	if name == s.failed {
+		return s.err
+	}
+	return s.IAMStorageAPI.loadPolicyDoc(ctx, name, out)
+}
+
+func TestBatchPolicySnapshotKeepsUnavailableDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "retry-allow", batchAuthzFullAccess)
+	batchAuthzPolicy(f.ctx, t, f.sys, "retry-deny", `{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "retry-user", "retry-allow,retry-deny")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	store := f.sys.store
+	original := store.IAMStorageAPI
+	defer func() { store.IAMStorageAPI = original }()
+	cache := store.lock()
+	delete(cache.iamPolicyDocsMap, "retry-deny")
+	store.unlock()
+	store.IAMStorageAPI = &batchPolicyOutage{IAMStorageAPI: original, failed: "retry-deny", err: errErasureReadQuorum}
+	request := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
+	if err := sub.authorize(f.ctx, request); !isBatchJobUnavailable(err) {
+		t.Fatalf("missing Deny must pause, got %v", err)
+	}
+	store.IAMStorageAPI = original
+	if err := sub.authorize(f.ctx, request); !isBatchJobAuthzError(err) {
+		t.Fatalf("recovered Deny was not enforced: %v", err)
+	}
+	// Root also waits for IAM readiness; startup must not bypass the pause gate.
+	root, err := newBatchJobSubmitter(globalActiveCred)
+	mustIAM(t, err)
+	f.sys.Lock()
+	f.sys.store = nil
+	f.sys.Unlock()
+	_, err = root.principal(f.ctx)
+	f.sys.Lock()
+	f.sys.store = store
+	f.sys.Unlock()
+	if !isBatchJobUnavailable(err) {
+		t.Fatalf("root before IAM readiness: %v", err)
+	}
+}
+
+func TestBatchPausedReportCompatibility(t *testing.T) {
+	before := &batchJobInfo{JobID: "paused", Object: "cursor", PausedReason: "IAM unavailable"}
+	data, err := before.MarshalMsg(nil)
+	mustIAM(t, err)
+	var after batchJobInfo
+	_, err = after.UnmarshalMsg(data)
+	mustIAM(t, err)
+	if after.PausedReason != before.PausedReason || after.clone().PausedReason != before.PausedReason || after.statusMetric().PausedReason != before.PausedReason {
+		t.Fatal("paused reason lost")
+	}
+	old := msgp.AppendMapHeader(nil, 2)
+	old = msgp.AppendString(old, "jid")
+	old = msgp.AppendString(old, "legacy")
+	old = msgp.AppendString(old, "cmp")
+	old = msgp.AppendBool(old, true)
+	var legacy batchJobInfo
+	_, err = legacy.UnmarshalMsg(old)
+	mustIAM(t, err)
+	if legacy.PausedReason != "" || !legacy.Complete {
+		t.Fatal("old report decode changed")
+	}
+	after.refuse(errBatchJobInvalid)
+	if after.PausedReason != "" || !after.Failed {
+		t.Fatal("terminal denial retained pause")
+	}
+}
+
+func TestBatchPersistenceFencesOlderExecutors(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+	if job == nil {
+		t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+	}
+	raw, err := readConfig(f.ctx, f.obj, getJobPath(*job))
+	mustIAM(t, err)
+	if !bytes.HasPrefix(raw, []byte(batchJobEnvelope)) {
+		t.Fatal("new manifest has no executor version fence")
+	}
+	var oldDecoder BatchJobRequest
+	if _, err := oldDecoder.UnmarshalMsg(raw); err == nil {
+		t.Fatal("old bare-msgpack executor could decode new manifest")
+	}
+	var loaded BatchJobRequest
+	mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+	if loaded.Submitter == nil {
+		t.Fatal("new executor lost submitter")
+	}
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started}
+	outage := batchUnavailable("test", errErasureReadQuorum)
+	if err := ri.savePaused(f.ctx, f.obj, *job, outage); !errors.Is(err, outage) {
+		t.Fatal(err)
+	}
+	reportPath, err := job.getJobReportPath()
+	mustIAM(t, err)
+	report, err := readConfig(f.ctx, f.obj, reportPath)
+	mustIAM(t, err)
+	if binary.LittleEndian.Uint16(report[2:4]) == batchExpireVersionV1 {
+		t.Fatal("old executor would accept paused report")
+	}
+	mustIAM(t, ri.load(f.ctx, f.obj, *job))
+	if ri.PausedReason == "" || ri.Failed {
+		t.Fatal("restart lost pause")
+	}
+	// A stale resume loaded the manifest before cancellation.
+	mustIAM(t, globalBatchJobPool.canceler(job.ID, true))
+	mustIAM(t, job.delete(f.ctx, f.obj))
+	if err := globalBatchJobPool.queueJob(&loaded); !errors.Is(err, context.Canceled) {
+		t.Fatalf("canceled job re-admitted: %v", err)
+	}
+	if err := loaded.load(f.ctx, f.obj, getJobPath(*job)); !errors.Is(err, errNoSuchJob) {
+		t.Fatalf("canceled manifest recreated: %v", err)
+	}
+}

cmd/batch-rotate.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"maps"
 	"math/rand"
@@ -35,6 +36,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/pgsty/silo-pkg/v3/env"
+	"github.com/pgsty/silo-pkg/v3/policy"
 	"github.com/pgsty/silo-pkg/v3/workers"
 )
 
@@ -161,8 +163,68 @@ func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error
 	return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
 }
 
-// KeyRotate rotates encryption key of an object
-func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
+// batchAuthoritativeVersion returns the version obj as all its copies
+// describe it. With several pools a version can have copies in more than one
+// pool, and a read returns one of them; the merged copy carries the newest
+// Object Lock and tag state. The caller holds the object's write lock.
+func batchAuthoritativeVersion(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
+	z, ok := api.(*erasureServerPools)
+	if !ok || z.SinglePool() {
+		return obj, nil
+	}
+	versionID := obj.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	return z.replicaObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{VersionID: versionID, NoLock: true, NoAuditLog: true})
+}
+
+// batchAuthoritativeRead is batchAuthoritativeVersion for a version that has
+// been opened for reading. It takes the object's read lock itself, since an
+// opened reader may hold none.
+func batchAuthoritativeRead(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
+	z, ok := api.(*erasureServerPools)
+	if !ok || z.SinglePool() {
+		return obj, nil
+	}
+	lk := z.NewNSLock(bucket, encodeDirObject(object))
+	lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.RUnlock(lkctx)
+	return batchAuthoritativeVersion(lkctx.Context(), api, bucket, object, obj)
+}
+
+// rotateAuthzRequests are the actions of rotating one version: it is read and
+// written again in place with the new key.
+func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzRequest {
+	versionID := oi.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	// The encryption the rewrite leaves: rotation keeps the kind of the
+	// version (rotateKey), and SSE-KMS takes the requested key.
+	values := map[string][]string{}
+	switch {
+	case crypto.S3KMS.IsEncrypted(oi.UserDefined):
+		values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
+		if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
+			values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
+		}
+	case crypto.S3.IsEncrypted(oi.UserDefined):
+		values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionAES}
+	}
+	return []batchAuthzRequest{
+		{action: policy.GetObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
+		{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
+		{action: policy.PutObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags, values: values},
+	}
+}
+
+// KeyRotate rotates encryption key of an object. A version that Object Lock
+// protects is not rewritten: it returns ObjectLocked.
+func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo, sub *batchJobSubmitter) error {
 	srcBucket := r.Bucket
 	srcObject := objInfo.Name
 
@@ -180,7 +242,8 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
 
-	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	// Lock the name writers lock: the stored name of a directory object.
+	lock := api.NewObjectNSLock(r.Bucket, encodeDirObject(objInfo.Name))
 	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err
@@ -194,10 +257,32 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 		VersionSuspended: versionSuspended,
 		NoLock:           true,
 	}
-	obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
+	obj, pool, err := batchRotateRead(ctx, api, r.Bucket, objInfo.Name, opts)
 	if err != nil {
 		return err
 	}
+	// Decide on the version as it is under the lock that the rewrite holds,
+	// before any key is requested, with the metadata of all its copies.
+	logical, err := batchAuthoritativeVersion(ctx, api, r.Bucket, objInfo.Name, obj)
+	if err != nil {
+		return err
+	}
+	if enforceRetentionForDeletion(ctx, logical) {
+		return ObjectLocked{Bucket: r.Bucket, Object: obj.Name, VersionID: obj.VersionID}
+	}
+	p, err := sub.principal(ctx)
+	if err != nil {
+		return err
+	}
+	// Authorize the copy that is read and rewritten, and the version as all
+	// its copies describe it.
+	for _, v := range []ObjectInfo{obj, logical} {
+		for _, req := range r.rotateAuthzRequests(v) {
+			if err := p.authorize(req); err != nil {
+				return err
+			}
+		}
+	}
 	oi := obj.Clone()
 	var (
 		newKeyID      string
@@ -225,16 +310,61 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	oi.metadataOnly = true
 	oi.keyRotation = true
 	maps.Copy(oi.UserDefined, encMetadata)
-	if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
-		VersionID: oi.VersionID,
-	}, ObjectOptions{
-		VersionID: oi.VersionID,
-		NoLock:    true,
-	}); err != nil {
-		return err
+	// The rewrite replaces the stored metadata with UserDefined, which a
+	// read returns without the tags and the expiry: keep them.
+	if oi.UserTags != "" {
+		oi.UserDefined[xhttp.AmzObjectTagging] = oi.UserTags
 	}
+	if !oi.Expires.IsZero() {
+		oi.UserDefined["expires"] = oi.ExpiresStr()
+	}
+	srcOpts := ObjectOptions{VersionID: oi.VersionID}
+	dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true}
+	if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
+		// Rewrite the copy that was read and authorized. The pools layer
+		// would route the rewrite by the latest version of the key, which
+		// can be in a pool that holds another write of this version ID.
+		//
+		// Check the pool's state again where the pools layer checks it,
+		// right before the write: a move can have started since the read.
+		if z.IsSuspended(pool) || z.IsPoolRebalancing(pool) {
+			return errBatchRotateMoving
+		}
+		dstOpts.replicaObjectInfo = z.replicaObjectInfo
+		name := encodeDirObject(oi.Name)
+		_, err = z.serverPools[pool].CopyObject(ctx, r.Bucket, name, r.Bucket, name, oi, srcOpts, dstOpts)
+	} else {
+		_, err = api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, srcOpts, dstOpts)
+	}
+	return err
+}
 
-	return nil
+// errBatchRotateMoving: the version is read from a pool that a decommission
+// or a rebalance is moving.
+var errBatchRotateMoving = errors.New("the version is being moved to another pool; rotate it once the move is done")
+
+// batchRotateRead reads the version that a rotation rewrites, as
+// GetObjectInfo does. With several pools it also returns the pool of the
+// copy it read, otherwise -1. The caller holds the object lock.
+//
+// A copy in a pool that a decommission or a rebalance is moving is the
+// source of that move, which must not change under it, and rewriting a copy
+// in another pool would change which copy the version's metadata comes
+// from. Such a version is left to a later run (errBatchRotateMoving).
+func batchRotateRead(ctx context.Context, api ObjectLayer, bucket, object string, opts ObjectOptions) (ObjectInfo, int, error) {
+	z, ok := api.(*erasureServerPools)
+	if !ok || z.SinglePool() {
+		oi, err := api.GetObjectInfo(ctx, bucket, object, opts)
+		return oi, -1, err
+	}
+	if err := checkGetObjArgs(ctx, bucket, object); err != nil {
+		return ObjectInfo{}, -1, err
+	}
+	oi, idx, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
+	if err == nil && (z.IsSuspended(idx) || z.IsPoolRebalancing(idx)) {
+		return ObjectInfo{}, -1, errBatchRotateMoving
+	}
+	return oi, idx, err
 }
 
 const (
@@ -255,11 +385,14 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		StartTime: job.Started,
 	}
 	if err := ri.loadOrInit(ctx, api, job); err != nil {
-		return err
+		return batchUnavailable("loading job progress", err)
 	}
 	if ri.Complete {
 		return nil
 	}
+	if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
+		return err
+	}
 
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
@@ -273,8 +406,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
 
-	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
-
 	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
@@ -355,65 +486,103 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(ctx)
+	ctx, cancelCause := context.WithCancelCause(ctx)
+	defer cancelCause(nil)
+	// refuse ends the job for its submitter: record why before stopping.
+	refuse := func(err error) {
+		if isBatchJobUnavailable(err) {
+			ri.pause(err)
+		} else {
+			ri.refuse(err)
+		}
+		cancelCause(err)
+	}
+	progress := newBatchProgress(ri, r.Bucket)
 
 	results := make(chan itemOrErr[ObjectInfo], 100)
 	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
 		Marker: lastObject,
 		Filter: selectObj,
 	}); err != nil {
-		cancel()
-		// Do not need to retry if we can't list objects on source.
-		return err
+		cause := batchListingError(err)
+		if isBatchJobUnavailable(cause) {
+			return ri.savePaused(ctx, api, job, cause)
+		}
+		return cause
 	}
 	failed := false
 	for res := range results {
+		if context.Cause(ctx) != nil {
+			// Refused or canceled: schedule nothing more.
+			continue
+		}
 		if res.Err != nil {
 			failed = true
+			cancelCause(batchListingError(res.Err))
 			batchLogIf(ctx, res.Err)
-			break
+			continue
 		}
 		result := res.Item
+		if !batchResumeAfter(lastObject, result.Name) {
+			// Completed by the run that set the cursor.
+			continue
+		}
 		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
 		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
 		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
 			continue
 		}
+		seq := progress.begin()
 		wk.Take()
 		go func() {
 			defer wk.Give()
+			var counts batchCounts
 			for attempts := 1; attempts <= retryAttempts; attempts++ {
 				stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
-				success := true
-				if err := r.KeyRotate(ctx, api, result); err != nil {
-					stopFn(result, err)
-					batchLogIf(ctx, err)
-					success = false
-					if attempts >= retryAttempts {
-						auditOptions := AuditLogOptions{
-							Event:     "KeyRotate",
-							APIName:   "StartBatchJob",
-							Bucket:    result.Bucket,
-							Object:    result.Name,
-							VersionID: result.VersionID,
-							Error:     err.Error(),
-						}
-						auditLogInternal(ctx, auditOptions)
-					}
-				} else {
-					stopFn(result, nil)
+				err := r.KeyRotate(ctx, api, result, job.Submitter)
+				if errors.Is(err, errBatchRotateMoving) {
+					err = batchUnavailable("object is being moved", err)
 				}
-				ri.trackCurrentBucketObject(r.Bucket, result, success, attempts)
-				globalBatchJobsMetrics.save(job.ID, ri)
-				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-				if success {
+				stopFn(result, err)
+				if err == nil {
+					counts.objects = 1
+					break
+				}
+				if isBatchJobStopped(err) {
+					refuse(err)
+					return
+				}
+				if isErrObjectLocked(err) {
+					// Left as it is, not retried and not a failure.
+					counts.versionsSkipped = 1
+					break
+				}
+				if ctx.Err() != nil {
+					// Not done; a later run rotates it.
+					return
+				}
+				batchLogIf(ctx, err)
+				if attempts >= retryAttempts {
+					auditOptions := AuditLogOptions{
+						Event:     "KeyRotate",
+						APIName:   "StartBatchJob",
+						Bucket:    result.Bucket,
+						Object:    result.Name,
+						VersionID: result.VersionID,
+						Error:     err.Error(),
+					}
+					auditLogInternal(ctx, auditOptions)
+					counts.objectsFailed = 1
 					break
 				}
 				if delay > 0 {
-					time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+					time.Sleep(delay + time.Duration(rand.Float64()*float64(delay)))
 				}
 			}
+			progress.done(seq, result.Name, counts)
+			globalBatchJobsMetrics.save(job.ID, ri)
+			// persist in-memory state to disk after every 10secs.
+			batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 			if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
 				time.Sleep(wait)
@@ -421,18 +590,34 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		}()
 	}
 	wk.Wait()
-
-	ri.Complete = !failed && ri.ObjectsFailed == 0
-	ri.Failed = failed || ri.ObjectsFailed > 0
-	globalBatchJobsMetrics.save(job.ID, ri)
-	// persist in-memory state to disk.
-	batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-
-	if err := r.Notify(ctx, ri); err != nil {
-		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	cause := context.Cause(ctx)
+	if !failed && cause == nil {
+		// The walk ended: its last key is done.
+		progress.finish()
+	}
+	if cause != nil && !ri.refused() {
+		if isBatchJobUnavailable(cause) {
+			return ri.savePaused(ctx, api, job, cause)
+		}
+		// Stopped before the end: keep the work after the cursor pending.
+		globalBatchJobsMetrics.save(job.ID, ri)
+		batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+		return cause
 	}
 
-	cancel()
+	if !ri.refused() {
+		ri.mu.Lock()
+		ri.Complete = !failed && ri.ObjectsFailed == 0
+		ri.Failed = failed || ri.ObjectsFailed > 0
+		ri.mu.Unlock()
+	}
+	globalBatchJobsMetrics.save(job.ID, ri)
+	// persist in-memory state to disk.
+	batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+
+	if err := r.Notify(GlobalContext, ri); err != nil {
+		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	}
 	return nil
 }
 

cmd/batch-submitter-authz_test.go

@@ -0,0 +1,456 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	"go.yaml.in/yaml/v3"
+)
+
+// batchAuthzFixture runs batch jobs submitted through the admin API, one job
+// at a time and in the test's goroutine.
+type batchAuthzFixture struct {
+	ctx    context.Context
+	sys    *IAMSys
+	obj    ObjectLayer
+	bucket string
+}
+
+func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture {
+	t.Helper()
+	ctx, sys := iamAdminFixture(t)
+	obj := newObjectLayerFn()
+	batchTestJobPool(ctx, t, obj)
+	f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
+	mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+	_, err := globalBucketMetadataSys.Update(ctx, f.bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+	mustIAM(t, err)
+	return f
+}
+
+// batchTestJobPool queues submitted jobs without running them.
+func batchTestJobPool(ctx context.Context, t *testing.T, obj ObjectLayer) {
+	t.Helper()
+	prevPool := globalBatchJobPool
+	globalBatchJobPool = &BatchJobPool{
+		ctx:       ctx,
+		objLayer:  obj,
+		jobCh:     make(chan *BatchJobRequest, 10),
+		jobOwners: make(map[string]*batchJobOwner),
+	}
+	globalBatchJobsMetrics.Lock()
+	globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo)
+	globalBatchJobsMetrics.Unlock()
+	t.Cleanup(func() { globalBatchJobPool = prevPool })
+}
+
+// putVersions writes two versions of key and returns the older one.
+func (f *batchAuthzFixture) putVersions(t *testing.T, key string) string {
+	t.Helper()
+	var older string
+	for i := range 2 {
+		data := []byte(fmt.Sprintf("version %d", i))
+		oi, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
+		mustIAM(t, err)
+		if i == 0 {
+			older = oi.VersionID
+		}
+	}
+	return older
+}
+
+func (f *batchAuthzFixture) versionExists(t *testing.T, key, versionID string) bool {
+	t.Helper()
+	_, err := f.obj.GetObjectInfo(f.ctx, f.bucket, key, ObjectOptions{VersionID: versionID})
+	if err == nil {
+		return true
+	}
+	if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+		return false
+	}
+	t.Fatal(err)
+	return false
+}
+
+func (f *batchAuthzFixture) expireYAML(prefix string) string {
+	return fmt.Sprintf(`expire:
+  apiVersion: v1
+  bucket: %s
+  prefix: %q
+  rules:
+    - type: object
+      purge:
+        retainVersions: 1
+`, f.bucket, prefix)
+}
+
+// submit sends a job definition to StartBatchJob as cred. An accepted job is
+// returned from the queue, not run.
+func (f *batchAuthzFixture) submit(t *testing.T, cred auth.Credentials, job string) (*httptest.ResponseRecorder, *BatchJobRequest) {
+	t.Helper()
+	headers := map[string]string{}
+	if cred.IsTemp() && !cred.IsServiceAccount() {
+		headers["X-Amz-Security-Token"] = cred.SessionToken
+	}
+	body := []byte(job)
+	req, err := newTestSignedRequestV4(http.MethodPost, "http://localhost/minio/admin/v3/start-job", int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
+	mustIAM(t, err)
+	mustIAM(t, req.ParseForm())
+	rec := httptest.NewRecorder()
+	req = req.WithContext(newContext(req, rec, "StartBatchJob"))
+	adminAPIHandlers{}.StartBatchJob(rec, req)
+	if rec.Code != http.StatusOK {
+		return rec, nil
+	}
+	select {
+	case queued := <-globalBatchJobPool.jobCh:
+		return rec, queued
+	default:
+		t.Fatal("accepted job was not queued")
+		return rec, nil
+	}
+}
+
+func (f *batchAuthzFixture) run(t *testing.T, job *BatchJobRequest) {
+	t.Helper()
+	ctx := job.ctx
+	if ctx == nil {
+		ctx = f.ctx
+	}
+	switch {
+	case job.Expire != nil:
+		if err := job.Expire.Start(ctx, f.obj, *job); err != nil {
+			t.Logf("start: %v", err)
+		}
+	default:
+		t.Fatalf("unexpected job type %s", job.Type())
+	}
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
+	if err := ri.load(f.ctx, f.obj, *job); err == nil {
+		t.Logf("job %s: %+v", job.ID, ri.clone())
+	}
+}
+
+func (f *batchAuthzFixture) jobInfo(t *testing.T, job *BatchJobRequest) *batchJobInfo {
+	t.Helper()
+	ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
+	mustIAM(t, ri.load(f.ctx, f.obj, *job))
+	return ri
+}
+
+func batchAuthzPolicy(ctx context.Context, t *testing.T, sys *IAMSys, name, statements string) {
+	t.Helper()
+	p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[` + statements + `]}`))
+	mustIAM(t, err)
+	_, err = sys.SetPolicy(ctx, name, *p)
+	mustIAM(t, err)
+}
+
+const batchAuthzFullAccess = `{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}`
+
+// A credential that may start batch jobs but has no access to the bucket
+// cannot delete through a job.
+func TestBatchJobSubmitterWithoutDataAccess(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-only", `{"Effect":"Allow","Action":["admin:StartBatchJob","admin:ListBatchJobs"]}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-only-user", "batch-only")
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job != nil {
+		f.run(t, job)
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job submitted without data access deleted a version")
+	}
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("submission status %d, want 403: %s", rec.Code, rec.Body.String())
+	}
+}
+
+// A job acts only while the credential that submitted it is the same valid
+// credential.
+func TestBatchJobSubmitterChangedBeforeStart(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		change func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials)
+	}{
+		{"unchanged", nil},
+		{"deleted", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+		}},
+		{"disabled", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			_, err := f.sys.SetUserStatus(f.ctx, cred.AccessKey, madmin.AccountDisabled)
+			mustIAM(t, err)
+		}},
+		{"secret-changed", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			mustIAM(t, f.sys.SetUserSecretKey(f.ctx, cred.AccessKey, "batch-authz-changed-secret"))
+		}},
+		{"access-key-reused", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+			mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+			time.Sleep(10 * time.Millisecond)
+			iamAdminUser(f.ctx, t, f.sys, cred.AccessKey, "batch-full")
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+			cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			older := f.putVersions(t, "data/a")
+			rec, job := f.submit(t, cred, f.expireYAML(""))
+			if job == nil {
+				t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+			}
+			if tc.change != nil {
+				tc.change(t, f, cred)
+			}
+			f.run(t, job)
+			exists := f.versionExists(t, "data/a", older)
+			if tc.change == nil && exists {
+				t.Fatal("control: an authorized job did not expire the older version")
+			}
+			if tc.change != nil && !exists {
+				t.Fatal("a job ran as a credential that changed after submission")
+			}
+		})
+	}
+}
+
+// An expired temporary credential does not keep its job running.
+func TestBatchJobSubmitterExpired(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(2 * time.Second).Unix()}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	time.Sleep(time.Until(cred.Expiration) + time.Second)
+	f.run(t, job)
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job ran as an expired credential")
+	}
+}
+
+// A session policy limits the job as it limits requests, also when an
+// authorization plugin decides the rest.
+func TestBatchJobSessionPolicyDeny(t *testing.T) {
+	session := `{"Version":"2012-10-17","Statement":[
+{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`
+	for _, tc := range []struct {
+		name   string
+		plugin bool
+	}{{"local", false}, {"plugin-allows", true}} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+			parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+			cred := iamAdminServiceAccount(f.ctx, t, f.sys, parent.AccessKey, "batch-restricted", session)
+			if tc.plugin {
+				iamAdminPlugin(t, true)
+			}
+			older := f.putVersions(t, "data/a")
+			rec, job := f.submit(t, cred, f.expireYAML(""))
+			if job != nil {
+				f.run(t, job)
+			} else {
+				t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+			}
+			if !f.versionExists(t, "data/a", older) {
+				t.Fatal("a job deleted a version its session policy denies")
+			}
+		})
+	}
+}
+
+// A temporary credential's session policy limits its job too.
+func TestBatchJobTemporarySessionPolicyDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+	session := `{"Version":"2012-10-17","Statement":[
+{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions","s3:GetObject","s3:GetObjectVersion","s3:DeleteObject"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}]}`
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+		parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
+		policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(session)),
+	}, globalActiveCred.SecretKey)
+	mustIAM(t, err)
+	cred.ParentUser = parent.AccessKey
+	_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+	mustIAM(t, err)
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job != nil {
+		f.run(t, job)
+	} else {
+		t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job deleted a version its session policy does not allow")
+	}
+}
+
+// A Deny that depends on request context the job does not have refuses the
+// action instead of being skipped.
+func TestBatchJobAbsentContextDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-context", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-context")
+	older := f.putVersions(t, "data/a")
+	rec, job := f.submit(t, cred, f.expireYAML(""))
+	if job != nil {
+		f.run(t, job)
+	} else {
+		t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	if !f.versionExists(t, "data/a", older) {
+		t.Fatal("a job skipped a Deny that depends on request context")
+	}
+}
+
+// Each object is authorized: a Deny under an allowed prefix keeps its objects.
+func TestBatchJobPerObjectDeny(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "batch-keep", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/data/keep*"]}`)
+	cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-keep")
+	kept := f.putVersions(t, "data/keep")
+	rec, job := f.submit(t, cred, f.expireYAML("data/"))
+	if job == nil {
+		t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+	}
+	f.run(t, job)
+	if !f.versionExists(t, "data/keep", kept) {
+		t.Fatal("a job deleted a version that a per-object Deny protects")
+	}
+	if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+		t.Fatalf("refused job: complete=%v failed=%v", ri.Complete, ri.Failed)
+	}
+}
+
+// A job queued without its submitter, as an earlier release stored it, fails
+// and stays failed across another start.
+func TestBatchJobWithoutSubmitter(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	older := f.putVersions(t, "data/a")
+	job := &BatchJobRequest{}
+	mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
+	job.ID = "expire-legacy" + getKeySeparator() + "0"
+	job.User = globalActiveCred.AccessKey
+	job.Started = time.Now()
+	mustIAM(t, job.save(f.ctx, f.obj))
+	for range 2 {
+		loaded := &BatchJobRequest{}
+		mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+		f.run(t, loaded)
+		if !f.versionExists(t, "data/a", older) {
+			t.Fatal("a job without a recorded submitter deleted a version")
+		}
+		if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+			t.Fatalf("job without submitter: complete=%v failed=%v", ri.Complete, ri.Failed)
+		}
+	}
+}
+
+// Root and an authorized user can still run jobs.
+func TestBatchJobAuthorizedControls(t *testing.T) {
+	for _, root := range []bool{true, false} {
+		t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
+			f := newBatchAuthzFixture(t)
+			cred := globalActiveCred
+			if !root {
+				batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+				cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+			}
+			older := f.putVersions(t, "data/a")
+			rec, job := f.submit(t, cred, f.expireYAML(""))
+			if job == nil {
+				t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+			}
+			f.run(t, job)
+			if f.versionExists(t, "data/a", older) {
+				t.Fatal("an authorized job did not expire the older version")
+			}
+			if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed {
+				t.Fatalf("authorized job: complete=%v failed=%v", ri.Complete, ri.Failed)
+			}
+		})
+	}
+}
+
+// The describe output has no secrets of the job.
+func TestBatchJobDescribeRedactsSecrets(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	secrets := []string{"describe-source-secret", "describe-source-token", "describe-target-secret", "describe-target-token", "describe-notify-token"}
+	for _, job := range []BatchJobRequest{
+		{ID: "replicate-describe" + getKeySeparator() + "0", Replicate: &BatchJobReplicateV1{
+			APIVersion: batchReplJobAPIVersion,
+			Source: BatchJobReplicateSource{Bucket: "remote", Endpoint: "https://remote.invalid", Creds: BatchJobReplicateCredentials{
+				AccessKey: "describe-source-access", SecretKey: secrets[0], SessionToken: secrets[1],
+			}},
+			Target: BatchJobReplicateTarget{Bucket: f.bucket, Creds: BatchJobReplicateCredentials{
+				AccessKey: "describe-target-access", SecretKey: secrets[2], SessionToken: secrets[3],
+			}},
+			Flags: BatchJobReplicateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
+		}},
+		{ID: "expire-describe" + getKeySeparator() + "0", Expire: &BatchJobExpire{
+			APIVersion: batchExpireAPIVersion, Bucket: f.bucket,
+			NotificationCfg: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]},
+		}},
+		{ID: "keyrotate-describe" + getKeySeparator() + "0", KeyRotate: &BatchJobKeyRotateV1{
+			APIVersion: batchKeyRotateAPIVersion, Bucket: f.bucket,
+			Flags: BatchJobKeyRotateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
+		}},
+	} {
+		data, err := job.MarshalMsg(nil)
+		mustIAM(t, err)
+		mustIAM(t, saveConfig(f.ctx, f.obj, getJobPath(job), data))
+		cred := globalActiveCred
+		req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/describe-job?jobId="+job.ID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
+		mustIAM(t, err)
+		mustIAM(t, req.ParseForm())
+		rec := httptest.NewRecorder()
+		req = req.WithContext(newContext(req, rec, "DescribeBatchJob"))
+		adminAPIHandlers{}.DescribeBatchJob(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("%s: describe status %d: %s", job.ID, rec.Code, rec.Body.String())
+		}
+		for _, s := range secrets {
+			if strings.Contains(rec.Body.String(), s) {
+				t.Errorf("%s: describe output contains %s", job.ID, s)
+			}
+		}
+		if job.Replicate != nil && !strings.Contains(rec.Body.String(), "describe-source-access") {
+			t.Errorf("%s: describe output lost the access key", job.ID)
+		}
+		// The stored job keeps what a run needs.
+		stored := &BatchJobRequest{}
+		mustIAM(t, stored.load(f.ctx, f.obj, getJobPath(job)))
+		if stored.Replicate != nil && stored.Replicate.Source.Creds.SecretKey != secrets[0] {
+			t.Errorf("%s: describe changed the stored job", job.ID)
+		}
+	}
+}

cmd/batch-submitter-edits_test.go

@@ -0,0 +1,124 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// Edits that keep the credential keep a job running; edits of the
+// credential itself, and its status, still end it.
+func TestBatchSubmitterCredentialEdits(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "edit-parent", batchAuthzFullAccess)
+	parent := iamAdminUser(f.ctx, t, f.sys, "edit-parent", "edit-parent")
+	soon := UTCNow().Add(time.Hour)
+	later := UTCNow().Add(24 * time.Hour)
+	newService := func(t *testing.T, expiring bool) (auth.Credentials, *batchJobSubmitter) {
+		t.Helper()
+		var opts newServiceAccountOpts
+		if expiring {
+			opts.expiration = &soon
+		}
+		created, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, opts)
+		mustIAM(t, err)
+		stored, ok := f.sys.GetUser(f.ctx, created.AccessKey)
+		if !ok {
+			t.Fatal("service account missing")
+		}
+		sub, err := newBatchJobSubmitter(stored.Credentials)
+		mustIAM(t, err)
+		mustIAM(t, sub.authorize(f.ctx, batchContextRead(f)))
+		return stored.Credentials, sub
+	}
+	for _, tc := range []struct {
+		name     string
+		expiring bool
+		edit     func(t *testing.T, cred auth.Credentials)
+		keep     bool
+	}{
+		{"description", false, func(t *testing.T, cred auth.Credentials) {
+			_, err := f.sys.UpdateServiceAccount(f.ctx, cred.AccessKey, updateServiceAccountOpts{description: "cosmetic"})
+			mustIAM(t, err)
+		}, true},
+		{"name", false, func(t *testing.T, cred auth.Credentials) {
+			_, err := f.sys.UpdateServiceAccount(f.ctx, cred.AccessKey, updateServiceAccountOpts{name: "renamed"})
+			mustIAM(t, err)
+		}, true},
+		{"expiry-extended", true, func(t *testing.T, cred auth.Credentials) {
+			_, err := f.sys.UpdateServiceAccount(f.ctx, cred.AccessKey, updateServiceAccountOpts{expiration: &later})
+			mustIAM(t, err)
+		}, true},
+		// An expiry makes a service account a temporary credential, which
+		// IsAllowed evaluates differently.
+		{"expiry-added", false, func(t *testing.T, cred auth.Credentials) {
+			_, err := f.sys.UpdateServiceAccount(f.ctx, cred.AccessKey, updateServiceAccountOpts{expiration: &later})
+			mustIAM(t, err)
+		}, false},
+		{"groups-refreshed", false, func(t *testing.T, cred auth.Credentials) {
+			// As the LDAP group refresh rewrites a derived credential.
+			u, ok := f.sys.GetUser(f.ctx, cred.AccessKey)
+			if !ok {
+				t.Fatal("service account missing")
+			}
+			wrote, err := f.sys.store.RefreshUserGroups(f.ctx, u.Credentials, append(u.Credentials.Groups, "refreshed-group"))
+			mustIAM(t, err)
+			if !wrote {
+				t.Fatal("control: the refresh did not write")
+			}
+		}, true},
+		{"secret-rotated", false, func(t *testing.T, cred auth.Credentials) {
+			_, err := f.sys.UpdateServiceAccount(f.ctx, cred.AccessKey, updateServiceAccountOpts{secretKey: "edit-rotated-secret-key"})
+			mustIAM(t, err)
+		}, false},
+		{"disabled", false, func(t *testing.T, cred auth.Credentials) {
+			_, err := f.sys.UpdateServiceAccount(f.ctx, cred.AccessKey, updateServiceAccountOpts{status: string(madmin.AccountDisabled)})
+			mustIAM(t, err)
+		}, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cred, sub := newService(t, tc.expiring)
+			before, _ := f.sys.GetUser(f.ctx, cred.AccessKey)
+			time.Sleep(2 * time.Millisecond) // a distinct UpdatedAt
+			tc.edit(t, cred)
+			after, _ := f.sys.GetUser(f.ctx, cred.AccessKey)
+			if after.UpdatedAt.Equal(before.UpdatedAt) {
+				t.Fatal("control: the edit did not change the identity record")
+			}
+			err := sub.authorize(f.ctx, batchContextRead(f))
+			if tc.keep && err != nil {
+				t.Errorf("an edit that keeps the credential ended the job: %v", err)
+			}
+			if !tc.keep && !isBatchJobAuthzError(err) {
+				t.Errorf("an edit of the credential must end the job, got %v", err)
+			}
+		})
+	}
+}
+
+// A user deleted and created again under its access key, with the same
+// secret, is not the credential that submitted the job.
+func TestBatchSubmitterRecreatedSameSecret(t *testing.T) {
+	f := newBatchAuthzFixture(t)
+	batchAuthzPolicy(f.ctx, t, f.sys, "recreate-full", batchAuthzFullAccess)
+	cred := iamAdminUser(f.ctx, t, f.sys, "recreate-user", "recreate-full")
+	sub, err := newBatchJobSubmitter(cred)
+	mustIAM(t, err)
+	mustIAM(t, sub.authorize(f.ctx, batchContextRead(f)))
+	mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+	time.Sleep(10 * time.Millisecond)
+	again := iamAdminUser(f.ctx, t, f.sys, cred.AccessKey, "recreate-full")
+	if again.SecretKey != cred.SecretKey {
+		t.Fatal("control: the user must be created again with the same secret")
+	}
+	err = sub.authorize(f.ctx, batchContextRead(f))
+	t.Logf("recreated: %v", err)
+	if !isBatchJobAuthzError(err) {
+		t.Errorf("a recreated user must not run the job of the deleted one: %v", err)
+	}
+}

cmd/bucket-cors-site-replication_test.go

@@ -511,14 +511,13 @@ func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ st
 	}))
 	defer remote.Close()
 
-	serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key")
+	serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "cors-status-service",
+		secretKey: "cors-status-service-secret-key",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	serviceCred.ParentUser = credentials.AccessKey
-	if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
-		t.Fatal(err)
-	}
 	defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
 
 	globalSiteReplicationSys.Lock()
@@ -585,14 +584,13 @@ func testSiteReplicationStatusCountsCorsPerSite(obj ObjectLayer, _ string, bucke
 	}))
 	defer remote.Close()
 
-	serviceCred, err := auth.CreateCredentials("cors-count-service", "cors-count-service-secret-key")
+	serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "cors-count-service",
+		secretKey: "cors-count-service-secret-key",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	serviceCred.ParentUser = credentials.AccessKey
-	if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
-		t.Fatal(err)
-	}
 	defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
 
 	globalSiteReplicationSys.Lock()
@@ -1190,14 +1188,13 @@ func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, buck
 	}))
 	defer remote.Close()
 
-	serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret")
+	serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
+		accessKey: "cors-integration-svc",
+		secretKey: "cors-integration-service-secret",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	serviceCred.ParentUser = credentials.AccessKey
-	if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
-		t.Fatal(err)
-	}
 	defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
 
 	globalSiteReplicationSys.Lock()

cmd/bucket-handlers.go

@@ -494,6 +494,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
+	duplicateOf := make(map[int]int)
 
 	vc, _ := globalBucketVersioningSys.Get(bucket)
 	oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
@@ -523,13 +524,32 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 		}
 
+		// All deletes on directory objects needs to be for `nullVersionID`.
+		// Decide this before the lookup below, so that the retention check
+		// reads the version the delete removes.
+		if isDirObject(object.ObjectName) && object.VersionID == "" {
+			object.VersionID = nullVersionID
+		}
+
 		opts := ObjectOptions{
 			VersionID:        object.VersionID,
 			Versioned:        vc.PrefixEnabled(object.ObjectName),
 			VersionSuspended: vc.Suspended(),
 		}
 
-		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
+		// Physical version deletes in a locked bucket are checked by the
+		// single-object path below, while its mutation lock is held. A lookup
+		// here could miss a later protection update or a protected pool copy.
+		if hasLockEnabled && object.VersionID != "" {
+			if first, ok := objectsToDelete[object]; ok {
+				duplicateOf[index] = first
+			} else {
+				objectsToDelete[object] = index
+			}
+			continue
+		}
+
+		if replicateDeletes || !globalTierConfigMgr.Empty() {
 			if !globalTierConfigMgr.Empty() && object.VersionID == "" && opts.VersionSuspended {
 				opts.VersionID = nullVersionID
 			}
@@ -541,11 +561,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			oss[index].SetTransitionState(goi.TransitionedObject)
 		}
 
-		// All deletes on directory objects needs to be for `nullVersionID`
-		if isDirObject(object.ObjectName) && object.VersionID == "" {
-			object.VersionID = nullVersionID
-		}
-
 		if replicateDeletes {
 			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectV: ObjectV{
@@ -563,21 +578,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				object.ReplicateDecisionStr = dsc.String()
 			}
 		}
-		if object.VersionID != "" && hasLockEnabled {
-			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); err != nil {
-				apiErr := toAPIError(ctx, err)
-				deleteResults[index].errInfo = DeleteError{
-					Code:      apiErr.Code,
-					Message:   apiErr.Description,
-					Key:       object.ObjectName,
-					VersionID: object.VersionID,
-				}
-				continue
-			}
-		}
-
 		// Avoid duplicate objects, we use map to filter them out.
-		if _, ok := objectsToDelete[object]; !ok {
+		if first, ok := objectsToDelete[object]; ok {
+			duplicateOf[index] = first
+		} else {
 			objectsToDelete[object] = index
 		}
 	}
@@ -596,10 +600,57 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	ctx = bgContext(ctx)
 
 	deleteList := toNames(objectsToDelete)
-	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		PrefixEnabledFn:  vc.PrefixEnabled,
-		VersionSuspended: vc.Suspended(),
-	})
+	dObjects := make([]DeletedObject, len(deleteList))
+	errs := make([]error, len(deleteList))
+	bulk := make([]ObjectToDelete, 0, len(deleteList))
+	bulkIndex := make([]int, 0, len(deleteList))
+	for i, object := range deleteList {
+		if !hasLockEnabled || object.VersionID == "" {
+			bulk = append(bulk, object)
+			bulkIndex = append(bulkIndex, i)
+			continue
+		}
+		reqInfo.ObjectName, reqInfo.VersionID = object.ObjectName, object.VersionID
+		opts := ObjectOptions{
+			VersionID:        object.VersionID,
+			Versioned:        vc.PrefixEnabled(object.ObjectName),
+			VersionSuspended: vc.Suspended(),
+		}
+		index := objectsToDelete[object]
+		if !globalTierConfigMgr.Empty() {
+			oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(object.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
+		}
+		opts.SetEvalRetentionBypassFn(func(oi ObjectInfo, err error) error {
+			return enforceRetentionBypassForDelete(ctx, r, bucket, object, oi, err)
+		})
+		opts.SetEvalMetadataFn(func(oi *ObjectInfo, err error) (ReplicateDecision, error) {
+			if err == nil && oss[index] != nil {
+				oss[index].SetTransitionState(oi.TransitionedObject)
+			}
+			if replicateDeletes {
+				return checkReplicateDelete(ctx, bucket, object, *oi, opts, err), nil
+			}
+			return ReplicateDecision{}, nil
+		})
+		// DeleteObject merges protection across pools before evaluating these
+		// callbacks and holds the same lock as retention and legal-hold writes.
+		oi, err := objectAPI.DeleteObject(ctx, bucket, object.ObjectName, opts)
+		dObjects[i] = DeletedObject{
+			ObjectName:       object.ObjectName,
+			VersionID:        object.VersionID,
+			ReplicationState: oi.ReplicationState(),
+		}
+		errs[i] = err
+	}
+	if len(bulk) > 0 {
+		deleted, bulkErrs := deleteObjectsFn(ctx, bucket, bulk, ObjectOptions{
+			PrefixEnabledFn:  vc.PrefixEnabled,
+			VersionSuspended: vc.Suspended(),
+		})
+		for i, index := range bulkIndex {
+			dObjects[index], errs[index] = deleted[i], bulkErrs[i]
+		}
+	}
 
 	// Are all objects saying bucket not found?
 	if isAllBucketsNotFound(errs) {
@@ -608,23 +659,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	for i := range errs {
-		// DeleteMarkerVersionID is not used specifically to avoid
-		// lookup errors, since DeleteMarkerVersionID is only
-		// created during DeleteMarker creation when client didn't
-		// specify a versionID.
-		objToDel := ObjectToDelete{
-			ObjectV: ObjectV{
-				ObjectName: decodeDirObject(dObjects[i].ObjectName),
-				VersionID:  dObjects[i].VersionID,
-			},
-			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus(),
-			VersionPurgeStatuses:          dObjects[i].ReplicationState.VersionPurgeStatusInternal,
-			DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal,
-			ReplicateDecisionStr:          dObjects[i].ReplicationState.ReplicateDecisionStr,
-		}
-		dindex := objectsToDelete[objToDel]
+		// Match by the input identity: replication decisions may have changed
+		// under the mutation lock, and a missing version has no stored identity.
+		dindex := objectsToDelete[deleteList[i]]
 		if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
-			if replicateDeletes {
+			if replicateDeletes && (!hasLockEnabled || deleteList[i].VersionID == "") {
 				dObjects[i].ReplicationState = deleteList[i].ReplicationState()
 			}
 			deleteResults[dindex].delInfo = dObjects[i]
@@ -638,20 +677,28 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			VersionID: deleteList[i].VersionID,
 		}
 	}
+	for index, first := range duplicateOf {
+		deleteResults[index] = deleteResults[first]
+	}
 
 	// Generate response
 	deleteErrors := make([]DeleteError, 0, len(deleteObjectsReq.Objects))
 	deletedObjects := make([]DeletedObject, 0, len(deleteObjectsReq.Objects))
-	for _, deleteResult := range deleteResults {
+	eventObjects := make([]DeletedObject, 0, len(deleteObjectsReq.Objects))
+	for index, deleteResult := range deleteResults {
+		// Storage normalizes a directory to the null version. The response
+		// retains the caller's version identity, including an explicit null.
+		if isDirObject(deleteObjectsReq.Objects[index].ObjectName) {
+			deleteResult.delInfo.VersionID = deleteObjectsReq.Objects[index].VersionID
+			deleteResult.errInfo.VersionID = deleteObjectsReq.Objects[index].VersionID
+		}
 		if deleteResult.errInfo.Code != "" {
 			deleteErrors = append(deleteErrors, deleteResult.errInfo)
 		} else {
-			// All deletes on directory objects was with `nullVersionID`.
-			// Remove it from response.
-			if isDirObject(deleteResult.delInfo.ObjectName) && deleteResult.delInfo.VersionID == nullVersionID {
-				deleteResult.delInfo.VersionID = ""
-			}
 			deletedObjects = append(deletedObjects, deleteResult.delInfo)
+			if _, duplicate := duplicateOf[index]; !duplicate {
+				eventObjects = append(eventObjects, deleteResult.delInfo)
+			}
 		}
 	}
 
@@ -660,7 +707,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
-	for _, dobj := range deletedObjects {
+	for _, dobj := range eventObjects {
 		if dobj.ObjectName == "" {
 			continue
 		}
@@ -703,8 +750,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Clean up transitioned objects from remote tier
-	for _, os := range oss {
-		if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
+	for index, os := range oss {
+		if os == nil || deleteResults[index].errInfo.Code != "" { // Sweep only successful deletions.
 			continue
 		}
 		os.Sweep()

cmd/bucket-lifecycle-incomplete.go

@@ -0,0 +1,174 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/event"
+)
+
+// An ILM expiry attempt that ends without removing what it addressed is
+// incomplete: the rule no longer applied when it was checked again under the
+// object lock, Object Lock protects the version, the history needed to decide
+// could not be read, or another error. It is counted by reason, traced with
+// the reason, and, unless it was refused, audited and logged. It never sends
+// a removal notification: what it addressed is still there.
+//
+// An expiry of all versions of a key can also be refused because a version
+// is in a remote tier, or because the latest version changed since a batch
+// matched it, and it can stop at its budget after it removed some versions.
+const (
+	ilmIncompleteNotApplicable      = "not_applicable"
+	ilmIncompleteLocked             = "locked"
+	ilmIncompleteHistoryUnavailable = "history_unavailable"
+	ilmIncompleteError              = "error"
+	ilmIncompleteTierUnsupported    = "tier_unsupported"
+	ilmIncompleteBudget             = "budget"
+	ilmIncompleteChanged            = "changed"
+	// ilmIncompleteTierSkipped counts the scanner's observations of a key
+	// whose expiry of all versions it did not queue, because a version it
+	// holds is in a remote tier and the purge would refuse it. Each drive's
+	// scanner of the key observes it, once per cycle.
+	ilmIncompleteTierSkipped = "tier_unsupported_skipped"
+)
+
+// ilmIncompleteReasons are the reasons expiry_incomplete_total counts, a
+// bounded set of label values.
+var ilmIncompleteReasons = []string{
+	ilmIncompleteNotApplicable,
+	ilmIncompleteLocked,
+	ilmIncompleteHistoryUnavailable,
+	ilmIncompleteError,
+	ilmIncompleteTierUnsupported,
+	ilmIncompleteBudget,
+	ilmIncompleteChanged,
+	ilmIncompleteTierSkipped,
+}
+
+// ilmIncompleteStats counts incomplete ILM expiries by reason.
+type ilmIncompleteStats struct {
+	counts map[string]*atomic.Int64
+}
+
+func newILMIncompleteStats(reasons ...string) *ilmIncompleteStats {
+	s := &ilmIncompleteStats{counts: make(map[string]*atomic.Int64, len(reasons))}
+	for _, r := range reasons {
+		s.counts[r] = &atomic.Int64{}
+	}
+	return s
+}
+
+func (s *ilmIncompleteStats) add(reason string) {
+	if c, ok := s.counts[reason]; ok {
+		c.Add(1)
+	}
+}
+
+// snapshot returns the counts by reason.
+func (s *ilmIncompleteStats) snapshot() map[string]int64 {
+	out := make(map[string]int64, len(s.counts))
+	for r, c := range s.counts {
+		out[r] = c.Load()
+	}
+	return out
+}
+
+var globalILMIncomplete = newILMIncompleteStats(ilmIncompleteReasons...)
+
+// ilmExpiryNotDone classifies the error of an ILM expiry that did not remove
+// what it addressed. A version that is already gone is not counted.
+func ilmExpiryNotDone(err error) (reason string, counted bool) {
+	switch {
+	case err == nil || isErrObjectNotFound(err) || isErrVersionNotFound(err):
+		return "", false
+	case errors.Is(err, errDeleteAllTiered):
+		return ilmIncompleteTierUnsupported, true
+	case errors.Is(err, errDeleteAllChanged):
+		return ilmIncompleteChanged, true
+	case errors.Is(err, errDeleteAllBudget):
+		return ilmIncompleteBudget, true
+	case errors.Is(err, errLifecycleNotApplicable):
+		return ilmIncompleteNotApplicable, true
+	}
+	var locked ObjectLocked
+	if errors.As(err, &locked) {
+		return ilmIncompleteLocked, true
+	}
+	if isErrReadQuorum(err) || isErrWriteQuorum(err) {
+		return ilmIncompleteHistoryUnavailable, true
+	}
+	return ilmIncompleteError, true
+}
+
+// ilmExpiryRefused reports whether reason is a refusal, which changes
+// nothing, rather than a failure.
+func ilmExpiryRefused(reason string) bool {
+	switch reason {
+	case ilmIncompleteNotApplicable, ilmIncompleteLocked, ilmIncompleteTierUnsupported, ilmIncompleteChanged:
+		return true
+	}
+	return false
+}
+
+// reportILMExpiryNotDone reports an ILM expiry of oi that ended with err. It
+// reports nothing and returns false when err says the version is already
+// gone; the caller reports that as it did before.
+func reportILMExpiryNotDone(ctx context.Context, oi ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, err error, logID string, traceFn func(event string, metadata map[string]string, err error)) bool {
+	reason, counted := ilmExpiryNotDone(err)
+	if !counted {
+		return false
+	}
+	globalILMIncomplete.add(reason)
+	if ilmExpiryRefused(reason) {
+		traceFn(ILMExpiry, map[string]string{"result": "refused", "reason": reason, "version-id": oi.VersionID}, err)
+		return true
+	}
+	traceFn(ILMExpiry, map[string]string{"result": "failed", "reason": reason, "version-id": oi.VersionID}, err)
+	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+	tags["version-id"] = oi.VersionID
+	tags["ilm-incomplete-reason"] = reason
+	auditLogInternal(ctx, AuditLogOptions{
+		Event:     ILMExpiry,
+		APIName:   "ILMExpiry",
+		Bucket:    oi.Bucket,
+		Object:    oi.Name,
+		VersionID: oi.VersionID,
+		Error:     err.Error(),
+		Tags:      tags,
+	})
+	ilmLogOnceIf(ctx, fmt.Errorf("ILM expiry of %s/%s (%s) not done: %w", oi.Bucket, oi.Name, oi.VersionID, err), logID)
+	return true
+}
+
+// ilmExpiryEventName is the notification of an expiry that completed. An
+// expiry that addressed no version ID (Expiration of a latest version in a
+// versioned or suspended bucket) and whose result is a delete marker created
+// that marker, also when the marker takes the null version ID of the version
+// it replaces. Any other expiry removed the version it addressed, also when
+// that version was itself a delete marker.
+func ilmExpiryEventName(requestedVersionID string, result ObjectInfo) event.Name {
+	if requestedVersionID == "" && result.DeleteMarker {
+		return event.ObjectRemovedDeleteMarkerCreated
+	}
+	return event.ObjectRemovedDelete
+}

cmd/bucket-lifecycle-incomplete_test.go

@@ -0,0 +1,364 @@
+// Copyright (c) 2026 PGSTY
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/logger"
+	loghttp "github.com/minio/minio/internal/logger/target/http"
+	"github.com/minio/minio/internal/pubsub"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestILMExpiryNotDoneClassification(t *testing.T) {
+	for _, tc := range []struct {
+		err     error
+		reason  string
+		counted bool
+	}{
+		{nil, "", false},
+		{ObjectNotFound{}, "", false},
+		{VersionNotFound{}, "", false},
+		{errLifecycleNotApplicable, ilmIncompleteNotApplicable, true},
+		{fmt.Errorf("wrapped: %w", errLifecycleNotApplicable), ilmIncompleteNotApplicable, true},
+		{ObjectLocked{}, ilmIncompleteLocked, true},
+		{InsufficientWriteQuorum{}, ilmIncompleteHistoryUnavailable, true},
+		{InsufficientReadQuorum{}, ilmIncompleteHistoryUnavailable, true},
+		{errFaultyDisk, ilmIncompleteError, true},
+	} {
+		reason, counted := ilmExpiryNotDone(tc.err)
+		if reason != tc.reason || counted != tc.counted {
+			t.Errorf("%v: %q %v, want %q %v", tc.err, reason, counted, tc.reason, tc.counted)
+		}
+	}
+}
+
+// fixedDeleteResults is an object layer whose bulk deletes return errs.
+type fixedDeleteResults struct {
+	ObjectLayer
+	errs []error
+}
+
+func (o fixedDeleteResults) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	deleted := make([]DeletedObject, len(objects))
+	for i, obj := range objects {
+		deleted[i] = DeletedObject{ObjectName: obj.ObjectName, VersionID: obj.VersionID}
+	}
+	return deleted, slices.Clone(o.errs)
+}
+
+// ilmTestAudit captures the ILMExpiry audit entries of bucket.
+func ilmTestAudit(t *testing.T, bucket string) <-chan map[string]any {
+	t.Helper()
+	if len(logger.AuditTargets()) != 0 {
+		t.Fatal("unexpected pre-existing test audit targets")
+	}
+	entries := make(chan map[string]any, 16)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var entry struct {
+			Error string         `json:"error"`
+			Tags  map[string]any `json:"tags"`
+			API   struct {
+				Name, Bucket string
+				Objects      []struct {
+					ObjectName string `json:"objectName"`
+				} `json:"objects"`
+			}
+		}
+		if err := json.NewDecoder(r.Body).Decode(&entry); err == nil && entry.API.Name == "ILMExpiry" && entry.API.Bucket == bucket {
+			tags := entry.Tags
+			if tags == nil {
+				tags = map[string]any{}
+			}
+			tags["error"] = entry.Error
+			if len(entry.API.Objects) > 0 {
+				tags["object"] = entry.API.Objects[0].ObjectName
+			}
+			entries <- tags
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	endpoint, err := xnet.ParseHTTPURL(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	if errs := logger.UpdateAuditWebhooks(ctx, map[string]loghttp.Config{"ilm": {Enabled: true, Name: "ilm", Endpoint: endpoint, BatchSize: 1, QueueSize: 128, MaxRetry: 1, RetryIntvl: time.Millisecond, HTTPTimeout: time.Second}}); len(errs) > 0 {
+		t.Fatal(errs)
+	}
+	targets := logger.AuditTargets()
+	t.Cleanup(func() {
+		logger.UpdateAuditWebhooks(ctx, nil)
+		for _, target := range targets {
+			target.Cancel()
+		}
+		server.Close()
+	})
+	return entries
+}
+
+// A refused or failed noncurrent purge is counted and traced with its reason,
+// a failure is audited, and only the version that was removed is announced.
+func TestDeleteObjectVersionsReportsRefusalAndFailure(t *testing.T) {
+	s := newSplitILM(t, versioning.Enabled, splitILMRules)
+	ctx := t.Context()
+	audits := ilmTestAudit(t, s.bucket)
+	done := make(chan struct{})
+	defer close(done)
+	events := make(chan event.Event, 8)
+	if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedAll), events, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	traces := make(chan madmin.TraceInfo, 8)
+	if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	before := globalILMIncomplete.snapshot()
+
+	layer := fixedDeleteResults{ObjectLayer: s.z, errs: []error{errLifecycleNotApplicable, InsufficientWriteQuorum{}, nil}}
+	toDel := []ObjectToDelete{
+		{ObjectV: ObjectV{ObjectName: "ncv-a", VersionID: "11111111-1111-1111-1111-111111111111"}},
+		{ObjectV: ObjectV{ObjectName: "ncv-b", VersionID: "22222222-2222-2222-2222-222222222222"}},
+		{ObjectV: ObjectV{ObjectName: "ncv-c", VersionID: "33333333-3333-3333-3333-333333333333"}},
+	}
+	evs := []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}}
+	deleteObjectVersions(ctx, layer, s.bucket, toDel, evs)
+
+	after := globalILMIncomplete.snapshot()
+	if d := after[ilmIncompleteNotApplicable] - before[ilmIncompleteNotApplicable]; d != 1 {
+		t.Errorf("not_applicable moved by %d, want 1", d)
+	}
+	if d := after[ilmIncompleteHistoryUnavailable] - before[ilmIncompleteHistoryUnavailable]; d != 1 {
+		t.Errorf("history_unavailable moved by %d, want 1", d)
+	}
+	var removed []string
+	results := map[string]string{}
+	for drained := false; !drained; {
+		select {
+		case ev := <-events:
+			removed = append(removed, ev.S3.Object.Key)
+		case tr := <-traces:
+			if r := tr.Custom["result"]; r != "" {
+				results[tr.Custom["version-id"]] = r
+				if tr.Error == "" {
+					t.Errorf("%s expiry of %s traced without its error", r, tr.Custom["version-id"])
+				}
+			}
+		default:
+			drained = true
+		}
+	}
+	if !slices.Equal(removed, []string{"ncv-c"}) {
+		t.Errorf("removal events for %v, want only ncv-c", removed)
+	}
+	if results[toDel[0].VersionID] != "refused" || results[toDel[1].VersionID] != "failed" {
+		t.Errorf("traces %v, want refused and failed", results)
+	}
+	// Drain every audit entry of the bucket: until the removal of ncv-c,
+	// audited after the refusal and the failure were reported, has arrived,
+	// and then while the target empties its queue.
+	var got []map[string]any
+	deadline := time.After(10 * time.Second)
+	for removal := false; !removal; {
+		select {
+		case tags := <-audits:
+			got = append(got, tags)
+			removal = removal || tags["object"] == "ncv-c"
+		case <-deadline:
+			t.Fatalf("audits %v: the removal of ncv-c was not audited", got)
+		}
+	}
+	for idle := time.After(time.Second); idle != nil; {
+		select {
+		case tags := <-audits:
+			got = append(got, tags)
+		case <-idle:
+			idle = nil
+		}
+	}
+	perObject := map[any][]map[string]any{}
+	for _, tags := range got {
+		perObject[tags["object"]] = append(perObject[tags["object"]], tags)
+	}
+	if a := perObject["ncv-a"]; len(a) != 0 {
+		t.Errorf("the refusal was audited: %v", a)
+	}
+	if b := perObject["ncv-b"]; len(b) != 1 || b[0]["ilm-incomplete-reason"] != ilmIncompleteHistoryUnavailable || b[0]["error"] == "" {
+		t.Errorf("audits of the failure: %v, want one with its reason and error", b)
+	}
+	if c := perObject["ncv-c"]; len(c) != 1 || c[0]["ilm-incomplete-reason"] != nil {
+		t.Errorf("audits of the removal: %v, want one", c)
+	}
+}
+
+// expiryEvent returns the removal events sent while run runs.
+func expiryEvents(t *testing.T, run func()) []event.Event {
+	t.Helper()
+	done := make(chan struct{})
+	defer close(done)
+	events := make(chan event.Event, 8)
+	if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedAll), events, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	run()
+	var got []event.Event
+	for {
+		select {
+		case ev := <-events:
+			got = append(got, ev)
+		default:
+			return got
+		}
+	}
+}
+
+// The notification of a completed expiry names what its delete did: the
+// delete marker it created, or the version it removed.
+func TestExpiryEventsNameTheResult(t *testing.T) {
+	type expiry struct {
+		name      string
+		status    versioning.State
+		key       string
+		versions  []splitVersion
+		expire    string // label of the version expired
+		action    lifecycle.Action
+		wantEvent event.Name
+		wantID    func(s *splitILM, key string) []string // acceptable version IDs
+	}
+	marker := func(s *splitILM, key string) []string {
+		oi, err := s.z.GetObjectInfo(s.t.Context(), s.bucket, key, ObjectOptions{})
+		if !oi.DeleteMarker {
+			s.t.Fatalf("latest of %s is not a marker: %+v %v", key, oi, err)
+		}
+		return []string{oi.VersionID}
+	}
+	label := func(l string) func(s *splitILM, key string) []string {
+		return func(s *splitILM, key string) []string { return []string{s.ids[key+" "+l]} }
+	}
+	null := func(*splitILM, string) []string { return []string{"", nullVersionID} }
+	cases := []expiry{
+		{"latest-versioned", versioning.Enabled, "exp-ev-latest", []splitVersion{splitAt("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, marker},
+		{"noncurrent", versioning.Enabled, "ncv-ev-noncurrent", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 3, 0)}, "v1", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("v1")},
+		{"expired-marker", versioning.Enabled, "edm-ev-marker", []splitVersion{splitMarker("dm", 5, 0)}, "dm", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("dm")},
+		{"latest-suspended-null", versioning.Suspended, "exp-ev-null", []splitVersion{splitNull("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, null},
+		{"tiered-latest-versioned", versioning.Enabled, "exp-ev-tiered", []splitVersion{splitTiered("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, marker},
+		{"tiered-noncurrent", versioning.Enabled, "ncv-ev-tiered", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 3, 0)}, "v1", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("v1")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			s := newSplitILM(t, tc.status, splitILMRules)
+			installExpiryTestTier(t, map[string][]byte{"remote-v1": []byte("v1")})
+			for _, v := range tc.versions {
+				s.write(tc.key, v)
+			}
+			vid := s.ids[tc.key+" "+tc.expire]
+			if vid == "" {
+				vid = nullVersionID
+			}
+			oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, tc.key, ObjectOptions{VersionID: vid})
+			if err != nil && !isErrMethodNotAllowed(err) {
+				t.Fatal(err)
+			}
+			oi.Bucket, oi.Name = s.bucket, tc.key
+			lcEvent := lifecycle.Event{Action: tc.action}
+			got := expiryEvents(t, func() {
+				if oi.TransitionedObject.Status != "" {
+					if !applyExpiryOnTransitionedObject(t.Context(), s.z, oi, lcEvent, lcEventSrc_Scanner) {
+						t.Fatal("transitioned expiry did not apply")
+					}
+					return
+				}
+				if !applyExpiryOnNonTransitionedObjects(t.Context(), s.z, oi, lcEvent, lcEventSrc_Scanner) {
+					t.Fatal("expiry did not apply")
+				}
+			})
+			if len(got) != 1 {
+				t.Fatalf("events %+v, want one", got)
+			}
+			if got[0].EventName != tc.wantEvent {
+				t.Errorf("event %s, want %s", got[0].EventName, tc.wantEvent)
+			}
+			if want := tc.wantID(s, tc.key); !slices.Contains(want, got[0].S3.Object.VersionID) {
+				t.Errorf("event names version %q, want one of %v", got[0].S3.Object.VersionID, want)
+			}
+		})
+	}
+}
+
+// expiry_incomplete_total is exported as a counter with a reason label by
+// both metrics versions.
+func TestILMExpiryIncompleteMetrics(t *testing.T) {
+	// Metrics v3 exports a counter once it is above zero.
+	for _, r := range ilmIncompleteReasons {
+		globalILMIncomplete.add(r)
+	}
+	// v3, through the collector the metrics handler registers.
+	families, err := newMetricGroups(prometheus.NewRegistry()).mgGatherers[ilmCollectorPath].Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := ilmCollectorPath.metricPrefix() + "_" + string(expiryIncompleteTotal)
+	reasons := map[string]bool{}
+	for _, family := range families {
+		if family.GetName() != want {
+			continue
+		}
+		for _, m := range family.Metric {
+			if m.Counter == nil || m.GetCounter().GetValue() <= 0 {
+				t.Errorf("v3 %s: %v, want a counter above zero", want, m)
+			}
+			for _, l := range m.Label {
+				if l.GetName() == "reason" {
+					reasons[l.GetValue()] = true
+				}
+			}
+		}
+	}
+	for _, r := range ilmIncompleteReasons {
+		if !reasons[r] {
+			t.Errorf("v3: %s{reason=%q} not exported", want, r)
+		}
+	}
+	reasons = map[string]bool{}
+	for _, mv := range getILMNodeMetrics().Get() {
+		if mv.Description.Name == expiryIncompleteTotal {
+			if mv.Description.Type != counterMetric {
+				t.Errorf("v2 type %v", mv.Description.Type)
+			}
+			reasons[mv.VariableLabels["reason"]] = true
+		}
+	}
+	for _, r := range ilmIncompleteReasons {
+		if !reasons[r] {
+			t.Errorf("v2: reason %q not exported", r)
+		}
+	}
+}

cmd/bucket-lifecycle-restore-boundary_test.go

@@ -0,0 +1,277 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+// Read the actual metadata from every disk without object-layer read repair.
+func restoreBoundarySnapshot(t *testing.T, r *restoreTest) [][]byte {
+	t.Helper()
+	var result [][]byte
+	for _, pool := range r.z.serverPools {
+		for _, disk := range pool.getHashedSet(r.object).getDisks() {
+			raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+			if err != nil && err != errFileNotFound {
+				t.Fatal(err)
+			}
+			result = append(result, bytes.Clone(raw.Buf))
+		}
+	}
+	return result
+}
+
+func TestExpireRestoredKeepsReplacementDeleteMarker(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, pools, versioning.Suspended)
+			r.put("old", false, false)
+			oi := r.put("null", true, true)
+			r.restore(oi)
+			stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts("")); err != nil {
+				t.Fatal(err)
+			}
+			if !r.fileInfo(nullVersionID).Deleted {
+				t.Fatal("ordinary DELETE did not replace null with a delete marker")
+			}
+			before := restoreBoundarySnapshot(t, r)
+			err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+			if !isErrVersionNotFound(err) {
+				t.Errorf("stale cleanup should reject the replacement marker: %v", err)
+			}
+			if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+				t.Error("stale cleanup changed stored versions or their data")
+			}
+			if _, _, err := r.read(""); !isErrObjectNotFound(err) {
+				t.Errorf("stale cleanup exposed the old version: %v", err)
+			}
+			// An explicit user purge must still remove that same marker.
+			if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts(nullVersionID)); err != nil {
+				t.Fatal(err)
+			}
+			if label, data, err := r.read(""); err != nil || label != "old" || !bytes.Equal(data, r.bodies["old"]) {
+				t.Fatalf("ordinary marker purge: label=%s err=%v", label, err)
+			}
+		})
+	}
+}
+
+func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, operation := range []string{"cleanup", "restore", "restore-cleanup", "lookup-cleanup", "lookup-restore", "get", "delete"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, operation), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				oi := r.put("target", false, true)
+				r.restore(oi)
+				fi := r.fileInfo(oi.VersionID)
+				// A readable latest version lets pools lookup finish before the
+				// set reads the specifically addressed, incomplete old version.
+				r.put("newer", false, false)
+				er := r.z.serverPools[0].getHashedSet(r.object)
+				disks := er.getDisks()
+				for _, disk := range disks[1:] {
+					raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					var meta xlMetaV2
+					if err := meta.Load(raw.Buf); err != nil {
+						t.Fatal(err)
+					}
+					del := FileInfo{VersionID: oi.VersionID}
+					del.SetSkipTierFreeVersion()
+					if _, err := meta.DeleteVersion(del); err != nil {
+						t.Fatal(err)
+					}
+					buf, err := meta.AppendTo(nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					// Remove only this metadata entry; retain every real shard.
+					if err := disk.WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), buf); err != nil {
+						t.Fatal(err)
+					}
+				}
+				before := restoreBoundarySnapshot(t, r)
+				ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+				defer cancel()
+				opts := r.opts(oi.VersionID)
+				var err error
+				switch operation {
+				case "cleanup":
+					err = expireTransitionedObject(ctx, r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredVersionAction}, lcEventSrc_Scanner)
+				case "restore":
+					opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+					err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, opts)
+				case "restore-cleanup":
+					// The initial read succeeded before remote download failed;
+					// cleanup must not purge a now-incomplete target on reread.
+					err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil)
+				case "lookup-cleanup", "lookup-restore":
+					// Version-sensitive lookup also covers native restore routing.
+					opts.MetadataChg = true
+					if operation == "lookup-cleanup" {
+						opts.Transition.ExpireRestored = true
+					} else {
+						opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+					}
+					_, _, err = r.z.getPoolInfoExistingWithOpts(ctx, r.bucket, r.object, opts)
+				case "get":
+					_, err = r.z.GetObjectInfo(ctx, r.bucket, r.object, opts)
+				case "delete":
+					_, err = r.z.DeleteObject(ctx, r.bucket, r.object, opts)
+				}
+				if ctx.Err() != nil {
+					t.Fatalf("%s timed out instead of returning its read error: %v", operation, err)
+				}
+				if err == nil {
+					t.Fatal("incomplete target unexpectedly readable")
+				}
+				if operation != "get" && operation != "delete" {
+					if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+						t.Error("failed restore read changed metadata, inline data or free versions")
+					}
+					if n := r.dataDirs(fi.DataDir); n != len(disks) {
+						t.Errorf("failed restore read removed shards: %d/%d directories remain", n, len(disks))
+					}
+					return
+				}
+				// Ordinary GET/DELETE retain their existing dangling cleanup.
+				raw, err := disks[0].ReadXL(t.Context(), r.bucket, r.object, false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				versions, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if len(versions.Versions) != 1 || len(versions.FreeVersions) != 1 || r.dataDirs(fi.DataDir) != len(disks)-1 {
+					t.Fatal("ordinary dangling cleanup no longer removed the minority version")
+				}
+			})
+		}
+	}
+}
+
+func TestNativeRestoreInitialMissingReadDoesNotWrite(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	r.put("live", false, false)
+	before := restoreBoundarySnapshot(t, r)
+	opts := r.opts(mustGetUUID())
+	opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+		t.Fatalf("missing restore target: %v", err)
+	}
+	if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+		t.Fatal("initial restore failure wrote metadata without a resolved version")
+	}
+}
+
+func TestExpireRestoredStorageRejectsOtherTypes(t *testing.T) {
+	for _, kind := range []string{"null-marker", "uuid-marker", "legacy", "legacy-json"} {
+		t.Run(kind, func(t *testing.T) {
+			disk, _, err := newXLStorageTestSetup(t)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx := t.Context()
+			const bucket, object = "restore-types", "obj"
+			if err := disk.MakeVol(ctx, bucket); err != nil {
+				t.Fatal(err)
+			}
+			var meta xlMetaV2
+			versionID, metaName, dataDir := nullVersionID, xlStorageFormatFile, ""
+			var buf []byte
+			switch kind {
+			case "null-marker", "uuid-marker":
+				if kind == "uuid-marker" {
+					versionID = mustGetUUID()
+				}
+				err = meta.AddVersion(FileInfo{VersionID: versionID, Deleted: true, ModTime: time.Now().UTC()})
+			case "legacy", "legacy-json":
+				legacy := getSampleXLMeta(1)
+				legacy.DataDir, dataDir = "legacy", "legacy"
+				if kind == "legacy-json" {
+					metaName = xlStorageFormatFileV1
+					buf, err = json.Marshal(legacy)
+					disk.storage.Lock()
+					disk.storage.formatLegacy = true
+					disk.storage.Unlock()
+				} else {
+					err = meta.AddLegacy(&legacy)
+				}
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if buf == nil {
+				buf, err = meta.AppendTo(nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := disk.WriteAll(ctx, bucket, pathJoin(object, metaName), buf); err != nil {
+				t.Fatal(err)
+			}
+			payload := []byte("sole legacy contents")
+			dataPath := filepath.Join(disk.Endpoint().Path, bucket, object, dataDir, "part.1")
+			if dataDir != "" {
+				if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(dataPath, payload, 0o600); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID, ExpireRestored: true, TransitionStatus: lifecycle.TransitionComplete}, false, DeleteOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			after, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName))
+			if err != nil || !bytes.Equal(buf, after) {
+				t.Errorf("restore cleanup changed %s metadata: %v", kind, err)
+			}
+			if dataDir != "" {
+				if data, err := os.ReadFile(dataPath); err != nil || !bytes.Equal(data, payload) {
+					t.Errorf("restore cleanup removed legacy contents: %v", err)
+				}
+			}
+			if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID}, false, DeleteOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName)); err != errFileNotFound {
+				t.Errorf("ordinary purge left %s: %v", kind, err)
+			}
+		})
+	}
+}

cmd/bucket-lifecycle-restore-safety_test.go

@@ -0,0 +1,284 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+const restoreSafetyRules = `<LifecycleConfiguration><Rule><ID>tier</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Transition><Days>1000</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`
+
+// Reuse the original restore-copy fixture with a small, isolated real cluster.
+func newRestoreSafetyTest(t *testing.T, pools int, state versioning.State) *restoreTest {
+	t.Helper()
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	dirs, err := getRandomDisks(4 * pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { removeRoots(dirs) })
+	var endpoints EndpointServerPools
+	for i := range pools {
+		endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+	}
+	obj, _, err := initObjectLayer(t.Context(), endpoints)
+	if err != nil {
+		t.Fatal(err)
+	}
+	z := obj.(*erasureServerPools)
+	t.Cleanup(func() { z.Shutdown(context.Background()) })
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+	r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}}
+	if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: state != ""}); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := globalBucketMetadataSys.Get(r.bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state == versioning.Suspended {
+		meta.VersioningConfigXML = []byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)
+	}
+	meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+	if err := meta.Save(t.Context(), z); err != nil {
+		t.Fatal(err)
+	}
+	globalBucketMetadataSys.Set(r.bucket, meta)
+	return r
+}
+
+type restoreCleanupFaultDisk struct {
+	consistencyReadFaultDisk
+	deletes *atomic.Int64
+}
+
+func (d restoreCleanupFaultDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+	if bucket == d.bucket && object == d.object {
+		d.deletes.Add(1)
+	}
+	return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func TestExpireRestoredRequiresFreshMetadata(t *testing.T) {
+	for _, restored := range []bool{true, false} {
+		t.Run(fmt.Sprintf("expire-restored=%v", restored), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, "")
+			oi := r.put("v1", true, true)
+			r.restore(oi)
+			before := r.fileInfo(oi.VersionID)
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			original := er.getDisks
+			disks := original()
+			var deletes atomic.Int64
+			faulty := make([]StorageAPI, len(disks))
+			for i, d := range disks {
+				faulty[i] = restoreCleanupFaultDisk{consistencyReadFaultDisk{d, r.bucket, r.object}, &deletes}
+			}
+			er.getDisks = func() []StorageAPI { return faulty }
+			// Restore the callback before any fixture reads and on a fatal assertion.
+			defer func() { er.getDisks = original }()
+			opts := ObjectOptions{VersionID: nullVersionID, Transition: TransitionOptions{ExpireRestored: restored}, Expiration: ExpirationOptions{Expire: restored}}
+			_, err := er.DeleteObject(t.Context(), r.bucket, r.object, opts)
+			er.getDisks = original
+			if restored {
+				var quorum InsufficientWriteQuorum
+				if !errors.As(err, &quorum) {
+					t.Errorf("cleanup without fresh metadata: got %v, want write-quorum error", err)
+				}
+				if n := deletes.Load(); n != 0 {
+					t.Errorf("cleanup attempted %d blind disk deletes", n)
+				}
+				if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before, after) {
+					t.Error("failed metadata read changed the stored version")
+				}
+				if n := r.dataDirs(before.DataDir); n != len(disks) {
+					t.Errorf("cleanup removed data: %d/%d directories remain", n, len(disks))
+				}
+			} else {
+				// Ordinary unversioned DELETE retains the existing tryDel behavior.
+				if err != nil || deletes.Load() != int64(len(disks)) {
+					t.Fatalf("ordinary DELETE: err=%v, deletes=%d", err, deletes.Load())
+				}
+				if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); !isErrObjectNotFound(err) {
+					t.Fatalf("ordinary DELETE left the object: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestExpireRestoredRenewedOrMissing(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi := r.put("v1", false, true)
+	r.restore(oi)
+	stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil {
+		t.Fatal(err)
+	}
+	before := r.fileInfo(oi.VersionID)
+	renewed := completedRestoreObj(time.Now().Add(48 * time.Hour)).String()
+	opts := r.opts(oi.VersionID)
+	opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = renewed
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+		t.Fatal(err)
+	}
+	err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+	if !errors.Is(err, errLifecycleNotApplicable) {
+		t.Errorf("stale task should be rejected, got %v", err)
+	}
+	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir {
+		t.Error("stale task changed renewed restore")
+	}
+	if n := r.dataDirs(before.DataDir); n != 4 {
+		t.Errorf("renewed copy has %d directories, want 4", n)
+	}
+	if _, data, err := r.read(oi.VersionID); err != nil || !bytes.Equal(data, r.bodies["v1"]) {
+		t.Fatalf("renewed copy not readable: %v", err)
+	}
+
+	for _, id := range []string{mustGetUUID(), nullVersionID} {
+		opts := r.opts(id)
+		opts.Expiration.Expire = true
+		opts.Transition.ExpireRestored = true
+		if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+			t.Errorf("missing version %s: %v", id, err)
+		}
+	}
+	missing := r.opts(nullVersionID)
+	missing.Expiration.Expire = true
+	missing.Transition.ExpireRestored = true
+	if _, err := r.z.DeleteObject(t.Context(), r.bucket, "missing-object", missing); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+		t.Errorf("missing object: %v", err)
+	}
+	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed {
+		t.Error("missing-target cleanup changed existing restore")
+	}
+}
+
+func TestExpireRestoredStorageBoundary(t *testing.T) {
+	for _, status := range []string{"", lifecycle.TransitionPending, lifecycle.TransitionComplete} {
+		for _, inline := range []bool{false, true} {
+			for _, requestStatus := range []string{"", lifecycle.TransitionComplete} {
+				t.Run(fmt.Sprintf("stored=%s/inline=%v/request=%s", status, inline, requestStatus), func(t *testing.T) {
+					disk, _, err := newXLStorageTestSetup(t)
+					if err != nil {
+						t.Fatal(err)
+					}
+					const bucket, object = "restore-storage", "obj"
+					ctx := t.Context()
+					if err := disk.MakeVol(ctx, bucket); err != nil {
+						t.Fatal(err)
+					}
+					fi := newFileInfo(object, 2, 2)
+					fi.VersionID, fi.DataDir = mustGetUUID(), mustGetUUID()
+					fi.ModTime, fi.Size = time.Now().UTC(), 1024
+					fi.TransitionStatus = status
+					fi.AddObjectPart(1, "", fi.Size, fi.Size, fi.ModTime, nil, nil)
+					fi.Metadata = map[string]string{xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), "x-amz-meta-preserve": "keep"}
+					payload := []byte("sole local contents")
+					if inline {
+						fi.Data = payload
+						fi.SetInlineData()
+					}
+					if err := disk.WriteMetadata(ctx, "", bucket, object, fi); err != nil {
+						t.Fatal(err)
+					}
+					dir := filepath.Join(disk.Endpoint().Path, bucket, object, fi.DataDir)
+					if err := os.MkdirAll(dir, 0o700); err != nil {
+						t.Fatal(err)
+					}
+					if err := os.WriteFile(filepath.Join(dir, "part.1"), payload, 0o600); err != nil {
+						t.Fatal(err)
+					}
+					raw, err := disk.ReadXL(ctx, bucket, object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					var xl xlMetaV2
+					if err := xl.Load(raw.Buf); err != nil {
+						t.Fatal(err)
+					}
+					del := FileInfo{VersionID: fi.VersionID, ExpireRestored: true, TransitionStatus: requestStatus}
+					dataDir, err := xl.DeleteVersion(del)
+					if err != nil {
+						t.Fatal(err)
+					}
+					wantDir := ""
+					if status == lifecycle.TransitionComplete {
+						wantDir = fi.DataDir
+					}
+					if dataDir != wantDir {
+						t.Errorf("format authorized directory %q, want %q", dataDir, wantDir)
+					}
+					before, err := getFileInfo(raw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if err := disk.DeleteVersion(ctx, bucket, object, del, false, DeleteOptions{}); err != nil {
+						t.Fatal(err)
+					}
+					afterRaw, err := disk.ReadXL(ctx, bucket, object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					after, err := getFileInfo(afterRaw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					data, dirErr := os.ReadFile(filepath.Join(dir, "part.1"))
+					if status != lifecycle.TransitionComplete {
+						if !reflect.DeepEqual(before, after) {
+							t.Error("non-complete cleanup changed metadata or inline data")
+						}
+						if dirErr != nil || !bytes.Equal(data, payload) {
+							t.Errorf("sole local contents removed: %v", dirErr)
+						}
+					} else {
+						if !os.IsNotExist(dirErr) {
+							t.Errorf("completed restore directory still present: %v", dirErr)
+						}
+						if len(after.Data) != 0 || after.Metadata[xhttp.AmzRestore] != "" {
+							t.Error("completed restore kept cached data or restore headers")
+						}
+						if after.TransitionStatus != status || after.VersionID != before.VersionID || after.Metadata["x-amz-meta-preserve"] != "keep" {
+							t.Error("cleanup changed the transitioned version identity or user metadata")
+						}
+					}
+				})
+			}
+		}
+	}
+}

cmd/bucket-lifecycle-restore_test.go

@@ -0,0 +1,453 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreTestTier is a remote tier held in memory that counts its reads.
+type restoreTestTier struct {
+	*tierExpiryTestBackend
+	gets atomic.Int64
+}
+
+func (b *restoreTestTier) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	b.gets.Add(1)
+	return b.tierExpiryTestBackend.Get(ctx, object, rv, opts)
+}
+
+// restoreTest is a cluster with a remote tier WARM and a bucket whose
+// versions are transitioned to it and restored.
+type restoreTest struct {
+	t      *testing.T
+	z      *erasureServerPools
+	bucket string
+	object string
+	queue  chan expiryOp
+	labels map[string]string // version ID ("null" for the null version) -> label
+	bodies map[string][]byte // label -> data
+}
+
+func restoreTestVersionKey(versionID string) string {
+	if versionID == "" {
+		return nullVersionID
+	}
+	return versionID
+}
+
+// put writes a version of the object to the first pool. A restored version
+// is 1 MiB, so that its restored copy has a data directory of its own.
+func (r *restoreTest) put(label string, null, restored bool) ObjectInfo {
+	r.t.Helper()
+	body := []byte(r.object + " " + label)
+	if restored {
+		body = bytes.Repeat(body, (1<<20)/len(body)+1)
+	}
+	opts := ObjectOptions{Versioned: !null}
+	if null {
+		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+	}
+	oi, err := r.z.serverPools[0].PutObject(r.t.Context(), r.bucket, r.object,
+		mustGetPutObjReader(r.t, bytes.NewReader(body), int64(len(body)), "", ""), opts)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	r.labels[restoreTestVersionKey(oi.VersionID)] = label
+	r.bodies[label] = body
+	return oi
+}
+
+// opts returns the options of a request on a version of the object, as the
+// S3 handlers set them.
+func (r *restoreTest) opts(versionID string) ObjectOptions {
+	return ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object),
+	}
+}
+
+// restore transitions a version to the tier, restores it, and moves the
+// expiry date of the restore into the past.
+func (r *restoreTest) restore(oi ObjectInfo) {
+	r.t.Helper()
+	ctx := r.t.Context()
+	vopts := r.opts(oi.VersionID)
+	topts := vopts
+	topts.MTime = oi.ModTime
+	topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+	if err := r.z.TransitionObject(ctx, r.bucket, r.object, topts); err != nil {
+		r.t.Fatal(err)
+	}
+	ropts := vopts
+	ropts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+	// Through the pool, as with a single pool: with more than one pool the
+	// pools layer holds the object lock that the write of the restored copy
+	// waits for.
+	if err := r.z.serverPools[0].RestoreTransitionedObject(ctx, r.bucket, r.object, ropts); err != nil {
+		r.t.Fatal(err)
+	}
+	expired := completedRestoreObj(time.Now().Add(-24 * time.Hour).UTC()).String()
+	mopts := vopts
+	mopts.MTime = oi.ModTime
+	mopts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = expired
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, mopts); err != nil {
+		r.t.Fatal(err)
+	}
+}
+
+// fileInfo returns the version of the object from a drive of the first pool.
+func (r *restoreTest) fileInfo(versionID string) FileInfo {
+	r.t.Helper()
+	infos, errs := readAllFileInfo(r.t.Context(), r.z.serverPools[0].getHashedSet(r.object).getDisks(), "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false)
+	if errs[0] != nil {
+		r.t.Fatal(errs[0])
+	}
+	return infos[0]
+}
+
+// dataDirs returns how many drives of the first pool hold the data
+// directory of a version.
+func (r *restoreTest) dataDirs(dataDir string) int {
+	r.t.Helper()
+	n := 0
+	for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() {
+		_, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, dataDir))
+		if err == nil {
+			n++
+		} else if !os.IsNotExist(err) {
+			r.t.Fatal(err)
+		}
+	}
+	return n
+}
+
+// versions returns the versions of the object in each pool, newest first,
+// and its free versions in all pools.
+func (r *restoreTest) versions() (views []string, free int) {
+	r.t.Helper()
+	for _, pool := range r.z.serverPools {
+		rf, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+		if errors.Is(err, errFileNotFound) {
+			views = append(views, "")
+			continue
+		}
+		if err != nil {
+			r.t.Fatal(err)
+		}
+		fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+		if err != nil {
+			r.t.Fatal(err)
+		}
+		var labels []string
+		for _, fi := range fivs.Versions {
+			if fi.Deleted {
+				labels = append(labels, "DM")
+			} else {
+				labels = append(labels, r.labels[restoreTestVersionKey(fi.VersionID)])
+			}
+		}
+		views = append(views, strings.Join(labels, " "))
+		free += len(fivs.FreeVersions)
+	}
+	return views, free
+}
+
+// read returns the label and the data of a version, or of the latest version
+// when versionID is empty, or the error of the read.
+func (r *restoreTest) read(versionID string) (string, []byte, error) {
+	r.t.Helper()
+	gr, err := r.z.GetObjectNInfo(r.t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: versionID})
+	if err != nil {
+		return "", nil, err
+	}
+	defer gr.Close()
+	data, err := io.ReadAll(gr)
+	if err != nil {
+		return "", nil, err
+	}
+	return r.labels[restoreTestVersionKey(gr.ObjInfo.VersionID)], data, nil
+}
+
+// runQueue runs the queued expiries with an expiry worker.
+func (r *restoreTest) runQueue() {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		globalExpiryState.Worker(r.queue)
+	}()
+	r.queue <- nil // the worker returns once it ran every expiry queued before
+	<-done
+}
+
+// objInfos returns the versions of the object as a drive of the
+// first pool holds them, as the scanner and the walks of decommission and
+// the drive heal see them.
+func (r *restoreTest) objInfos() []ObjectInfo {
+	r.t.Helper()
+	rf, err := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	versioned := globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object) || globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+	objInfos := make([]ObjectInfo, len(fivs.Versions))
+	for i, fi := range fivs.Versions {
+		objInfos[i] = fi.ToObjectInfo(r.bucket, r.object, versioned)
+	}
+	return objInfos
+}
+
+// The restored copy of a transitioned version expires on the expiry date of
+// its restore. The expiry removes the copy and nothing else: the version
+// stays where it is in the version stack, transitioned, and is read from the
+// tier again. Every path that applies ILM runs the expiry: the scanner and a
+// GET or HEAD queue it for an expiry worker, decommission and the drive heal
+// run it at once. The expiry of the restored copy of the latest version used
+// to delete the object instead: a versioned bucket got a delete marker that
+// hid it. With versioning suspended it went to the null version, removed its
+// data if there was one and failed if not, and the restored copy stayed.
+func TestExpireRestoredCopy(t *testing.T) {
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	prevExpiry := globalExpiryState
+	t.Cleanup(func() { globalExpiryState = prevExpiry })
+
+	// Nothing here is old enough to transition; the rule makes the bucket
+	// one with a tier, as restored versions are in.
+	const rules = `<LifecycleConfiguration><Rule><ID>tier</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
+		`<Transition><Days>1000</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`
+
+	type version struct {
+		label    string
+		null     bool // the null version
+		restored bool // transitioned and restored, the restore has expired
+	}
+	testCases := []struct {
+		name       string
+		versioning versioning.State // "", Enabled or Suspended
+		versions   []version        // oldest first
+	}{
+		{name: "versioned", versioning: versioning.Enabled, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+		{name: "versioned-noncurrent", versioning: versioning.Enabled, versions: []version{{label: "v1", restored: true}, {label: "v2"}}},
+		{name: "versioned-null", versioning: versioning.Enabled, versions: []version{{label: "v1", null: true, restored: true}}},
+		{name: "suspended", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true}, {label: "v2", restored: true}}},
+		{name: "suspended-no-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+		{name: "suspended-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", null: true, restored: true}}},
+		{name: "suspended-null-noncurrent", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true, restored: true}, {label: "v2"}}},
+		{name: "unversioned", versions: []version{{label: "v1", null: true, restored: true}}},
+	}
+
+	sources := []struct {
+		name   string
+		expire func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event)
+	}{
+		{"scanner", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+			lc, err := globalLifecycleSys.Get(r.bucket)
+			if err != nil {
+				r.t.Fatal(err)
+			}
+			lr, err := globalBucketObjectLockSys.Get(r.bucket)
+			if err != nil {
+				r.t.Fatal(err)
+			}
+			item := scannerItem{bucket: r.bucket, objectName: r.object, lifeCycle: lc}
+			item.applyActions(r.t.Context(), r.z, r.objInfos(), lr, &sizeSummary{}, nil)
+			r.runQueue()
+		}},
+		{"head", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+			applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi)
+			r.runQueue()
+		}},
+		{"list", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+			// ListObjects of the key with max-keys=1 reads its latest
+			// version without the versioning of the bucket, a listing of
+			// versions reads them all.
+			if _, err := r.z.ListObjects(r.t.Context(), r.bucket, r.object, "", "", 1); err != nil {
+				r.t.Fatal(err)
+			}
+			if _, err := r.z.ListObjectVersions(r.t.Context(), r.bucket, r.object, "", "", "", 100); err != nil {
+				r.t.Fatal(err)
+			}
+			r.runQueue()
+		}},
+		{"decom", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+			for _, o := range r.objInfos() {
+				if o.VersionID == oi.VersionID {
+					applyExpiryOnTransitionedObject(r.t.Context(), r.z, o, evt, lcEventSrc_Decom)
+					return
+				}
+			}
+			r.t.Fatal("version not found on the drive")
+		}},
+	}
+
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			ctx := t.Context()
+			dirs, err := getRandomDisks(4 * pools)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { removeRoots(dirs) })
+			var endpoints EndpointServerPools
+			for i := range pools {
+				endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+			}
+			obj, _, err := initObjectLayer(ctx, endpoints)
+			if err != nil {
+				t.Fatal(err)
+			}
+			z := obj.(*erasureServerPools)
+			t.Cleanup(func() { z.Shutdown(context.Background()) })
+			// After initObjectLayer, which sets up a new tier config.
+			tier := &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+			globalTierConfigMgr = NewTierConfigMgr()
+			globalTierConfigMgr.drivercache["WARM"] = tier
+
+			for _, tc := range testCases {
+				for _, src := range sources {
+					t.Run(tc.name+"/"+src.name, func(t *testing.T) {
+						r := &restoreTest{
+							t: t, z: z,
+							bucket: "restore-" + tc.name + "-" + src.name, object: "obj",
+							queue:  make(chan expiryOp, 100),
+							labels: map[string]string{}, bodies: map[string][]byte{},
+						}
+						if err := z.MakeBucket(ctx, r.bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
+							t.Fatal(err)
+						}
+						meta, err := globalBucketMetadataSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						if tc.versioning == versioning.Suspended {
+							meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+						}
+						meta.LifecycleConfigXML = []byte(rules)
+						if err := meta.Save(ctx, z); err != nil {
+							t.Fatal(err)
+						}
+						globalBucketMetadataSys.Set(r.bucket, meta)
+						es := &expiryState{ctx: ctx, objAPI: z}
+						workers := []chan expiryOp{r.queue}
+						es.workers.Store(&workers)
+						globalExpiryState = es
+
+						var restored ObjectInfo
+						for _, v := range tc.versions {
+							oi := r.put(v.label, v.null, v.restored)
+							if v.restored {
+								r.restore(oi)
+								restored = oi
+							}
+						}
+						rlabel := r.labels[restoreTestVersionKey(restored.VersionID)]
+
+						// The fixture: the restored copy is on the drives,
+						// its restore has expired, ILM expires it.
+						fi := r.fileInfo(restored.VersionID)
+						if fi.TransitionStatus != lifecycle.TransitionComplete || fi.DataDir == "" || r.dataDirs(fi.DataDir) == 0 {
+							t.Fatalf("fixture: %s transition status %q, restored copy on %d drives", rlabel, fi.TransitionStatus, r.dataDirs(fi.DataDir))
+						}
+						oi, err := z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+						if err != nil {
+							t.Fatal(err)
+						}
+						lc, err := globalLifecycleSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						lr, err := globalBucketObjectLockSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
+						if !evt.Action.DeleteRestored() || oi.RestoreExpires.IsZero() {
+							t.Fatalf("fixture: %s restore expires %v, action %v", rlabel, oi.RestoreExpires, evt.Action)
+						}
+						before, _ := r.versions()
+						for key, label := range r.labels {
+							if _, data, err := r.read(key); err != nil || !bytes.Equal(data, r.bodies[label]) {
+								t.Fatalf("fixture: GET %s: %v", label, err)
+							}
+						}
+
+						src.expire(r, oi, evt)
+
+						after, free := r.versions()
+						if fmt.Sprint(after) != fmt.Sprint(before) || free != 0 {
+							t.Errorf("versions in each pool: before %q, after %q with %d free versions", before, after, free)
+						}
+						if label, _, err := r.read(""); err != nil || label != tc.versions[len(tc.versions)-1].label {
+							t.Errorf("GET reads %q, %v; want %q", label, err, tc.versions[len(tc.versions)-1].label)
+						}
+						if n := r.dataDirs(fi.DataDir); n != 0 {
+							t.Errorf("restored copy of %s still on %d drives", rlabel, n)
+						}
+						oi, err = z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+						if err != nil {
+							t.Fatalf("HEAD %s: %v", rlabel, err)
+						}
+						if oi.TransitionedObject.Status != lifecycle.TransitionComplete || oi.UserDefined[xhttp.AmzRestore] != "" || !oi.RestoreExpires.IsZero() {
+							t.Errorf("HEAD %s: transition status %q, x-amz-restore %q", rlabel, oi.TransitionedObject.Status, oi.UserDefined[xhttp.AmzRestore])
+						}
+						for _, v := range tc.versions {
+							for key, label := range r.labels {
+								if label != v.label {
+									continue
+								}
+								gets := tier.gets.Load()
+								_, data, err := r.read(key)
+								switch {
+								case err != nil:
+									t.Errorf("GET %s: %v", v.label, err)
+								case !bytes.Equal(data, r.bodies[v.label]):
+									t.Errorf("GET %s reads %d bytes that differ", v.label, len(data))
+								case v.restored && tier.gets.Load() == gets:
+									t.Errorf("GET %s does not read the tier", v.label)
+								}
+							}
+						}
+					})
+				}
+			}
+		})
+	}
+}

cmd/bucket-lifecycle.go

@@ -37,7 +37,8 @@ import (
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
@@ -161,7 +162,7 @@ func (n noncurrentVersionsTask) OpHash() uint64 {
 }
 
 func (j jentry) OpHash() uint64 {
-	return xxh3.HashString(j.TierName + j.ObjName)
+	return xxh3.HashString(j.Bucket + j.Object + j.TierName + j.RemoteObject + j.RemoteVersionID)
 }
 
 func (e expiryTask) OpHash() uint64 {
@@ -353,7 +354,16 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 			case noncurrentVersionsTask:
 				deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.events)
 			case jentry:
-				transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+				z, ok := es.objAPI.(*erasureServerPools)
+				if !ok {
+					transitionLogIf(es.ctx, errTierOwnershipUnavailable)
+					continue
+				}
+				transitionLogIf(es.ctx, z.deleteTierTuple(es.ctx, v.Bucket, v.Object, tierRemoteTuple{
+					Tier:      v.TierName,
+					Object:    v.RemoteObject,
+					VersionID: v.RemoteVersionID,
+				}))
 			case freeVersionTask:
 				oi := v.ObjectInfo
 				traceFn := globalLifecycleSys.trace(oi)
@@ -369,19 +379,26 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 					}
 					return err
 				}
-				// Remove the remote object
-				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
+				z, ok := es.objAPI.(*erasureServerPools)
+				if !ok {
+					transitionLogIf(es.ctx, errTierOwnershipUnavailable)
+					traceFn(ILMFreeVersionDelete, nil, errTierOwnershipUnavailable)
+					continue
+				}
+
+				// Check ownership, remove the remote object, and remove only the
+				// matching free record while the object lock is held.
+				err := z.deleteTierTuple(es.ctx, oi.Bucket, oi.Name, tierRemoteTuple{
+					Tier:      oi.TransitionedObject.Tier,
+					Object:    oi.TransitionedObject.Name,
+					VersionID: oi.TransitionedObject.VersionID,
+				})
 				if ignoreNotFoundErr(err) != nil {
 					transitionLogIf(es.ctx, err)
 					traceFn(ILMFreeVersionDelete, nil, err)
 					continue
 				}
 
-				// Remove this free version
-				_, err = es.objAPI.DeleteObject(es.ctx, oi.Bucket, oi.Name, ObjectOptions{
-					VersionID:        oi.VersionID,
-					InclFreeVersions: true,
-				})
 				if err == nil {
 					auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn)
 				}
@@ -610,8 +627,9 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error {
 	traceFn := globalLifecycleSys.trace(*oi)
 	opts := ObjectOptions{
-		Versioned:  globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
-		Expiration: ExpirationOptions{Expire: true},
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		Expiration:       ExpirationOptions{Expire: true},
 	}
 	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = oi.VersionID
@@ -621,6 +639,17 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
+		//
+		// The restored copy belongs to this version, also when it is the
+		// latest one (DeleteRestoredAction). Without its version ID the
+		// delete applies to the object: a versioned bucket gets a delete
+		// marker that hides it, a suspended one strips the null version.
+		// An empty version ID is the null version, as callers that read
+		// it without the versioning of the bucket leave it.
+		opts.VersionID = oi.VersionID
+		if opts.VersionID == "" {
+			opts.VersionID = nullVersionID
+		}
 		opts.Transition.ExpireRestored = true
 		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 		if err == nil {
@@ -631,32 +660,54 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		return err
 	}
 
-	// Delete remote object from warm-tier
-	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
-	if err == nil {
-		// Skip adding free version since we successfully deleted the
-		// remote object
-		opts.SkipFreeVersion = true
-	} else {
-		transitionLogIf(ctx, err)
+	// Recheck the lifecycle decision and create the hot-tier tombstone before
+	// touching the remote tier. If the lifecycle configuration changed after
+	// the scanner queued this task, DeleteObject rejects the stale action while
+	// the remote object is still protected.
+	keepRemote := opts.VersionID == "" &&
+		(opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID))
+
+	// Delete the hot-tier namespace first. For a transitioned version this
+	// leaves a durable free-version record until the remote delete succeeds;
+	// that record is the retry point if the process stops between the two
+	// operations.
+	dobj, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	if err != nil {
+		return err
 	}
 
-	// Now, delete object from hot-tier namespace
-	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-		return err
+	if !keepRemote {
+		z, ok := objectAPI.(*erasureServerPools)
+		if !ok {
+			transitionLogIf(ctx, errTierOwnershipUnavailable)
+			return nil // The completed local expiry owns a durable free record.
+		}
+		if err := z.deleteTierTuple(ctx, oi.Bucket, oi.Name, tierRemoteTuple{
+			Tier:      oi.TransitionedObject.Tier,
+			Object:    oi.TransitionedObject.Name,
+			VersionID: oi.TransitionedObject.VersionID,
+		}); err != nil {
+			// The hot delete has already created a durable free record. Keep
+			// the expiry successful while the scanner retries remote cleanup;
+			// ownership or tier errors must never cause a second remote delete
+			// without that record.
+			transitionLogIf(ctx, err)
+		}
 	}
 
 	// Send audit for the lifecycle delete operation
 	defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
 
-	eventName := event.ObjectRemovedDelete
-	if oi.DeleteMarker {
-		eventName = event.ObjectRemovedDeleteMarkerCreated
-	}
+	// Name what the delete did: a marker it created, or the version it
+	// removed.
+	eventName := ilmExpiryEventName(opts.VersionID, dobj)
 	objInfo := ObjectInfo{
 		Name:         oi.Name,
-		VersionID:    oi.VersionID,
-		DeleteMarker: oi.DeleteMarker,
+		VersionID:    dobj.VersionID,
+		DeleteMarker: dobj.DeleteMarker,
+	}
+	if objInfo.VersionID == "" {
+		objInfo.VersionID = opts.VersionID
 	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
@@ -931,8 +982,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
 	}, nil
 }
 
-// set ObjectOptions for PUT call to restore temporary copy of transitioned data
-func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data.
+// opts are the restore's options, with the versioning POST captured: the
+// bucket's versioning may have changed since, and must not mint a new version.
+func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) {
 	meta := make(map[string]string)
 	sc := rreq.OutputLocation.S3.StorageClass
 	if sc == "" {
@@ -955,8 +1008,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
-			Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+			Versioned:        opts.Versioned,
+			VersionSuspended: opts.VersionSuspended,
 			UserDefined:      meta,
 		}
 	}
@@ -964,19 +1017,37 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	if len(objInfo.UserTags) != 0 {
 		meta[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-	// Set restore object status
+	// Set restore object status; a completed restore has no attempt.
 	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
 	meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
+	delete(meta, restoreAttemptKey)
 	return ObjectOptions{
-		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
 		UserDefined:      meta,
-		VersionID:        objInfo.VersionID,
+		VersionID:        restoreVersionID(objInfo.VersionID),
 		MTime:            objInfo.ModTime,
 		Expires:          objInfo.Expires,
 	}
 }
 
+// restoredCopyExpires reports whether the restored copy of oi, as read now,
+// expires by its own reason: lifecycle expires the restored copy of this
+// version, its restore deadline is valid and has passed, its transition is
+// complete and no restore is running. Another rule falling due is no reason.
+func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool {
+	if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() {
+		return false
+	}
+	status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore])
+	if err != nil {
+		return false
+	}
+	expiry, completed := status.Expiry()
+	return completed && !expiry.IsZero() && UTCNow().After(expiry) &&
+		oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing
+}
+
 var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
 
 // IsRemote returns true if this object version's contents are in its remote

cmd/bucket-lifecycle_test.go

@@ -19,11 +19,17 @@ package cmd
 
 import (
 	"bytes"
+	"context"
+	"io"
 	"net/http"
+	"slices"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	xhttp "github.com/minio/minio/internal/http"
 )
 
@@ -245,3 +251,180 @@ func TestValidateTransitionTier(t *testing.T) {
 		}
 	}
 }
+
+// tierExpiryTestBackend is a remote tier held in memory.
+type tierExpiryTestBackend struct {
+	mu      sync.Mutex
+	objects map[string][]byte
+	removed []string
+}
+
+func (b *tierExpiryTestBackend) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
+	return b.PutWithMeta(ctx, object, r, length, nil)
+}
+
+func (b *tierExpiryTestBackend) PutWithMeta(_ context.Context, object string, r io.Reader, _ int64, _ map[string]string) (remoteVersionID, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return "", err
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.objects[object] = data
+	return "", nil
+}
+
+func (b *tierExpiryTestBackend) Get(_ context.Context, object string, _ remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	data, ok := b.objects[object]
+	if !ok {
+		return nil, ObjectNotFound{Bucket: "tier", Object: object}
+	}
+	if opts.length > 0 {
+		data = data[opts.startOffset : opts.startOffset+opts.length]
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b *tierExpiryTestBackend) Remove(_ context.Context, object string, _ remoteVersionID) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	delete(b.objects, object)
+	b.removed = append(b.removed, object)
+	return nil
+}
+
+func (b *tierExpiryTestBackend) InUse(context.Context) (bool, error) { return false, nil }
+
+// The expiry of a transitioned latest version that only puts a delete marker
+// on top of it must keep its remote object, so the version stays readable as
+// a noncurrent version. The remote object goes when the version is deleted.
+func TestExpireTransitionedObjectKeepsNoncurrentData(t *testing.T) {
+	z, _ := migrationTestPools(t)
+	ctx := t.Context()
+	tier := &tierExpiryTestBackend{objects: map[string][]byte{}}
+	prevTier := globalTierConfigMgr
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.drivercache["WARM"] = tier
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+
+	const expire = `<LifecycleConfiguration><Rule><ID>exp</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
+		`<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration></Rule></LifecycleConfiguration>`
+
+	testCases := []struct {
+		name       string
+		versioning string   // "", Enabled or Suspended
+		nullLatest bool     // the latest version is the null version
+		keep       bool     // the latest version survives as noncurrent
+		want       []string // versions after the expiry, newest first
+	}{
+		{name: "versioned", versioning: "Enabled", keep: true, want: []string{"DM", "v2", "v1"}},
+		{name: "suspended", versioning: "Suspended", keep: true, want: []string{"DM", "v2", "v1"}},
+		{name: "suspended-null", versioning: "Suspended", nullLatest: true, want: []string{"DM", "v1"}},
+		{name: "unversioned", nullLatest: true, want: nil},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			bucket := "tier-expiry-" + tc.name
+			if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
+				t.Fatal(err)
+			}
+			meta, err := globalBucketMetadataSys.Get(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tc.versioning == "Suspended" {
+				meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+			}
+			meta.LifecycleConfigXML = []byte(expire)
+			if err := meta.Save(ctx, z); err != nil {
+				t.Fatal(err)
+			}
+			globalBucketMetadataSys.Set(bucket, meta)
+
+			// tiered writes a version whose data is in the remote tier.
+			labels := map[string]string{}
+			tiered := func(label string, opts ObjectOptions) ObjectInfo {
+				t.Helper()
+				oi := putConsistencyObject(t, z, bucket, "obj", 0, "obj "+label, opts)
+				vid := migrationTestVersion(oi)
+				infos, errs := readAllFileInfo(ctx, z.serverPools[0].getHashedSet("obj").getDisks(), "", bucket, "obj", vid, false, false)
+				if errs[0] != nil {
+					t.Fatal(errs[0])
+				}
+				fi := infos[0]
+				fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", bucket+"/"+label
+				if _, err := tier.Put(ctx, fi.TransitionedObjName, strings.NewReader("obj "+label), -1); err != nil {
+					t.Fatal(err)
+				}
+				if err := z.serverPools[0].DecomTieredObject(ctx, bucket, "obj", fi, ObjectOptions{}); err != nil {
+					t.Fatal(err)
+				}
+				labels[vid] = label
+				return oi
+			}
+			var latest ObjectInfo
+			if tc.versioning != "" {
+				tiered("v1", ObjectOptions{Versioned: true})
+			}
+			if tc.nullLatest {
+				latest = tiered("v2", ObjectOptions{})
+			} else {
+				latest = tiered("v2", ObjectOptions{Versioned: true})
+			}
+
+			// Expire the latest version as the expiry worker does for the scanner.
+			getOpts := ObjectOptions{Versioned: tc.versioning == "Enabled", VersionSuspended: tc.versioning == "Suspended"}
+			oi, err := z.GetObjectInfo(ctx, bucket, "obj", getOpts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			lc, err := globalLifecycleSys.Get(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			evt := evalActionFromLifecycle(ctx, *lc, objectlock.Retention{}, nil, oi)
+			if evt.Action != lifecycle.DeleteAction || oi.TransitionedObject.Status != lifecycle.TransitionComplete {
+				t.Fatalf("fixture: action %v, transition status %q", evt.Action, oi.TransitionedObject.Status)
+			}
+			if err := expireTransitionedObject(ctx, z, &oi, evt, lcEventSrc_Scanner); err != nil {
+				t.Errorf("expiry: %v", err)
+			}
+
+			var got []string
+			for _, set := range z.serverPools[0].sets {
+				err := set.listObjectsToDecommission(ctx, decomBucketInfo{Name: bucket}, func(entry metaCacheEntry) {
+					fivs, err := entry.fileInfoVersions(bucket)
+					if err != nil {
+						t.Error(err)
+						return
+					}
+					for _, fi := range fivs.Versions {
+						if fi.Deleted {
+							got = append(got, "DM")
+						} else {
+							got = append(got, labels[migrationTestVersion(fi.ToObjectInfo(bucket, "obj", true))])
+						}
+					}
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			if !slices.Equal(got, tc.want) {
+				t.Errorf("versions after expiry: got %v, want %v", got, tc.want)
+			}
+			remote := bucket + "/v2"
+			if removed := slices.Contains(tier.removed, remote); removed == tc.keep {
+				t.Errorf("remote object of the expired version removed: %v, want %v", removed, !tc.keep)
+			}
+			if tc.keep {
+				body := migrationTestBody(t, z.GetObjectNInfo, bucket, "obj", migrationTestVersion(latest))
+				if body != "obj v2" {
+					t.Errorf("v2 reads %q after the expiry, want %q", body, "obj v2")
+				}
+			}
+		})
+	}
+}

cmd/bucket-object-lock.go

@@ -291,6 +291,53 @@ func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, g
 	return nil
 }
 
+// checkVersionLockedForOverwrite re-checks under the object write lock
+// whether a write would replace the stored version versionID (the null
+// version if empty) while Object Lock protects it. It reads the version's own
+// metadata, whatever the bucket configuration, and a failed read is an error.
+func checkVersionLockedForOverwrite(ctx context.Context, bucket, object, versionID string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error {
+	if isMinioMetaBucketName(bucket) {
+		return nil
+	}
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
+		VersionID:  versionID,
+		Transition: transition,
+		NoLock:     true,
+		NoAuditLog: true,
+	})
+	if err != nil {
+		// A delete marker is not protected.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
+			return nil
+		}
+		return err
+	}
+	if enforceRetentionForDeletion(ctx, objInfo) {
+		return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+	}
+	return nil
+}
+
+// refuseLockedMarkerOverwrite refuses a delete marker write that would replace
+// the stored version it names while Object Lock protects it. goi and gerr are
+// that version as read under the write lock. A new marker, or one replacing a
+// marker, is allowed.
+func refuseLockedMarkerOverwrite(ctx context.Context, bucket, object, versionID string, goi ObjectInfo, gerr error) error {
+	if gerr != nil {
+		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) || errors.As(gerr, &MethodNotAllowed{}) {
+			return nil
+		}
+		return gerr
+	}
+	if enforceRetentionForDeletion(ctx, goi) {
+		return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+	}
+	return nil
+}
+
 // checkPutObjectLockAllowed enforces object retention policy and legal hold policy
 // for requests with WORM headers
 // See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec.

cmd/bucket-replication-checksum_test.go

@@ -0,0 +1,257 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"maps"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// checksumTestObject builds the xl.meta checksum of an object the way
+// PutObject and CompleteMultipartUpload store it.
+func checksumTestObject(t *testing.T, alg hash.ChecksumType, full bool, parts ...[]byte) FileInfo {
+	t.Helper()
+	fi := FileInfo{Volume: "bucket", Name: "object"}
+	if len(parts) == 1 {
+		fi.Size = int64(len(parts[0]))
+		fi.Checksum = hash.NewChecksumFromData(alg, parts[0]).AppendTo(nil, nil)
+		return fi
+	}
+	var combined, all []byte
+	for i, p := range parts {
+		combined = append(combined, hash.NewChecksumFromData(alg, p).Raw...)
+		all = append(all, p...)
+		fi.Parts = append(fi.Parts, ObjectPartInfo{Number: i + 1, Size: int64(len(p)), ActualSize: int64(len(p))})
+		fi.Size += int64(len(p))
+	}
+	typ := alg | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
+	if full {
+		typ |= hash.ChecksumFullObject
+	}
+	cs := hash.NewChecksumFromData(typ, combined)
+	if typ.FullObjectRequested() {
+		// Full object CRC, as merged from the parts on completion.
+		whole := hash.NewChecksumFromData(alg, all)
+		cs = &hash.Checksum{Type: typ, Raw: whole.Raw, Encoded: whole.Encoded}
+	}
+	fi.Checksum = cs.AppendTo(nil, combined)
+	return fi
+}
+
+// TestGetCRCMetaChecksumType checks the checksum headers that replication
+// sends for an object: a FULL_OBJECT checksum type only for multipart full
+// object CRCs, where the target needs it to create a matching upload.
+func TestGetCRCMetaChecksumType(t *testing.T) {
+	p1, p2 := []byte("first part"), []byte("second part")
+	enc := func(alg hash.ChecksumType, b []byte) string { return hash.NewChecksumFromData(alg, b).Encoded }
+	whole := append(append([]byte{}, p1...), p2...)
+	composite := func(alg hash.ChecksumType) string {
+		var combined []byte
+		combined = append(combined, hash.NewChecksumFromData(alg, p1).Raw...)
+		combined = append(combined, hash.NewChecksumFromData(alg, p2).Raw...)
+		return enc(alg, combined) + "-2"
+	}
+	full := map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject}
+	tests := []struct {
+		name  string
+		fi    FileInfo
+		alg   hash.ChecksumType
+		isMP  bool
+		value string
+		extra map[string]string
+	}{
+		{"single-part SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1), hash.ChecksumSHA256, false, enc(hash.ChecksumSHA256, p1), nil},
+		{"single-part CRC32", checksumTestObject(t, hash.ChecksumCRC32, false, p1), hash.ChecksumCRC32, false, enc(hash.ChecksumCRC32, p1), nil},
+		{"single-part CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1), hash.ChecksumCRC64NVME, false, enc(hash.ChecksumCRC64NVME, p1), nil},
+		{"multipart COMPOSITE CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, false, p1, p2), hash.ChecksumCRC32C, true, composite(hash.ChecksumCRC32C), nil},
+		{"multipart COMPOSITE SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1, p2), hash.ChecksumSHA256, true, composite(hash.ChecksumSHA256), nil},
+		{"multipart FULL_OBJECT CRC32", checksumTestObject(t, hash.ChecksumCRC32, true, p1, p2), hash.ChecksumCRC32, true, enc(hash.ChecksumCRC32, whole), full},
+		{"multipart FULL_OBJECT CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, true, p1, p2), hash.ChecksumCRC32C, true, enc(hash.ChecksumCRC32C, whole), full},
+		{"multipart CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1, p2), hash.ChecksumCRC64NVME, true, enc(hash.ChecksumCRC64NVME, whole), full},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			oi := tt.fi.ToObjectInfo("bucket", "object", true)
+			want := map[string]string{tt.alg.Key(): tt.value, xhttp.AmzChecksumAlgo: tt.alg.String()}
+			maps.Copy(want, tt.extra)
+			got, isMP := getCRCMeta(oi, 0, nil)
+			if isMP != tt.isMP {
+				t.Errorf("isMP: got %t, want %t", isMP, tt.isMP)
+			}
+			if !maps.Equal(got, want) {
+				t.Errorf("object headers: got %v, want %v", got, want)
+			}
+			if !tt.isMP {
+				return
+			}
+			// Part headers carry the part checksum and never a type.
+			want = map[string]string{tt.alg.Key(): enc(tt.alg, p2), xhttp.AmzChecksumAlgo: tt.alg.String()}
+			if got, _ := getCRCMeta(oi, 2, nil); !maps.Equal(got, want) {
+				t.Errorf("part 2 headers: got %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+// TestReplicateObjectChecksumTypes replicates objects with checksums through
+// the S3 API of a test server, and checks that the replica gets the same
+// checksum as the source.
+func TestReplicateObjectChecksumTypes(t *testing.T) {
+	ctx := context.Background()
+	// Leave no object layer behind: unit tests that run later in the package
+	// rely on newObjectLayerFn() returning nil.
+	defer resetTestGlobals()
+	ts := StartTestServer(t, ErasureSDStr)
+	defer ts.Stop()
+
+	clnt, err := minio.New(strings.TrimPrefix(ts.Server.URL, "http://"), &minio.Options{
+		Creds: credentials.NewStaticV4(ts.AccessKey, ts.SecretKey, ""),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	core := &minio.Core{Client: clnt}
+	for _, bucket := range []string{"src", "dst"} {
+		if err := clnt.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
+			t.Fatal(err)
+		}
+		if err := clnt.EnableVersioning(ctx, bucket); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	small := []byte("single part object")
+	p1 := bytes.Repeat([]byte("a"), 5<<20)
+	p2 := []byte("last part")
+	tests := []struct {
+		name      string
+		alg       hash.ChecksumType
+		multipart bool
+		full      bool
+		copyOf    string // server-side copy of an object created above
+	}{
+		{"put-SHA256", hash.ChecksumSHA256, false, false, ""},
+		{"put-CRC32", hash.ChecksumCRC32, false, false, ""},
+		{"mp-COMPOSITE-CRC32C", hash.ChecksumCRC32C, true, false, ""},
+		{"mp-COMPOSITE-SHA256", hash.ChecksumSHA256, true, false, ""},
+		{"mp-FULL_OBJECT-CRC32", hash.ChecksumCRC32, true, true, ""},
+		{"mp-FULL_OBJECT-CRC32C", hash.ChecksumCRC32C, true, true, ""},
+		{"mp-CRC64NVME", hash.ChecksumCRC64NVME, true, false, ""},
+		// The copy keeps the multipart FULL_OBJECT checksum of its source
+		// but is not multipart, so it replicates with a single PUT.
+		{"copy-FULL_OBJECT-CRC32", hash.ChecksumCRC32, false, true, "mp-FULL_OBJECT-CRC32"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			object := tt.name
+			// Upload the source object like an S3 client does.
+			switch {
+			case tt.copyOf != "":
+				_, err := core.CopyObject(ctx, "src", tt.copyOf, "src", object, nil, minio.CopySrcOptions{}, minio.PutObjectOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+			case !tt.multipart:
+				_, err := core.PutObject(ctx, "src", object, bytes.NewReader(small), int64(len(small)), "", "", minio.PutObjectOptions{
+					UserMetadata: map[string]string{tt.alg.Key(): hash.NewChecksumFromData(tt.alg, small).Encoded},
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+			default:
+				meta := map[string]string{xhttp.AmzChecksumAlgo: tt.alg.String()}
+				if tt.full {
+					meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
+				}
+				uploadID, err := core.NewMultipartUpload(ctx, "src", object, minio.PutObjectOptions{UserMetadata: meta})
+				if err != nil {
+					t.Fatal(err)
+				}
+				var parts []minio.CompletePart
+				for i, p := range [][]byte{p1, p2} {
+					h := http.Header{}
+					h.Set(tt.alg.Key(), hash.NewChecksumFromData(tt.alg, p).Encoded)
+					pi, err := core.PutObjectPart(ctx, "src", object, uploadID, i+1, bytes.NewReader(p), int64(len(p)), minio.PutObjectPartOptions{CustomHeader: h})
+					if err != nil {
+						t.Fatal(err)
+					}
+					parts = append(parts, minio.CompletePart{
+						PartNumber: pi.PartNumber, ETag: pi.ETag,
+						ChecksumCRC32: pi.ChecksumCRC32, ChecksumCRC32C: pi.ChecksumCRC32C,
+						ChecksumSHA1: pi.ChecksumSHA1, ChecksumSHA256: pi.ChecksumSHA256,
+						ChecksumCRC64NVME: pi.ChecksumCRC64NVME,
+					})
+				}
+				if _, err := core.CompleteMultipartUpload(ctx, "src", object, uploadID, parts, minio.PutObjectOptions{}); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			// Replicate it the way replicateObject does.
+			gr, err := ts.Obj.GetObjectNInfo(ctx, "src", object, nil, http.Header{}, ObjectOptions{Versioned: true, ReplicationRequest: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer gr.Close()
+			objInfo := gr.ObjInfo
+			putOpts, isMP, err := putReplicationOpts(ctx, "", objInfo)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if isMP != tt.multipart {
+				t.Fatalf("multipart replication: got %t, want %t", isMP, tt.multipart)
+			}
+			if isMP {
+				err = replicateObjectWithMultipart(ctx, core, "dst", object, gr, objInfo, putOpts)
+			} else {
+				size, serr := objInfo.GetActualSize()
+				if serr != nil {
+					t.Fatal(serr)
+				}
+				_, err = core.PutObject(ctx, "dst", object, gr, size, "", "", putOpts)
+			}
+			if err != nil {
+				t.Fatalf("replication failed: %v", err)
+			}
+
+			src, err := clnt.StatObject(ctx, "src", object, minio.StatObjectOptions{Checksum: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			dst, err := clnt.StatObject(ctx, "dst", object, minio.StatObjectOptions{Checksum: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			checksums := func(oi minio.ObjectInfo) string {
+				return strings.Join([]string{oi.ChecksumMode, oi.ChecksumCRC32, oi.ChecksumCRC32C, oi.ChecksumSHA1, oi.ChecksumSHA256, oi.ChecksumCRC64NVME}, "|")
+			}
+			if checksums(src) == "|||||" || checksums(dst) != checksums(src) {
+				t.Errorf("replica checksum %q, source %q", checksums(dst), checksums(src))
+			}
+		})
+	}
+}

cmd/bucket-replication-metrics.go

@@ -119,7 +119,10 @@ func (a *ActiveWorkerStat) update() {
 	if a == nil {
 		return
 	}
-	a.Curr = globalReplicationPool.Get().ActiveWorkers()
+	a.Curr = 0
+	if pool := globalReplicationPool.Get(); pool != nil {
+		a.Curr = pool.ActiveWorkers()
+	}
 	a.hist.Update(int64(a.Curr))
 	a.Avg = float32(a.hist.Mean())
 	a.Max = int(a.hist.Max())

cmd/bucket-replication-metrics_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package cmd
+
+import (
+	"testing"
+
+	"github.com/minio/minio/internal/once"
+	"github.com/rcrowley/go-metrics"
+)
+
+func TestReplicationWorkerMetricsWithoutPool(t *testing.T) {
+	previous := globalReplicationPool
+	t.Cleanup(func() { globalReplicationPool = previous })
+
+	globalReplicationPool = once.NewSingleton[ReplicationPool]()
+	globalReplicationPool.Set(&ReplicationPool{activeWorkers: 8})
+	stat := newActiveWorkerStat(metrics.NewRegistry())
+	stat.update()
+	if got := stat.get(); got.Curr != 8 || got.Avg != 8 || got.Max != 8 {
+		t.Fatalf("active pool metrics = %+v, want current/average/maximum 8", got)
+	}
+
+	// A collector may still run after replication has been disabled.
+	globalReplicationPool = once.NewSingleton[ReplicationPool]()
+	globalReplicationPool.Set(nil)
+	stat.update()
+	if got := stat.get(); got.Curr != 0 || got.Avg != 4 || got.Max != 8 {
+		t.Fatalf("disabled pool metrics = %+v, want current 0, average 4, maximum 8", got)
+	}
+}

cmd/bucket-replication.go

@@ -4250,6 +4250,11 @@ func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string
 		if cktype.IsSet() {
 			meta[cktype.Key()] = v
 			meta[xhttp.AmzChecksumAlgo] = cktype.String()
+			// Without the type the target creates a COMPOSITE multipart
+			// upload and rejects the full object CRC on completion.
+			if isMP && cktype.CanMerge() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject {
+				meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
+			}
 		}
 	}
 	return meta, isMP

cmd/bulk-delete-invalid-name_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+// A bulk delete joins each name to the request bucket on the drives, so a name
+// with a ".." segment names a path in another bucket or in the meta bucket.
+// Such a name is the error of its own entry and reaches no drive, and the
+// other entries of the request are deleted as usual. This holds for the one
+// bulk lock of a single node, for a pool that locks per set, and for more than
+// one pool.
+func TestPoolsBulkDeleteInvalidNames(t *testing.T) {
+	for _, topology := range []struct {
+		name string
+		new  func(t *testing.T) (*erasureServerPools, string)
+	}{
+		{"single node", func(t *testing.T) (*erasureServerPools, string) { return bulkDeletePools(t, 1) }},
+		{"set locks", func(t *testing.T) (*erasureServerPools, string) { return ldPool(t, ldLocal) }},
+		{"two pools", func(t *testing.T) (*erasureServerPools, string) { return bulkDeletePools(t, 2) }},
+	} {
+		t.Run(topology.name, func(t *testing.T) {
+			z, bucket := topology.new(t)
+			ctx := t.Context()
+			other := "bulk-delete-other"
+			if err := z.MakeBucket(ctx, other, MakeBucketOptions{VersioningEnabled: true}); err != nil {
+				t.Fatal(err)
+			}
+			old := batchPutVersion(t, z, other, "kept", nil)
+			latest := batchPutVersion(t, z, other, "kept", nil)
+			batchPutVersion(t, z, other, "visible", nil)
+			metaKey := pathJoin(bucketMetaPrefix, other, bucketMetadataFile)
+			if !batchVersionExists(t, z, minioMetaBucket, metaKey, "") {
+				t.Fatal("the bucket metadata is not an object of the meta bucket")
+			}
+			for _, key := range []string{"own", "dir/"} {
+				if _, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{}); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			names := []ObjectToDelete{
+				{ObjectV: ObjectV{ObjectName: "../" + other + "/kept", VersionID: old.VersionID}}, // purge in another bucket
+				{ObjectV: ObjectV{ObjectName: "own"}},
+				{ObjectV: ObjectV{ObjectName: "a/../../" + other + "/visible"}},         // marker in another bucket
+				{ObjectV: ObjectV{ObjectName: "../" + minioMetaBucket + "/" + metaKey}}, // marker on bucket metadata
+				{ObjectV: ObjectV{ObjectName: "dir/"}},
+				{ObjectV: ObjectV{ObjectName: ""}},
+			}
+			invalid := map[int]bool{0: true, 2: true, 3: true, 5: true}
+			// Without a version ID these options ask for a delete marker, the
+			// one write a bulk delete makes.
+			deleted, errs := z.DeleteObjects(ctx, bucket, slices.Clone(names), ObjectOptions{Versioned: true})
+			for i, name := range names {
+				_, rejected := errs[i].(ObjectNameInvalid)
+				if rejected != invalid[i] || !invalid[i] && errs[i] != nil {
+					t.Errorf("%d %q: error = %v, rejected as invalid = %v, want %v", i, name.ObjectName, errs[i], rejected, invalid[i])
+				}
+				if deleted[i].ObjectName != name.ObjectName || deleted[i].found == invalid[i] {
+					t.Errorf("%d %q: result = %+v", i, name.ObjectName, deleted[i])
+				}
+			}
+
+			for _, version := range []string{old.VersionID, latest.VersionID} {
+				if !batchVersionExists(t, z, other, "kept", version) {
+					t.Errorf("version %q of the object in the other bucket was removed", version)
+				}
+			}
+			if !batchVersionExists(t, z, other, "visible", "") {
+				t.Error("the object of the other bucket is hidden by a marker")
+			}
+			if !batchVersionExists(t, z, minioMetaBucket, metaKey, "") {
+				t.Error("the metadata of the other bucket is hidden by a marker")
+			}
+			for _, key := range []string{"own", "dir/"} {
+				if batchVersionExists(t, z, bucket, key, "") {
+					t.Errorf("%q of the request bucket was not deleted", key)
+				}
+			}
+		})
+	}
+}
+
+// The names of an S3 DeleteObjects request come from its body, which the path
+// checks of the request URL do not see. A user allowed to delete in one bucket
+// reaches nothing outside it through such a name: a version under COMPLIANCE
+// retention in another bucket stays, and no delete marker lands on that object
+// or on the stored metadata of its bucket.
+func TestBulkDeleteObjectLockNameOutsideBucket(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"DeleteMultipleObjects"},
+		makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
+			ctx := t.Context()
+			user := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:DeleteObject","s3:DeleteObjectVersion"`)
+			locked := getRandomBucketName()
+			if err := obj.MakeBucket(ctx, locked, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true}); err != nil {
+				t.Fatal(err)
+			}
+			enableBucketObjectLock(t, locked)
+			worm := batchPutVersion(t, obj, locked, "secret", batchLockMeta("COMPLIANCE"))
+			batchPutVersion(t, obj, bucket, "own", nil)
+			metaKey := pathJoin(bucketMetaPrefix, locked, bucketMetadataFile)
+
+			outside := []ObjectToDelete{
+				{ObjectV: ObjectV{ObjectName: "../" + locked + "/secret", VersionID: worm.VersionID}},
+				{ObjectV: ObjectV{ObjectName: "../" + locked + "/secret"}},
+				{ObjectV: ObjectV{ObjectName: "../" + minioMetaBucket + "/" + metaKey}},
+			}
+			body := encodeResponse(DeleteObjectsRequest{Objects: append(slices.Clone(outside), ObjectToDelete{ObjectV: ObjectV{ObjectName: "own"}})})
+			req, err := newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucket), int64(len(body)), bytes.NewReader(body),
+				user.AccessKey, user.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := httptest.NewRecorder()
+			router.ServeHTTP(rec, req)
+			var resp DeleteObjectsResponse
+			if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &resp) != nil {
+				t.Fatalf("%s: status %d: %s", instance, rec.Code, rec.Body.String())
+			}
+
+			if len(resp.DeletedObjects) != 1 || resp.DeletedObjects[0].ObjectName != "own" || len(resp.Errors) != len(outside) {
+				t.Errorf("%s: want the three names outside the bucket refused and \"own\" deleted: %s", instance, rec.Body.String())
+			}
+			for _, e := range resp.Errors {
+				if e.Code != errorCodes[ErrInvalidObjectName].Code {
+					t.Errorf("%s: %q: code %s, want %s", instance, e.Key, e.Code, errorCodes[ErrInvalidObjectName].Code)
+				}
+			}
+			if !batchVersionExists(t, obj, locked, "secret", worm.VersionID) {
+				t.Errorf("%s: the version under COMPLIANCE retention in the other bucket was removed", instance)
+			}
+			if !batchVersionExists(t, obj, locked, "secret", "") {
+				t.Errorf("%s: a delete marker hides the object of the other bucket", instance)
+			}
+			if !batchVersionExists(t, obj, minioMetaBucket, metaKey, "") {
+				t.Errorf("%s: a delete marker hides the stored metadata of the other bucket", instance)
+			}
+			if batchVersionExists(t, obj, bucket, "own", "") {
+				t.Errorf("%s: the object of the request bucket was not deleted", instance)
+			}
+		},
+	})
+}

cmd/bulk-delete-object-lock-race_test.go

@@ -0,0 +1,346 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/mux"
+)
+
+// Schedule a protection update immediately before the storage mutation. Both
+// public delete entry points are gated so the test is independent of whether
+// the handler batches a version or uses the single-object deletion path.
+type bulkDeleteLockBarrier struct {
+	ObjectLayer
+	entered chan struct{}
+	release chan struct{}
+	once    sync.Once
+}
+
+func (g *bulkDeleteLockBarrier) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	g.wait()
+	return g.ObjectLayer.DeleteObjects(ctx, bucket, objects, opts)
+}
+
+func (g *bulkDeleteLockBarrier) wait() {
+	g.once.Do(func() { close(g.entered); <-g.release })
+}
+
+func (g *bulkDeleteLockBarrier) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	g.wait()
+	return g.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+}
+
+func bulkDeleteLockRequest(t *testing.T, bucket, key, version string) *http.Request {
+	t.Helper()
+	body := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: version}}}})
+	req, err := newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucket), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return mux.SetURLVars(req, map[string]string{"bucket": bucket})
+}
+
+func TestBulkDeleteObjectLockInterleaving(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, directory := range []bool{false, true} {
+			for _, kind := range []string{"legal-hold", "retention"} {
+				t.Run(fmt.Sprintf("pools=%d/directory=%v/%s", pools, directory, kind), func(t *testing.T) {
+					z, _ := bulkDeletePools(t, pools)
+					bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					key := "protection-update"
+					if directory {
+						key += "/"
+					}
+					var oi ObjectInfo
+					if directory {
+						oi, err = z.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{Versioned: true, VersionID: nullVersionID})
+						if err != nil {
+							t.Fatal(err)
+						}
+					} else {
+						oi = batchPutVersion(t, z, bucket, key, nil)
+					}
+					gate := &bulkDeleteLockBarrier{ObjectLayer: z, entered: make(chan struct{}), release: make(chan struct{})}
+					api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return gate }}
+					requestVersion := oi.VersionID
+					if directory {
+						requestVersion = ""
+					}
+					req := bulkDeleteLockRequest(t, bucket, key, requestVersion)
+					rec := httptest.NewRecorder()
+					done := make(chan struct{})
+					go func() { defer close(done); api.DeleteMultipleObjectsHandler(rec, req) }()
+					var release sync.Once
+					unblock := func() { release.Do(func() { close(gate.release) }) }
+					defer func() {
+						unblock()
+						select {
+						case <-done:
+						case <-time.After(10 * time.Second):
+							t.Error("delete did not finish")
+						}
+					}()
+					select {
+					case <-gate.entered:
+					case <-time.After(10 * time.Second):
+						t.Fatal("bulk handler did not reach deletion after validation")
+					}
+					body := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>ON</Status></LegalHold>`)
+					if kind == "retention" {
+						body = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+					}
+					holdReq, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/"+key+"?"+kind+"&versionId="+url.QueryEscape(oi.VersionID), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					holdRec := httptest.NewRecorder()
+					router.ServeHTTP(holdRec, holdReq)
+					protected, getErr := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: oi.VersionID})
+					unblock()
+					select {
+					case <-done:
+					case <-time.After(10 * time.Second):
+						t.Fatal("delete did not finish")
+					}
+					if holdRec.Code != http.StatusOK || getErr != nil {
+						t.Fatalf("protection setup failed: status=%d body=%s err=%v", holdRec.Code, holdRec.Body.String(), getErr)
+					}
+					if kind == "legal-hold" && protected.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] != "ON" {
+						t.Fatal("legal hold was not stored")
+					}
+					if kind == "retention" && protected.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "COMPLIANCE" {
+						t.Fatal("retention was not stored")
+					}
+					var response DeleteObjectsResponse
+					if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+						t.Fatal(err)
+					}
+					exists := batchVersionExists(t, z, bucket, key, oi.VersionID)
+					t.Logf("protection PUT=%d; delete POST=%d response=%s; protected_version_exists=%v", holdRec.Code, rec.Code, rec.Body.String(), exists)
+					if rec.Code != http.StatusOK || !exists || len(response.Errors) != 1 || len(response.DeletedObjects) != 0 {
+						t.Errorf("bulk deletion removed a version after its protection update succeeded")
+					}
+				})
+			}
+		}
+	}
+}
+
+func TestBulkDeleteObjectLockPoolCopies(t *testing.T) {
+	for _, kind := range []string{"COMPLIANCE", "LEGALHOLD"} {
+		t.Run(kind, func(t *testing.T) {
+			z, _ := bulkDeletePools(t, 2)
+			bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, bulk := range []bool{false, true} {
+				key := "single-control"
+				if bulk {
+					key = "bulk"
+				}
+				id := mustGetUUID()
+				mtime := UTCNow().Add(-time.Hour)
+				putConsistencyObject(t, z, bucket, key, 0, "same-data", ObjectOptions{Versioned: true, VersionID: id, MTime: mtime, UserDefined: nil})
+				putConsistencyObject(t, z, bucket, key, 1, "same-data", ObjectOptions{Versioned: true, VersionID: id, MTime: mtime, UserDefined: batchLockMeta(kind)})
+				selected, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id})
+				if err != nil {
+					t.Fatal(err)
+				}
+				if selected.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "" || selected.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] != "" {
+					t.Fatal("fixture did not select the unprotected pool copy")
+				}
+				req := bulkDeleteLockRequest(t, bucket, key, id)
+				if !bulk {
+					req, err = newTestSignedRequestV4(http.MethodDelete, getDeleteObjectURL("", bucket, key)+"?versionId="+id, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+				rec := httptest.NewRecorder()
+				router.ServeHTTP(rec, req)
+				if !strings.Contains(rec.Body.String(), "Object is WORM protected") {
+					t.Fatalf("expected Object Lock refusal: HTTP %d: %s", rec.Code, rec.Body.String())
+				}
+				remaining := 0
+				for _, pool := range z.serverPools {
+					_, err := pool.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id})
+					if err == nil {
+						remaining++
+					} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+						t.Fatal(err)
+					}
+				}
+				t.Logf("bulk=%v HTTP=%d response=%s remaining_pool_copies=%d", bulk, rec.Code, rec.Body.String(), remaining)
+				if remaining != 2 {
+					t.Errorf("protected version copies removed by bulk=%v", bulk)
+				}
+			}
+		})
+	}
+}
+
+// One response must retain the identity and outcome of every requested item,
+// even when physical version removals and marker creation take different paths.
+func TestBulkDeleteObjectLockMixedResponse(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, quiet := range []bool{false, true} {
+			t.Run(fmt.Sprintf("pools=%d/quiet=%v", pools, quiet), func(t *testing.T) {
+				z, _ := bulkDeletePools(t, pools)
+				bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
+				if err != nil {
+					t.Fatal(err)
+				}
+				locked := batchPutVersion(t, z, bucket, "locked", batchLockMeta("COMPLIANCE"))
+				free := batchPutVersion(t, z, bucket, "free", nil)
+				marker := batchPutVersion(t, z, bucket, "marker", batchLockMeta("LEGALHOLD"))
+				if _, err := z.PutObject(t.Context(), bucket, "directory/", mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{Versioned: true, VersionID: nullVersionID}); err != nil {
+					t.Fatal(err)
+				}
+				missing := mustGetUUID()
+				objects := []ObjectToDelete{
+					{ObjectV: ObjectV{ObjectName: "locked", VersionID: locked.VersionID}},
+					{ObjectV: ObjectV{ObjectName: "free", VersionID: free.VersionID}},
+					{ObjectV: ObjectV{ObjectName: "missing", VersionID: missing}},
+					{ObjectV: ObjectV{ObjectName: "marker"}},
+					{ObjectV: ObjectV{ObjectName: "directory/"}},
+					{ObjectV: ObjectV{ObjectName: "directory/", VersionID: nullVersionID}},
+					{ObjectV: ObjectV{ObjectName: "invalid", VersionID: "invalid"}},
+				}
+				objects = append(objects, objects[0], objects[1])
+				body := encodeResponse(DeleteObjectsRequest{Quiet: quiet, Objects: objects})
+				rec := r5Request(t, router, globalActiveCred, http.MethodPost, getDeleteMultipleObjectsURL("", bucket), string(body), nil)
+				var response DeleteObjectsResponse
+				if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &response) != nil {
+					t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
+				}
+				wantErrors := []ObjectV{objects[0].ObjectV, objects[6].ObjectV, objects[0].ObjectV}
+				if len(response.Errors) != len(wantErrors) {
+					t.Fatalf("errors: %+v", response.Errors)
+				}
+				for i, e := range response.Errors {
+					if e.Key != wantErrors[i].ObjectName || e.VersionID != wantErrors[i].VersionID {
+						t.Errorf("error identity %d: %+v", i, e)
+					}
+				}
+				if quiet {
+					if len(response.DeletedObjects) != 0 {
+						t.Errorf("quiet successes: %+v", response.DeletedObjects)
+					}
+				} else {
+					want := []ObjectV{objects[1].ObjectV, objects[2].ObjectV, objects[3].ObjectV, objects[4].ObjectV, objects[5].ObjectV, objects[1].ObjectV}
+					if len(response.DeletedObjects) != len(want) {
+						t.Fatalf("successes: %+v", response.DeletedObjects)
+					}
+					for i, d := range response.DeletedObjects {
+						if d.ObjectName != want[i].ObjectName || d.VersionID != want[i].VersionID {
+							t.Errorf("success identity %d: %+v", i, d)
+						}
+						if i == 2 && (!d.DeleteMarker || d.DeleteMarkerVersionID == "") {
+							t.Errorf("missing delete marker: %+v", d)
+						}
+					}
+				}
+				if !batchVersionExists(t, z, bucket, "locked", locked.VersionID) || !batchVersionExists(t, z, bucket, "marker", marker.VersionID) || batchVersionExists(t, z, bucket, "free", free.VersionID) || batchVersionExists(t, z, bucket, "directory/", nullVersionID) {
+					t.Fatal("unexpected stored versions after mixed deletion")
+				}
+			})
+		}
+	}
+}
+
+func TestBulkDeleteObjectLockGovernance(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			z, _ := bulkDeletePools(t, pools)
+			bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			deleteOnly := newObjectAttributesAuthzUser(t, "bulk-governance", bucket, `"s3:DeleteObjectVersion"`)
+			for _, allowed := range []bool{false, true} {
+				for _, bypass := range []bool{false, true} {
+					key := fmt.Sprintf("allowed-%v-bypass-%v", allowed, bypass)
+					oi := batchPutVersion(t, z, bucket, key, batchLockMeta("GOVERNANCE"))
+					cred := deleteOnly
+					if allowed {
+						cred = globalActiveCred
+					}
+					body := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: oi.VersionID}}}})
+					header := map[string]string{}
+					if bypass {
+						header[xhttp.AmzObjectLockBypassGovernance] = "true"
+					}
+					rec := r5Request(t, router, cred, http.MethodPost, getDeleteMultipleObjectsURL("", bucket), string(body), header)
+					var response DeleteObjectsResponse
+					if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &response) != nil {
+						t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
+					}
+					deleted := allowed && bypass
+					if batchVersionExists(t, z, bucket, key, oi.VersionID) == deleted || (len(response.Errors) == 0) != deleted {
+						t.Fatalf("%s: %s", key, rec.Body.String())
+					}
+				}
+			}
+		})
+	}
+}
+
+// The locked path computes replication decisions while holding the mutation
+// lock. Keep pending purges and their queue entries, including duplicate input.
+func TestBulkDeleteObjectLockReplication(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			z, _ := bulkDeletePools(t, pools)
+			bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			client, err := minio.New("localhost:1", &minio.Options{Region: "us-east-1", MaxRetries: 1})
+			if err != nil {
+				t.Fatal(err)
+			}
+			worker, cleanup := r5ReplicationFixture(t, z, bucket, client)
+			defer cleanup()
+			free := batchPutVersion(t, z, bucket, "free", nil)
+			locked := batchPutVersion(t, z, bucket, "locked", batchLockMeta("LEGALHOLD"))
+			objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: "free", VersionID: free.VersionID}}, {ObjectV: ObjectV{ObjectName: "locked", VersionID: locked.VersionID}}}
+			objects = append(objects, objects[0])
+			body := encodeResponse(DeleteObjectsRequest{Objects: objects})
+			rec := r5Request(t, router, globalActiveCred, http.MethodPost, getDeleteMultipleObjectsURL("", bucket), string(body), nil)
+			var response DeleteObjectsResponse
+			if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &response) != nil || len(response.Errors) != 1 || len(response.DeletedObjects) != 2 {
+				t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
+			}
+			if len(worker) != 1 {
+				t.Fatalf("want one purge queued, got %d", len(worker))
+			}
+			op, ok := (<-worker).(DeletedObjectReplicationInfo)
+			if !ok || op.ObjectName != "free" || op.VersionID != free.VersionID || op.VersionPurgeStatus() != replication.VersionPurgePending || op.ReplicationState.ReplicateDecisionStr == "" {
+				t.Fatalf("lost purge identity/state: %+v", op)
+			}
+			if !batchVersionExists(t, z, bucket, "locked", locked.VersionID) {
+				t.Fatal("protected version was deleted")
+			}
+		})
+	}
+}

cmd/bulk-delete-object-lock_test.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+// A directory object always has the null version, and a bulk delete without a
+// version ID removes it. The retention check must read that version's lock.
+func TestBulkDeleteDirectoryObjectLock(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+		t: t, endpoints: []string{"DeleteMultipleObjects"},
+		objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+			enableBucketObjectLock(t, bucket)
+			for _, kind := range append([]string{"unlocked"}, batchLockKinds...) {
+				key := "dir-" + kind + "/"
+				if _, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""),
+					ObjectOptions{Versioned: true, VersionID: nullVersionID, UserDefined: batchLockMeta(kind)}); err != nil {
+					t.Fatal(err)
+				}
+				body := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key}}}})
+				req, err := newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucket), int64(len(body)), bytes.NewReader(body),
+					cred.AccessKey, cred.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				rec := httptest.NewRecorder()
+				router.ServeHTTP(rec, req)
+				var resp DeleteObjectsResponse
+				if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &resp) != nil {
+					t.Fatalf("%s: %s: status %d: %s", instance, key, rec.Code, rec.Body.String())
+				}
+				exists := batchVersionExists(t, obj, bucket, key, nullVersionID)
+				if kind == "unlocked" {
+					if exists || len(resp.DeletedObjects) != 1 {
+						t.Errorf("%s: %s: unlocked directory object was not deleted: %s", instance, key, rec.Body.String())
+					}
+					continue
+				}
+				if !exists || len(resp.Errors) != 1 || len(resp.DeletedObjects) != 0 ||
+					resp.Errors[0].Key != key || !strings.Contains(resp.Errors[0].Message, "WORM") {
+					t.Errorf("%s: %s: want the WORM error and the object kept (exists=%v): %s", instance, key, exists, rec.Body.String())
+				}
+			}
+		},
+	})
+}

cmd/common-main.go

@@ -883,7 +883,7 @@ func serverHandleEnvVars() {
 	}
 	globalInplaceUpdateDisabled = true
 	if env.IsSet(api.EnvAPIMultipartListing) {
-		logger.Warning("%s is ignored: multipart listing always uses persistent metadata; uploads without identity fields are omitted", api.EnvAPIMultipartListing)
+		logger.Warning("%s is ignored: multipart listing uses the legacy cache/exact-key behavior; persistent listing is deferred", api.EnvAPIMultipartListing)
 	}
 
 	// Check if the supported credential env vars,

cmd/config-current.go

@@ -609,7 +609,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 			loggerCfg.HTTP[n] = l
 		}
-		if errs := logger.UpdateHTTPWebhooks(ctx, loggerCfg.HTTP); len(errs) > 0 {
+		// Webhook workers outlive the bounded configuration reload.
+		if errs := logger.UpdateHTTPWebhooks(GlobalContext, loggerCfg.HTTP); len(errs) > 0 {
 			configLogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
@@ -627,7 +628,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			loggerCfg.AuditWebhook[n] = l
 		}
 
-		if errs := logger.UpdateAuditWebhooks(ctx, loggerCfg.AuditWebhook); len(errs) > 0 {
+		// Both the queue worker and disk replay follow the server lifetime.
+		if errs := logger.UpdateAuditWebhooks(GlobalContext, loggerCfg.AuditWebhook); len(errs) > 0 {
 			configLogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
@@ -644,7 +646,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 {
+		// The Kafka disk replay follows the server lifetime as well.
+		if errs := logger.UpdateAuditKafkaTargets(GlobalContext, loggerCfg); len(errs) > 0 {
 			configLogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:

cmd/config-current_test.go

@@ -19,12 +19,246 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
+	"github.com/IBM/sarama"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/logger"
+	httpTarget "github.com/minio/minio/internal/logger/target/http"
+	types "github.com/minio/minio/internal/logger/target/loggertypes"
 )
 
+type webhookConfigObjectLayer struct {
+	ObjectLayer
+}
+
+func (webhookConfigObjectLayer) SetDriveCounts() []int { return []int{1} }
+
+// TestDynamicWebhookOutlivesConfigContext applies logger targets the way a
+// peer reload does: under a bounded context that ends when the handler returns.
+func TestDynamicWebhookOutlivesConfigContext(t *testing.T) {
+	for _, subSys := range []string{config.AuditWebhookSubSys, config.LoggerWebhookSubSys, config.AuditKafkaSubSys} {
+		for _, queue := range []struct {
+			name     string
+			disk     bool
+			failInit bool
+		}{
+			{name: "disk=false"},
+			{name: "disk=true", disk: true},
+			{name: "disk=unavailable", disk: true, failInit: true},
+		} {
+			if subSys == config.AuditKafkaSubSys && queue.failInit {
+				// An unopenable Kafka queue starts no goroutine.
+				continue
+			}
+			t.Run(subSys+"/"+queue.name, func(t *testing.T) {
+				// Isolate global target registries and wait for all workers to exit
+				// before the parent removes the disk queue.
+				if os.Getenv("_SILO_WEBHOOK_CONTEXT_TEST") != t.Name() {
+					executable, err := os.Executable()
+					if err != nil {
+						t.Fatal(err)
+					}
+					queueDir := ""
+					if queue.disk {
+						queueDir = t.TempDir()
+					}
+					ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+					defer cancel()
+					child := exec.CommandContext(ctx, executable,
+						"-test.run=^"+strings.ReplaceAll(t.Name(), "/", "$/^")+"$",
+						"-test.v", "-test.timeout=25s")
+					// Logger targets from the environment would join the registries.
+					for _, kv := range os.Environ() {
+						if !strings.HasPrefix(kv, "MINIO_AUDIT_") && !strings.HasPrefix(kv, "MINIO_LOGGER_") {
+							child.Env = append(child.Env, kv)
+						}
+					}
+					child.Env = append(child.Env,
+						"_SILO_WEBHOOK_CONTEXT_TEST="+t.Name(),
+						"_SILO_WEBHOOK_QUEUE_DIR="+queueDir)
+					output, err := child.CombinedOutput()
+					if err != nil {
+						t.Fatalf("isolated webhook test: %v\n%s", err, output)
+					}
+					t.Logf("%s", output)
+					return
+				}
+
+				if subSys == config.AuditKafkaSubSys {
+					testDynamicKafkaOutlivesConfigContext(t, os.Getenv("_SILO_WEBHOOK_QUEUE_DIR"))
+					return
+				}
+
+				targets := logger.AuditTargets
+				if subSys == config.LoggerWebhookSubSys {
+					targets = logger.SystemTargets
+				}
+				received := make(chan string, 8)
+				receiver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					var entry struct{ Marker string }
+					if err := json.NewDecoder(r.Body).Decode(&entry); err != nil {
+						t.Errorf("decode webhook: %v", err)
+					}
+					received <- entry.Marker
+					w.WriteHeader(http.StatusOK)
+				}))
+				t.Cleanup(receiver.Close)
+
+				queueDir := os.Getenv("_SILO_WEBHOOK_QUEUE_DIR")
+				if queue.failInit {
+					queueDir = filepath.Join(queueDir, "queue")
+					if err := os.WriteFile(queueDir, []byte("not a directory"), 0o600); err != nil {
+						t.Fatal(err)
+					}
+				}
+				cfg := newServerConfig()
+				_, err := cfg.SetKVS(fmt.Sprintf("%s:context-test enable=on endpoint=%s queue_size=16 batch_size=1 queue_dir=%s http_timeout=1s retry_interval=100ms",
+					subSys, receiver.URL, queueDir), config.DefaultKVS)
+				if err != nil {
+					t.Fatal(err)
+				}
+				// A reload cancels the target it replaces; clean up only the last one.
+				var current *httpTarget.Target
+				t.Cleanup(func() {
+					if current != nil {
+						current.Cancel()
+					}
+				})
+				// reload applies the subsystem the way a peer does and returns the
+				// registered target once the bounded reload context has ended.
+				reload := func() *httpTarget.Target {
+					t.Helper()
+					ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+					defer cancel()
+					if err := applyDynamicConfigForSubSys(ctx, webhookConfigObjectLayer{}, cfg, subSys); err != nil {
+						t.Fatal(err)
+					}
+					var target *httpTarget.Target
+					for _, candidate := range targets() {
+						if candidate.Endpoint() == receiver.URL {
+							if target != nil {
+								t.Fatal("configured webhook was registered twice")
+							}
+							target = candidate.(*httpTarget.Target)
+						}
+					}
+					if target == nil {
+						t.Fatal("configured webhook was not registered")
+					}
+					current = target
+					return target
+				}
+
+				target := reload()
+				if err := target.Send(t.Context(), struct{ Marker string }{"after-reload"}); err != nil {
+					t.Fatal(err)
+				}
+				if queue.failInit {
+					// A failed initialization starts no worker: the record stays queued.
+					select {
+					case got := <-received:
+						t.Fatalf("received %q before the queue was initialized", got)
+					case <-time.After(250 * time.Millisecond):
+					}
+					if got := target.Stats().QueueLength; got != 1 {
+						t.Fatalf("failed queue consumed an event: queue length %d", got)
+					}
+					// After the operator repairs queue_dir, applying the configuration
+					// again replaces the target, which must take over the record.
+					if err := os.Remove(queueDir); err != nil {
+						t.Fatal(err)
+					}
+					target = reload()
+				}
+				select {
+				case got := <-received:
+					if got != "after-reload" {
+						t.Fatalf("received marker %q, want after-reload", got)
+					}
+				case <-time.After(10 * time.Second):
+					t.Fatalf("webhook stopped after reload returned; stats: %+v", target.Stats())
+				}
+			})
+		}
+	}
+}
+
+// testDynamicKafkaOutlivesConfigContext runs in the isolated child process.
+// With a queue_dir, Kafka records reach the broker only through the disk
+// replay, which polls the queue every few seconds.
+func testDynamicKafkaOutlivesConfigContext(t *testing.T, queueDir string) {
+	const topic = "audit-context-test"
+	broker := sarama.NewMockBroker(t, 1)
+	t.Cleanup(broker.Close)
+	broker.SetHandlerByMap(map[string]sarama.MockResponse{
+		"ApiVersionsRequest": sarama.NewMockApiVersionsResponse(t).SetApiKeys([]sarama.ApiVersionsResponseKey{
+			{ApiKey: 0, MinVersion: 0, MaxVersion: 8},  // Produce
+			{ApiKey: 3, MinVersion: 0, MaxVersion: 11}, // Metadata
+			{ApiKey: 18, MinVersion: 0, MaxVersion: 3}, // ApiVersions
+		}),
+		"MetadataRequest": sarama.NewMockMetadataResponse(t).
+			SetBroker(broker.Addr(), broker.BrokerID()).
+			SetLeader(topic, 0, broker.BrokerID()),
+		"ProduceRequest": sarama.NewMockProduceResponse(t),
+	})
+	produced := func() (n int) {
+		for _, exchange := range broker.History() {
+			if _, ok := exchange.Request.(*sarama.ProduceRequest); ok {
+				n++
+			}
+		}
+		return n
+	}
+
+	cfg := newServerConfig()
+	_, err := cfg.SetKVS(fmt.Sprintf("%s:context-test enable=on brokers=%s topic=%s queue_size=16 queue_dir=%s",
+		config.AuditKafkaSubSys, broker.Addr(), topic, queueDir), config.DefaultKVS)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	if err := applyDynamicConfigForSubSys(ctx, webhookConfigObjectLayer{}, cfg, config.AuditKafkaSubSys); err != nil {
+		t.Fatal(err)
+	}
+	var target logger.Target
+	for _, candidate := range logger.AuditTargets() {
+		if candidate.Type() == types.TargetKafka {
+			if target != nil {
+				t.Fatal("more than one Kafka target was registered")
+			}
+			target = candidate
+		}
+	}
+	if target == nil {
+		t.Fatal("configured Kafka target was not registered")
+	}
+	t.Cleanup(target.Cancel)
+	// Returning from the peer reload cancels its bounded context.
+	cancel()
+
+	before := produced()
+	if err := target.Send(t.Context(), struct{ Marker string }{"after-reload"}); err != nil {
+		t.Fatal(err)
+	}
+	for deadline := time.Now().Add(10 * time.Second); produced() == before; time.Sleep(50 * time.Millisecond) {
+		if time.Now().After(deadline) {
+			t.Fatalf("Kafka target stopped after reload returned; stats: %+v", target.Stats())
+		}
+	}
+}
+
 func TestServerConfig(t *testing.T) {
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()

cmd/config-migrate.go

@@ -91,7 +91,7 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
 			return nil, err
 		}
 
-		data, err = decryptData(data, configFile)
+		data, err = decryptData(ctx, data, configFile)
 		if err != nil {
 			return nil, err
 		}

cmd/config.go

@@ -68,7 +68,7 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 					continue
 				}
 
-				data, err = decryptData(data, obj.Name)
+				data, err = decryptData(ctx, data, obj.Name)
 				if err != nil {
 					// ignore history file that cannot be loaded.
 					continue
@@ -110,7 +110,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
 		return nil, err
 	}
 
-	return decryptData(data, historyFile)
+	return decryptData(ctx, data, historyFile)
 }
 
 func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@@ -162,7 +162,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con
 			return nil, err
 		}
 
-		data, err = decryptData(data, configFile)
+		data, err = decryptData(ctx, data, configFile)
 		if err != nil {
 			lookupConfigs(srvCfg, objAPI)
 			return nil, err

cmd/data-scanner.go

@@ -24,10 +24,12 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"maps"
 	"math"
 	"math/rand"
 	"os"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -776,7 +778,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 
 					// We got an entry which we should be able to heal.
-					fiv, err := entry.fileInfoVersions(bucket)
+					fiv, err := entry.allFileInfoVersions(bucket)
 					if err != nil {
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
@@ -1118,8 +1120,31 @@ eventLoop:
 		size := actualSz
 		switch event.Action {
 		case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
-			remainingVersions = 0
-			applyExpiryRule(event, lcEventSrc_Scanner, oi)
+			// An expiry of a hot latest version purges the key, which refuses
+			// a key with a version in a remote tier: do not queue it when this
+			// pool holds one. A tiered version only another pool holds is
+			// not known here; the purge decides on it. A transitioned latest
+			// version takes the transitioned expiry, as before.
+			refused := oi.TransitionedObject.Status == "" && slices.ContainsFunc(objInfos, ObjectInfo.deleteAllTierRefuses)
+			if refused {
+				globalILMIncomplete.add(ilmIncompleteTierSkipped)
+			} else {
+				remainingVersions = 0
+				applyExpiryRule(event, lcEventSrc_Scanner, oi)
+			}
+			// The versions hold their space until the purge removes them,
+			// which can refuse the key or stop at its budget: account each
+			// at its size, with no other action. The alerts on many or
+			// large versions count those of a key that is not queued.
+			for _, v := range objInfos[idx:] {
+				sz, _ := v.GetActualSize()
+				if fn != nil {
+					fn(v, sz, sz, sizeS)
+				}
+				if refused {
+					cumulativeSize += sz
+				}
+			}
 			break eventLoop
 
 		case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1130,9 +1155,9 @@ eventLoop:
 			applyExpiryRule(event, lcEventSrc_Scanner, oi)
 
 		case lifecycle.DeleteVersionAction: // noncurrent versions expiration
-			opts := objOpts[idx]
 			remainingVersions--
 			size = 0
+			opts := objOpts[idx]
 			toDel = append(toDel, ObjectToDelete{
 				ObjectV: ObjectV{
 					ObjectName: opts.Name,
@@ -1210,10 +1235,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) {
 	timeILM := globalScannerMetrics.timeILM(lcEvent.Action)
 	if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-			return false
-		}
-		ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err))
+		reportILMExpiryNotDone(ctx, obj, lcEvent, src, err, "transition-expiry"+obj.Name, globalLifecycleSys.trace(obj))
 		return false
 	}
 	timeILM(1)
@@ -1235,11 +1257,18 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
 	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 
+	// Purge every version of exactly this key, by version ID and under the
+	// object lock, after the pools approved it on the versions they hold;
+	// see erasureServerPools.deleteAllVersions.
+	var purge deleteAllResult
 	if lcEvent.Action.DeleteAll() {
-		opts.DeletePrefix = true
-		// use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-		opts.DeletePrefixObject = true
+		opts = ObjectOptions{
+			Expiration:        ExpirationOptions{Expire: true},
+			DeleteAllVersions: true,
+			DeleteAllResult:   &purge,
+		}
 	}
+
 	var (
 		dobj ObjectInfo
 		err  error
@@ -1254,22 +1283,21 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		if lcEvent.Action != lifecycle.NoneAction {
 			numVersions := uint64(1)
 			if lcEvent.Action.DeleteAll() {
-				numVersions = uint64(obj.NumVersions)
+				numVersions = uint64(purge.Removed)
 			}
 			timeILM(numVersions)
 		}
 	}()
 
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
+	if lcEvent.Action.DeleteAll() && (err != nil || purge.Removed == 0) {
+		return deleteAllNotCompleted(ctx, obj, lcEvent, src, purge, err, traceFn)
+	}
 	if err != nil {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+		// Assume it is still there, unless it is gone already.
+		if !reportILMExpiryNotDone(ctx, obj, lcEvent, src, err, "non-transition-expiry"+obj.Name, traceFn) {
 			traceFn(ILMExpiry, nil, nil)
-			return false
 		}
-		// Assume it is still there.
-		err := fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err)
-		ilmLogOnceIf(ctx, err, "non-transition-expiry"+obj.Name)
-		traceFn(ILMExpiry, nil, err)
 		return false
 	}
 	if dobj.Name == "" {
@@ -1278,14 +1306,14 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	tags["version-id"] = dobj.VersionID
+	if lcEvent.Action.DeleteAll() {
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+	}
 
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
 
-	eventName := event.ObjectRemovedDelete
-	if obj.DeleteMarker {
-		eventName = event.ObjectRemovedDeleteMarkerCreated
-	}
+	eventName := ilmExpiryEventName(opts.VersionID, dobj)
 	switch lcEvent.Action {
 	case lifecycle.DeleteAllVersionsAction:
 		eventName = event.ObjectRemovedDeleteAllVersions
@@ -1304,6 +1332,65 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	return true
 }
 
+// ilmVersionsRemoved is the audit tag with the number of versions that an
+// expiry of all versions removed; ilmPartial marks one that began to delete
+// and did not finish.
+const (
+	ilmVersionsRemoved = "ilm-versions-removed"
+	ilmPartial         = "ilm-partial"
+)
+
+// deleteAllNotCompleted reports an expiry of all versions of obj that did
+// not remove them all. A purge that had nothing to remove, or that failed
+// before it deleted anything, reports what a rejected expiry does. One that
+// failed or stopped after it began to delete is audited as partial, with the
+// versions it removed, but sends no removal event: the key is still there.
+//
+// Each is counted and traced with its reason, as reportILMExpiryNotDone
+// reports other expiries. A refusal because a version is in a remote tier is
+// also logged, since it lasts until such versions can be purged.
+func deleteAllNotCompleted(ctx context.Context, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, purge deleteAllResult, err error, traceFn func(event string, metadata map[string]string, err error)) bool {
+	logID := "non-transition-expiry" + obj.Name
+	switch {
+	case err == nil: // there was nothing to remove
+		traceFn(ILMExpiry, nil, nil)
+		return false
+	case purge.Started:
+		reason, _ := ilmExpiryNotDone(err)
+		globalILMIncomplete.add(reason)
+		tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+		tags["version-id"] = purge.Latest.VersionID
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+		tags[ilmPartial] = "true"
+		tags["ilm-incomplete-reason"] = reason
+		auditLogInternal(ctx, AuditLogOptions{
+			Event:     ILMExpiry,
+			APIName:   "ILMExpiry",
+			Bucket:    obj.Bucket,
+			Object:    obj.Name,
+			VersionID: purge.Latest.VersionID,
+			Error:     err.Error(),
+			Tags:      tags,
+		})
+		trace := maps.Clone(tags)
+		trace["result"], trace["reason"] = "partial", reason
+		traceFn(ILMExpiry, trace, err)
+		if errors.Is(err, errDeleteAllBudget) {
+			return false // a long history; the next expiry continues
+		}
+		// Assume it is still there.
+		ilmLogOnceIf(ctx, fmt.Errorf("ILM expiry of all versions of %s/%s stopped after it removed %d: %w", obj.Bucket, obj.Name, purge.Removed, err), logID)
+		return false
+	case errors.Is(err, errDeleteAllTiered):
+		// Kept until tiered versions can be purged; see deleteAllVersions.
+		ilmLogOnceIf(ctx, fmt.Errorf("ILM expiry of all versions of %s/%s refused: a version is in a remote tier, and a purge of all versions does not remove tiered versions yet", obj.Bucket, obj.Name), logID)
+	}
+	if !reportILMExpiryNotDone(ctx, obj, lcEvent, src, err, logID, traceFn) {
+		traceFn(ILMExpiry, nil, nil) // gone already
+	}
+	return false
+}
+
 // Apply object, object version, restored object or restored object version action on the given object
 func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
 	globalExpiryState.enqueueByDays(obj, event, src)

cmd/data-scanner_test.go

@@ -240,8 +240,10 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 			},
 		},
 		{
+			// Every version is accounted at its size until the purge
+			// removes it.
 			objInfos:    allVersExpObjInfos,
-			wants:       nil,
+			wants:       allVersExpObjInfos,
 			wantExpired: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: obj, VersionID: allVersExpObjInfos[0].VersionID}}},
 		},
 		{

cmd/endpoint-ellipses_test.go

@@ -27,6 +27,7 @@ import (
 
 // Tests create endpoints with ellipses and without.
 func TestCreateServerEndpoints(t *testing.T) {
+	useStaticTestEndpoints(t)
 	testCases := []struct {
 		serverAddr string
 		args       []string

cmd/endpoint_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -25,9 +26,42 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/minio/dnscache"
 )
 
+type endpointTestResolver struct {
+	dnscache.DNSResolver
+}
+
+func (r endpointTestResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
+	// These endpoint fixtures test locality and parsing, not public DNS.
+	for i, name := range []string{"example.org", "example.com", "example.net", "example.edu"} {
+		if host == name {
+			return []string{fmt.Sprintf("203.0.113.%d", i+1)}, nil
+		}
+	}
+	return r.DNSResolver.LookupHost(ctx, host)
+}
+
+func useTestEndpointDNS(t *testing.T) {
+	t.Helper()
+	previous := globalDNSCache
+	globalDNSCache = &dnscache.Resolver{Resolver: endpointTestResolver{net.DefaultResolver}}
+	t.Cleanup(func() { globalDNSCache = previous })
+}
+
+func useStaticTestEndpoints(t *testing.T) {
+	t.Helper()
+	useTestEndpointDNS(t)
+	// These fixtures deliberately use localhost, including inside containers.
+	previous := orchestrated
+	orchestrated = false
+	t.Cleanup(func() { orchestrated = previous })
+}
+
 func TestNewEndpoint(t *testing.T) {
+	useTestEndpointDNS(t)
 	u2, _ := url.Parse("https://example.org/path")
 	u4, _ := url.Parse("http://192.168.253.200/path")
 	rootSlashFoo, _ := filepath.Abs("/foo")
@@ -130,6 +164,7 @@ func TestNewEndpoints(t *testing.T) {
 }
 
 func TestCreateEndpoints(t *testing.T) {
+	useStaticTestEndpoints(t)
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {
 		globalMinioPort = tempGlobalMinioPort
@@ -353,6 +388,7 @@ func TestCreateEndpoints(t *testing.T) {
 // So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
 // is considered a remote service from localhost:9000 perspective.
 func TestGetLocalPeer(t *testing.T) {
+	useStaticTestEndpoints(t)
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {
 		globalMinioPort = tempGlobalMinioPort
@@ -393,6 +429,7 @@ func TestGetLocalPeer(t *testing.T) {
 }
 
 func TestGetRemotePeers(t *testing.T) {
+	useStaticTestEndpoints(t)
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {
 		globalMinioPort = tempGlobalMinioPort

cmd/erasure-healing-native-audit_test.go

@@ -0,0 +1,244 @@
+// Copyright (c) 2026 Jasper Bernhardt
+//
+// This file is part of Silo Object Storage.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+// Unimplemented operations panic: the dry-run tests only allow reads and checks.
+type nativeAuditDisk struct {
+	StorageAPI
+	info    FileInfo
+	readErr error
+	verify  func() (*CheckPartsResp, error)
+	deletes atomic.Int32
+}
+
+func (d *nativeAuditDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
+	return d.info, d.readErr
+}
+
+func (d *nativeAuditDisk) VerifyFile(context.Context, string, string, FileInfo) (*CheckPartsResp, error) {
+	return d.verify()
+}
+
+func (d *nativeAuditDisk) DeleteVersion(context.Context, string, string, FileInfo, bool, DeleteOptions) error {
+	d.deletes.Add(1)
+	return nil
+}
+
+func nativeAuditMetadata() []FileInfo {
+	meta := make([]FileInfo, 4)
+	for i := range meta {
+		meta[i] = newFileInfo("object", 2, 2)
+		meta[i].Volume, meta[i].Name = "bucket", "object"
+		meta[i].DataDir = "data"
+		meta[i].ModTime = time.Unix(123, 0).UTC()
+		meta[i].Size = 1024
+		meta[i].Erasure.Distribution = []int{1, 2, 3, 4}
+		meta[i].Erasure.Index = i + 1
+		meta[i].Parts = []ObjectPartInfo{{Number: 1, Size: 512}, {Number: 2, Size: 512}}
+	}
+	return meta
+}
+
+func nativeAuditEndpoints() []Endpoint {
+	endpoints := make([]Endpoint, 4)
+	for i := range endpoints {
+		endpoints[i].URL = &url.URL{Path: fmt.Sprintf("/synthetic/disk%d", i)}
+	}
+	return endpoints
+}
+
+func TestNativeAuditDryRunPreservesFaults(t *testing.T) {
+	for _, fault := range []string{"metadata-quorum", "corrupt-metadata", "missing-parts", "corrupt-parts", "inline-corrupt", "delete-marker-quorum"} {
+		t.Run(fault, func(t *testing.T) {
+			meta := nativeAuditMetadata()
+			disks := make([]StorageAPI, len(meta))
+			for i := range disks {
+				disk := &nativeAuditDisk{info: meta[i], verify: func() (*CheckPartsResp, error) {
+					return &CheckPartsResp{Results: []int{checkPartSuccess, checkPartSuccess}}, nil
+				}}
+				if i < 3 {
+					switch fault {
+					case "metadata-quorum", "delete-marker-quorum":
+						disk.readErr = errFileVersionNotFound
+					case "corrupt-metadata":
+						disk.readErr = errFileCorrupt
+					case "missing-parts":
+						disk.verify = func() (*CheckPartsResp, error) { return nil, errFileNotFound }
+					case "corrupt-parts":
+						disk.verify = func() (*CheckPartsResp, error) { return nil, errFileCorrupt }
+					}
+				}
+				if fault == "delete-marker-quorum" {
+					disk.info.Deleted = true
+					disk.info.Size = 0
+				}
+				if fault == "inline-corrupt" {
+					disk.info.Data = []byte("corrupt")
+					disk.info.Parts = disk.info.Parts[:1]
+					disk.info.SetInlineData()
+					disk.info.Erasure.Checksums = []ChecksumInfo{{PartNumber: 1, Algorithm: SHA256, Hash: make([]byte, sha256.Size)}}
+				}
+				disks[i] = disk
+			}
+			er := erasureObjects{
+				setDriveCount: 4, defaultParityCount: 2,
+				getDisks:     func() []StorageAPI { return disks },
+				getEndpoints: nativeAuditEndpoints,
+			}
+			result, err := er.HealObject(t.Context(), "bucket", "object", "version", madmin.HealOpts{DryRun: true, NoLock: true, Remove: true, ScanMode: madmin.HealDeepScan})
+			for _, disk := range disks {
+				if got := disk.(*nativeAuditDisk).deletes.Load(); got != 0 {
+					t.Fatalf("dry-run deleted data %d times", got)
+				}
+			}
+			quorumFault := fault == "metadata-quorum" || fault == "corrupt-metadata" || fault == "delete-marker-quorum"
+			if quorumFault && err == nil {
+				t.Fatal("metadata quorum failure reported success")
+			}
+			if !quorumFault && err != nil {
+				t.Fatal(err)
+			}
+			if len(result.Before.Drives) != len(disks) || !reflect.DeepEqual(result.Before, result.After) {
+				t.Fatalf("dry-run states must preserve observed damage: %+v", result)
+			}
+			switch fault {
+			case "metadata-quorum", "delete-marker-quorum":
+				if !isErrVersionNotFound(err) {
+					t.Fatalf("dry-run must return the reduced metadata error, got %v", err)
+				}
+			case "corrupt-metadata":
+				if !errors.Is(err, errFileCorrupt) {
+					t.Fatalf("dry-run must return the reduced metadata error, got %v", err)
+				}
+			}
+			unhealthy := 0
+			for _, drive := range result.Before.Drives {
+				if drive.State != madmin.DriveStateOk {
+					unhealthy++
+				}
+			}
+			if unhealthy < 3 {
+				t.Fatalf("faults were not reported: %+v", result.Before)
+			}
+		})
+	}
+}
+
+// This control proves the missing-metadata fixture reaches dangling deletion.
+func TestNativeAuditNonDryRunStillDeletesDangling(t *testing.T) {
+	meta := nativeAuditMetadata()
+	disks := make([]StorageAPI, len(meta))
+	for i := range disks {
+		disk := &nativeAuditDisk{info: meta[i]}
+		if i < 3 {
+			disk.readErr = errFileVersionNotFound
+		}
+		disks[i] = disk
+	}
+	er := erasureObjects{
+		setDriveCount: 4, defaultParityCount: 2,
+		getDisks:     func() []StorageAPI { return disks },
+		getEndpoints: nativeAuditEndpoints,
+	}
+	_, err := er.HealObject(t.Context(), "bucket", "object", "version", madmin.HealOpts{NoLock: true, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err == nil {
+		t.Fatal("dangling deletion must report the object removed")
+	}
+	for _, disk := range disks {
+		if got := disk.(*nativeAuditDisk).deletes.Load(); got != 1 {
+			t.Fatalf("control did not reach dangling deletion: %d", got)
+		}
+	}
+}
+
+func TestNativeAuditDryRunPreservesDanglingFiles(t *testing.T) {
+	obj, roots, err := prepareErasure(t.Context(), 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	setObjectLayer(obj)
+	defer removeRoots(roots)
+	defer obj.Shutdown(t.Context())
+	const bucket, object = "native-audit", "dangling-object"
+	if err = obj.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	data := bytes.Repeat([]byte("x"), 1024*1024)
+	if _, err = obj.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	for _, root := range roots[:3] {
+		if err = os.Remove(filepath.Join(root, bucket, object, xlStorageFormatFile)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	snapshot := func() map[string]string {
+		files := make(map[string]string)
+		for _, root := range roots {
+			err := filepath.Walk(filepath.Join(root, bucket, object), func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() {
+					return nil
+				}
+				data, err := os.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				files[path] = fmt.Sprintf("%d/%d/%x", info.Size(), info.ModTime().UnixNano(), sha256.Sum256(data))
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+		return files
+	}
+	before := snapshot()
+	if len(before) < 2 {
+		t.Fatal("fixture has no surviving metadata and payload files")
+	}
+	result, err := obj.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{DryRun: true, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err == nil {
+		t.Fatal("missing metadata quorum reported success")
+	}
+	if !reflect.DeepEqual(result.Before, result.After) {
+		t.Fatalf("dry-run reported changes: %+v", result)
+	}
+	if after := snapshot(); !reflect.DeepEqual(before, after) {
+		t.Fatalf("dry-run changed surviving object files: before=%v after=%v", before, after)
+	}
+}

cmd/erasure-healing.go

@@ -343,6 +343,11 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
+		if dryRun {
+			// A metadata quorum failure must not delete dangling data in dry-run.
+			return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+				errs, bucket, object, versionID), err
+		}
 		m, derr := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, nil, ObjectOptions{
 			VersionID: versionID,
 		})
@@ -373,6 +378,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if err != nil {
 		return result, err
 	}
+	// The restore fields fence native restores and restored-copy cleanup.
+	// Heal copies them only as a quorum of the drives holds them; while the
+	// drives disagree, it waits rather than create a quorum for one side.
+	latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum))
+	if err != nil {
+		return result, toObjectErr(err, bucket, object, versionID)
+	}
 
 	// No modtime quorum
 	filterDisksByETag := quorumETag != ""

cmd/erasure-multipart-cleanup_test.go

@@ -126,8 +126,8 @@ func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, stri
 	return FileInfo{ModTime: d.modTime}, d.readErr
 }
 
-func (d *multipartCleanupDisk) StatVol(context.Context, string) (VolInfo, error) {
-	return VolInfo{Created: time.Now()}, nil
+func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) {
+	return (&xlStorage{drivePath: d.root}).StatVol(ctx, volume)
 }
 
 func TestCleanupStaleUploadsOnDisk(t *testing.T) {
@@ -157,7 +157,13 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
 			}
 			disk := &multipartCleanupDisk{root: t.TempDir(), modTime: modTime, readErr: tc.readErr}
 			uploadDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha", id)
-			if err := os.MkdirAll(uploadDir, 0o755); err != nil {
+			for _, dir := range []string{uploadDir, filepath.Join(disk.root, minioMetaTmpDeletedBucket)} {
+				if err := os.MkdirAll(dir, 0o755); err != nil {
+					t.Fatal(err)
+				}
+			}
+			// An old parent must not bypass a skipped upload's age or read error.
+			if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil {
 				t.Fatal(err)
 			}
 			erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
@@ -169,9 +175,60 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
 				if !os.IsNotExist(err) {
 					t.Errorf("expired upload must be removed: %v", err)
 				}
+				if _, err := os.Stat(filepath.Dir(uploadDir)); !os.IsNotExist(err) {
+					t.Errorf("empty hash directory must be removed in the same pass: %v", err)
+				}
 			} else if err != nil {
 				t.Errorf("upload must be preserved: %v", err)
 			}
 		})
 	}
 }
+
+func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) {
+	for _, age := range []time.Duration{0, 2 * globalAPIConfig.getStaleUploadsExpiry()} {
+		t.Run(age.String(), func(t *testing.T) {
+			disk := &multipartCleanupDisk{root: t.TempDir()}
+			shaDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha")
+			for _, dir := range []string{shaDir, filepath.Join(disk.root, minioMetaTmpDeletedBucket)} {
+				if err := os.MkdirAll(dir, 0o755); err != nil {
+					t.Fatal(err)
+				}
+			}
+			modTime := time.Now().Add(-age)
+			if err := os.Chtimes(shaDir, modTime, modTime); err != nil {
+				t.Fatal(err)
+			}
+			erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+			if _, err := os.Stat(shaDir); !os.IsNotExist(err) {
+				t.Fatalf("empty hash directory must be removed regardless of age: %v", err)
+			}
+		})
+	}
+}
+
+func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) {
+	disk := &multipartCleanupDisk{root: t.TempDir(), readErr: errFileNotFound}
+	shaDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha")
+	now := time.Now()
+	expiredID := fmt.Sprintf("d16ad42a-6fe5-4a3b-8092-69983142238fx%d", now.Add(-2*globalAPIConfig.getStaleUploadsExpiry()).UnixNano())
+	recentID := fmt.Sprintf("4eacd477-e733-4851-b821-6fd42785ff87x%d", now.UnixNano())
+	for _, id := range []string{expiredID, recentID} {
+		if err := os.MkdirAll(filepath.Join(shaDir, id), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	partPath := filepath.Join(shaDir, recentID, "part.1")
+	const part = "recent upload part"
+	if err := os.WriteFile(partPath, []byte(part), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+	if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) {
+		t.Errorf("expired upload must be removed: %v", err)
+	}
+	if got, err := os.ReadFile(partPath); err != nil || string(got) != part {
+		t.Errorf("recent upload must remain intact: got %q, err %v", got, err)
+	}
+}

cmd/erasure-multipart-listing-benchmark_test.go

@@ -14,7 +14,7 @@ import (
 // Run explicitly with -benchtime=1x. Fixture creation is excluded from timing;
 // each request still discovers real on-disk metadata. The 16-drive/10k case
 // deliberately exceeds the existing aggregate directory-entry budget.
-func BenchmarkMultipartListingCapacity(b *testing.B) {
+func BenchmarkMultipartDiagnosticScanCapacity(b *testing.B) {
 	for _, tc := range []struct {
 		drives, uploads int
 		mixed, limited  bool
@@ -67,7 +67,7 @@ func BenchmarkMultipartListingCapacity(b *testing.B) {
 			}
 			b.ResetTimer()
 			for range b.N {
-				got, err := z.ListMultipartUploads(b.Context(), bucket, "dir/", "", "", "", 1000)
+				got, err := z.scanMultipartUploadsForTest(b.Context(), bucket, "dir/", "", "", "", 1000)
 				if tc.limited {
 					var limited SlowDown
 					if !errors.As(err, &limited) || len(got.Uploads) != 0 {
@@ -83,3 +83,88 @@ func BenchmarkMultipartListingCapacity(b *testing.B) {
 		})
 	}
 }
+
+// The public path is measured with a populated cache as well as after cache
+// loss. An empty post-restart response alone is not capacity acceptance.
+func BenchmarkMultipartPublicListingCapacity(b *testing.B) {
+	for _, tc := range []struct {
+		drives, uploads int
+		sameKey         bool
+	}{
+		{4, 10_000, false},
+		{16, 3_126, false},
+		{16, 10_000, true},
+	} {
+		b.Run(fmt.Sprintf("drives=%d/uploads=%d/same-key=%t", tc.drives, tc.uploads, tc.sameKey), func(b *testing.B) {
+			obj, dirs, err := prepareErasure(b.Context(), tc.drives)
+			if err != nil {
+				b.Fatal(err)
+			}
+			z := obj.(*erasureServerPools)
+			b.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
+			const bucket, other = "multipart-public-capacity", "multipart-public-other"
+			for _, name := range []string{bucket, other} {
+				if err := z.MakeBucket(b.Context(), name, MakeBucketOptions{}); err != nil {
+					b.Fatal(err)
+				}
+			}
+			wantCached, wantExact := 0, 1
+			for n := range tc.uploads {
+				target, key := bucket, fmt.Sprintf("dir/%05d", n)
+				if tc.sameKey {
+					key = "dir/00000"
+				} else if n%2 != 0 {
+					target = other
+				}
+				if target == bucket {
+					wantCached++
+				}
+				if _, err := z.NewMultipartUpload(b.Context(), target, key, ObjectOptions{}); err != nil {
+					b.Fatal(err)
+				}
+			}
+			if tc.sameKey {
+				wantExact = 1000
+			}
+			if tc.drives == 16 && !tc.sameKey {
+				_, err := z.scanMultipartUploadsForTest(b.Context(), bucket, "", "", "", "", 1000)
+				var limited SlowDown
+				if !errors.As(err, &limited) {
+					b.Fatalf("fixture did not reproduce the previous global budget failure: %v", err)
+				}
+			}
+			var directoryCalls, metadataCalls atomic.Int64
+			for _, set := range z.serverPools[0].sets {
+				original := set.getDisks
+				set.getDisks = func() []StorageAPI {
+					disks := append([]StorageAPI(nil), original()...)
+					for i, disk := range disks {
+						disks[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
+					}
+					return disks
+				}
+				b.Cleanup(func() { set.getDisks = original })
+			}
+			measure := func(name, prefix string, want int) {
+				b.Run(name, func(b *testing.B) {
+					directoryCalls.Store(0)
+					metadataCalls.Store(0)
+					b.ResetTimer()
+					for range b.N {
+						got, err := z.ListMultipartUploads(b.Context(), bucket, prefix, "", "", "", 1000)
+						if err != nil || len(got.Uploads) != want {
+							b.Fatalf("public listing: count=%d want=%d err=%v", len(got.Uploads), want, err)
+						}
+					}
+					b.StopTimer()
+					b.ReportMetric(float64(directoryCalls.Load())/float64(b.N), "ListDir/op")
+					b.ReportMetric(float64(metadataCalls.Load())/float64(b.N), "ReadVersion/op")
+				})
+			}
+			measure("populated-cache", "", wantCached)
+			z.mpCache.Clear()
+			measure("lost-cache", "", 0)
+			measure("known-key", "dir/00000", wantExact)
+		})
+	}
+}

cmd/erasure-multipart-listing.go

@@ -359,7 +359,7 @@ func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartP
 		return multipartPreflightReport{}, err
 	}
 	defer s.close()
-	report := multipartPreflightReport{Complete: true, Mode: "persistent"}
+	report := multipartPreflightReport{Complete: true, Mode: "legacy"}
 	for p, pool := range z.serverPools {
 		for i, set := range pool.sets {
 			_, _ = set.scanMultipartUploads(s, "", p, i)

cmd/erasure-multipart-listing_test.go

@@ -22,10 +22,53 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 )
 
-// Check the real handler and storage path, including a successful abort between
-// pages. Choose IDs increasing in both initiation time and lexical order so the
-// failure does not depend on differing interpretations of S3 marker ordering.
-func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
+// The persistent scanner is retained for diagnostics. This test-only wrapper
+// preserves its pagination coverage without exposing it through public listing.
+func (z *erasureServerPools) scanMultipartUploadsForTest(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+	if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
+		return ListMultipartsInfo{}, err
+	}
+	if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		return ListMultipartsInfo{}, toObjectErr(err, bucket)
+	}
+	scan, err := startMultipartScan(ctx, false)
+	if err != nil {
+		return ListMultipartsInfo{}, err
+	}
+	defer scan.close()
+
+	var uploads []MultipartInfo
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
+		poolUploads, err := pool.scanMultipartUploadsForTest(scan, bucket, idx)
+		if err != nil {
+			return ListMultipartsInfo{}, err
+		}
+		uploads = append(uploads, poolUploads...)
+	}
+
+	// Confirmed historical records lack a recoverable bucket/key and are
+	// omitted by the scan. They do not block listing identifiable uploads.
+	return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
+}
+
+func (s *erasureSets) scanMultipartUploadsForTest(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, error) {
+	var uploads []MultipartInfo
+	for i, set := range s.sets {
+		setUploads, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
+		if err != nil {
+			return nil, err
+		}
+		uploads = append(uploads, setUploads...)
+	}
+	return uploads, nil
+}
+
+// Check that a successful HTTP abort removes the cached upload. The restored
+// cache path still ignores page limits; this is a documented legacy limitation.
+func TestMultipartListingAbortHTTPUpdatesCache(t *testing.T) {
 	z, _ := consistencyPools(t)
 	bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{})
 	if err != nil {
@@ -81,8 +124,8 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
 		return result
 	}
 	first := list("", "", 1)
-	if len(first.Uploads) != 1 || first.Uploads[0].UploadID != firstID || !first.IsTruncated {
-		t.Fatalf("unexpected first page: %+v", first)
+	if len(first.Uploads) != 3 || first.Uploads[0].UploadID != firstID || first.IsTruncated {
+		t.Fatalf("unexpected legacy cache view: %+v", first)
 	}
 	rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID))
 	if rec.Code != http.StatusNoContent {
@@ -93,12 +136,12 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
 	for _, upload := range rest.Uploads {
 		keys = append(keys, upload.Key)
 	}
-	t.Logf("GET first page=200; DELETE marker=204; GET next page=200 keys=%v truncated=%v", keys, rest.IsTruncated)
+	t.Logf("GET cache=200; DELETE upload=204; GET cache=200 keys=%v truncated=%v", keys, rest.IsTruncated)
 	if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil {
 		t.Fatalf("remaining upload is not valid: %v", err)
 	}
 	if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID {
-		t.Fatalf("valid remaining upload for key a is missing from continuation: %+v", rest.Uploads)
+		t.Fatalf("valid remaining upload for key a is missing from cache: %+v", rest.Uploads)
 	}
 }
 
@@ -149,7 +192,7 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
 	}
 	z.mpCache.Clear()
 	for _, target := range []string{bucket, other} {
-		got, err := z.ListMultipartUploads(t.Context(), target, "", "", "", "", 10)
+		got, err := z.scanMultipartUploadsForTest(t.Context(), target, "", "", "", "", 10)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -159,14 +202,14 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
 			requireMultipartUploadKeys(t, got)
 		}
 	}
-	// Exact-looking prefixes now have the same durable semantics.
-	exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10)
+	// Diagnostic discovery cannot recover an unidentified historical key.
+	exact, err := z.scanMultipartUploadsForTest(t.Context(), other, "old", "", "", "", 10)
 	if err != nil {
 		t.Fatal(err)
 	}
 	requireMultipartUploadKeys(t, exact)
 	report, err := z.multipartPreflight(t.Context())
-	if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" {
+	if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "legacy" {
 		t.Fatalf("legacy preflight: %+v %v", report, err)
 	}
 	if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
@@ -176,14 +219,14 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
 	if err != nil || !report.Ready || report.LegacyUploads != 0 {
 		t.Fatalf("drained preflight: %+v %v", report, err)
 	}
-	got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+	got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 	if err != nil {
 		t.Fatal(err)
 	}
 	requireMultipartUploadKeys(t, got, "a")
 }
 
-func TestMultipartListingIdentityFallback(t *testing.T) {
+func TestMultipartDiagnosticIdentityFallback(t *testing.T) {
 	for _, kind := range []string{"old", "partial", "corrupt", "read-failure", "wrong-bucket"} {
 		t.Run(kind, func(t *testing.T) {
 			z, set, bucket := multipartListingFixture(t)
@@ -220,7 +263,7 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
 				}
 				return disks
 			}
-			got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+			got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -469,7 +512,7 @@ func (d multipartListingCountingDisk) ReadVersion(ctx context.Context, original,
 }
 
 // Counts storage API calls; this is not a deployment throughput benchmark.
-func TestMultipartListingScanCosts(t *testing.T) {
+func TestMultipartDiagnosticScanCosts(t *testing.T) {
 	obj, dirs, err := prepareErasure(t.Context(), 4)
 	if err != nil {
 		t.Fatal(err)
@@ -507,7 +550,7 @@ func TestMultipartListingScanCosts(t *testing.T) {
 		t.Helper()
 		directoryCalls.Store(0)
 		metadataCalls.Store(0)
-		result, err := z.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 1)
+		result, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "dir/", "", "", "", 1)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -589,7 +632,7 @@ func TestMultipartListingMissingMarkers(t *testing.T) {
 	}
 }
 
-func TestMultipartListingCoverage(t *testing.T) {
+func TestMultipartDiagnosticCoverage(t *testing.T) {
 	z, set, bucket := multipartListingFixture(t)
 	mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
 	if err != nil {
@@ -605,12 +648,12 @@ func TestMultipartListingCoverage(t *testing.T) {
 		}
 	}
 	set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} }
-	_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+	_, err = z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 	if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
 		t.Fatalf("incomplete discovery returned success: %v", err)
 	}
 	set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} }
-	got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+	got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -621,7 +664,7 @@ func TestMultipartListingCoverage(t *testing.T) {
 	}
 }
 
-func TestMultipartListingBudgetAndAdmission(t *testing.T) {
+func TestMultipartDiagnosticBudgetAndAdmission(t *testing.T) {
 	_, set, bucket := multipartListingFixture(t)
 	if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
 		t.Fatal(err)
@@ -651,7 +694,7 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
 	}
 }
 
-func TestMultipartListingAdmissionHTTP(t *testing.T) {
+func TestMultipartPublicListingAdmissionHTTP(t *testing.T) {
 	z, _, _ := multipartListingFixture(t)
 	bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
 	if err != nil {
@@ -673,12 +716,12 @@ func TestMultipartListingAdmissionHTTP(t *testing.T) {
 	}
 	rec := httptest.NewRecorder()
 	router.ServeHTTP(rec, req)
-	var response APIErrorResponse
+	var response ListMultipartUploadsResponse
 	if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
 		t.Fatal(err)
 	}
-	if rec.Code != http.StatusServiceUnavailable || response.Code != "SlowDown" {
-		t.Fatalf("admission returned %d %s", rec.Code, rec.Body.String())
+	if rec.Code != http.StatusOK || len(response.Uploads) != 0 {
+		t.Fatalf("diagnostic admission blocked public listing: %d %s", rec.Code, rec.Body.String())
 	}
 }
 
@@ -705,12 +748,12 @@ func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
 		}
 	}
 	report, err := z.multipartPreflight(t.Context())
-	if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" {
+	if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "legacy" {
 		t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
 	}
 }
 
-func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
+func TestMultipartDiagnosticCancellationRetainsAdmission(t *testing.T) {
 	z, set, bucket := multipartListingFixture(t)
 	for i := range 40 {
 		if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil {
@@ -749,7 +792,7 @@ func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
 	defer cancel()
 	done := make(chan error, 1)
 	go func() {
-		_, err := z.ListMultipartUploads(ctx, bucket, "", "", "", "", 10)
+		_, err := z.scanMultipartUploadsForTest(ctx, bucket, "", "", "", "", 10)
 		done <- err
 	}()
 	for range 16 {

cmd/erasure-multipart-public_test.go

@@ -0,0 +1,106 @@
+// Copyright (c) 2026 Ruohang Feng
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/xml"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+)
+
+type multipartRootScanGuard struct {
+	StorageAPI
+	roots *atomic.Int64
+}
+
+func (d multipartRootScanGuard) ListDir(ctx context.Context, original, volume, dir string, count int) ([]string, error) {
+	if volume == minioMetaMultipartBucket && dir == "" {
+		d.roots.Add(1)
+		return nil, errFaultyDisk
+	}
+	return d.StorageAPI.ListDir(ctx, original, volume, dir, count)
+}
+
+// A busy or unreadable diagnostic namespace must not become a prerequisite for
+// ordinary listing, including the lower-level exact-object entry points.
+func TestMultipartPublicListingIndependentOfDiagnostics(t *testing.T) {
+	for _, busy := range []bool{false, true} {
+		t.Run(map[bool]string{false: "unreadable-root", true: "busy-diagnostics"}[busy], func(t *testing.T) {
+			z, _ := consistencyPools(t)
+			bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			const key = "dir/known"
+			mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			var roots atomic.Int64
+			for _, pool := range z.serverPools {
+				for _, set := range pool.sets {
+					original := set.getDisks
+					set.getDisks = func() []StorageAPI {
+						disks := append([]StorageAPI(nil), original()...)
+						for i, disk := range disks {
+							if disk != nil {
+								disks[i] = multipartRootScanGuard{disk, &roots}
+							}
+						}
+						return disks
+					}
+					t.Cleanup(func() { set.getDisks = original })
+				}
+			}
+			if busy {
+				for range cap(multipartScanSlots) {
+					scan, err := startMultipartScan(t.Context(), true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					defer scan.close()
+				}
+			}
+			for _, prefix := range []string{"", key, "dir/"} {
+				req, err := newTestSignedRequestV4(http.MethodGet,
+					getListMultipartUploadsURLWithParams("", bucket, prefix, "", "", "", "1"),
+					0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				rec := httptest.NewRecorder()
+				router.ServeHTTP(rec, req)
+				if rec.Code != http.StatusOK {
+					t.Fatalf("prefix %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
+				}
+				var got ListMultipartUploadsResponse
+				if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+					t.Fatal(err)
+				}
+				if prefix == "dir/" {
+					if len(got.Uploads) != 0 {
+						t.Fatalf("partial prefix must retain the documented exact-key limitation: %+v", got)
+					}
+				} else if len(got.Uploads) != 1 || got.Uploads[0].UploadID != mp.UploadID {
+					t.Fatalf("known upload missing: %+v", got)
+				}
+			}
+			for _, pool := range z.serverPools {
+				for _, list := range []func(context.Context, string, string, string, string, string, int) (ListMultipartsInfo, error){
+					pool.ListMultipartUploads, pool.getHashedSet(key).ListMultipartUploads,
+				} {
+					if _, err := list(t.Context(), bucket, key, "", "", "", 10); err != nil {
+						t.Fatalf("lower-level exact listing: %v", err)
+					}
+				}
+			}
+			if roots.Load() != 0 {
+				t.Fatalf("public listing attempted %d global multipart scans", roots.Load())
+			}
+		})
+	}
+}

cmd/erasure-multipart-unified_test.go

@@ -4,11 +4,10 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
-	"encoding/xml"
 	"errors"
 	"net/http"
-	"net/http/httptest"
 	"slices"
 	"testing"
 )
@@ -33,14 +32,10 @@ func removeMultipartIdentity(metadata map[string]string) {
 	delete(metadata, multipartMetaObject)
 }
 
-// The actual S3 handler must use the same semantics for every prefix and page,
-// even with old records and a stale, misleading node-local cache.
-func TestMultipartListingMixedHTTP(t *testing.T) {
-	z, set, _ := multipartListingFixture(t)
-	bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
+// The retained diagnostic scanner handles mixed records independently of a
+// stale cache. Public HTTP behavior is tested separately.
+func TestMultipartDiagnosticMixed(t *testing.T) {
+	z, set, bucket := multipartListingFixture(t)
 	for _, key := range []string{"dir/old", "dir/a", "dir/sub/b", "outside"} {
 		mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
 		if err != nil {
@@ -54,22 +49,11 @@ func TestMultipartListingMixedHTTP(t *testing.T) {
 	z.mpCache.Store("stale", MultipartInfo{Bucket: bucket, Object: "ghost", UploadID: "stale"})
 	list := func(prefix, keyMarker, uploadMarker, delimiter string) ListMultipartUploadsResponse {
 		t.Helper()
-		req, err := newTestSignedRequestV4(http.MethodGet,
-			getListMultipartUploadsURLWithParams("", bucket, prefix, keyMarker, uploadMarker, delimiter, "1"),
-			0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+		got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, prefix, keyMarker, uploadMarker, delimiter, 1)
 		if err != nil {
 			t.Fatal(err)
 		}
-		rec := httptest.NewRecorder()
-		router.ServeHTTP(rec, req)
-		var got ListMultipartUploadsResponse
-		if rec.Code != http.StatusOK {
-			t.Fatalf("list %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
-		}
-		if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
-			t.Fatal(err)
-		}
-		return got
+		return generateListMultipartUploadsResponse(bucket, got, "")
 	}
 	var keys []string
 	keyMarker, uploadMarker := "", ""
@@ -103,7 +87,7 @@ func TestMultipartListingMixedHTTP(t *testing.T) {
 	}
 }
 
-func TestMultipartListingInvalidIdentity(t *testing.T) {
+func TestMultipartDiagnosticInvalidIdentity(t *testing.T) {
 	for _, kind := range []string{"missing-bucket", "missing-object", "empty-bucket", "empty-object", "empty-both", "wrong-hash", "read-failure"} {
 		t.Run(kind, func(t *testing.T) {
 			z, set, bucket := multipartListingFixture(t)
@@ -142,7 +126,7 @@ func TestMultipartListingInvalidIdentity(t *testing.T) {
 					return disks
 				}
 			}
-			got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+			got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 			if err == nil || len(got.Uploads) != 0 || toAPIError(t.Context(), err).HTTPStatusCode != http.StatusServiceUnavailable {
 				t.Fatalf("invalid/unreadable record returned a successful page: %+v %v", got, err)
 			}
@@ -185,7 +169,14 @@ func TestMultipartListingRestart(t *testing.T) {
 	if _, cached := obj.(*erasureServerPools).mpCache.Load(mp.UploadID); cached {
 		t.Fatal("restart unexpectedly preserved the upload cache")
 	}
-	got, err := obj.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 10)
+	for _, prefix := range []string{"", "dir/"} {
+		got, err := obj.ListMultipartUploads(t.Context(), bucket, prefix, "", "", "", 10)
+		if err != nil {
+			t.Fatal(err)
+		}
+		requireMultipartUploadKeys(t, got)
+	}
+	got, err := obj.ListMultipartUploads(t.Context(), bucket, key, "", "", "", 10)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -193,9 +184,18 @@ func TestMultipartListingRestart(t *testing.T) {
 	if got.Uploads[0].UploadID != mp.UploadID {
 		t.Fatal("restart changed upload identity")
 	}
+	part, err := obj.PutObjectPart(t.Context(), bucket, key, mp.UploadID, 1,
+		mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
+	if err != nil {
+		t.Fatalf("continue known upload after restart: %v", err)
+	}
+	if _, err := obj.CompleteMultipartUpload(t.Context(), bucket, key, mp.UploadID,
+		[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}); err != nil {
+		t.Fatalf("complete known upload after restart: %v", err)
+	}
 }
 
-func TestMultipartListingConflictingIdentityCopies(t *testing.T) {
+func TestMultipartDiagnosticConflictingIdentityCopies(t *testing.T) {
 	for _, oldCopies := range []int{1, 3} {
 		z, set, bucket := multipartListingFixture(t)
 		mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
@@ -213,14 +213,14 @@ func TestMultipartListingConflictingIdentityCopies(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
-		got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+		got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 		if !errors.Is(err, errMultipartListingIdentity) || len(got.Uploads) != 0 {
 			t.Fatalf("%d contradictory copies returned a successful page: %+v %v", oldCopies, got, err)
 		}
 	}
 }
 
-func TestMultipartListingUnsupportedStoredID(t *testing.T) {
+func TestMultipartDiagnosticUnsupportedStoredID(t *testing.T) {
 	for _, historical := range []bool{false, true} {
 		z, set, bucket := multipartListingFixture(t)
 		mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
@@ -244,7 +244,7 @@ func TestMultipartListingUnsupportedStoredID(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
-		got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
+		got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
 		if historical {
 			if err != nil || len(got.Uploads) != 0 {
 				t.Fatalf("unidentified historical ID: %+v %v", got, err)

cmd/erasure-multipart.go

@@ -197,6 +197,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 	drivePath := disk.Endpoint().Path
 
 	readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
+		hasSkippedUploads := false
 		readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
 			modTime := extractModTimeFromUploadIDDir(uploadIDDir)
@@ -205,12 +206,14 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				wait := deleteMultipartCleanupSleeper.Timer(ctx)
 				fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
 				if err != nil {
+					hasSkippedUploads = true
 					return nil
 				}
 				modTime = fi.ModTime
 				wait()
 			}
 			if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() {
+				hasSkippedUploads = true
 				return nil
 			}
 			w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -223,24 +226,15 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				return nil
 			})
 		})
-		// Get the modtime of the shaDir.
-		vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir))
-		if err != nil {
-			return nil
-		}
-		// Modtime is returned in the Created field. See (*xlStorage).StatVol
-		if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() {
+		if hasSkippedUploads {
 			return nil
 		}
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 		return w.Run(func() error {
 			wait := deleteMultipartCleanupSleeper.Timer(ctx)
-			pathUUID := mustGetUUID()
-			targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
-
-			// We are not deleting shaDir recursively here, if shaDir is empty
-			// and its older then we can happily delete it.
-			Rename(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), targetPath)
+			// Removing an upload updates the parent's mtime. Remove the parent
+			// immediately if it is empty; other uploads keep it in place.
+			_ = Remove(pathJoin(drivePath, minioMetaMultipartBucket, shaDir))
 			wait()
 			return nil
 		})
@@ -414,7 +408,7 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload
 }
 
 // listMultipartUploadsExact preserves the hashed exact-object lookup used by
-// multipart write placement. Public listing always uses the durable scan.
+// multipart write placement and nonempty public listing prefixes.
 func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
 
@@ -522,16 +516,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefi
 	if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
 		return ListMultipartsInfo{}, err
 	}
-	scan, err := startMultipartScan(ctx, false)
-	if err != nil {
-		return ListMultipartsInfo{}, err
-	}
-	defer scan.close()
-	uploads, err := er.scanMultipartUploads(scan, bucket, 0, 0)
-	if err != nil {
-		return ListMultipartsInfo{}, err
-	}
-	return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
+	return er.listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 }
 
 // newMultipartUpload - wrapper for initializing a new multipart
@@ -1702,6 +1687,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		defer lk.Unlock(lkctx)
 	}
 
+	if opts.RefuseLockedOverwrite {
+		getObjectInfo := er.getObjectInfo
+		if opts.replicaObjectInfo != nil {
+			getObjectInfo = opts.replicaObjectInfo
+		}
+		if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil {
+			return oi, toObjectErr(err, bucket, object)
+		}
+	}
 	if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement {
 		getObjectInfo := er.getObjectInfo
 		if opts.replicaObjectInfo != nil {

cmd/erasure-object.go

@@ -45,7 +45,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/grid"
-	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
@@ -509,6 +508,11 @@ func joinErrs(errs []error) string {
 }
 
 func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) {
+	// Restore and restored-copy cleanup must preserve the version on read
+	// failure, including lookups and failure cleanup that reread metadata.
+	if opts.Transition.ExpireRestored || opts.Transition.RestoreRequest != nil {
+		return FileInfo{}, errErasureReadQuorum
+	}
 	m, ok := isObjectDangling(metaArr, errs, dataErrsByPart)
 	if !ok {
 		// We only come here if we cannot figure out if the object
@@ -656,9 +660,15 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 	meta := &xlMetaV2{versions: mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)}
 	lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers, true)
 	if err != nil {
+		// No version in quorum proves absence only when every drive answered.
+		// Otherwise a drive holding a version keeps the read retryable.
+		partial := errors.Is(err, errFileNotFound) && !allDrivesAnswered(errs)
 		for i := range errs {
 			if errs[i] == nil {
 				errs[i] = err
+				if partial && hasVersion(metadataArray[i], inclFreeVers) {
+					errs[i] = errErasureReadQuorum
+				}
 			}
 		}
 		return metaFileInfos, errs
@@ -702,6 +712,33 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 	return metaFileInfos, errs
 }
 
+// allDrivesAnswered reports whether every drive returned its metadata or a
+// definitive answer that it has none. Offline drives keep errDiskOngoingReq
+// in getObjectFileInfo; unreachable ones return errDiskNotFound or a
+// transport error.
+func allDrivesAnswered(errs []error) bool {
+	for _, err := range errs {
+		if err != nil && !IsErr(err, errFileNotFound, errFileVersionNotFound, errFileCorrupt, errVolumeNotFound) {
+			return false
+		}
+	}
+	return true
+}
+
+// hasVersion reports whether a drive's xl.meta holds a version a read can
+// return.
+func hasVersion(xl *xlMetaV2, inclFreeVers bool) bool {
+	if xl == nil {
+		return false
+	}
+	for _, ver := range xl.versions {
+		if inclFreeVers || !ver.header.FreeVersion() {
+			return true
+		}
+	}
+	return false
+}
+
 // Checking if an object is dangling costs some IOPS; hence implementing this function
 // which decides which condition it is useful to check if an object is dangling
 //
@@ -1002,8 +1039,10 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		}
 		// when we have insufficient read quorum and inconsistent metadata return
 		// file not found, since we can't possibly have a way to recover this object
-		// anyway.
-		if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta {
+		// anyway. That holds only when every drive answered: a drive that is
+		// offline or not yet reconnected may hold the copies that make quorum,
+		// and a 404 for an existing object is final to clients.
+		if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta && allDrivesAnswered(errs) {
 			if opts.VersionID != "" {
 				err = errFileVersionNotFound
 			} else {
@@ -1672,6 +1711,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 	}
+	if opts.RefuseLockedOverwrite {
+		getObjectInfo := er.getObjectInfo
+		if opts.replicaObjectInfo != nil {
+			getObjectInfo = opts.replicaObjectInfo
+		}
+		if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil {
+			return ObjectInfo{}, toObjectErr(err, bucket, object)
+		}
+	}
 
 	// Rename the successfully written temporary object to final location.
 	onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
@@ -1769,6 +1817,13 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	if opts.DeleteAllVersions {
+		errs := make([]error, len(objects))
+		for i := range errs {
+			errs[i] = errDeleteAllUnsupported
+		}
+		return make([]DeletedObject, len(objects)), errs
+	}
 	if !opts.NoAuditLog {
 		for _, obj := range objects {
 			auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectName, &er)
@@ -1891,6 +1946,23 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 				if err == nil {
 					continue
 				}
+				if err == errFileNotFound || err == errFileVersionNotFound {
+					// A retry after partial physical deletion may mix removals
+					// with already-absent replies. Both prove the desired outcome.
+					// Marker creation and replication-state updates still require
+					// successful writes; absence cannot acknowledge those operations.
+					physical := true
+					for _, v := range dedupVersions[i].Versions {
+						if v.Deleted || v.MarkDeleted || v.ReplicationState.ReplicationStatusInternal != "" ||
+							v.ReplicationState.VersionPurgeStatusInternal != "" || v.ReplicationState.ReplicaStatus != "" {
+							physical = false
+							break
+						}
+					}
+					if physical {
+						continue
+					}
+				}
 				for _, v := range dedupVersions[i].Versions {
 					delObjErrs[index][v.Idx] = err
 				}
@@ -2002,6 +2074,9 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if opts.DeleteAllVersions {
+		return objInfo, errDeleteAllUnsupported
+	}
 	if !opts.NoAuditLog {
 		auditObjectErasureSet(ctx, "DeleteObject", object, &er)
 	}
@@ -2028,6 +2103,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	// expiration attempted on a bucket with no lifecycle
 	// rules shall be rejected.
 	if lc == nil && opts.Expiration.Expire {
+		if !opts.Transition.ExpireRestored {
+			return objInfo, errLifecycleNotApplicable
+		}
 		if opts.VersionID != "" {
 			return objInfo, VersionNotFound{
 				Bucket:    bucket,
@@ -2072,6 +2150,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 					isErr = true
 				}
 				if isErr {
+					if !opts.Transition.ExpireRestored {
+						return goi, errLifecycleNotApplicable
+					}
 					if goi.VersionID != "" {
 						return goi, VersionNotFound{
 							Bucket:    bucket,
@@ -2090,6 +2171,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
 	}
 
+	// An ILM version expiry that the pools checked with the versions in all
+	// pools; the set still guards the protection it reads, see
+	// checkedExpiryRefused.
+	checkedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && opts.VersionID != "" && !opts.Transition.ExpireRestored
+
 	storageDisks := er.getDisks()
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
@@ -2097,7 +2183,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	tryDel := false
 	if gerr != nil && goi.Name == "" {
 		if _, ok := gerr.(InsufficientReadQuorum); ok {
-			if opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
+			// A restored-copy expiry needs fresh metadata to confirm its target
+			// and deadline, a checked expiry to guard its protection; ordinary
+			// unversioned DELETE may still try a blind delete.
+			if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
 				// Add an MRF heal for next time.
 				er.addPartial(bucket, object, opts.VersionID)
 				return objInfo, InsufficientWriteQuorum{}
@@ -2117,6 +2206,26 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
+	// A queued null restore may now address a replacement delete marker.
+	// Such errors carry ObjectInfo, but never authorize cache cleanup.
+	if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) {
+		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+	}
+	// A restored copy expires only by its own reason, decided on the
+	// version's authoritative generation: a drive that missed a renewal or a
+	// later POST must not authorize removing the cache, nor drives that
+	// disagree on the restore status. Another lifecycle deletion falling due,
+	// or the all-pools check of an ordinary expiry, does not stand in for it.
+	if opts.Transition.ExpireRestored {
+		gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+		if err != nil || gen.split || gen.fi.Deleted || lc == nil {
+			return goi, errLifecycleNotApplicable
+		}
+		if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) {
+			return goi, errLifecycleNotApplicable
+		}
+	}
+
 	if opts.EvalMetadataFn != nil {
 		dsc, err := opts.EvalMetadataFn(&goi, gerr)
 		if err != nil {
@@ -2134,7 +2243,14 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	if opts.Expiration.Expire {
+	// A delete marker that names a stored version replaces it.
+	if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" {
+		if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil {
+			return ObjectInfo{}, err
+		}
+	}
+
+	if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools {
 		if gerr == nil {
 			evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 			var isErr bool
@@ -2145,6 +2261,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 				isErr = true
 			}
 			if isErr {
+				if !opts.Transition.ExpireRestored {
+					return goi, errLifecycleNotApplicable
+				}
 				if goi.VersionID != "" {
 					return goi, VersionNotFound{
 						Bucket:    bucket,
@@ -2158,6 +2277,8 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 				}
 			}
 		}
+	} else if checkedExpiry && checkedExpiryRefused(*lc, rcfg, replcfg, goi, opts.Expiration.ApprovedProtection) {
+		return goi, errLifecycleNotApplicable
 	}
 
 	//  Determine whether to mark object deleted for replication
@@ -2195,6 +2316,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
+	if opts.DataMovement && opts.DeleteMarker {
+		// Moving a null marker replaces any older null data/marker. Moving
+		// a UUID marker preserves it on retries instead of purging it.
+		deleteMarker = true
+		markDelete = opts.VersionID != nullVersionID
+	}
+
 	purge := opts.isVersionPurge()
 	if purge {
 		// A delete marker describes the version being removed; it is not an
@@ -2597,107 +2725,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	return err
 }
 
-// RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-// This is similar to PostObjectRestore from AWS GLACIER
-// storage class. When PostObjectRestore API is called, a temporary copy of the object
-// is restored locally to the bucket on source cluster until the restore expiry date.
-// The copy that was transitioned continues to reside in the transitioned tier.
-func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
-	return er.restoreTransitionedObject(ctx, bucket, object, opts)
-}
-
-// update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
-	oi := objInfo.Clone()
-	oi.metadataOnly = true // Perform only metadata updates.
-
-	// allow retry in the case of failure to restore
-	delete(oi.UserDefined, xhttp.AmzRestore)
-
-	if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
-		VersionID: oi.VersionID,
-	}, ObjectOptions{
-		VersionID: oi.VersionID,
-	}); err != nil {
-		storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
-		return err
-	}
-	return nil
-}
-
-// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts
-// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
-func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
-	setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
-		if rerr == nil {
-			return nil // nothing to do; restore object was successful
-		}
-		er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
-		return rerr
-	}
-	var oi ObjectInfo
-	// get the file info on disk for transitioned object
-	actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
-	if err != nil {
-		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-	}
-
-	oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
-	ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
-	if len(oi.Parts) == 1 {
-		var rs *HTTPRangeSpec
-		gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
-		if err != nil {
-			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-		}
-		defer gr.Close()
-		hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
-		if err != nil {
-			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-		}
-		pReader := NewPutObjReader(hashReader)
-		_, err = er.PutObject(ctx, bucket, object, pReader, ropts)
-		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-	}
-
-	res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
-	if err != nil {
-		return setRestoreHeaderFn(oi, err)
-	}
-
-	var uploadedParts []CompletePart
-	var rs *HTTPRangeSpec
-	// get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted.
-	gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
-	if err != nil {
-		return setRestoreHeaderFn(oi, err)
-	}
-	defer gr.Close()
-
-	// rehydrate the parts back on disk as per the original xl.meta prior to transition
-	for _, partInfo := range oi.Parts {
-		hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
-		if err != nil {
-			return setRestoreHeaderFn(oi, err)
-		}
-		pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{})
-		if err != nil {
-			return setRestoreHeaderFn(oi, err)
-		}
-		if pInfo.Size != partInfo.Size {
-			return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object})
-		}
-		uploadedParts = append(uploadedParts, CompletePart{
-			PartNumber: pInfo.PartNumber,
-			ETag:       pInfo.ETag,
-		})
-	}
-	_, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{
-		MTime: oi.ModTime,
-	})
-	return setRestoreHeaderFn(oi, err)
-}
-
 // DecomTieredObject - moves tiered object to another pool during decommissioning.
 func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
 	if opts.UserDefined == nil {

cmd/erasure-object_test.go

@@ -28,10 +28,15 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strconv"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config/storageclass"
 )
 
@@ -1302,3 +1307,247 @@ func TestJoinErrs(t *testing.T) {
 		}
 	}
 }
+
+type stagedDataGoneDisk struct {
+	StorageAPI
+}
+
+func (d stagedDataGoneDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
+	if err := d.Delete(ctx, srcVolume, srcPath, DeleteOptions{Recursive: true, Immediate: true}); err != nil {
+		return RenameDataResp{}, err
+	}
+	return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
+}
+
+// stagedDataGoneDisks makes every drive of the only set lose the staged data
+// of a write right before RenameData, as when the caller has already given up
+// and removed it. It returns a function that restores the drives.
+func stagedDataGoneDisks(z *erasureServerPools) func() {
+	// Swap the drives behind the set's getDisks under the mutex that
+	// getDisks takes, so that background readers do not race the swap.
+	sets := z.serverPools[0]
+	sets.erasureDisksMu.Lock()
+	disks := sets.erasureDisks[0]
+	original := slices.Clone(disks)
+	for i, disk := range disks {
+		disks[i] = stagedDataGoneDisk{StorageAPI: disk}
+	}
+	sets.erasureDisksMu.Unlock()
+	return func() {
+		sets.erasureDisksMu.Lock()
+		copy(sets.erasureDisks[0], original)
+		sets.erasureDisksMu.Unlock()
+	}
+}
+
+// A write whose staged data is gone before the drives rename it fails. The
+// object directory the drives create for it must not stay behind: it is
+// invisible while its parent holds other objects, but once they are deleted
+// the parent can no longer be removed and is listed as an empty prefix.
+func TestRenameDataStagedDataGoneLeavesNoPrefix(t *testing.T) {
+	ctx := t.Context()
+	obj, dirs, err := prepareErasure16(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer obj.Shutdown(context.Background())
+	defer removeRoots(dirs)
+	z := obj.(*erasureServerPools)
+	const bucket = "bucket"
+	const sibling, putObject, mpuObject = "day=1/b=9/sibling", "day=1/b=9/put", "day=1/b=9/mpu"
+	if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	body := bytes.Repeat([]byte{'a'}, 8<<20) // above the inline limit: data goes to a data dir
+	if _, err := z.PutObject(ctx, bucket, sibling, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	res, err := z.NewMultipartUpload(ctx, bucket, mpuObject, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	pi, err := z.PutObjectPart(ctx, bucket, mpuObject, res.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	restore := stagedDataGoneDisks(z)
+	if _, err := z.PutObject(ctx, bucket, putObject, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}); err == nil {
+		t.Error("PutObject succeeded without its staged data")
+	}
+	if _, err := z.CompleteMultipartUpload(ctx, bucket, mpuObject, res.UploadID, []CompletePart{{PartNumber: 1, ETag: pi.ETag}}, ObjectOptions{}); err == nil {
+		t.Error("CompleteMultipartUpload succeeded without its staged data")
+	}
+	restore()
+
+	if _, err := z.DeleteObject(ctx, bucket, sibling, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	for _, prefix := range []string{"", "day=1/", "day=1/b=9/"} {
+		res, err := z.ListObjectsV2(ctx, bucket, prefix, "", "/", 1000, false, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(res.Objects) != 0 || len(res.Prefixes) != 0 {
+			t.Errorf("ListObjectsV2 %q: %d objects, prefixes %v, want none", prefix, len(res.Objects), res.Prefixes)
+		}
+	}
+	for _, dir := range dirs {
+		if _, err := os.Stat(filepath.Join(dir, bucket, "day=1")); !os.IsNotExist(err) {
+			t.Errorf("%s: day=1 is still on the drive (%v)", dir, err)
+		}
+	}
+}
+
+// A failed overwrite whose staged data is gone must keep the existing object.
+func TestRenameDataStagedDataGoneKeepsExistingObject(t *testing.T) {
+	ctx := t.Context()
+	obj, dirs, err := prepareErasure16(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer obj.Shutdown(context.Background())
+	defer removeRoots(dirs)
+	z := obj.(*erasureServerPools)
+	const bucket, object = "bucket", "day=1/b=9/object"
+	if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	body := bytes.Repeat([]byte{'a'}, 8<<20) // above the inline limit: data goes to a data dir
+	if _, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
+	restore := stagedDataGoneDisks(z)
+	other := bytes.Repeat([]byte{'b'}, 8<<20)
+	if _, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(other), int64(len(other)), "", ""), ObjectOptions{}); err == nil {
+		t.Error("overwrite succeeded without its staged data")
+	}
+	restore()
+
+	var buf bytes.Buffer
+	if err := GetObject(ctx, z, bucket, object, 0, int64(len(body)), &buf, "", ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(buf.Bytes(), body) {
+		t.Error("existing object content changed")
+	}
+}
+
+// A failed RenameData that runs late removes its object directory while it is
+// empty. When it does so on a drive between the directory creation and the
+// rename of another write of the same key, that write must create the
+// directory again and succeed instead of failing on the drive.
+func TestRenameDataSurvivesConcurrentEmptyDirCleanup(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip("directory creation is not traced on Windows")
+	}
+	for _, tc := range []struct {
+		name      string
+		size      int
+		multipart bool
+	}{
+		{"put", 8 << 20, false},
+		{"inline_put", 1 << 10, false},
+		{"multipart", 8 << 20, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := t.Context()
+			obj, dirs, err := prepareErasure16(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer obj.Shutdown(context.Background())
+			defer removeRoots(dirs)
+			z := obj.(*erasureServerPools)
+			const bucket, object = "bucket", "day=1/b=9/object"
+			if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			body := bytes.Repeat([]byte{'a'}, tc.size)
+			var uploadID, etag string
+			if tc.multipart {
+				res, err := z.NewMultipartUpload(ctx, bucket, object, ObjectOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+				pi, err := z.PutObjectPart(ctx, bucket, object, res.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+				uploadID, etag = res.UploadID, pi.ETag
+			}
+
+			drives := make(map[string]*xlStorage)
+			for _, disk := range z.serverPools[0].sets[0].getDisks() {
+				s := disk.(*xlStorageDiskIDCheck).storage
+				drives[filepath.Join(s.drivePath, bucket, filepath.FromSlash(object))] = s
+			}
+			var mu sync.Mutex
+			hit := make(map[string]bool)
+			var hits atomic.Int32
+			var lateErrs []error
+			traceOSCalls(t, func(tr madmin.TraceInfo) {
+				if tr.FuncName != "os.Mkdir" || tr.Error != "" {
+					return
+				}
+				s := drives[tr.Path]
+				if s == nil {
+					return
+				}
+				mu.Lock()
+				first := !hit[tr.Path]
+				hit[tr.Path] = true
+				mu.Unlock()
+				if !first {
+					return
+				}
+				// The write has just created the object directory on this
+				// drive. A late RenameData of the same key, whose staged data
+				// is gone, fails now and removes the empty directory.
+				fi := newFileInfo(object, 8, 8)
+				fi.DataDir, fi.VersionID = mustGetUUID(), mustGetUUID()
+				fi.Erasure.Index = 1
+				fi.ModTime = time.Now().UTC()
+				fi.Size = 1
+				fi.AddObjectPart(1, "", 1, 1, fi.ModTime, nil, nil)
+				_, err := s.RenameData(context.Background(), minioMetaTmpBucket, mustGetUUID(), fi, bucket, object, RenameOptions{})
+				if _, serr := os.Stat(tr.Path); !os.IsNotExist(serr) {
+					err = fmt.Errorf("late RenameData left %s (%v, rename error %v)", tr.Path, serr, err)
+				} else if errors.Is(err, errFileNotFound) {
+					err = nil
+				}
+				mu.Lock()
+				if err != nil {
+					lateErrs = append(lateErrs, err)
+				}
+				mu.Unlock()
+				hits.Add(1)
+			})
+
+			if tc.multipart {
+				_, err = z.CompleteMultipartUpload(ctx, bucket, object, uploadID, []CompletePart{{PartNumber: 1, ETag: etag}}, ObjectOptions{})
+			} else {
+				_, err = z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+			}
+			if n := hits.Load(); n != int32(len(dirs)) {
+				t.Fatalf("late cleanup ran on %d drives, want %d", n, len(dirs))
+			}
+			mu.Lock()
+			for _, lateErr := range lateErrs {
+				t.Error(lateErr)
+			}
+			mu.Unlock()
+			if err != nil {
+				t.Fatalf("write failed after a concurrent cleanup: %v", err)
+			}
+			var buf bytes.Buffer
+			if err := GetObject(ctx, z, bucket, object, 0, int64(len(body)), &buf, "", ObjectOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(buf.Bytes(), body) {
+				t.Fatal("object content differs")
+			}
+		})
+	}
+}

cmd/erasure-partial-view-404_test.go

@@ -0,0 +1,315 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+// downTier is a remote tier that is unreachable, as a warm backend reports a
+// refused connection after ErrorRespToObjectError.
+type downTier struct{ *tierExpiryTestBackend }
+
+func (downTier) Get(context.Context, string, remoteVersionID, WarmBackendGetOpts) (io.ReadCloser, error) {
+	return nil, BackendDown{Err: "dial tcp 127.0.0.1:9710: connect: connection refused"}
+}
+
+// partialView is an 8-drive, EC 4+4 erasure set (the layout of the 4-node x
+// 2-drive acceptance cluster: positions 0,1 = n1; 2,3 = n2; 4,5 = n3;
+// 6,7 = n4) with an in-memory tier WARM.
+type partialView struct {
+	*restoreTest
+	er       *erasureObjects
+	original func() []StorageAPI
+	all      []StorageAPI
+}
+
+func newPartialView(t *testing.T, state versioning.State) *partialView {
+	t.Helper()
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	dirs, err := getRandomDisks(8)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { removeRoots(dirs) })
+	obj, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	z := obj.(*erasureServerPools)
+	t.Cleanup(func() { z.Shutdown(context.Background()) })
+	if n := len(z.serverPools[0].sets); n != 1 {
+		t.Fatalf("want one erasure set, got %d", n)
+	}
+	globalTierConfigMgr = NewTierConfigMgr()
+	backend := &tierExpiryTestBackend{objects: map[string][]byte{}}
+	globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: backend}
+	r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "t1", labels: map[string]string{}, bodies: map[string][]byte{}}
+	if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: state == versioning.Enabled}); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := globalBucketMetadataSys.Get(r.bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+	if err := meta.Save(t.Context(), z); err != nil {
+		t.Fatal(err)
+	}
+	globalBucketMetadataSys.Set(r.bucket, meta)
+	er := z.serverPools[0].getHashedSet(r.object)
+	if er.setDriveCount != 8 || er.defaultParityCount != 4 {
+		t.Fatalf("want EC 4+4 on 8 drives, got %d drives parity %d", er.setDriveCount, er.defaultParityCount)
+	}
+	p := &partialView{restoreTest: r, er: er, original: er.getDisks, all: er.getDisks()}
+	t.Cleanup(func() { er.getDisks = p.original })
+	return p
+}
+
+// see makes only the drives at the given positions reachable from the
+// erasure set, as for a node whose connections to the others are not yet up.
+func (p *partialView) see(positions ...int) {
+	visible := make([]StorageAPI, len(p.all))
+	for _, i := range positions {
+		visible[i] = p.all[i]
+	}
+	p.er.getDisks = func() []StorageAPI { return visible }
+}
+
+func (p *partialView) seeAll() { p.er.getDisks = p.original }
+
+func (p *partialView) tierDown() {
+	globalTierConfigMgr.drivercache["WARM"] = downTier{&tierExpiryTestBackend{objects: map[string][]byte{}}}
+}
+
+// drives describes the version on each drive: R restored, T transitioned
+// (not restored), L local, - missing.
+func (p *partialView) drives(versionID string) string {
+	p.t.Helper()
+	infos, errs := readAllFileInfo(p.t.Context(), p.all, "", p.bucket, p.object, restoreTestVersionKey(versionID), false, false)
+	var b bytes.Buffer
+	for i := range infos {
+		switch {
+		case errs[i] != nil:
+			b.WriteByte('-')
+		case infos[i].IsRemote():
+			b.WriteByte('T')
+		case infos[i].TransitionStatus != "":
+			b.WriteByte('R')
+		default:
+			b.WriteByte('L')
+		}
+	}
+	return b.String()
+}
+
+func (p *partialView) transition(oi ObjectInfo) {
+	p.t.Helper()
+	topts := p.opts(oi.VersionID)
+	topts.MTime = oi.ModTime
+	topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+	if err := p.z.TransitionObject(p.t.Context(), p.bucket, p.object, topts); err != nil {
+		p.t.Fatal(err)
+	}
+}
+
+func (p *partialView) restoreObject(oi ObjectInfo) {
+	p.t.Helper()
+	ropts := p.opts(oi.VersionID)
+	ropts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+	if err := p.z.serverPools[0].RestoreTransitionedObject(p.t.Context(), p.bucket, p.object, ropts); err != nil {
+		p.t.Fatal(err)
+	}
+}
+
+// getAndHead reads the latest version as an S3 GET and HEAD without a
+// version ID would.
+func (p *partialView) getAndHead() (data []byte, getErr, headErr error) {
+	opts := p.opts("")
+	gr, getErr := p.z.GetObjectNInfo(p.t.Context(), p.bucket, p.object, nil, nil, opts)
+	if getErr == nil {
+		data, getErr = io.ReadAll(gr)
+		gr.Close()
+	}
+	_, headErr = p.z.GetObjectInfo(p.t.Context(), p.bucket, p.object, opts)
+	return data, getErr, headErr
+}
+
+// wantRetryable requires the answer of a read that cannot reach quorum from a
+// partial view: a read-quorum error, which the API returns as 503 SlowDownRead,
+// never a 404 for an object that exists.
+func wantRetryable(t *testing.T, what string, err error) {
+	t.Helper()
+	var rq InsufficientReadQuorum
+	if !errors.As(err, &rq) {
+		t.Errorf("%s: got %T %v, want InsufficientReadQuorum", what, err, err)
+		return
+	}
+	if code := toAPIError(context.Background(), err).HTTPStatusCode; code != http.StatusServiceUnavailable {
+		t.Errorf("%s: HTTP %d, want 503", what, code)
+	}
+}
+
+// The acceptance failure: node1 restores t1 while node2's drives are not
+// reachable, so the restore commits on 6 of 8 drives (write quorum 5). node2
+// then restarts and reaches only its own drives and node4's (n1 and n3 not yet
+// reconnected), with the tier down. The 4 visible xl.meta are 2 transitioned
+// and 2 restored copies of the same version and mod time.
+func TestPartialViewRestoreSplitIsRetryable(t *testing.T) {
+	for _, state := range []versioning.State{versioning.Enabled, ""} {
+		t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+			p := newPartialView(t, state)
+			oi := p.put("v1", state == "", true)
+			p.transition(oi)
+			p.see(0, 1, 4, 5, 6, 7) // node1's view: n2 not reachable
+			p.restoreObject(oi)
+			p.seeAll()
+			if got, want := p.drives(oi.VersionID), "RRTTRRRR"; got != want {
+				t.Fatalf("drive states %s, want %s", got, want)
+			}
+
+			p.tierDown()
+			p.see(2, 3, 6, 7) // node2's view after its restart
+			_, getErr, headErr := p.getAndHead()
+			wantRetryable(t, "GET", getErr)
+			wantRetryable(t, "HEAD", headErr)
+
+			// Every view that contains a quorum of the restored copy reads it.
+			p.see(0, 1, 4, 5, 6, 7)
+			data, getErr, headErr := p.getAndHead()
+			if getErr != nil || headErr != nil || !bytes.Equal(data, p.bodies["v1"]) {
+				t.Errorf("node1 view: GET err=%v HEAD err=%v bytes=%d", getErr, headErr, len(data))
+			}
+		})
+	}
+}
+
+// The same split, made by a transition instead of a restore: a node that sees
+// 2 local and 2 transitioned copies.
+func TestPartialViewTransitionSplitIsRetryable(t *testing.T) {
+	p := newPartialView(t, versioning.Enabled)
+	oi := p.put("v1", false, true)
+	p.see(0, 1, 4, 5, 6, 7)
+	p.transition(oi)
+	p.seeAll()
+	if got, want := p.drives(oi.VersionID), "TTLLTTTT"; got != want {
+		t.Fatalf("drive states %s, want %s", got, want)
+	}
+	p.see(2, 3, 6, 7)
+	_, getErr, headErr := p.getAndHead()
+	wantRetryable(t, "GET", getErr)
+	wantRetryable(t, "HEAD", headErr)
+}
+
+// Generic, without tiering: an overwrite committed on 6 of 8 drives, read by
+// a node that sees 2 old and 2 new copies.
+func TestPartialViewOverwriteSplit(t *testing.T) {
+	for _, state := range []versioning.State{versioning.Enabled, ""} {
+		t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+			p := newPartialView(t, state)
+			p.put("old", state == "", true)
+			p.see(0, 1, 4, 5, 6, 7)
+			p.put("new", state == "", true)
+			p.seeAll()
+			p.see(2, 3, 6, 7)
+			data, getErr, headErr := p.getAndHead()
+			if state == "" {
+				wantRetryable(t, "GET", getErr)
+				wantRetryable(t, "HEAD", headErr)
+				return
+			}
+			// A versioned overwrite can be served from the older version
+			// through such a view (existing behavior, out of scope here),
+			// but must never be reported missing.
+			for what, err := range map[string]error{"GET": getErr, "HEAD": headErr} {
+				if err != nil {
+					wantRetryable(t, what, err)
+				}
+			}
+			if getErr == nil && !bytes.Equal(data, p.bodies["old"]) && !bytes.Equal(data, p.bodies["new"]) {
+				t.Errorf("GET returned neither version (%d bytes)", len(data))
+			}
+		})
+	}
+}
+
+// Controls: what must not change.
+func TestPartialViewControls(t *testing.T) {
+	p := newPartialView(t, versioning.Enabled)
+	// A key that never existed is still 404 from a partial view.
+	p.see(2, 3, 6, 7)
+	if _, getErr, headErr := p.getAndHead(); !isErrObjectNotFound(getErr) || !isErrObjectNotFound(headErr) {
+		t.Errorf("absent key: GET err=%v HEAD err=%v, want ObjectNotFound", getErr, headErr)
+	}
+	// A consistent object is readable from exactly a read quorum of drives.
+	p.seeAll()
+	p.put("v1", false, true)
+	p.see(2, 3, 6, 7)
+	if data, getErr, headErr := p.getAndHead(); getErr != nil || headErr != nil || !bytes.Equal(data, p.bodies["v1"]) {
+		t.Errorf("consistent object: GET err=%v HEAD err=%v", getErr, headErr)
+	}
+}
+
+// Upstream behavior that must stay: when every drive answered and no
+// metadata reaches read quorum, the object is reported missing (#20522).
+func TestAllDrivesAnsweredInconsistentStays404(t *testing.T) {
+	wipe := func(p *partialView, positions ...int) {
+		for _, i := range positions {
+			if err := p.all[i].Delete(t.Context(), p.bucket, p.object, DeleteOptions{Recursive: true, Immediate: true}); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	t.Run("restore", func(t *testing.T) {
+		p := newPartialView(t, versioning.Enabled)
+		oi := p.put("v1", false, true)
+		p.transition(oi)
+		p.see(0, 1, 5, 6, 7)
+		p.restoreObject(oi)
+		p.seeAll()
+		wipe(p, 0, 1)
+		if got, want := p.drives(oi.VersionID), "--TTTRRR"; got != want {
+			t.Fatalf("drive states %s, want %s", got, want)
+		}
+		_, getErr, headErr := p.getAndHead()
+		if !isErrObjectNotFound(getErr) || !isErrObjectNotFound(headErr) {
+			t.Errorf("GET err=%v HEAD err=%v, want ObjectNotFound", getErr, headErr)
+		}
+	})
+	t.Run("unversioned-overwrite", func(t *testing.T) {
+		p := newPartialView(t, "")
+		p.put("old", true, true)
+		p.see(0, 1, 5, 6, 7)
+		p.put("new", true, true)
+		p.seeAll()
+		wipe(p, 0, 1)
+		_, getErr, headErr := p.getAndHead()
+		if !isErrObjectNotFound(getErr) || !isErrObjectNotFound(headErr) {
+			t.Errorf("GET err=%v HEAD err=%v, want ObjectNotFound", getErr, headErr)
+		}
+	})
+}

cmd/erasure-read-generation_test.go

@@ -0,0 +1,309 @@
+//go:build consistency_strict
+
+// Preserved diagnostic for the deferred strict-read policy. This is not a
+// maintenance release gate; failures record intentionally unresolved risks.
+
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	madmin "github.com/minio/madmin-go/v3"
+)
+
+// All metadata is valid and readable. Only the completed generations differ.
+// No timing, missing files, synthetic read error or malformed metadata is needed.
+func TestReadGenerationConflictIsNotAbsence(t *testing.T) {
+	raw := make([]RawFileInfo, 8)
+	for i := range raw {
+		fi := newFileInfo("object", 4, 4)
+		fi.Erasure.Index = i + 1
+		fi.ModTime = time.Unix(100+int64(i%4), 0)
+		fi.DataDir = mustGetUUID()
+		fi.Metadata = map[string]string{"etag": fmt.Sprint(i % 4)}
+		var meta xlMetaV2
+		if err := meta.AddVersion(fi); err != nil {
+			t.Fatal(err)
+		}
+		buf, err := meta.AppendTo(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		raw[i] = RawFileInfo{Buf: buf}
+	}
+	_, errs := pickLatestQuorumFilesInfo(t.Context(), raw, make([]error, len(raw)), "bucket", "object", true, false)
+	err := reduceReadQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 4)
+	if api := toAPIError(t.Context(), toObjectErr(err, "bucket", "object")); api.HTTPStatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("readable conflicting generations reported as absence: %v (%+v)", err, api)
+	}
+}
+
+func TestPoolsReadGenerationConflictDoesNotExposeOldValue(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
+	for _, tc := range []struct {
+		pool int
+		mode string
+	}{
+		{0, "split"},
+		{1, "split"},
+		{0, "older-quorum"},
+		{1, "older-quorum"},
+		{0, "newer-quorum"},
+		{1, "newer-quorum"},
+	} {
+		t.Run(fmt.Sprintf("%s/pool-%d", tc.mode, tc.pool), func(t *testing.T) {
+			broken := tc.pool
+			object := fmt.Sprintf("conflict-%s-%d", tc.mode, broken)
+			base := UTCNow().Add(-time.Hour)
+			putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{MTime: base})
+			set := z.serverPools[broken].getHashedSet(object)
+			disks := set.getDisks()
+			saved := make([][]byte, len(disks))
+			savedParts := make([]map[string][]byte, len(disks))
+			for generation := range 4 {
+				putConsistencyObject(t, z, bucket, object, broken, fmt.Sprintf("newer-%d", generation),
+					ObjectOptions{MTime: base.Add(time.Duration(generation+1) * time.Minute)})
+				for i, disk := range disks {
+					g := i % 4
+					switch tc.mode {
+					case "older-quorum":
+						g = 0
+						if i >= len(disks)-3 {
+							g = i - (len(disks) - 3) + 1
+						}
+					case "newer-quorum":
+						g = 3
+						if i < 3 {
+							g = i
+						}
+					}
+					if g != generation {
+						continue
+					}
+					buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
+					if err != nil {
+						t.Fatal(err)
+					}
+					saved[i] = bytes.Clone(buf)
+					savedParts[i] = readGenerationParts(t, disk, bucket, object, buf)
+				}
+			}
+			for i, disk := range disks {
+				for path, data := range savedParts[i] {
+					if err := disk.WriteAll(ctx, bucket, path, data); err != nil {
+						t.Fatal(err)
+					}
+				}
+				if err := disk.WriteAll(ctx, bucket, pathJoin(object, xlStorageFormatFile), saved[i]); err != nil {
+					t.Fatal(err)
+				}
+			}
+			for _, version := range []string{"", nullVersionID} {
+				for _, method := range []string{http.MethodHead, http.MethodGet} {
+					rec := consistencyRequest(t, router, method, bucket, object, version)
+					want := http.StatusServiceUnavailable
+					if tc.mode == "newer-quorum" {
+						want = http.StatusOK
+					}
+					if rec.Code != want {
+						t.Errorf("%s version=%q: want status=%d, got=%d body=%q", method, version, want, rec.Code, rec.Body.String())
+					}
+					if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != "newer-3" {
+						t.Errorf("latest quorum did not return its payload: %q", rec.Body.String())
+					}
+				}
+			}
+			if tc.mode != "newer-quorum" {
+				for _, version := range []string{"", nullVersionID} {
+					_, err := set.CopyObject(ctx, bucket, object, bucket, object,
+						ObjectInfo{metadataOnly: true, UserDefined: map[string]string{"copy-test": "conflicting-source"}},
+						ObjectOptions{VersionID: version}, ObjectOptions{})
+					if !isErrReadQuorum(err) {
+						t.Errorf("COPY-to-self version=%q: want read-quorum error, got %v", version, err)
+					}
+					for _, scan := range []madmin.HealScanMode{madmin.HealNormalScan, madmin.HealDeepScan} {
+						_, err := set.HealObject(ctx, bucket, object, version, madmin.HealOpts{ScanMode: scan, Remove: true})
+						if !isErrReadQuorum(err) {
+							t.Errorf("heal version=%q scan=%v: want read-quorum error, got %v", version, scan, err)
+						}
+					}
+				}
+			}
+			for i, disk := range disks {
+				buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
+				if err != nil || !bytes.Equal(buf, saved[i]) {
+					t.Errorf("read or heal destroyed conflicting generation on disk %d: %v", i, err)
+				}
+				if tc.mode != "newer-quorum" {
+					for path, data := range savedParts[i] {
+						got, err := disk.ReadAll(ctx, bucket, path)
+						if err != nil || !bytes.Equal(got, data) {
+							t.Errorf("read or heal destroyed generation shard on disk %d: %s: %v", i, path, err)
+						}
+					}
+				}
+			}
+		})
+	}
+}
+
+func readGenerationParts(t *testing.T, disk StorageAPI, bucket, object string, metadata []byte) map[string][]byte {
+	t.Helper()
+	fi, err := fileInfoFromRaw(RawFileInfo{Buf: metadata}, bucket, object, false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	parts := make(map[string][]byte)
+	if fi.InlineData() || fi.Size == 0 {
+		return parts
+	}
+	// Later PUTs remove the previous data directory. Save its real bitrot
+	// shards too, so restoring metadata never fabricates missing-part damage.
+	for _, part := range fi.Parts {
+		path := pathJoin(object, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
+		data, err := disk.ReadAll(t.Context(), bucket, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		parts[path] = data
+	}
+	return parts
+}
+
+func TestPoolsReadUnavailablePoolDoesNotExposeOldValue(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for broken := range 2 {
+		t.Run(fmt.Sprintf("broken-pool-%d", broken), func(t *testing.T) {
+			object := fmt.Sprintf("unavailable-%d", broken)
+			putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{})
+			set := z.serverPools[broken].getHashedSet(object)
+			disks, switchDisks := installListingDiskSwitch(set)
+			defer switchDisks(disks)
+			faulty := append([]StorageAPI(nil), disks...)
+			for i := range faulty {
+				faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+			}
+			switchDisks(faulty)
+			if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil || isErrObjectNotFound(err) {
+				t.Errorf("unknown pool state allowed an old value: %v", err)
+			}
+		})
+	}
+}
+
+type generationReadGateDisk struct {
+	StorageAPI
+	ready   chan<- struct{}
+	release <-chan struct{}
+}
+
+func (d generationReadGateDisk) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
+	fi, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
+	d.ready <- struct{}{}
+	if d.release != nil {
+		<-d.release
+	}
+	return fi, err
+}
+
+func (d generationReadGateDisk) ReadVersion(ctx context.Context, orig, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+	fi, err := d.StorageAPI.ReadVersion(ctx, orig, volume, path, version, opts)
+	d.ready <- struct{}{}
+	if d.release != nil {
+		<-d.release
+	}
+	return fi, err
+}
+
+func TestReadGenerationFastInfoWaitsForSlowDisk(t *testing.T) {
+	for _, version := range []string{"", nullVersionID} {
+		t.Run("version="+version, func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			const object = "slow-newer-generation"
+			set := z.serverPools[0].getHashedSet(object)
+			disks, switchDisks := installListingDiskSwitch(set)
+			defer switchDisks(disks)
+			base := UTCNow().Add(-time.Hour)
+			putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: base})
+			saved := make([][]byte, len(disks)-1)
+			for i := range saved {
+				saved[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
+			}
+			putConsistencyObject(t, z, bucket, object, 0, "new", ObjectOptions{MTime: base.Add(time.Minute)})
+			for i, meta := range saved {
+				if err := disks[i].WriteAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), meta); err != nil {
+					t.Fatal(err)
+				}
+			}
+			ready, release := make(chan struct{}, len(disks)), make(chan struct{})
+			var once sync.Once
+			unblock := func() { once.Do(func() { close(release) }) }
+			defer unblock()
+			wrapped := append([]StorageAPI(nil), disks...)
+			for i, disk := range disks {
+				gate := generationReadGateDisk{StorageAPI: disk, ready: ready}
+				if i == len(disks)-1 {
+					gate.release = release
+				}
+				wrapped[i] = gate
+			}
+			switchDisks(wrapped)
+			done := make(chan error, 1)
+			go func() {
+				_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true, VersionID: version}, false)
+				done <- err
+			}()
+			for range disks {
+				select {
+				case <-ready:
+				case <-time.After(5 * time.Second):
+					t.Fatal("metadata reads did not reach the barrier")
+				}
+			}
+			early := false
+			select {
+			case err := <-done:
+				early = true
+				t.Errorf("fast read returned before the newer disk: %v", err)
+			case <-time.After(100 * time.Millisecond):
+			}
+			unblock()
+			if !early {
+				select {
+				case err := <-done:
+					if !isErrReadQuorum(err) {
+						t.Fatalf("fast read exposed the old generation: %v", err)
+					}
+				case <-time.After(5 * time.Second):
+					t.Fatal("fast read did not finish after the newer disk")
+				}
+			}
+		})
+	}
+}

cmd/erasure-server-pool-consistency.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"maps"
 	"sort"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	madmin "github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/sync/errgroup"
 )
 
 // objectPoolInfos reads the addressed version in every pool, including pools
@@ -62,6 +64,292 @@ func (z *erasureServerPools) objectPoolInfos(ctx context.Context, bucket, object
 	return copies, nil
 }
 
+// checkCurrentPreconditions evaluates the public write conditions in
+// opts.CheckPrecondFn against the logical current object across pools,
+// including draining pools. The pool that receives a write may be empty or
+// hold a stale copy, so its local state cannot decide the condition. A
+// current delete marker reads as an absent key: If-Match then fails with
+// ObjectNotFound and If-None-Match may proceed. An unreadable pool may hold
+// the newest object; it is not absence, and its error is returned without
+// evaluating the condition. The caller must hold the pools-layer object lock
+// and, once the condition is accepted, clear the callback so the destination
+// does not evaluate it again against its local copy.
+func (z *erasureServerPools) checkCurrentPreconditions(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{
+		VersionID:        "", // Compare the current object, not the write's version.
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
+		NoAuditLog:       true,
+	})
+	if err == nil && copies[0].ObjInfo.DeleteMarker {
+		err = toObjectErr(errFileNotFound, bucket, decodeDirObject(object))
+	}
+	switch {
+	case err == nil:
+		if opts.CheckPrecondFn(copies[0].ObjInfo) {
+			return PreConditionFailed{}
+		}
+		return nil
+	case !isErrObjectNotFound(err) && !isErrVersionNotFound(err):
+		return err
+	case opts.HasIfMatch:
+		return err
+	default:
+		return nil
+	}
+}
+
+// checkDataMovementCopies rejects moving a version when another pool holds a
+// newer copy of it, which reads return instead. A cross-pool overwrite can
+// leave such a copy, and a client write can add one while the move copies.
+// Equal timestamps normally identify an interrupted move. Conflicting ties
+// retain the same pool-order winner as reads. While the caller holds the
+// pools-layer object lock, the returned resolver serves the set layer's
+// reconciliation of this version from the same reads. Without opts.NoLock
+// it takes the read lock itself.
+func (z *erasureServerPools) checkDataMovementCopies(ctx context.Context, bucket, object string, opts ObjectOptions) (GetObjectInfoFn, error) {
+	if opts.MTime.IsZero() {
+		// The set layer would stamp the moved copy as the newest one.
+		return nil, errInvalidArgument
+	}
+	if !opts.NoLock {
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return nil, err
+		}
+		ctx = lkctx.Context()
+		defer lk.RUnlock(lkctx)
+	}
+	versionID := opts.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+	copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: versionID, NoAuditLog: true})
+	if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+		return nil, err
+	}
+	// The snapshot may have been purged or replaced after the reader opened.
+	// In that case the walker must leave the current source state alone.
+	sourcePresent := false
+	for _, copy := range copies {
+		if copy.Index == opts.SrcPoolIdx && copy.ObjInfo.ModTime.Equal(opts.MTime) && copy.ObjInfo.DeleteMarker == opts.DeleteMarker {
+			sourcePresent = true
+		}
+	}
+	if !sourcePresent {
+		return nil, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+	}
+	for _, copy := range copies {
+		if copy.Index != opts.SrcPoolIdx && copy.ObjInfo.ModTime.After(opts.MTime) {
+			return nil, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSuperseded}
+		}
+	}
+	if winner := copies[0]; winner.Index != opts.SrcPoolIdx && winner.ObjInfo.ModTime.Equal(opts.MTime) {
+		for _, copy := range copies {
+			if copy.Index == opts.SrcPoolIdx && (copy.ObjInfo.ETag != winner.ObjInfo.ETag || copy.ObjInfo.DeleteMarker != winner.ObjInfo.DeleteMarker || copy.ObjInfo.Size != winner.ObjInfo.Size) {
+				return nil, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSuperseded}
+			}
+		}
+	}
+	return func(ctx context.Context, b, o string, ropts ObjectOptions) (ObjectInfo, error) {
+		if ropts.VersionID == "" {
+			ropts.VersionID = nullVersionID
+		}
+		if b != bucket || o != object || ropts.VersionID != versionID {
+			return z.replicaObjectInfo(ctx, b, o, ropts)
+		}
+		return replicaCopiesInfo(copies, err, bucket, object, versionID)
+	}, nil
+}
+
+type migrationExpiredVersion struct {
+	version            FileInfo
+	event              lifecycle.Event
+	lifecycleUpdatedAt time.Time
+	source             lcEventSrc
+}
+
+// applyMigrationExpiry runs the logical expiry path before migration retires
+// any remaining source copies. This keeps replication and tier cleanup under
+// their normal ownership and makes lifecycle configuration changes fail safe.
+func (z *erasureServerPools) applyMigrationExpiry(ctx context.Context, bucket, object string, source int, expired []migrationExpiredVersion) error {
+	if len(expired) == 0 {
+		return nil
+	}
+
+	lc, lifecycleUpdatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+	if err != nil {
+		return PreConditionFailed{}
+	}
+	for _, pending := range expired {
+		if !pending.lifecycleUpdatedAt.Equal(lifecycleUpdatedAt) {
+			return PreConditionFailed{}
+		}
+	}
+	lr, err := globalBucketObjectLockSys.Get(bucket)
+	if err != nil {
+		return err
+	}
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	encodedObject := encodeDirObject(object)
+	seenDeleteAll := false
+	eventSource := lcEventSrc_Decom
+	for _, pending := range expired {
+		if pending.source != lcEventSrc_None {
+			eventSource = pending.source
+			break
+		}
+	}
+
+	for _, pending := range expired {
+		if pending.event.Action.DeleteAll() {
+			if seenDeleteAll {
+				continue
+			}
+			seenDeleteAll = true
+		}
+
+		versionID := pending.version.VersionID
+		if versionID == "" {
+			versionID = nullVersionID
+		}
+		copies, err := z.objectPoolInfos(ctx, bucket, encodedObject, ObjectOptions{
+			VersionID:  versionID,
+			NoAuditLog: true,
+		})
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			continue
+		}
+		if err != nil {
+			return err
+		}
+
+		current := copies[0].ObjInfo
+		for _, copy := range copies {
+			if copy.Index == source {
+				current = copy.ObjInfo
+				break
+			}
+		}
+		current.Bucket = bucket
+		current.Name = object
+		currentEvent := evalActionFromLifecycle(ctx, *lc, lr, rcfg, current)
+		if currentEvent.Action != pending.event.Action {
+			return PreConditionFailed{}
+		}
+
+		var ok bool
+		if current.TransitionedObject.Status != "" && !pending.event.Action.DeleteAll() {
+			ok = applyExpiryOnTransitionedObject(ctx, z, current, pending.event, eventSource)
+		} else {
+			ok = applyExpiryOnNonTransitionedObjects(ctx, z, current, pending.event, eventSource)
+		}
+		if ok {
+			continue
+		}
+
+		// A concurrent expiry may have removed the addressed version after
+		// the first lookup. Anything still present means the logical delete
+		// was not committed and migration must stop before purging its source.
+		if _, err := z.objectPoolInfos(ctx, bucket, encodedObject, ObjectOptions{VersionID: versionID, NoAuditLog: true}); err == nil {
+			return PreConditionFailed{}
+		} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// removeMovedVersions retires only the source versions in the walker's
+// snapshot. A concurrent delete can create a new marker in the source pool;
+// deleting the whole prefix would remove that marker and expose moved data.
+// Versions in expired were not moved because ILM deletes them. They must stay
+// in the source until the ordinary expiry path has deleted them: that path
+// owns replication state and tier cleanup, and it re-evaluates the current
+// lifecycle rule. A migration cleanup must never turn an asynchronous ILM
+// decision into an unconditional physical purge.
+func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo, expired []migrationExpiredVersion) error {
+	if err := z.applyMigrationExpiry(ctx, bucket, object, source, expired); err != nil {
+		return err
+	}
+	object = encodeDirObject(object)
+	lk := z.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx)
+	for _, version := range versions {
+		vid := version.VersionID
+		if vid == "" {
+			vid = nullVersionID
+		}
+		copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: vid, NoAuditLog: true})
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		var src *ObjectInfo
+		var retained []PoolObjInfo
+		for _, copy := range copies {
+			if copy.Index == source {
+				src = &copy.ObjInfo
+			} else {
+				retained = append(retained, copy)
+			}
+		}
+		if src == nil {
+			continue
+		}
+		if !src.ModTime.Equal(version.ModTime) || src.DeleteMarker != version.Deleted {
+			return PreConditionFailed{}
+		}
+		if isExpiredVersion(expired, version) {
+			// The logical expiry path owns this version. Keeping the source is
+			// also the retry point if that path failed or the pool check races it.
+			continue
+		}
+		// Successful movement is not enough if the destination changed again.
+		// Keep the source unless another copy still preserves its read result.
+		if len(retained) == 0 || retained[0].ObjInfo.ModTime.Before(src.ModTime) {
+			return PreConditionFailed{}
+		}
+		_, err = z.serverPools[source].DeleteObject(ctx, bucket, object, ObjectOptions{
+			VersionID: vid, NoLock: true, NoAuditLog: true,
+			SkipFreeVersion: true, // preserve the old prefix cleanup's tier-GC behavior
+		})
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ilmDeletesVersionOnly reports whether the ILM action queued for a version
+// that data movement skipped deletes that version and nothing else, rather
+// than every version of the object or only the restored copy of a
+// transitioned version. Such a version is passed to removeMovedVersions.
+func ilmDeletesVersionOnly(action lifecycle.Action) bool {
+	return action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction
+}
+
+func isExpiredVersion(expired []migrationExpiredVersion, version FileInfo) bool {
+	for _, pending := range expired {
+		fi := pending.version
+		if fi.VersionID == version.VersionID && fi.ModTime.Equal(version.ModTime) && fi.Deleted == version.Deleted {
+			return true
+		}
+	}
+	return false
+}
+
 // Healing also commits object metadata. Both queued healing and drive healing
 // must use the same namespace as pooled writes, rather than racing them under
 // a destination set's independent namespace.
@@ -137,11 +425,15 @@ func (z *erasureServerPools) replicaObjectInfo(ctx context.Context, bucket, obje
 		opts.VersionID = nullVersionID
 	}
 	copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
+	return replicaCopiesInfo(copies, err, bucket, object, opts.VersionID)
+}
+
+func replicaCopiesInfo(copies []PoolObjInfo, err error, bucket, object, versionID string) (ObjectInfo, error) {
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	if copies[0].ObjInfo.DeleteMarker {
-		return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
+		return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
 	}
 	return mergedPoolObjectInfo(copies), nil
 }
@@ -410,3 +702,124 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
 	oi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
 	return oi, err
 }
+
+// deleteObjectsReconciled removes data while retaining each item's authoritative
+// copy until every other pool has completed its cleanup. The caller holds the
+// pools-layer write locks and separates marker creation from this batch.
+// approved holds the Object Lock state each checked ILM expiry was approved
+// with, see ExpirationOptions.ApprovedProtection.
+func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, approved []objectLockState, opts ObjectOptions) ([]DeletedObject, []error) {
+	deleted := make([]DeletedObject, len(objects))
+	errs := make([]error, len(objects))
+	primary := make([]int, len(objects))
+	tiered := make([]bool, len(objects))
+	lookup := errgroup.WithNErrs(len(objects)).WithConcurrency(32)
+	for i, obj := range objects {
+		// Preserve the full request identity even if lookup or cleanup fails.
+		// The HTTP handler includes replication state in its response-index key.
+		deleted[i] = DeletedObject{
+			ObjectName:       obj.ObjectName,
+			VersionID:        obj.VersionID,
+			ReplicationState: obj.ReplicationState(),
+		}
+		primary[i] = -1
+		lookup.Go(func() error {
+			copies, err := z.objectPoolInfos(ctx, bucket, obj.ObjectName, ObjectOptions{VersionID: obj.VersionID, NoAuditLog: true})
+			if err == nil {
+				primary[i] = copies[0].Index
+				for _, copy := range copies {
+					tiered[i] = tiered[i] || copy.ObjInfo.TransitionedObject.Status == lifecycle.TransitionComplete
+				}
+			} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+				if isErrReadQuorum(err) {
+					err = InsufficientWriteQuorum{Bucket: bucket, Object: decodeDirObject(obj.ObjectName)}
+				}
+				errs[i] = err
+			}
+			return nil
+		}, i)
+	}
+	lookup.Wait()
+
+	type poolBatch struct {
+		objects []ObjectToDelete
+		indices []int
+		deleted []DeletedObject
+		errs    []error
+	}
+	for phase := 0; phase < 2; phase++ {
+		batches := make([]poolBatch, len(z.serverPools))
+		for i, obj := range objects {
+			if errs[i] != nil || tiered[i] {
+				continue
+			}
+			for p := range z.serverPools {
+				if (p == primary[i]) != (phase == 1) {
+					continue
+				}
+				batches[p].objects = append(batches[p].objects, obj)
+				batches[p].indices = append(batches[p].indices, i)
+			}
+		}
+		// Each phase still uses the set layer's bulk disk I/O and runs pools
+		// concurrently. Include apparently absent pools in cleanup: read-quorum
+		// absence is not proof that an offline drive holds no remaining copy.
+		group := errgroup.WithNErrs(len(z.serverPools))
+		for p, pool := range z.serverPools {
+			if len(batches[p].objects) == 0 {
+				continue
+			}
+			group.Go(func() error {
+				batches[p].deleted, batches[p].errs = pool.DeleteObjects(ctx, bucket, batches[p].objects, opts)
+				return nil
+			}, p)
+		}
+		group.Wait()
+		for _, batch := range batches {
+			for j, i := range batch.indices {
+				if errs[i] != nil {
+					continue
+				}
+				if batch.errs[j] != nil {
+					errs[i] = batch.errs[j]
+				} else if phase == 1 || primary[i] == -1 {
+					deleted[i] = batch.deleted[j]
+				}
+			}
+		}
+	}
+	// Tiered copies can share one remote object, including restored copies.
+	// Reuse the existing reconciliation which suppresses premature tier GC.
+	// Serializing this uncommon path also keeps duplicate key/version entries
+	// in one request from racing their metadata updates under the shared lock.
+	for i, obj := range objects {
+		if !tiered[i] || errs[i] != nil {
+			continue
+		}
+		deleteOpts := opts
+		deleteOpts.VersionID, deleteOpts.NoLock = obj.VersionID, true
+		// DeleteObjects already checked ordinary ILM UUID/explicit null
+		// versions against all pools. Empty IDs and user deletes have no such
+		// authorization; restored-copy expiry uses the single-object path.
+		deleteOpts.Expiration.CheckedAllPools = opts.Expiration.Expire && obj.VersionID != ""
+		if deleteOpts.Expiration.CheckedAllPools {
+			deleteOpts.Expiration.ApprovedProtection = approved[i]
+		}
+		deleteOpts.DeleteReplication = obj.ReplicationState()
+		if opts.PrefixEnabledFn != nil {
+			deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName)
+		}
+		_, err := z.deleteObjectReconciled(ctx, bucket, obj.ObjectName, deleteOpts)
+		if errors.Is(err, errLifecycleNotApplicable) {
+			// A later lifecycle rejection did not commit a local purge. Do
+			// not let the worker replicate the request's pending decision.
+			deleted[i].ReplicationState = ReplicationState{}
+		}
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			errs[i] = err
+		} else {
+			deleted[i].found = true
+		}
+	}
+	return deleted, errs
+}

cmd/erasure-server-pool-consistency_test.go

@@ -522,10 +522,20 @@ func TestPoolsDeleteVersionSpecialCalls(t *testing.T) {
 			opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID, InclFreeVersions: !expiration, Expiration: ExpirationOptions{Expire: expiration}}
 			_, err := z.DeleteObject(t.Context(), bucket, object, opts)
 			if expiration {
-				// No lifecycle rule authorizes expiration. Preserve its existing
-				// version-not-found result, even with an unrelated unreadable pool.
-				if !isErrVersionNotFound(err) {
-					t.Errorf("expiration changed its scanner contract: %v", err)
+				// No rule authorizes expiration. Distinguish refusal from real
+				// absence, even with an unrelated unreadable pool, so a worker
+				// cannot acknowledge or replicate a purge that never happened.
+				if !errors.Is(err, errLifecycleNotApplicable) {
+					t.Errorf("expiration did not preserve its rejection: %v", err)
+				}
+				gr, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: oi.VersionID})
+				if err != nil {
+					t.Fatal(err)
+				}
+				data, err := io.ReadAll(gr)
+				gr.Close()
+				if err != nil || string(data) != "payload" || gr.ObjInfo.VersionID != oi.VersionID {
+					t.Errorf("rejected expiration changed the stored version: %v", err)
 				}
 			} else if err != nil {
 				t.Errorf("free-version cleanup was forced through all-pool resolution: %v", err)
@@ -1071,6 +1081,8 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		restored          bool
 		unconditional     bool
 		skipFreeVersion   bool
+		bulk              bool
+		unversioned       bool
 	}{
 		{name: "metadata-copy"},
 		{name: "restored-metadata-copy", restored: true},
@@ -1081,6 +1093,14 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		{name: "ordinary-failed-primary-delete", deleting: true, unconditional: true, failPrimaryDelete: true},
 		{name: "ordinary-successful-delete", deleting: true, unconditional: true},
 		{name: "ordinary-skip-free-version", deleting: true, unconditional: true, skipFreeVersion: true},
+		{name: "bulk-failed-primary-delete", deleting: true, bulk: true, failPrimaryDelete: true},
+		{name: "bulk-restored-failed-primary-delete", deleting: true, bulk: true, restored: true, failPrimaryDelete: true},
+		{name: "bulk-successful-delete", deleting: true, bulk: true},
+		{name: "bulk-distinct-reference", deleting: true, bulk: true, differentRemote: true},
+		{name: "bulk-skip-free-version", deleting: true, bulk: true, skipFreeVersion: true},
+		{name: "unversioned-failed-primary-delete", deleting: true, unconditional: true, unversioned: true, failPrimaryDelete: true},
+		{name: "unversioned-successful-delete", deleting: true, unconditional: true, unversioned: true},
+		{name: "bulk-unversioned-failed-primary-delete", deleting: true, bulk: true, unversioned: true, failPrimaryDelete: true},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			z, bucket := consistencyPools(t)
@@ -1094,13 +1114,13 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 			if test.restored {
 				metadata[xhttp.AmzRestore] = completedRestoreObj(time.Now().Add(time.Hour)).String()
 			}
-			oi := putConsistencyObject(t, z, bucket, object, 0, "data", ObjectOptions{Versioned: true, UserDefined: metadata})
+			oi := putConsistencyObject(t, z, bucket, object, 0, "data", ObjectOptions{Versioned: !test.unversioned, UserDefined: metadata})
 			secondaryMetadata := maps.Clone(metadata)
 			if test.differentRemote {
 				secondaryMetadata[ReservedMetadataPrefixLower+TransitionedObjectName] = "other-remote-object"
 			}
 			putConsistencyObject(t, z, bucket, object, 1, "data", ObjectOptions{
-				Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: secondaryMetadata,
+				Versioned: !test.unversioned, VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: secondaryMetadata,
 			})
 			current, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
 			if err != nil || current.TransitionedObject.Status != "complete" || current.IsRemote() == test.restored {
@@ -1113,7 +1133,11 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 				set := z.serverPools[0].getHashedSet(object)
 				getDisks := set.getDisks
 				faulty := append([]StorageAPI(nil), getDisks()...)
+				var failed, attempted atomic.Int32
 				for i := range faulty {
+					if test.bulk {
+						faulty[i] = bulkDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
+					}
 					faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
 				}
 				set.getDisks = func() []StorageAPI { return faulty }
@@ -1121,13 +1145,20 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 			}
 			if test.deleting {
 				opts := ObjectOptions{
-					Versioned: true, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
+					Versioned: !test.unversioned, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
 					CheckPrecondFn: func(info ObjectInfo) bool { return info.ETag != current.ETag },
 				}
 				if test.unconditional {
 					opts.CheckPrecondFn = nil
 				}
-				_, err := z.DeleteObject(t.Context(), bucket, object, opts)
+				var err error
+				if test.bulk {
+					opts.CheckPrecondFn = nil
+					_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: oi.VersionID}}}, opts)
+					err = errs[0]
+				} else {
+					_, err = z.DeleteObject(t.Context(), bucket, object, opts)
+				}
 				if (err != nil) != test.failPrimaryDelete {
 					t.Fatalf("unexpected authoritative delete result: %v", err)
 				}
@@ -1145,7 +1176,7 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 			_, err = z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
 			primaryDeleted := test.deleting && !test.failPrimaryDelete
 			if primaryDeleted {
-				if !isErrVersionNotFound(err) {
+				if !isErrVersionNotFound(err) && (!test.unversioned || !isErrObjectNotFound(err)) {
 					t.Fatalf("authoritative copy survived successful delete: %v", err)
 				}
 			} else if err != nil {
@@ -1306,3 +1337,182 @@ func TestPoolsDeleteDirectoryMarker(t *testing.T) {
 		}
 	}
 }
+
+// An unconditional DELETE of an object without versions removes its copies in
+// every pool. When the delete fails in one pool, the latest copy must stay in
+// place: removing it anyway serves an older write of the object.
+func TestPoolsDeleteUnversionedReportsOtherPoolFailure(t *testing.T) {
+	for _, offline := range []bool{false, true} {
+		for latest := range 2 {
+			t.Run(fmt.Sprintf("offline=%v/latest=%d", offline, latest), func(t *testing.T) {
+				z, bucket := consistencyPools(t)
+				const object = "unversioned-failure"
+				older := 1 - latest
+				putConsistencyObject(t, z, bucket, object, older, "older", ObjectOptions{MaxParity: true, MTime: time.Now().Add(-time.Hour)})
+				oi := putConsistencyObject(t, z, bucket, object, latest, "latest", ObjectOptions{MaxParity: true})
+				set := z.serverPools[older].getHashedSet(object)
+				getDisks := set.getDisks
+				disks := append([]StorageAPI(nil), getDisks()...)
+				for i := range disks {
+					switch {
+					case !offline:
+						disks[i] = consistencyDeleteFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
+					case i < len(disks)/2:
+						// An EC 8+8 copy stays readable, but a delete needs 9 drives.
+						disks[i] = nil
+					}
+				}
+				set.getDisks = func() []StorageAPI { return disks }
+				defer func() { set.getDisks = getDisks }()
+				if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err == nil {
+					t.Error("delete reported success despite the other pool's write failure")
+				}
+				set.getDisks = getDisks
+				if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err != nil || got.ETag != oi.ETag {
+					t.Errorf("after the failed delete the object reads ETag %q, %v; want the latest write %q", got.ETag, err, oi.ETag)
+				}
+			})
+		}
+	}
+}
+
+// consistencyBulkDeleteFaultDisk fails the multi-object deletes of one object.
+type consistencyBulkDeleteFaultDisk struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d consistencyBulkDeleteFaultDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
+	errs := make([]error, len(versions))
+	var rest []FileInfoVersions
+	var restIdx []int
+	for i, v := range versions {
+		if volume == d.bucket && v.Name == d.object {
+			errs[i] = errDiskFull
+			continue
+		}
+		rest = append(rest, v)
+		restIdx = append(restIdx, i)
+	}
+	if len(rest) > 0 {
+		for i, err := range d.StorageAPI.DeleteVersions(ctx, volume, rest, opts) {
+			errs[restIdx[i]] = err
+		}
+	}
+	return errs
+}
+
+// A multi-object delete must not report an object or version as deleted while
+// a pool that failed to delete it still holds a copy.
+func TestPoolsDeleteObjectsReportsOtherPoolFailure(t *testing.T) {
+	for _, versioned := range []bool{false, true} {
+		t.Run(fmt.Sprintf("versioned=%v", versioned), func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			const object = "bulk-delete-failure"
+			opts := ObjectOptions{Versioned: versioned}
+			oi := putConsistencyObject(t, z, bucket, object, 0, "latest", opts)
+			if versioned {
+				// The copy a data movement leaves until it removes the source.
+				putConsistencyObject(t, z, bucket, object, 1, "latest", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime})
+			} else {
+				putConsistencyObject(t, z, bucket, object, 1, "older", ObjectOptions{MTime: time.Now().Add(-time.Hour)})
+			}
+			set := z.serverPools[1].getHashedSet(object)
+			getDisks := set.getDisks
+			faulty := append([]StorageAPI(nil), getDisks()...)
+			for i := range faulty {
+				faulty[i] = consistencyBulkDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+			}
+			set.getDisks = func() []StorageAPI { return faulty }
+			defer func() { set.getDisks = getDisks }()
+			_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: oi.VersionID}}}, opts)
+			if errs[0] == nil {
+				t.Error("multi-object delete reported success despite the other pool's write failure")
+			}
+		})
+	}
+}
+
+// Pools delete a multi-object batch in parallel. With half the drives of the
+// older copy's set offline, an EC 8+8 copy stays readable but cannot be
+// deleted: the batch must fail the object and keep its latest copy, instead of
+// deleting it and serving the older write once the drives are back.
+func TestPoolsDeleteObjectsKeepsLatestWithoutDeleteQuorum(t *testing.T) {
+	for latest := range 2 {
+		t.Run(fmt.Sprintf("latest=%d", latest), func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			const object = "bulk-delete-quorum"
+			older := 1 - latest
+			putConsistencyObject(t, z, bucket, object, older, "older", ObjectOptions{MaxParity: true, MTime: time.Now().Add(-time.Hour)})
+			oi := putConsistencyObject(t, z, bucket, object, latest, "latest", ObjectOptions{MaxParity: true})
+			set := z.serverPools[older].getHashedSet(object)
+			getDisks := set.getDisks
+			disks := append([]StorageAPI(nil), getDisks()...)
+			for i := range len(disks) / 2 {
+				disks[i] = nil
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			defer func() { set.getDisks = getDisks }()
+			_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
+			if !errors.As(errs[0], &InsufficientWriteQuorum{}) {
+				t.Errorf("multi-object delete: got %v, want InsufficientWriteQuorum", errs[0])
+			}
+			set.getDisks = getDisks
+			if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err != nil || got.ETag != oi.ETag {
+				t.Errorf("after the failed delete the object reads ETag %q, %v; want the latest write %q", got.ETag, err, oi.ETag)
+			}
+			// Nothing was deleted, so the retry removes every copy.
+			if _, errs = z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{}); errs[0] != nil {
+				t.Fatal(errs[0])
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
+					t.Errorf("pool %d retained the object after the retry: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+// A delete marker hides the object in every pool. When the versions of an
+// object are in two pools and the set of one of them cannot take a write, the
+// marker goes to the other pool instead of failing the delete. The versions
+// stay below the marker.
+func TestPoolsDeleteMarkerSkipsSetWithoutQuorum(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		degraded int // pool whose set loses drives
+		offline  int
+	}{
+		{"older version readable", 0, 8},
+		{"older version unreadable", 0, 9},
+		{"latest version readable", 1, 8},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			const object = "marker-quorum"
+			v1 := putConsistencyObject(t, z, bucket, object, 0, "v1", ObjectOptions{Versioned: true, MaxParity: true, MTime: time.Now().Add(-time.Hour)})
+			v2 := putConsistencyObject(t, z, bucket, object, 1, "v2", ObjectOptions{Versioned: true, MaxParity: true})
+			set := z.serverPools[tc.degraded].getHashedSet(object)
+			getDisks := set.getDisks
+			disks := append([]StorageAPI(nil), getDisks()...)
+			for i := range tc.offline {
+				disks[i] = nil
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			defer func() { set.getDisks = getDisks }()
+			if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true}); err != nil {
+				t.Fatalf("delete with the set of pool %d without quorum: %v", tc.degraded, err)
+			}
+			set.getDisks = getDisks
+			if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) && !isErrMethodNotAllowed(err) {
+				t.Errorf("object still readable after the delete: %v", err)
+			}
+			for pool, v := range []ObjectInfo{v1, v2} {
+				if _, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: v.VersionID}); err != nil {
+					t.Errorf("pool %d lost version %s: %v", pool, v.VersionID, err)
+				}
+			}
+		})
+	}
+}

cmd/erasure-server-pool-decom.go

@@ -182,7 +182,9 @@ func (p *poolMeta) returnResumablePools() []PoolStatus {
 }
 
 func (p *poolMeta) DecommissionComplete(idx int) bool {
-	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Complete {
+	if p.Pools[idx].Decommission != nil &&
+		!p.Pools[idx].Decommission.Complete &&
+		!p.Pools[idx].Decommission.Canceled {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.Complete = true
 		p.Pools[idx].Decommission.Failed = false
@@ -193,7 +195,9 @@ func (p *poolMeta) DecommissionComplete(idx int) bool {
 }
 
 func (p *poolMeta) DecommissionFailed(idx int) bool {
-	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Failed {
+	if p.Pools[idx].Decommission != nil &&
+		!p.Pools[idx].Decommission.Failed &&
+		!p.Pools[idx].Decommission.Canceled {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.StartTime = time.Time{}
 		p.Pools[idx].Decommission.Complete = false
@@ -616,6 +620,12 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
 	}
 
 	if objInfo.isMultipart() {
+		// Completion repeats this check under the object write lock.
+		if _, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(objInfo.Name), ObjectOptions{
+			VersionID: objInfo.VersionID, MTime: objInfo.ModTime, SrcPoolIdx: idx,
+		}); err != nil {
+			return err
+		}
 		res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:    objInfo.VersionID,
 			UserDefined:  migrationObjectMetadata(objInfo),
@@ -656,6 +666,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
+			VersionID:    objInfo.VersionID,
 			SrcPoolIdx:   idx,
 			DataMovement: true,
 			MTime:        objInfo.ModTime,
@@ -762,6 +773,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	var vc *versioning.Versioning
 	var lc *lifecycle.Lifecycle
+	var lifecycleUpdatedAt time.Time
 	var lr objectlock.Retention
 	var rcfg *replication.Config
 	if bi.Name != minioMetaBucket {
@@ -771,7 +783,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		}
 
 		// Check if the current bucket has a configured lifecycle policy
-		lc, err = globalLifecycleSys.Get(bi.Name)
+		lc, lifecycleUpdatedAt, err = globalBucketMetadataSys.GetLifecycleConfig(bi.Name)
 		if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
 			return err
 		}
@@ -789,23 +801,41 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	}
 
 	for setIdx, set := range pool.sets {
-		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
+		// filterLifecycle returns an ILM deletion that migration must handle
+		// before retiring the source copy, or NoneAction.
+		filterLifecycle := func(bucket, object string, fi FileInfo, history *migrationHistory) lifecycle.Action {
 			if lc == nil {
-				return false
+				return lifecycle.NoneAction
 			}
+			history.loadTier(ctx, z, bucket, object)
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
 			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			switch {
-			case evt.Action.DeleteRestored(): // if restored copy has expired, delete it synchronously
+			case evt.Action.DeleteRestored():
+				// A restored copy past its deadline is removed now, before the
+				// version moves, as the scanner would. When that removal is
+				// refused, the version moves like any other, with its tier
+				// metadata and its deadline, and the scanner of the pool that
+				// holds it then removes the copy. A version the walk neither
+				// moves nor expires leaves the key behind, and fails it.
 				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
-				return false
+				return lifecycle.NoneAction
+			case evt.Action == lifecycle.DeleteAction && versioned:
+				// Expiring the latest version of a versioned bucket only puts
+				// a delete marker on top of it. The version itself survives as
+				// a noncurrent version and has to be moved like any other. The
+				// scanner applies the expiry once it lives in another pool.
+				return lifecycle.NoneAction
+			case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt, history):
+				// Another pool holds versions with which the rule does not
+				// apply. The version moves like any other.
+				return lifecycle.NoneAction
 			case evt.Action.Delete():
-				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom)
-				return true
+				return evt.Action
 			default:
-				return false
+				return lifecycle.NoneAction
 			}
 		}
 
@@ -825,27 +855,32 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
-			var decommissioned, expired int
+			var decommissioned int
+			var expired []migrationExpiredVersion
+			// The key's history in all pools, read once for its versions.
+			var history migrationHistory
 			for _, version := range fivs.Versions {
 				stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 				// Apply lifecycle rules on the objects that are expired.
-				if filterLifecycle(bi.Name, version.Name, version) {
-					expired++
+				if action := filterLifecycle(bi.Name, version.Name, version, &history); action.Delete() {
 					decommissioned++
+					if ilmDeletesVersionOnly(action) || (action.DeleteAll() && !version.IsRemote()) {
+						// Run the logical expiry before retiring the source.
+						// A full-object expiry is safe here only for hot-tier
+						// metadata; transitioned objects retain their tier cleanup
+						// ownership in the ordinary expiry worker.
+						expired = append(expired, migrationExpiredVersion{
+							version:            version,
+							event:              lifecycle.Event{Action: action},
+							lifecycleUpdatedAt: lifecycleUpdatedAt,
+							source:             lcEventSrc_Decom,
+						})
+					}
 					stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
 					continue
 				}
 
-				// any object with only single DEL marker we don't need
-				// to decommission, just skip it, this also includes
-				// any other versions that have already expired.
-				remainingVersions := len(fivs.Versions) - expired
-				if version.Deleted && remainingVersions == 1 && rcfg == nil {
-					decommissioned++
-					stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
-					continue
-				}
-
+				// Preserve every marker: older versions may live in another pool.
 				versionID := version.VersionID
 				if versionID == "" {
 					versionID = nullVersionID
@@ -870,6 +905,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 							NoAuditLog:         true,
 						})
+					if isDataMovementSuperseded(err) {
+						err = nil
+					}
 					if err != nil {
 						// This can happen when rebalance stop races with ongoing rebalance workers.
 						// These rebalance failures can be ignored.
@@ -898,17 +936,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				// gr.Close() is ensured by decommissionObject().
 				for range 3 {
 					if version.IsRemote() {
-						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
+						err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
 							VersionID:    versionID,
 							MTime:        version.ModTime,
 							UserDefined:  version.Metadata,
 							SrcPoolIdx:   idx,
 							DataMovement: true,
-						}); err != nil {
-							if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
-								ignore = true
-								stopFn(0, nil)
-							}
+						})
+						if isDataMovementSuperseded(err) {
+							err = nil
+						}
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
 						}
 						if !ignore {
 							stopFn(version.Size, err)
@@ -949,7 +989,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						stopFn(version.Size, err)
 						continue
 					}
-					if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil {
+					if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil && !isDataMovementSuperseded(err) {
 						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
 							ignore = true
 							stopFn(0, nil)
@@ -979,15 +1019,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// if all versions were decommissioned, then we can delete the object versions.
 			if decommissioned == len(fivs.Versions) {
 				stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
-				_, err := set.DeleteObject(ctx,
-					bi.Name,
-					encodeDirObject(entry.name),
-					ObjectOptions{
-						DeletePrefix:       true, // use prefix delete to delete all versions at once.
-						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-						NoAuditLog:         true,
-					},
-				)
+				err := z.removeMovedVersions(ctx, bi.Name, entry.name, idx, fivs.Versions, expired)
+				if err == nil {
+					err = z.removeFreeVersions(ctx, bi.Name, entry.name, idx, fivs.FreeVersions)
+				}
 				stopFn(0, err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
@@ -1072,12 +1107,26 @@ func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(z i
 	}
 }
 
+// decommissionInBackground decommissions the pending buckets of pool idx. A
+// bucket that cannot be finished stays queued and does not stop the others:
+// the first such error is returned after the last bucket, so the
+// decommission fails and a retry resumes with what is left. A canceled
+// context returns at once.
 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
 	z.poolMetaMutex.RLock()
 	pending := z.poolMeta.PendingBuckets(idx)
 	z.poolMetaMutex.RUnlock()
 
+	var firstErr error
+	failed := func(bucket decomBucketInfo, err error) {
+		if firstErr == nil {
+			firstErr = err
+			return
+		}
+		// Only the first error is returned; log the others.
+		decomLogIf(ctx, fmt.Errorf("decommission of bucket %s: %w", bucket, err))
+	}
 	for _, bucket := range pending {
 		z.poolMetaMutex.RLock()
 		isDecommissioned := z.poolMeta.isBucketDecommissioned(idx, bucket.String())
@@ -1101,7 +1150,24 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name)
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
 			stopFn(0, err)
-			return err
+			if contextCanceled(ctx) {
+				return err
+			}
+			failed(bucket, err)
+			continue
+		}
+		// Do not retire the bucket from the resumable queue until every
+		// source version and free version is gone. In particular, an ILM
+		// action queued during the walk may still be pending; marking the
+		// bucket complete here would allow the source pool to be removed
+		// before that action performs its logical and tier cleanup.
+		if err := z.checkAfterDecomBucket(ctx, idx, bucket); err != nil {
+			stopFn(0, err)
+			if contextCanceled(ctx) {
+				return err
+			}
+			failed(bucket, err)
+			continue
 		}
 		stopFn(0, nil)
 
@@ -1111,6 +1177,67 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		}
 		z.poolMetaMutex.Unlock()
 	}
+	return firstErr
+}
+
+// removeFreeVersions removes the free versions fvs of object from the pool
+// idx being decommissioned, as the scanner removes free versions: first the
+// remote object of each, then the free version. A free version is not moved.
+// In another pool it would be a delete marker on top of the object, and once
+// this pool is gone nothing would remove its remote object. A remote object
+// that is already gone is not an error.
+func (z *erasureServerPools) removeFreeVersions(ctx context.Context, bucket, object string, idx int, fvs []FileInfo) error {
+	if len(fvs) == 0 {
+		return nil
+	}
+	for _, fv := range fvs {
+		err := z.deleteTierTuple(ctx, bucket, object, tierRemoteTuple{
+			Tier:      fv.TransitionTier,
+			Object:    fv.TransitionedObjName,
+			VersionID: fv.TransitionVersionID,
+		})
+		if errors.Is(err, errTierNotFound) {
+			// The free version is the only record of its remote object. It
+			// stays until the tier can be reached again.
+			return fmt.Errorf("unable to remove the free version %s of %s/%s: its remote object %s is in tier %s, which is not configured; "+
+				"configure tier %s again under the same name so that the remote object and the free version can be removed: %w",
+				fv.VersionID, bucket, object, fv.TransitionedObjName, fv.TransitionTier, fv.TransitionTier, err)
+		}
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return fmt.Errorf("unable to remove the remote object %s of a free version of %s/%s from tier %s: %w",
+				fv.TransitionedObjName, bucket, object, fv.TransitionTier, err)
+		}
+	}
+	return nil
+}
+
+// checkAfterDecomBucket verifies that a source bucket is physically empty
+// before it is removed from the resumable decommission queue. ILM decisions
+// are deliberately not treated as an exemption: an expired version remains
+// until the expiry path has performed its logical delete, replication, and
+// tier cleanup.
+func (z *erasureServerPools) checkAfterDecomBucket(ctx context.Context, idx int, bi decomBucketInfo) error {
+	versionsFound := 0
+	for _, set := range z.serverPools[idx].sets {
+		if err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
+			if !entry.isObject() {
+				return
+			}
+			if bi.Name == minioMetaBucket && strings.Contains(entry.name, dataUsageCacheName) {
+				return
+			}
+			fivs, err := entry.fileInfoVersions(bi.Name)
+			if err != nil {
+				return
+			}
+			versionsFound += len(fivs.Versions) + len(fivs.FreeVersions)
+		}); err != nil {
+			return err
+		}
+	}
+	if versionsFound > 0 {
+		return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name)
+	}
 	return nil
 }
 
@@ -1119,100 +1246,11 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 	if err != nil {
 		return err
 	}
-
-	pool := z.serverPools[idx]
-	for _, set := range pool.sets {
-		for _, bi := range buckets {
-			var vc *versioning.Versioning
-			var lc *lifecycle.Lifecycle
-			var lr objectlock.Retention
-			var rcfg *replication.Config
-			if bi.Name != minioMetaBucket {
-				vc, err = globalBucketVersioningSys.Get(bi.Name)
-				if err != nil {
-					return err
-				}
-
-				// Check if the current bucket has a configured lifecycle policy
-				lc, err = globalLifecycleSys.Get(bi.Name)
-				if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
-					return err
-				}
-
-				// Check if bucket is object locked.
-				lr, err = globalBucketObjectLockSys.Get(bi.Name)
-				if err != nil {
-					return err
-				}
-
-				rcfg, err = getReplicationConfig(ctx, bi.Name)
-				if err != nil {
-					return err
-				}
-			}
-
-			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
-				if lc == nil {
-					return false
-				}
-				versioned := vc != nil && vc.Versioned(object)
-				objInfo := fi.ToObjectInfo(bucket, object, versioned)
-
-				evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
-				switch {
-				case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
-					applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
-					return false
-				case evt.Action.Delete():
-					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom)
-					return true
-				default:
-					return false
-				}
-			}
-
-			var versionsFound int
-			if err = set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
-				if !entry.isObject() {
-					return
-				}
-
-				// `.usage-cache.bin` still exists, must be not readable ignore it.
-				if bi.Name == minioMetaBucket && strings.Contains(entry.name, dataUsageCacheName) {
-					// skipping bucket usage cache name, as its autogenerated.
-					return
-				}
-
-				fivs, err := entry.fileInfoVersions(bi.Name)
-				if err != nil {
-					return
-				}
-
-				var ignored int
-
-				for _, version := range fivs.Versions {
-					// Apply lifecycle rules on the objects that are expired.
-					if filterLifecycle(bi.Name, version.Name, version) {
-						ignored++
-						continue
-					}
-					if version.Deleted {
-						ignored++
-						continue
-					}
-				}
-
-				versionsFound += len(fivs.Versions) - ignored
-			}); err != nil {
-				return err
-			}
-
-			if versionsFound > 0 {
-				return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name)
-			}
+	for _, bi := range buckets {
+		if err := z.checkAfterDecomBucket(ctx, idx, bi); err != nil {
+			return err
 		}
 	}
-
 	return nil
 }
 

cmd/erasure-server-pool-decom_test.go

@@ -197,3 +197,29 @@ func TestPoolMetaValidate(t *testing.T) {
 		})
 	}
 }
+
+func TestPoolMetaDecommissionCancellationIsTerminal(t *testing.T) {
+	t.Parallel()
+
+	canceled := poolMeta{Pools: []PoolStatus{{Decommission: &PoolDecommissionInfo{}}}}
+	if !canceled.DecommissionCancel(0) {
+		t.Fatal("failed to record cancellation")
+	}
+	if canceled.DecommissionFailed(0) {
+		t.Fatal("a canceled decommission was changed to failed")
+	}
+	if canceled.DecommissionComplete(0) {
+		t.Fatal("a canceled decommission was changed to complete")
+	}
+	if got := canceled.returnResumablePools(); len(got) != 0 {
+		t.Fatalf("canceled decommission is resumable: %+v", got)
+	}
+
+	failed := poolMeta{Pools: []PoolStatus{{Decommission: &PoolDecommissionInfo{}}}}
+	if !failed.DecommissionFailed(0) {
+		t.Fatal("failed to record failure")
+	}
+	if got := failed.returnResumablePools(); len(got) != 1 {
+		t.Fatalf("failed decommission is not resumable: %+v", got)
+	}
+}