docs: link reviewed runbook sources before website deployment

Signed-off-by: Feng Ruohang <rh@vonng.com>

.github/workflows/docker-release.yml

@@ -140,7 +140,7 @@ jobs:
             mkdir -p "${context}/dockerscripts"
             tar -xzf "${archive}" -C "${context}" silo
             cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
-            cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
+            cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
               "${context}/dockerscripts/"
           done
 

.github/workflows/test-release.yml

@@ -10,7 +10,7 @@ on:
       - "Dockerfile.distroless"
       - "cmd/healthcheck-main.go"
       - "cmd/main.go"
-      - "dockerscripts/download-static-curl.sh"
+      - "dockerscripts/build-static-curl.sh"
       - "dockerscripts/docker-entrypoint.sh"
       - "dockerscripts/docker-entrypoint_test.sh"
       - "silo.service"
@@ -41,6 +41,28 @@ permissions:
   contents: read
 
 jobs:
+  curl:
+    name: Static curl (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-latest
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v7
+      - name: Build and exercise curl in an empty runtime
+        run: |
+          docker build --build-arg TARGETARCH=${{ matrix.arch }} \
+            --target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
+          docker run --rm silo-curl-test --version | tee curl-version.txt
+          grep -F 'curl 8.22.0 ' curl-version.txt
+          grep -F 'HTTP2' curl-version.txt
+          docker run --rm silo-curl-test --fail --silent --show-error \
+            --connect-timeout 15 --max-time 60 https://curl.se/robots.txt
+
   validate:
     runs-on: ubuntu-latest
     steps:
@@ -490,7 +512,7 @@ jobs:
           bash -n buildscripts/package/lifecycle_test.sh
           buildscripts/package/lifecycle_test.sh
           bash -n dockerscripts/docker-entrypoint_test.sh
-          bash -n dockerscripts/download-static-curl.sh
+          bash -n dockerscripts/build-static-curl.sh
           dockerscripts/docker-entrypoint_test.sh
           go run ./buildscripts/rebrand-guard
           buildscripts/verify-rebrand.sh

.github/workflows/vulncheck.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Install govulncheck
         run: |
-          go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
+          go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
 
       - name: Run govulncheck

CHANGELOG.md

@@ -0,0 +1,151 @@
+# Changelog
+
+## Unreleased
+
+The entries below describe source changes on main since the latest published Server.
+**The latest published Server remains 20260903.** These changes are not in its
+binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
+and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
+
+### Authorization and security
+
+- Persist IAM deletion revisions and parent revocation boundaries so stale site
+  events cannot restore deleted identities, policies or their older grants
+  (#191, #192). Peer deletion notifications reload committed storage; deliberate
+  recreation requires a newer revision, and credentials issued before the
+  parent's revocation remain invalid.
+  **Coordinated upgrade required:** upgrade every participating node and site.
+  Mixed old/new nodes sharing an IAM backend and rolling downgrade are
+  unsupported. Back up complete IAM storage and encryption material; an admin
+  export of live records omits deletion history. Reissue credentials for
+  recreated parents and explicitly reconcile pre-upgrade revocations whose
+  history is already lost. Restoring an older backup can lose later revocations;
+  keep affected sites isolated until reconciliation/rekeying is complete. See
+  [the operator runbook](https://github.com/pgsty/silo.pgsty.com/blob/29c7f220b3acc556ad570694056d35e11246f1b9/content/operations/replication/iam-upgrade.md).
+- Enforce an absolute HTTP/1 request-header deadline through the connection
+  wrapper (#196). Repeated small reads no longer extend that deadline, and
+  `--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
+  server. HTTP/1 request bodies retain the rolling idle timeout; this does not
+  impose a total upload/download duration. A shorter setting also constrains
+  TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
+- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
+  copy of another object accessible to the signer (SN-2026-011). The latest
+  public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
+- Align signed request fields with policy conditions and enforce header-only
+  presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
+- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
+  from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
+  both denies if the previous combined restriction must survive upgrades or
+  rollback. Saved policies are not rewritten. Deploy with the matching Console
+  and pkg; see [the migration guide](docs/iam/password-permissions.md).
+
+### Object storage and replication
+
+- Preserve object tags during multi-pool metadata reconciliation by reading the
+  resolved tag field together with its revision (#189). Previously, reconciliation
+  could replace existing tags with an empty value.
+- Preserve the tag revision on SSE-KMS metadata replication (#193), and advance
+  tag revisions monotonically on local PUT/DELETE tagging (#196). Empty tags
+  participate in reconciliation as an ordered deletion, preventing older
+  events from restoring removed tags. SSE-C key rotation also retains the tag
+  revision. Malformed historical revisions can fail and retry; their missing
+  history is not reconstructed by the upgrade.
+- Complete delete-marker version purges and preserve their identity and retry
+  state through MRF recovery (#196). Recovery accepts a 405 marker response only
+  when its version, bucket, object name and modification time match the task.
+  Purge audit status is normalized from `COMPLETE` to `COMPLETED`.
+  Thanks to Julien Laurenceau (@julienlau) for the investigation and proposed
+  fix in #184 that helped shape this follow-up.
+- Restore only the six replication-specific metadata fields after ordinary
+  request metadata extraction (#194). This prevents transport-only `aws-chunked`
+  from being stored as Content-Encoding while preserving the signed-header
+  protections. Trusted Snowball entries no longer inherit the outer archive's
+  ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
+  **Existing data:** these repairs prevent new errors; they do not scan or rewrite
+  historical object metadata, recover lost tags or prove that old purge work has
+  converged. Follow the [read-only audit procedure](https://github.com/pgsty/silo.pgsty.com/blob/29c7f220b3acc556ad570694056d35e11246f1b9/content/operations/replication/replica-metadata-audit.md)
+  before planning any repair of stored state.
+
+- Evaluate conditional multipart completion against the logical current object
+  across all pools while holding the existing object lock. A stale `If-Match`
+  can no longer replace newer data in another pool, and the current ETag is no
+  longer rejected because the upload resides next to an older copy. Conditions
+  are evaluated once; a current delete marker counts as an absent object.
+  **Availability change:** if any pool's metadata cannot be read, conditional
+  completion fails even when another pool can still serve GET/HEAD. This also
+  applies when the unreadable pool may not hold the object: absence cannot be
+  verified. Retry after the pool recovers. Unconditional completion and the
+  single-pool path retain their existing behavior.
+  Ordinary conditional PUT has a separate cross-pool precondition gap tracked
+  in [#199](https://github.com/pgsty/silo/issues/199); the multipart repair does
+  not resolve it.
+
+- Reconcile ordinary single-object version DELETE across all pools, including
+  null versions, delete markers and unqualified directory-marker DELETE. This
+  applies the deletion to every resolved pool copy under existing quorum
+  rules. Pending outbound delete replication retains versions until the
+  existing replication worker completes their purge; a successful response
+  does not imply immediate physical removal from every drive. Unreadable
+  pools now consistently return 503 instead of depending on pool traversal
+  order; insufficient read quorum returns `SlowDownRead`. This extends the
+  existing failure surface. Retry after recovery.
+  Cleanup failures also return an error. Batch deletion already fans out across
+  pools; replication and scanner cleanup keep their existing contracts. See
+  [scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
+
+- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
+  its tracker, mover, scanner hooks, configuration, XML actions and metrics.
+  Accept and ignore retired configuration/XML and preserve ordinary statistics
+  when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
+  The [decision record](docs/investigations/access-tiering-revert.md) preserves
+  the feature's introduction, subsequent fixes, rollback scope and review history.
+- Preserve the independent multi-pool write, metadata, healing and conditional
+  deletion fixes from PR #178, including shared remote-tier reference protection.
+- Enforce `If-Match` on DELETE, preserve retention and independently ordered
+  Object Lock/tag updates, and correctly retransmit encrypted replicas.
+- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
+  compression, honor key-rotation checksums, and complete attributes pagination.
+- Repair federated CopyObject checksums, destination timestamps, reserved
+  metadata, encrypted-object forwarding, legal hold and KMS context.
+- Make resync counters, target selection, cancellation and worker lifetimes
+  reflect actual work, and report bounded MRF drops.
+- Converge bucket metadata with deterministic source state, deletion tombstones,
+  creation time recovery and diagnostics. The mixed-version export gate requires
+  coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
+- Include per-bucket CORS in metadata export/import, close metadata publication
+  and logger races, and report effective bucket quotas in metrics.
+
+### Console, dependencies and delivery
+
+- Restore embedded Console login over loopback TLS, trusted-proxy handling and
+  all four WebSocket connection limits. Preserve Go TLS defaults across transports.
+- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
+  `v0.0.0-20260913015128-417559bb2c97` and MC
+  `v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
+- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
+  modules and security fixes including bounded AMQP frame handling. Keep Go
+  1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
+- Refresh container base digests and build static curl 8.22.0 from verified
+  source for both Linux architectures. Pin the actual mcli 20260913 archives and
+  hashes. Helm's client image follows that release; its Server image still names
+  the latest published Server 20260903.
+
+The dependency update passed the final candidate's Go, vulnerability and Test
+Release workflows; native curl builds passed on both architectures. A local
+ARM64 image passed startup, health, S3 transfer and embedded Console checks.
+These checks do not publish a Server tag or production image and do not replace
+cluster upgrade/rollback acceptance for the next release. Dated investigations
+retain the exact source and runtime boundaries they tested.
+
+## RELEASE.2026-09-03T13-18-01Z
+
+Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
+[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
+[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
+
+This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
+mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
+Installing the newer standalone mcli or Console does not replace components
+inside this existing Server binary or image.
+
+Earlier releases: [release archive](https://github.com/pgsty/silo/releases).

CONTRIBUTORS.md

@@ -62,7 +62,7 @@ the Git history and [NOTICE](NOTICE); this record covers activity in the PGSTY r
 | :-- | :-- | :-- |
 | [@Vonng](https://github.com/Vonng) | Maintains SILO, Console, mcli, shared packages, releases, and documentation | 85 merged PRs across five repositories; [complete maintainer record](#maintainer-record) |
 | [@h5vx](https://github.com/h5vx) | Implemented per-bucket CORS configuration and enforcement | [pgsty/silo#71](https://github.com/pgsty/silo/pull/71) — Merged ([e4e3007da](https://github.com/pgsty/silo/commit/e4e3007da6d7d1198a6a050e34f84566d40a9654)) |
-| [@mrjavadseydi](https://github.com/mrjavadseydi) | Fixed effective bucket quota metrics; proposed access-frequency ILM | [pgsty/silo#132](https://github.com/pgsty/silo/pull/132) — Merged ([ad873c735](https://github.com/pgsty/silo/commit/ad873c73571b121293791f13f8dc46ddf5264d60))<br>[pgsty/silo#60](https://github.com/pgsty/silo/pull/60) — Open |
+| [@mrjavadseydi](https://github.com/mrjavadseydi) | Fixed effective bucket quota metrics; proposed access-frequency ILM | [pgsty/silo#132](https://github.com/pgsty/silo/pull/132) — Merged ([ad873c735](https://github.com/pgsty/silo/commit/ad873c73571b121293791f13f8dc46ddf5264d60))<br>[pgsty/silo#60](https://github.com/pgsty/silo/pull/60) — Merged; access-frequency feature subsequently removed |
 | [@Dansyuqri](https://github.com/Dansyuqri) | Added ChecksumType to multipart completion responses | [pgsty/silo#57](https://github.com/pgsty/silo/pull/57) — Merged ([a96116b12](https://github.com/pgsty/silo/commit/a96116b128bbf2aa42f85eafbf75eb6636cd36ee)) |
 | [@ycjlin](https://github.com/ycjlin) | Fixed missing-bucket ListObjects semantics | [pgsty/silo#37](https://github.com/pgsty/silo/pull/37) — Merged ([49c8aeac4](https://github.com/pgsty/silo/commit/49c8aeac403916f52f8588bbe8ee42753d86eeef)) |
 | [@pinginfo](https://github.com/pinginfo) | Repaired bucket notification streaming | [pgsty/silo#34](https://github.com/pgsty/silo/pull/34) — Merged ([b7f52ca43](https://github.com/pgsty/silo/commit/b7f52ca4336bc45a48b81b39c8983a5e6882a6fa)) |

Dockerfile.goreleaser

@@ -1,4 +1,15 @@
-FROM golang:1.27.1-alpine AS build
+FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
+ARG TARGETARCH
+COPY dockerscripts/build-static-curl.sh /build/build-static-curl
+RUN /bin/sh /build/build-static-curl
+
+# Exercise the exact shipped curl without a dynamic loader or shared libraries.
+FROM scratch AS curl-runtime
+COPY --from=curl-build /go/bin/curl /curl
+COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ENTRYPOINT ["/curl"]
+
+FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
 
 ARG TARGETARCH
 
@@ -6,9 +17,9 @@ ENV GOPATH=/go
 ENV CGO_ENABLED=0
 
 ARG MC_REPO=pgsty/mc
-ARG MC_VERSION=RELEASE.2026-09-03T07-13-05Z
-ARG MC_AMD64_SHA256=cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8
-ARG MC_ARM64_SHA256=7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8
+ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
+ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
+ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
 
 RUN apk add -U --no-cache \
     ca-certificates \
@@ -56,18 +67,14 @@ RUN apk add -U --no-cache \
     chmod +x /go/bin/mcli && \
     ln -sf mcli /go/bin/mc
 
-COPY dockerscripts/download-static-curl.sh /build/download-static-curl
-RUN chmod +x /build/download-static-curl && \
-    /build/download-static-curl
-
-FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
+FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
 RUN dnf -y install ca-certificates && \
     update-ca-trust && \
     cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
-FROM registry.access.redhat.com/ubi9/ubi-micro:latest
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
 
 LABEL org.opencontainers.image.title="Silo" \
       org.opencontainers.image.description="S3-Interface Libre Object Storage" \
@@ -88,7 +95,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY silo /usr/bin/silo
 COPY --from=build /go/bin/mcli /usr/bin/mcli
-COPY --from=build /go/bin/curl* /usr/bin/
+COPY --from=curl-build /go/bin/curl /usr/bin/curl
+COPY --from=curl-build /go/share/curl /licenses/curl
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY LICENSE /licenses/LICENSE
 COPY NOTICE /licenses/NOTICE

Makefile

@@ -216,7 +216,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
 			--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
 		mkdir -p "$$context/dockerscripts"; \
 		cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
-		cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
+		cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
 			"$$context/dockerscripts/"; \
 		docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
 			-f "$$context/Dockerfile.goreleaser" "$$context"

README.md

@@ -35,6 +35,14 @@
 > [!NOTE]
 > Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
 
+## Current release and main branch
+
+The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
+As of 2026-09-13, the main branch has newer security, storage, Console and
+shared-package changes that have not shipped in a Server release. See
+[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
+for the exact release/source boundary, including SN-2026-011 and password-policy migration.
+
 ## Overview
 
 PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.

README_ZH.md

@@ -35,6 +35,13 @@
 > [!NOTE]
 > 2026-08-06，本仓库由 `pgsty/minio` 更名为 `pgsty/silo`，默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支，以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
 
+## 当前发行版与主分支
+
+最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
+截至 2026-09-13，主分支已合入更新的安全、存储、Console 与共享包改动，但尚未发布新 Server。
+准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/)，
+其中包括 SN-2026-011 修复状态与密码权限迁移要求。
+
 ## 概述
 
 上游停止社区发行后，Silo 为开源 MinIO 服务端维护一条持续可用的版本线：构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。

buildscripts/install-mcli.sh

@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
 	exit 0
 fi
 
-release=${MCLI_RELEASE:-RELEASE.2026-09-03T07-13-05Z}
+release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
 version_hyphen=${release#RELEASE.}
 package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
 if [ "${package_version}" = "${version_hyphen}" ]; then

buildscripts/rebrand-guard/compat-baseline.json

@@ -289,16 +289,6 @@
     "MINIO_IDENTITY_TLS_SKIP_VERIFY",
     "MINIO_IDENTITY_TLS_STS_EXPIRY",
     "MINIO_IDLE_TIMEOUT",
-    "MINIO_ILM_ACCESS_BINS",
-    "MINIO_ILM_ACCESS_BIN_WIDTH",
-    "MINIO_ILM_ACCESS_FLUSH",
-    "MINIO_ILM_ACCESS_MAX_SIZE",
-    "MINIO_ILM_ACCESS_MAX_TRACKED",
-    "MINIO_ILM_ACCESS_MIN_RESIDENCY",
-    "MINIO_ILM_ACCESS_POOLS",
-    "MINIO_ILM_ACCESS_PROMOTE_WATERMARK",
-    "MINIO_ILM_ACCESS_TIERING",
-    "MINIO_ILM_ACCESS_WORKERS",
     "MINIO_ILM_EXPIRATION_WORKERS",
     "MINIO_ILM_TRANSITION_WORKERS",
     "MINIO_INTERFACE",
@@ -512,6 +502,7 @@
     "MINIO_SITE_COMMENT",
     "MINIO_SITE_NAME",
     "MINIO_SITE_REGION",
+    "MINIO_SITE_REPLICATION_METADATA_TOMBSTONES",
     "MINIO_STORAGE_CLASS_COMMENT",
     "MINIO_STORAGE_CLASS_INLINE_BLOCK",
     "MINIO_STORAGE_CLASS_OPTIMIZE",
@@ -744,7 +735,6 @@
     "/idp/ldap/policy/{operation}",
     "/idp/openid/list-access-keys-bulk",
     "/ilm",
-    "/ilm/access",
     "/import-bucket-metadata",
     "/import-iam",
     "/import-iam-v2",
@@ -839,6 +829,7 @@
     "/site-replication/peer/bucket-ops",
     "/site-replication/peer/edit",
     "/site-replication/peer/iam-item",
+    "/site-replication/peer/iam-revisions",
     "/site-replication/peer/idp-settings",
     "/site-replication/peer/join",
     "/site-replication/peer/remove",
@@ -887,6 +878,7 @@
     "/v2/metrics/cluster",
     "/v2/metrics/node",
     "/v2/metrics/resource",
+    "/v3/site-replication/peer/iam-revisions",
     "/var/vcap/bosh",
     "/verifybinary",
     "/version",
@@ -914,6 +906,7 @@
     ".minio.sys/config/config.json",
     ".minio.sys/config/hello.txt",
     ".minio.sys/config/iam/${username}/identity.json",
+    ".minio.sys/config/ilm/access",
     ".minio.sys/format.json",
     ".minio.sys/multipart",
     ".minio.sys/multipart/bucket/object/uploads.json",
@@ -941,6 +934,7 @@
     "arn:minio:kms:::this-is-disregarded",
     "arn:minio:kms:::xyz-test-key",
     "arn:minio:replication:",
+    "arn:minio:replication::",
     "arn:minio:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket",
     "arn:minio:replication:::",
     "arn:minio:replication:::dest-bucket",

buildscripts/verify-rebrand.sh

@@ -36,7 +36,7 @@ for file in \
 	buildscripts/verify-helm-migration.sh \
 	Dockerfile.goreleaser \
 	Dockerfile.distroless \
-	dockerscripts/download-static-curl.sh \
+	dockerscripts/build-static-curl.sh \
 	dockerscripts/docker-entrypoint.sh \
 	helm/silo/Chart.yaml \
 	helm/silo/values.yaml \
@@ -101,7 +101,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
 require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
 require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
 require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
-require_text dockerscripts/download-static-curl.sh "sha256sum -c"
+require_text dockerscripts/build-static-curl.sh "sha256sum -c"
 require_text helm/silo/Chart.yaml "name: silo"
 require_text helm/silo/values.yaml "repository: pgsty/silo"
 require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"

cmd/admin-bucket-handlers.go

@@ -76,7 +76,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	quotaConfig, err := parseBucketQuota(bucket, data)
+	_, err = parseBucketQuota(bucket, data)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -94,9 +94,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		Quota:     data,
 		UpdatedAt: updatedAt,
 	}
-	if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
-		bucketMeta.Quota = nil
-	}
 
 	// Call site replication hook.
 	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
@@ -438,7 +435,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				configData, err := json.Marshal(config)
+				configData, err := canonicalBucketPolicy(config)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
@@ -922,7 +919,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			configData, err := json.Marshal(bucketPolicy)
+			configData, err := canonicalBucketPolicy(bucketPolicy)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
 				continue
@@ -1081,18 +1078,39 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		var merged BucketMetadata
+		var commitAt time.Time
 		err := func() error {
 			lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
 			if err != nil {
 				return err
 			}
 			defer unlock()
-			merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, true)
+			merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, false)
 			if err != nil {
 				return err
 			}
+			if err := ensureBucketMetadataCreated(lockCtx, objectAPI, &merged); err != nil {
+				return err
+			}
+			commitAt = UTCNow()
+			for _, file := range replicatedBucketConfigs {
+				if _, ok := fields[file]; ok {
+					commitAt = localBucketConfigUpdatedAt(merged, file, commitAt)
+				}
+			}
 			applyImportedBucketMetadata(&merged, *meta, fields)
-			return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, merged)
+			for _, file := range replicatedBucketConfigs {
+				if _, ok := fields[file]; !ok {
+					continue
+				}
+				data, at := replicatedBucketConfig(&merged, file)
+				payload, _, err := bucketConfigPayload(bucket, file, *data, len(merged.ObjectLockConfigXML) != 0)
+				if err != nil {
+					return err
+				}
+				*data, *at = payload, commitAt
+			}
+			return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, &merged)
 		}()
 		if err != nil {
 			rpt.SetStatus(bucket, "", err)
@@ -1100,7 +1118,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		*meta = merged
 		globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
-		hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: updatedAt}
+		hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: commitAt}
 		var hookNeeded bool
 		if _, ok := fields[bucketQuotaConfigFile]; ok {
 			hook.Quota = meta.QuotaConfigJSON
@@ -1108,7 +1126,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		if _, ok := fields[bucketPolicyConfig]; ok {
 			hook.Policy = meta.PolicyConfigJSON
-			hookNeeded = true
+			hookNeeded = hookNeeded || len(hook.Policy) != 0
 		}
 		if _, ok := fields[bucketVersioningConfig]; ok {
 			hook.Versioning = enc(meta.VersioningConfigXML)
@@ -1129,6 +1147,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		if hookNeeded {
 			err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
 		}
+		if _, ok := fields[bucketPolicyConfig]; ok && len(meta.PolicyConfigJSON) == 0 {
+			// An omitted bulk Policy cannot express deletion.
+			err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type: madmin.SRBucketMetaTypePolicy, Bucket: bucket, UpdatedAt: commitAt,
+			}))
+		}
 		if _, ok := fields[bucketCorsConfig]; ok {
 			// CORS carries its own timestamp, so it replicates through the
 			// dedicated event rather than the shared bucket metadata hook. It

cmd/admin-router.go

@@ -388,6 +388,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
+		adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))

cmd/bucket-cors-site-replication_test.go

@@ -467,25 +467,6 @@ func testAdversarialHealCorsPropagatesNewerEqualValueTimestamp(obj ObjectLayer,
 	}
 }
 
-func TestAdversarialBucketMetadataComparisonIsBase64CaseSensitive(t *testing.T) {
-	upper := "QQ=="
-	lower := "qQ=="
-	upperBytes, err := base64.StdEncoding.Strict().DecodeString(upper)
-	if err != nil {
-		t.Fatal(err)
-	}
-	lowerBytes, err := base64.StdEncoding.Strict().DecodeString(lower)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if string(upperBytes) == string(lowerBytes) {
-		t.Fatal("test inputs unexpectedly decode to the same bytes")
-	}
-	if isBucketMetadataEqual(&upper, &lower) {
-		t.Fatal("different decoded payloads were treated as equal")
-	}
-}
-
 func TestSiteReplicationStatusDetectsCorsTimestampMismatch(t *testing.T) {
 	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
 		t:          t,

cmd/bucket-lifecycle-audit.go

@@ -39,7 +39,6 @@ const (
 	lcEventSrc_s3PutObject
 	lcEventSrc_s3CopyObject
 	lcEventSrc_s3CompleteMultipartUpload
-	lcEventSrc_AccessTier
 )
 
 //revive:enable:var-naming

cmd/bucket-metadata-replication.go

@@ -0,0 +1,330 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"sort"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/tags"
+	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// Only these fields share the site-replication source-time ordering contract.
+// Bulk apply/import process Object Lock before Versioning, whose effective
+// document depends on it. Periodic heal retains its existing type order.
+var replicatedBucketConfigs = [...]string{
+	objectLockConfig, bucketVersioningConfig, bucketPolicyConfig,
+	bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile,
+}
+
+func replicatedBucketConfig(meta *BucketMetadata, file string) (*[]byte, *time.Time) {
+	switch file {
+	case bucketPolicyConfig:
+		return &meta.PolicyConfigJSON, &meta.PolicyConfigUpdatedAt
+	case bucketTaggingConfig:
+		return &meta.TaggingConfigXML, &meta.TaggingConfigUpdatedAt
+	case bucketSSEConfig:
+		return &meta.EncryptionConfigXML, &meta.EncryptionConfigUpdatedAt
+	case bucketQuotaConfigFile:
+		return &meta.QuotaConfigJSON, &meta.QuotaConfigUpdatedAt
+	case bucketVersioningConfig:
+		return &meta.VersioningConfigXML, &meta.VersioningConfigUpdatedAt
+	case objectLockConfig:
+		return &meta.ObjectLockConfigXML, &meta.ObjectLockConfigUpdatedAt
+	}
+	return nil, nil
+}
+
+// Callers that only need to know whether a file is under the contract must not
+// probe replicatedBucketConfig with a throwaway BucketMetadata.
+func isReplicatedBucketConfig(file string) bool {
+	for _, replicated := range replicatedBucketConfigs {
+		if replicated == file {
+			return true
+		}
+	}
+	return false
+}
+
+func bucketConfigUpdateOnly(file string) bool {
+	return file == bucketVersioningConfig || file == objectLockConfig
+}
+
+// Reuse the persistence rule before comparison, so an accepted Versioning
+// event and the document Save actually writes have the same comparison key.
+func effectiveBucketVersioning(data []byte, lockEnabled bool) []byte {
+	if lockEnabled {
+		config, err := versioning.ParseConfig(bytes.NewReader(data))
+		if err != nil || !config.Enabled() || config.PrefixesExcluded() {
+			return enabledBucketVersioningConfig
+		}
+	}
+	return data
+}
+
+// A parsed policy still contains map-backed sets with nondeterministic Marshal
+// order. Sort every set array recursively, including statements and conditions.
+// RawMessage keeps integer values intact; decoding through float64 would not.
+func canonicalBucketPolicyJSON(data json.RawMessage) (json.RawMessage, error) {
+	data = bytes.TrimSpace(data)
+	if len(data) == 0 {
+		return nil, nil
+	}
+	switch data[0] {
+	case '{':
+		var obj map[string]json.RawMessage
+		if err := json.Unmarshal(data, &obj); err != nil {
+			return nil, err
+		}
+		for key, value := range obj {
+			var err error
+			obj[key], err = canonicalBucketPolicyJSON(value)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return json.Marshal(obj)
+	case '[':
+		var arr []json.RawMessage
+		if err := json.Unmarshal(data, &arr); err != nil {
+			return nil, err
+		}
+		for i := range arr {
+			var err error
+			arr[i], err = canonicalBucketPolicyJSON(arr[i])
+			if err != nil {
+				return nil, err
+			}
+		}
+		sort.Slice(arr, func(i, j int) bool { return bytes.Compare(arr[i], arr[j]) < 0 })
+		return json.Marshal(arr)
+	default:
+		var compact bytes.Buffer
+		if err := json.Compact(&compact, data); err != nil {
+			return nil, err
+		}
+		return compact.Bytes(), nil
+	}
+}
+
+// Encode the validated policy fields explicitly: BPStatement's required
+// Action/Resource tags otherwise try to marshal empty sets for the supported
+// NotAction/NotResource alternatives. This stays within the existing schema.
+func canonicalBucketPolicy(cfg *policy.BucketPolicy) ([]byte, error) {
+	if cfg.IsEmpty() {
+		return nil, nil
+	}
+	doc := map[string]any{"Version": cfg.Version}
+	if cfg.ID != "" {
+		doc["ID"] = cfg.ID
+	}
+	statements := make([]map[string]any, 0, len(cfg.Statements))
+	for _, st := range cfg.Statements {
+		statement := map[string]any{"Effect": st.Effect, "Principal": st.Principal}
+		if st.SID != "" {
+			statement["Sid"] = st.SID
+		}
+		if len(st.Actions) != 0 {
+			statement["Action"] = st.Actions
+		}
+		if len(st.NotActions) != 0 {
+			statement["NotAction"] = st.NotActions
+		}
+		if len(st.Resources) != 0 {
+			statement["Resource"] = st.Resources
+		}
+		if len(st.NotResources) != 0 {
+			statement["NotResource"] = st.NotResources
+		}
+		if len(st.Conditions) != 0 {
+			statement["Condition"] = st.Conditions
+		}
+		statements = append(statements, statement)
+	}
+	doc["Statement"] = statements
+	data, err := json.Marshal(doc)
+	if err != nil {
+		return nil, err
+	}
+	return canonicalBucketPolicyJSON(data)
+}
+
+// Validate with the same parsers as Save. Policy's established empty-policy
+// semantics are deletion; a parsed zero quota is still a live document.
+func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]byte, []byte, error) {
+	if len(data) == 0 {
+		return nil, nil, nil
+	}
+	var err error
+	key := data
+	switch file {
+	case bucketPolicyConfig:
+		var cfg *policy.BucketPolicy
+		cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
+		if err == nil {
+			if cfg.IsEmpty() {
+				return nil, nil, nil
+			}
+			key, err = canonicalBucketPolicy(cfg)
+		}
+	case bucketQuotaConfigFile:
+		cfg, parseErr := parseBucketQuota(bucket, data)
+		err = parseErr
+		if err == nil {
+			key, err = json.Marshal(cfg)
+		}
+	case bucketTaggingConfig:
+		_, err = tags.ParseBucketXML(bytes.NewReader(data))
+	case bucketSSEConfig:
+		_, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(data))
+	case objectLockConfig:
+		_, err = objectlock.ParseObjectLockConfig(bytes.NewReader(data))
+	case bucketVersioningConfig:
+		data = effectiveBucketVersioning(data, lockEnabled)
+		key = data
+		_, err = versioning.ParseConfig(bytes.NewReader(data))
+	}
+	return data, key, err
+}
+
+type bucketConfigState struct {
+	data, key   []byte
+	at          time.Time
+	real, valid bool
+}
+
+func newBucketConfigState(bucket, file string, data []byte, at, created time.Time, lockEnabled bool) (bucketConfigState, error) {
+	data, key, err := bucketConfigPayload(bucket, file, data, lockEnabled)
+	if err != nil {
+		return bucketConfigState{}, err
+	}
+	if at.IsZero() {
+		at = created
+	}
+	valid := !created.IsZero() && !at.Before(created)
+	modified := valid && at.After(created)
+	if bucketConfigUpdateOnly(file) && len(data) == 0 {
+		modified = false
+	}
+	return bucketConfigState{data: data, key: key, at: at, real: modified, valid: valid}, nil
+}
+
+func (s bucketConfigState) candidate() bool {
+	return s.valid && (s.real || len(s.data) != 0)
+}
+
+func compareBucketConfigStates(a, b bucketConfigState) int {
+	if a.valid != b.valid {
+		if a.valid {
+			return 1
+		}
+		return -1
+	}
+	if a.real != b.real {
+		if a.real {
+			return 1
+		}
+		return -1
+	}
+	if a.real {
+		if n := a.at.Compare(b.at); n != 0 {
+			return n
+		}
+		// At equal source time a real deletion wins, preventing resurrection.
+		if (len(a.data) == 0) != (len(b.data) == 0) {
+			if len(a.data) == 0 {
+				return 1
+			}
+			return -1
+		}
+	}
+	return bytes.Compare(a.key, b.key)
+}
+
+func localBucketConfigUpdatedAt(meta BucketMetadata, file string, now time.Time) time.Time {
+	_, at := replicatedBucketConfig(&meta, file)
+	for _, lower := range []time.Time{meta.Created, *at} {
+		if !now.After(lower) {
+			now = lower.Add(time.Nanosecond)
+		}
+	}
+	return now.UTC()
+}
+
+func ensureBucketMetadataCreated(ctx context.Context, obj ObjectLayer, meta *BucketMetadata) error {
+	if !meta.Created.IsZero() {
+		return nil
+	}
+	info, err := obj.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true})
+	if err != nil {
+		return err
+	}
+	if info.Created.IsZero() {
+		return errors.New("bucket metadata creation time is unknown")
+	}
+	meta.Created = info.Created.UTC()
+	return nil
+}
+
+// applyBucketConfig runs under metadata.lock, on freshly loaded metadata. It
+// changes only the selected field; the caller persists once after all checks.
+func applyBucketConfig(meta *BucketMetadata, file string, data []byte, at time.Time) (bool, error) {
+	if bucketConfigUpdateOnly(file) && len(data) == 0 {
+		return false, nil
+	}
+	current, currentAt := replicatedBucketConfig(meta, file)
+	lockEnabled := len(meta.ObjectLockConfigXML) != 0
+	incoming, err := newBucketConfigState(meta.Name, file, data, at, meta.Created, lockEnabled)
+	if err != nil {
+		return false, err
+	}
+	if !incoming.candidate() {
+		return false, nil
+	}
+	local, err := newBucketConfigState(meta.Name, file, *current, *currentAt, meta.Created, lockEnabled)
+	if err != nil {
+		return false, err
+	}
+	if compareBucketConfigStates(incoming, local) <= 0 {
+		return false, nil
+	}
+	*current, *currentAt = bytes.Clone(incoming.data), incoming.at.UTC()
+	return true, nil
+}
+
+func rebaseBucketConfigDefaults(meta *BucketMetadata, oldCreated time.Time) {
+	if meta.Created.Equal(oldCreated) {
+		return
+	}
+	// These six fields alone use Created to distinguish a baseline from a
+	// tombstone. Preserve actual source times when adopting an existing bucket.
+	for _, file := range replicatedBucketConfigs {
+		_, at := replicatedBucketConfig(meta, file)
+		if at.IsZero() || at.Equal(oldCreated) {
+			*at = meta.Created
+		}
+	}
+}

cmd/bucket-metadata-sys.go

@@ -126,19 +126,42 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 	}
 }
 
-func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool) (updatedAt time.Time, err error) {
+// bucketMetadataUpdate returns the committed snapshot to the caller. meta and
+// updatedAt hold the saved state only when changed is true. Local writes always
+// change state, because localBucketConfigUpdatedAt is strictly greater than the
+// current field time, so their handlers can broadcast meta without rechecking.
+type bucketMetadataUpdate struct {
+	meta      BucketMetadata
+	updatedAt time.Time
+	changed   bool
+}
+
+func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket, configFile string, configData []byte, parse, lifecycleDelete bool) (time.Time, error) {
+	result, err := sys.updateAndParseMetadata(ctx, bucket, configFile, configData, parse, lifecycleDelete, nil)
+	return result.updatedAt, err
+}
+
+func (sys *BucketMetadataSys) updateAndParseMetadata(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool, sourceTime *time.Time) (result bucketMetadataUpdate, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return updatedAt, errServerNotInitialized
+		return result, errServerNotInitialized
 	}
 
 	if isMinioMetaBucketName(bucket) {
-		return updatedAt, errInvalidArgument
+		return result, errInvalidArgument
+	}
+	// Load deletions without parsed caches (notably quota), and compare the
+	// six replicated fields against the raw document under the same lock.
+	if isReplicatedBucketConfig(configFile) {
+		parse = false
+		if bucketConfigUpdateOnly(configFile) && len(configData) == 0 {
+			return result, nil
+		}
 	}
 	notifyCtx := ctx
 	ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
-		return updatedAt, err
+		return result, err
 	}
 
 	err = func() error {
@@ -158,55 +181,73 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 				return err
 			}
 		}
-		updatedAt = UTCNow()
-		switch configFile {
-		case bucketPolicyConfig:
-			meta.PolicyConfigJSON = configData
-			meta.PolicyConfigUpdatedAt = updatedAt
-		case bucketNotificationConfig:
-			meta.NotificationConfigXML = configData
-			meta.NotificationConfigUpdatedAt = updatedAt
-		case bucketLifecycleConfig:
-			meta.LifecycleConfigXML = configData
-			meta.LifecycleConfigUpdatedAt = updatedAt
-		case bucketSSEConfig:
-			meta.EncryptionConfigXML = configData
-			meta.EncryptionConfigUpdatedAt = updatedAt
-		case bucketTaggingConfig:
-			meta.TaggingConfigXML = configData
-			meta.TaggingConfigUpdatedAt = updatedAt
-		case bucketQuotaConfigFile:
-			meta.QuotaConfigJSON = configData
-			meta.QuotaConfigUpdatedAt = updatedAt
-		case objectLockConfig:
-			meta.ObjectLockConfigXML = configData
-			meta.ObjectLockConfigUpdatedAt = updatedAt
-		case bucketVersioningConfig:
-			meta.VersioningConfigXML = configData
-			meta.VersioningConfigUpdatedAt = updatedAt
-		case bucketReplicationConfig:
-			meta.ReplicationConfigXML = configData
-			meta.ReplicationConfigUpdatedAt = updatedAt
-		case bucketTargetsFile:
-			meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
-				bucket:            meta.Name,
-				bucketTargetsFile: bucketTargetsFile,
-			})
-			if err != nil {
-				return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+		updatedAt := UTCNow()
+		if isReplicatedBucketConfig(configFile) {
+			if err := ensureBucketMetadataCreated(ctx, objAPI, &meta); err != nil {
+				var at time.Time
+				if sourceTime != nil {
+					at = *sourceTime
+				}
+				logBucketConfigReplication(ctx, bucket, configFile, "indeterminate", at, meta.Created, err.Error())
+				return err
+			}
+			if sourceTime == nil || sourceTime.IsZero() {
+				updatedAt = localBucketConfigUpdatedAt(meta, configFile, updatedAt)
+				if sourceTime != nil {
+					logBucketConfigReplication(ctx, bucket, configFile, "legacy-zero", *sourceTime, meta.Created, "assigned local source time")
+				}
+			} else {
+				updatedAt = sourceTime.UTC()
+			}
+			if updatedAt.Before(meta.Created) {
+				logBucketConfigReplication(ctx, bucket, configFile, "before-created", updatedAt, meta.Created, "peer event")
+				return nil
+			}
+			changed, err := applyBucketConfig(&meta, configFile, configData, updatedAt)
+			if err != nil {
+				return err
+			}
+			if !changed {
+				return nil
+			}
+		} else {
+			switch configFile {
+			case bucketNotificationConfig:
+				meta.NotificationConfigXML = configData
+				meta.NotificationConfigUpdatedAt = updatedAt
+			case bucketLifecycleConfig:
+				meta.LifecycleConfigXML = configData
+				meta.LifecycleConfigUpdatedAt = updatedAt
+			case bucketReplicationConfig:
+				meta.ReplicationConfigXML = configData
+				meta.ReplicationConfigUpdatedAt = updatedAt
+			case bucketTargetsFile:
+				meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
+					bucket:            meta.Name,
+					bucketTargetsFile: bucketTargetsFile,
+				})
+				if err != nil {
+					return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+				}
+				meta.BucketTargetsConfigUpdatedAt = updatedAt
+				meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
+			default:
+				return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 			}
-			meta.BucketTargetsConfigUpdatedAt = updatedAt
-			meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
-		default:
-			return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 		}
-		return sys.saveMetadata(ctx, objAPI, meta)
+		if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
+			return err
+		}
+		result = bucketMetadataUpdate{meta: meta, updatedAt: updatedAt, changed: true}
+		return nil
 	}()
 	if err != nil {
-		return updatedAt, err
+		return result, err
 	}
-	globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
-	return updatedAt, nil
+	if result.changed {
+		globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket)
+	}
+	return result, nil
 }
 
 func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
@@ -219,7 +260,7 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
 		return errInvalidArgument
 	}
 
-	if err := sys.saveMetadata(ctx, objAPI, meta); err != nil {
+	if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
 		return err
 	}
 
@@ -229,7 +270,7 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
 
 // saveMetadata persists and publishes metadata locally. Callers performing a
 // read-modify-write must hold metadata.lock and release it before peer fan-out.
-func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta BucketMetadata) error {
+func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta *BucketMetadata) error {
 	// A writer may have queued for metadata.lock before DeleteBucket completed.
 	// Recheck the physical bucket under that lock, before recreating metadata.
 	if _, err := objAPI.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true}); err != nil {
@@ -238,7 +279,7 @@ func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLay
 	if err := meta.Save(ctx, objAPI); err != nil {
 		return err
 	}
-	sys.Set(meta.Name, meta)
+	sys.Set(meta.Name, *meta)
 	return nil
 }
 
@@ -351,7 +392,7 @@ func (sys *BucketMetadataSys) UpdateExpiryLCConfig(ctx context.Context, bucket s
 		}
 		meta.LifecycleConfigXML = configData
 		meta.LifecycleConfigUpdatedAt = UTCNow()
-		return sys.saveMetadata(ctx, objAPI, meta)
+		return sys.saveMetadata(ctx, objAPI, &meta)
 	}()
 	if err != nil {
 		return err

cmd/bucket-metadata.go

@@ -388,15 +388,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	} else {
 		b.objectLockConfig = nil
 	}
-	if b.objectLockConfig != nil {
-		// Object Lock requires every object to be versioned. Whatever the lock
-		// document contains, a suspended or prefix-excluded versioning document
-		// is replaced by plain Enabled versioning; Save persists the result.
-		config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
-		if versioningErr != nil || !config.Enabled() || config.PrefixesExcluded() {
-			b.VersioningConfigXML = enabledBucketVersioningConfig
-		}
-	}
+	b.VersioningConfigXML = effectiveBucketVersioning(b.VersioningConfigXML, b.objectLockConfig != nil)
 
 	if len(b.VersioningConfigXML) != 0 {
 		b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))

cmd/bucket-metadata_test.go

@@ -39,3 +39,28 @@ func TestBucketMetadataCorsRoundTrip(t *testing.T) {
 		t.Fatalf("CorsConfigUpdatedAt not preserved")
 	}
 }
+
+// A persisted retired extension must not prevent the whole bucket's metadata
+// from loading, including unrelated versioning and ordinary lifecycle rules.
+func TestBucketMetadataRetiredAccessTiering(t *testing.T) {
+	meta := newBucketMetadata("retired-access")
+	meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><AccessTierQuota>500GiB</AccessTierQuota><Rule><ID>access</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AccessTransition><Window>10m</Window><PromoteAfterAccesses>10</PromoteAfterAccesses></AccessTransition></Rule><Rule><ID>ordinary</ID><Status>Enabled</Status><Filter><Prefix>expired/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
+	meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
+	data, err := meta.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := newBucketMetadata(meta.Name)
+	if _, err := got.UnmarshalMsg(data); err != nil {
+		t.Fatal(err)
+	}
+	if err := got.parseAllConfigs(t.Context(), nil); err != nil {
+		t.Fatalf("bucket metadata failed to load: %v", err)
+	}
+	if got.lifecycleConfig == nil || got.lifecycleConfig.HasActiveRules("logs/") || !got.lifecycleConfig.HasActiveRules("expired/") {
+		t.Fatal("unexpected lifecycle behavior")
+	}
+	if got.versioningConfig == nil || !got.versioningConfig.Enabled() {
+		t.Fatal("unrelated versioning lost")
+	}
+}

cmd/bucket-policy-handlers.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"encoding/json"
 	"io"
 	"net/http"
 
@@ -100,13 +99,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	configData, err := json.Marshal(bucketPolicy)
+	configData, err := canonicalBucketPolicy(bucketPolicy)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+	result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketPolicyConfig, configData, false, false, nil)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -116,8 +115,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
-		Policy:    bucketPolicyBytes,
-		UpdatedAt: updatedAt,
+		Policy:    result.meta.PolicyConfigJSON,
+		UpdatedAt: result.updatedAt,
 	}))
 
 	// Success.
@@ -200,7 +199,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	configData, err := json.Marshal(config)
+	configData, err := canonicalBucketPolicy(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-replication-utils.go

@@ -418,6 +418,9 @@ func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID
 
 	for _, rinfo := range rinfos.Targets {
 		if rinfo.ResyncTimestamp != "" {
+			if rs.ResetStatusesMap == nil {
+				rs.ResetStatusesMap = make(map[string]string)
+			}
 			rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
 		}
 	}

cmd/bucket-replication.go

@@ -425,6 +425,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // the mere presence or absence of the target version.
 func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
 	var replicationStatus replication.StatusType
+	isPurge := dobj.isVersionPurge()
 	bucket := dobj.Bucket
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
@@ -484,6 +485,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
+		dobj.RetryCount++
 		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 		sendEvent(eventArgs{
 			BucketName: bucket,
@@ -546,25 +548,38 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	replicationStatus = rinfos.ReplicationStatus()
 	prevStatus := dobj.DeleteMarkerReplicationStatus()
 
-	if dobj.VersionID != "" {
-		prevStatus = replication.StatusType(dobj.VersionPurgeStatus())
-		replicationStatus = replication.StatusType(rinfos.VersionPurgeStatus())
+	if isPurge {
+		prevStatus = purgeReplicationStatus(dobj.VersionPurgeStatus())
+		replicationStatus = purgeReplicationStatus(rinfos.VersionPurgeStatus())
 	}
 
 	// to decrement pending count later.
 	for _, rinfo := range rinfos.Targets {
-		if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
-			globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus,
-				prevStatus)
+		status, previous := rinfo.ReplicationStatus, rinfo.PrevReplicationStatus
+		if isPurge {
+			status = purgeReplicationStatus(rinfo.VersionPurgeStatus)
+			previous = purgeReplicationStatus(dobj.ReplicationState.PurgeTargets[rinfo.Arn])
+		}
+		if status != previous {
+			globalReplicationStats.Load().Update(dobj.Bucket, rinfo, status, previous)
 		}
 	}
 
 	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
+		dobj.RetryCount++
 		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
+	if isPurge {
+		// A purge must not rewrite the marker's creation/replica metadata.
+		// Multiple serialized empty target statuses can parse as nonempty,
+		// so explicitly send an empty creation update to the metadata writer.
+		drs.ReplicationStatusInternal = ""
+		drs.Targets = nil
+		drs.ReplicaStatus = ""
+	}
 	if replicationStatus != prevStatus {
 		drs.ReplicationTimeStamp = UTCNow()
 	}
@@ -606,6 +621,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 }
 
 func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+	isPurge := dobj.isVersionPurge()
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
@@ -615,42 +631,50 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	rinfo.OpType = dobj.OpType
 	rinfo.endpoint = tgt.EndpointURL().Host
 	rinfo.secure = tgt.EndpointURL().Scheme == "https"
+	// A purge leaves ReplicationStatus empty: the metadata writer interprets
+	// that as preserving the entire creation-status block, including targets
+	// outside this fan-out. Only VersionPurgeStatus records the purge outcome.
 	defer func() {
-		if rinfo.ReplicationStatus == replication.Completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
+		completed := rinfo.ReplicationStatus == replication.Completed
+		if isPurge {
+			completed = rinfo.VersionPurgeStatus == replication.VersionPurgeComplete
+		}
+		if completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
 			rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID)
 		}
 	}()
 
-	if dobj.VersionID == "" && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType {
+	if !isPurge && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType {
 		rinfo.ReplicationStatus = rinfo.PrevReplicationStatus
 		return rinfo
 	}
-	if dobj.VersionID != "" && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
+	if isPurge && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete {
 		return rinfo
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
+		rinfo.Err = fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN)
+		replLogOnceIf(ctx, rinfo.Err, "replication-target-offline-delete-"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
 				Bucket:       dobj.Bucket,
 				Name:         dobj.ObjectName,
-				VersionID:    dobj.VersionID,
+				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
 			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
-		if dobj.VersionID == "" {
-			rinfo.ReplicationStatus = replication.Failed
-		} else {
+		if isPurge {
 			rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
+		} else {
+			rinfo.ReplicationStatus = replication.Failed
 		}
 		return rinfo
 	}
 	// early return if already replicated delete marker for existing object replication/ healing delete markers
-	if dobj.DeleteMarkerVersionID != "" {
+	if !isPurge && dobj.DeleteMarkerVersionID != "" {
 		toi, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, minio.StatObjectOptions{
 			VersionID: versionID,
 			Internal: minio.AdvancedGetOptions{
@@ -662,16 +686,10 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		switch {
 		case isErrMethodNotAllowed(serr):
 			// delete marker already replicated
-			if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() {
-				rinfo.ReplicationStatus = replication.Completed
-				return rinfo
-			}
+			rinfo.ReplicationStatus = replication.Completed
+			return rinfo
 		case isErrObjectNotFound(serr), isErrVersionNotFound(serr):
-			// version being purged is already not found on target.
-			if !rinfo.VersionPurgeStatus.Empty() {
-				rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
-				return rinfo
-			}
+			// The marker still needs to be created on the target.
 		case isErrReadQuorum(serr), isErrWriteQuorum(serr):
 			// destination has some quorum issues, perform removeObject() anyways
 			// to complete the operation.
@@ -691,7 +709,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{
 		VersionID: versionID,
 		Internal: minio.AdvancedRemoveOptions{
-			ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "",
+			ReplicationDeleteMarker: !isPurge && dobj.DeleteMarkerVersionID != "",
 			ReplicationMTime:        dobj.DeleteMarkerMTime.Time,
 			ReplicationStatus:       minio.ReplicationStatusReplica,
 			ReplicationRequest:      true, // always set this to distinguish between `mc mirror` replication and serverside
@@ -699,20 +717,20 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	})
 	if rmErr != nil {
 		rinfo.Err = rmErr
-		if dobj.VersionID == "" {
-			rinfo.ReplicationStatus = replication.Failed
-		} else {
+		if isPurge {
 			rinfo.VersionPurgeStatus = replication.VersionPurgeFailed
+		} else {
+			rinfo.ReplicationStatus = replication.Failed
 		}
 		replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
 		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
 		}
 	} else {
-		if dobj.VersionID == "" {
-			rinfo.ReplicationStatus = replication.Completed
-		} else {
+		if isPurge {
 			rinfo.VersionPurgeStatus = replication.VersionPurgeComplete
+		} else {
+			rinfo.ReplicationStatus = replication.Completed
 		}
 	}
 	return rinfo
@@ -781,6 +799,18 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
+// replicationTaggingTimestamp carries a recorded removal even when tags are
+// empty. Only legacy nonempty tags use ModTime; absence is not a tombstone.
+func replicationTaggingTimestamp(objInfo ObjectInfo) (time.Time, error) {
+	if stamp, ok := caseInsensitiveMap(objInfo.UserDefined).Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
+		return time.Parse(time.RFC3339Nano, stamp)
+	}
+	if objInfo.UserTags != "" {
+		return objInfo.ModTime, nil
+	}
+	return time.Time{}, nil
+}
+
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) {
 	meta := make(map[string]string)
 	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
@@ -850,17 +880,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
 		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
 		if tag != nil {
 			putOpts.UserTags = tag.ToMap()
-			// set tag timestamp in opts
-			tagTimestamp := objInfo.ModTime
-			if tagTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
-				tagTimestamp, err = time.Parse(time.RFC3339Nano, tagTmstampStr)
-				if err != nil {
-					return putOpts, false, err
-				}
-			}
-			putOpts.Internal.TaggingTimestamp = tagTimestamp
 		}
 	}
+	putOpts.Internal.TaggingTimestamp, err = replicationTaggingTimestamp(objInfo)
+	if err != nil {
+		return putOpts, false, err
+	}
 
 	lkMap := caseInsensitiveMap(objInfo.UserDefined)
 	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
@@ -1003,6 +1028,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2Map, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
 		return replicateMetadata
 	}
+	// HEAD does not report the tag revision. Equal values can hide a newer
+	// deletion or re-addition, so scheduled metadata/heal work must deliver it.
+	// Completed objects are still excluded by the existing scanner gates.
+	if _, ok := caseInsensitiveMap(oi1.UserDefined).Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
+		return replicateMetadata
+	}
 
 	// Compare only necessary headers
 	compareKeys := []string{
@@ -1269,9 +1300,6 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 						oi.UserDefined[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
 					}
 				}
-				if ri.UserTags != "" {
-					oi.UserDefined[xhttp.AmzObjectTagging] = ri.UserTags
-				}
 				return dsc, nil
 			},
 		}
@@ -1689,14 +1717,11 @@ applyAction:
 		if _, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
 			dstOpts.Internal.RetentionTimestamp = objInfo.ModTime
 		}
-		if objInfo.UserTags != "" {
-			dstOpts.Internal.TaggingTimestamp = objInfo.ModTime
-		}
-		if tagTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok {
-			ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
-			if err == nil {
-				dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
-			}
+		dstOpts.Internal.TaggingTimestamp, rinfo.Err = replicationTaggingTimestamp(objInfo)
+		if rinfo.Err != nil {
+			rinfo.ReplicationStatus = replication.Failed
+			replLogIf(ctx, fmt.Errorf("invalid tagging timestamp for object %s/%s(%s): %w", bucket, object, objInfo.VersionID, rinfo.Err))
+			return rinfo
 		}
 		if retTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp); ok {
 			ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
@@ -1916,11 +1941,26 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 // DeletedObjectReplicationInfo has info on deleted object
 type DeletedObjectReplicationInfo struct {
 	DeletedObject
-	Bucket    string
-	EventType string
-	OpType    replication.Type
-	ResetID   string
-	TargetArn string
+	Bucket     string
+	EventType  string
+	OpType     replication.Type
+	ResetID    string
+	TargetArn  string
+	RetryCount int
+}
+
+// isVersionPurge also recognizes the old marker-shaped purge task. Use the
+// operation's state, rather than one target's possibly missing purge entry.
+func (di DeletedObjectReplicationInfo) isVersionPurge() bool {
+	return di.VersionID != "" || di.DeleteMarkerVersionID != "" && !di.VersionPurgeStatus().Empty()
+}
+
+// Purge metadata uses COMPLETE; operation statistics and audit use COMPLETED.
+func purgeReplicationStatus(status VersionPurgeStatusType) replication.StatusType {
+	if replication.StatusType(status) == replication.CompletedLegacy {
+		return replication.Completed
+	}
+	return replication.StatusType(status)
 }
 
 // ToMRFEntry returns the relevant info needed by MRF
@@ -1930,9 +1970,10 @@ func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry {
 		versionID = di.VersionID
 	}
 	return MRFReplicateEntry{
-		Bucket:    di.Bucket,
-		Object:    di.ObjectName,
-		versionID: versionID,
+		Bucket:     di.Bucket,
+		Object:     di.ObjectName,
+		versionID:  versionID,
+		RetryCount: di.RetryCount,
 	}
 }
 
@@ -2424,6 +2465,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 	case <-p.ctx.Done():
 	case ch <- doi:
 	default:
+		doi.RetryCount++
 		p.queueMRFSave(doi.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
@@ -3787,9 +3829,10 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},
-			Bucket:    roi.Bucket,
-			OpType:    replication.HealReplicationType,
-			EventType: ReplicateHealDelete,
+			Bucket:     roi.Bucket,
+			OpType:     replication.HealReplicationType,
+			EventType:  ReplicateHealDelete,
+			RetryCount: retryCount,
 		}
 		// heal delete marker replication failure or versioned delete replication failure
 		if roi.ReplicationStatus == replication.Pending ||
@@ -4072,7 +4115,12 @@ func (p *ReplicationPool) queueMRFHeal() error {
 				VersionID: vID,
 			})
 			cancel()
-			if err != nil {
+			// A versioned marker lookup returns its metadata with a 405. Only
+			// accept that error with a real, matching marker identity.
+			validMarker := isErrMethodNotAllowed(err) && oi.DeleteMarker &&
+				vID != "" && oi.VersionID == vID && !oi.ModTime.IsZero() &&
+				oi.Bucket == e.Bucket && oi.Name != "" && oi.Name == decodeDirObject(e.Object)
+			if err != nil && !validMarker || oi.Name == "" {
 				continue
 			}
 

cmd/bucket-versioning-handler.go

@@ -97,7 +97,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+	result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketVersioningConfig, configData, false, false, nil)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -107,12 +107,12 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 	//
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
-	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	cfgStr := base64.StdEncoding.EncodeToString(result.meta.VersioningConfigXML)
 	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,
-		UpdatedAt:  updatedAt,
+		UpdatedAt:  result.updatedAt,
 	}))
 
 	writeSuccessResponseHeadersOnly(w)

cmd/common-main.go

@@ -445,6 +445,7 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.SendBufSize = ctx.Int("send-buf-size")
 	ctxt.RecvBufSize = ctx.Int("recv-buf-size")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
+	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
 	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
 
 	if conf := ctx.String("config"); len(conf) > 0 {
@@ -900,6 +901,7 @@ func serverHandleEnvVars() {
 	}
 
 	globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
+	globalSiteReplicationMetadataTombstones = env.Get("MINIO_SITE_REPLICATION_METADATA_TOMBSTONES", config.EnableOff) == config.EnableOn
 }
 
 func loadRootCredentials() auth.Credentials {

cmd/config-current.go

@@ -227,7 +227,7 @@ func initHelp() {
 		},
 		config.HelpKV{
 			Key:         config.ILMSubSys,
-			Description: "manage ILM settings for expiration, transition, and access-tier workers",
+			Description: "manage ILM settings for expiration and transition workers",
 			Optional:    true,
 		},
 	}
@@ -704,9 +704,6 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			if globalExpiryState != nil {
 				globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
 			}
-			if globalAccessTierState != nil {
-				globalAccessTierState.UpdateWorkers(ilmCfg.AccessWorkers)
-			}
 			globalILMConfig.update(ilmCfg)
 		}
 	}

cmd/data-scanner.go

@@ -899,7 +899,6 @@ type scannerItem struct {
 	objectName  string // Only the object name without prefixes.
 	replication replicationConfig
 	lifeCycle   *lifecycle.Lifecycle
-	poolIdx     int
 	Typ         fs.FileMode
 	heal        struct {
 		enabled bool
@@ -910,7 +909,6 @@ type scannerItem struct {
 
 type sizeSummary struct {
 	totalSize       int64
-	hotTierSize     int64
 	versions        uint64
 	deleteMarkers   uint64
 	replicatedSize  int64
@@ -1161,14 +1159,6 @@ eventLoop:
 		globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents)
 	}
 	i.alertExcessiveVersions(remainingVersions, cumulativeSize)
-	if globalILMConfig.accessTieringEnabled() {
-		for idx, oi := range objInfos {
-			if oi.IsLatest && events[idx].Action == lifecycle.NoneAction {
-				applyAccessTransition(ctx, i, oi)
-				break
-			}
-		}
-	}
 }
 
 func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
@@ -1484,8 +1474,6 @@ const (
 	ILMFreeVersionDelete = "ilm:free-version-delete"
 	// ILMTransition - audit trail for ILM transitioning.
 	ILMTransition = " ilm:transition"
-	// ILMAccessTier - audit trail for moving objects between server pools.
-	ILMAccessTier = "ilm:access-tier"
 )
 
 func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
@@ -1497,8 +1485,6 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
 		apiName = "ILMFreeVersionDelete"
 	case ILMTransition:
 		apiName = "ILMTransition"
-	case ILMAccessTier:
-		apiName = "ILMAccessTier"
 	}
 	auditLogInternal(ctx, AuditLogOptions{
 		Event:     event,

cmd/data-usage-cache-v9_test.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"reflect"
+	"testing"
+)
+
+// Read a real pre-removal scanner cache with nonzero hot-tier bytes. A v8
+// payload with a relabeled header would not exercise the ignored hts field.
+func TestDataUsageCacheReadV9(t *testing.T) {
+	raw, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.bin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(raw) == 0 || raw[0] != 9 {
+		t.Fatal("fixture is not v9")
+	}
+	expected, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var want, got dataUsageCache
+	if err := json.Unmarshal(expected, &want); err != nil {
+		t.Fatal(err)
+	}
+	if err := got.deserialize(bytes.NewReader(raw)); err != nil {
+		t.Fatal(err)
+	}
+	if !got.Info.LastUpdate.Equal(want.Info.LastUpdate) {
+		t.Fatal("cache timestamp changed")
+	}
+	// msgp restores local time while JSON preserves the UTC representation.
+	want.Info.LastUpdate = got.Info.LastUpdate
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("v9 ordinary cache fields changed:\ngot: %#v\nwant: %#v", got, want)
+	}
+	flat := got.flatten(*got.root())
+	if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 {
+		t.Fatalf("statistics changed: %+v", flat)
+	}
+	if flat.AllTierStats == nil || flat.AllTierStats.Tiers["COLD"] != (tierStats{TotalSize: 65536, NumVersions: 1, NumObjects: 1}) {
+		t.Fatalf("remote tier statistics changed: %+v", flat.AllTierStats)
+	}
+	buckets := []BucketInfo{{Name: "v9-bucket"}}
+	if gotInfo, wantInfo := got.dui(dataUsageRoot, buckets), want.dui(dataUsageRoot, buckets); !reflect.DeepEqual(gotInfo, wantInfo) {
+		t.Fatalf("bucket usage aggregation changed: %+v != %+v", gotInfo, wantInfo)
+	}
+	var buf bytes.Buffer
+	if err := got.serializeTo(&buf); err != nil {
+		t.Fatal(err)
+	}
+	if buf.Bytes()[0] != 8 {
+		t.Fatalf("wrote cache version %d, want 8", buf.Bytes()[0])
+	}
+	var roundtrip dataUsageCache
+	if err := roundtrip.deserialize(&buf); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, roundtrip) {
+		t.Fatal("v8 round trip lost ordinary fields")
+	}
+	if err := roundtrip.deserialize(bytes.NewReader(raw[:len(raw)/2])); err == nil {
+		t.Fatal("truncated v9 cache accepted")
+	}
+	raw[0] = 10
+	if err := roundtrip.deserialize(bytes.NewReader(raw)); err == nil {
+		t.Fatal("unknown cache version accepted")
+	}
+}

cmd/data-usage-cache.go

@@ -61,7 +61,6 @@ type dataUsageEntry struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
 	Size          int64             `msg:"sz"`
-	HotTierSize   int64             `msg:"hts"`
 	Objects       uint64            `msg:"os"`
 	Versions      uint64            `msg:"vs"` // Versions that are not delete markers.
 	DeleteMarkers uint64            `msg:"dms"`
@@ -135,8 +134,8 @@ func (ts tierStats) add(u tierStats) tierStats {
 	}
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -200,30 +199,14 @@ type dataUsageEntryV7 struct {
 	Compacted     bool              `msg:"c"`
 }
 
-// dataUsageEntryV8 is the on-disk shape before access-tier accounting was
-// introduced. Keep it so caches written by the previous release decode
-// without being discarded.
-type dataUsageEntryV8 struct {
-	Children dataUsageHashMap `msg:"ch"`
-	// These fields do no include any children.
-	Size          int64             `msg:"sz"`
-	Objects       uint64            `msg:"os"`
-	Versions      uint64            `msg:"vs"`
-	DeleteMarkers uint64            `msg:"dms"`
-	ObjSizes      sizeHistogram     `msg:"szs"`
-	ObjVersions   versionsHistogram `msg:"vh"`
-	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
-	Compacted     bool              `msg:"c"`
-}
-
 // dataUsageCache contains a cache of data usage entries latest version.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Cache map[string]dataUsageEntry
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -261,12 +244,6 @@ type dataUsageCacheV7 struct {
 	Cache map[string]dataUsageEntryV7
 }
 
-// dataUsageCacheV8 contains a cache of data usage entries version 8.
-type dataUsageCacheV8 struct {
-	Info  dataUsageCacheInfo
-	Cache map[string]dataUsageEntryV8
-}
-
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -295,7 +272,6 @@ type dataUsageCacheInfo struct {
 
 func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Size += summary.totalSize
-	e.HotTierSize += summary.hotTierSize
 	e.Versions += summary.versions
 	e.DeleteMarkers += summary.deleteMarkers
 	e.ObjSizes.add(summary.totalSize)
@@ -315,7 +291,6 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Versions += other.Versions
 	e.DeleteMarkers += other.DeleteMarkers
 	e.Size += other.Size
-	e.HotTierSize += other.HotTierSize
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
@@ -456,7 +431,6 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	flat := d.flatten(*e)
 	dui := DataUsageInfo{
 		LastUpdate:              d.Info.LastUpdate,
-		ScannerCycle:            d.Info.NextCycle,
 		ObjectsTotalCount:       flat.Objects,
 		VersionsTotalCount:      flat.Versions,
 		DeleteMarkersTotalCount: flat.DeleteMarkers,
@@ -807,7 +781,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		flat := d.flatten(*e)
 		bui := BucketUsageInfo{
 			Size:                    uint64(flat.Size),
-			HotTierSize:             uint64(max(flat.HotTierSize, 0)),
 			VersionsCount:           flat.Versions,
 			ObjectsCount:            flat.Objects,
 			DeleteMarkersCount:      flat.DeleteMarkers,
@@ -1007,8 +980,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 9
-	dataUsageCacheVerV8      = 8
+	dataUsageCacheVerCurrent = 8
+	dataUsageCacheVerV9      = 9 // Retired access-tier cache; only adds the ignored "hts" entry key.
 	dataUsageCacheVerV7      = 7
 	dataUsageCacheVerV6      = 6
 	dataUsageCacheVerV5      = 5
@@ -1210,34 +1183,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		}
 
 		return nil
-	case dataUsageCacheVerV8:
-		// Zstd compressed.
-		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
-		if err != nil {
-			return err
-		}
-		defer dec.Close()
-		dold := &dataUsageCacheV8{}
-		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
-			return err
-		}
-		d.Info = dold.Info
-		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
-		for k, v := range dold.Cache {
-			d.Cache[k] = dataUsageEntry{
-				Children:      v.Children,
-				Size:          v.Size,
-				Objects:       v.Objects,
-				Versions:      v.Versions,
-				DeleteMarkers: v.DeleteMarkers,
-				ObjSizes:      v.ObjSizes,
-				ObjVersions:   v.ObjVersions,
-				AllTierStats:  v.AllTierStats,
-				Compacted:     v.Compacted,
-			}
-		}
-		return nil
-	case dataUsageCacheVerCurrent:
+	case dataUsageCacheVerCurrent, dataUsageCacheVerV9:
 		// Zstd compressed.
 		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
 		if err != nil {

cmd/data-usage-cache_gen.go

@@ -1591,145 +1591,6 @@ func (z *dataUsageCacheV7) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *dataUsageCacheV8) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Info":
-			err = z.Info.DecodeMsg(dc)
-			if err != nil {
-				err = msgp.WrapError(err, "Info")
-				return
-			}
-		case "Cache":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Cache")
-				return
-			}
-			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV8, zb0002)
-			} else if len(z.Cache) > 0 {
-				clear(z.Cache)
-			}
-			for zb0002 > 0 {
-				zb0002--
-				var za0001 string
-				za0001, err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Cache")
-					return
-				}
-				var za0002 dataUsageEntryV8
-				err = za0002.DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
-					return
-				}
-				z.Cache[za0001] = za0002
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageCacheV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Info":
-			bts, err = z.Info.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Info")
-				return
-			}
-		case "Cache":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Cache")
-				return
-			}
-			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV8, zb0002)
-			} else if len(z.Cache) > 0 {
-				clear(z.Cache)
-			}
-			for zb0002 > 0 {
-				var za0002 dataUsageEntryV8
-				zb0002--
-				var za0001 string
-				za0001, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache")
-					return
-				}
-				bts, err = za0002.UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
-					return
-				}
-				z.Cache[za0001] = za0002
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageCacheV8) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
-	if z.Cache != nil {
-		for za0001, za0002 := range z.Cache {
-			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
-		}
-	}
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -1762,12 +1623,6 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Size")
 				return
 			}
-		case "hts":
-			z.HotTierSize, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "HotTierSize")
-				return
-			}
 		case "os":
 			z.Objects, err = dc.ReadUint64()
 			if err != nil {
@@ -1946,12 +1801,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// check for omitted fields
-	zb0001Len := uint32(10)
-	var zb0001Mask uint16 /* 10 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x100
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -1981,16 +1836,6 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Size")
 			return
 		}
-		// write "hts"
-		err = en.Append(0xa3, 0x68, 0x74, 0x73)
-		if err != nil {
-			return
-		}
-		err = en.WriteInt64(z.HotTierSize)
-		if err != nil {
-			err = msgp.WrapError(err, "HotTierSize")
-			return
-		}
 		// write "os"
 		err = en.Append(0xa2, 0x6f, 0x73)
 		if err != nil {
@@ -2055,7 +1900,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 				return
 			}
 		}
-		if (zb0001Mask & 0x100) == 0 { // if not omitted
+		if (zb0001Mask & 0x80) == 0 { // if not omitted
 			// write "ats"
 			err = en.Append(0xa3, 0x61, 0x74, 0x73)
 			if err != nil {
@@ -2136,12 +1981,12 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// check for omitted fields
-	zb0001Len := uint32(10)
-	var zb0001Mask uint16 /* 10 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x100
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -2158,9 +2003,6 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 		// string "sz"
 		o = append(o, 0xa2, 0x73, 0x7a)
 		o = msgp.AppendInt64(o, z.Size)
-		// string "hts"
-		o = append(o, 0xa3, 0x68, 0x74, 0x73)
-		o = msgp.AppendInt64(o, z.HotTierSize)
 		// string "os"
 		o = append(o, 0xa2, 0x6f, 0x73)
 		o = msgp.AppendUint64(o, z.Objects)
@@ -2182,7 +2024,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 		for za0002 := range z.ObjVersions {
 			o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 		}
-		if (zb0001Mask & 0x100) == 0 { // if not omitted
+		if (zb0001Mask & 0x80) == 0 { // if not omitted
 			// string "ats"
 			o = append(o, 0xa3, 0x61, 0x74, 0x73)
 			if z.AllTierStats == nil {
@@ -2246,12 +2088,6 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Size")
 				return
 			}
-		case "hts":
-			z.HotTierSize, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "HotTierSize")
-				return
-			}
 		case "os":
 			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
 			if err != nil {
@@ -2429,7 +2265,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
 	if z.AllTierStats == nil {
 		s += msgp.NilSize
 	} else {
@@ -3422,438 +3258,6 @@ func (z *dataUsageEntryV7) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *dataUsageEntryV8) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var zb0001Mask uint8 /* 1 bits */
-	_ = zb0001Mask
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "ch":
-			err = z.Children.DecodeMsg(dc)
-			if err != nil {
-				err = msgp.WrapError(err, "Children")
-				return
-			}
-		case "sz":
-			z.Size, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "Size")
-				return
-			}
-		case "os":
-			z.Objects, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "Objects")
-				return
-			}
-		case "vs":
-			z.Versions, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "Versions")
-				return
-			}
-		case "dms":
-			z.DeleteMarkers, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "DeleteMarkers")
-				return
-			}
-		case "szs":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "ObjSizes")
-				return
-			}
-			if zb0002 != uint32(dataUsageBucketLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
-				return
-			}
-			for za0001 := range z.ObjSizes {
-				z.ObjSizes[za0001], err = dc.ReadUint64()
-				if err != nil {
-					err = msgp.WrapError(err, "ObjSizes", za0001)
-					return
-				}
-			}
-		case "vh":
-			var zb0003 uint32
-			zb0003, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "ObjVersions")
-				return
-			}
-			if zb0003 != uint32(dataUsageVersionLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
-				return
-			}
-			for za0002 := range z.ObjVersions {
-				z.ObjVersions[za0002], err = dc.ReadUint64()
-				if err != nil {
-					err = msgp.WrapError(err, "ObjVersions", za0002)
-					return
-				}
-			}
-		case "ats":
-			if dc.IsNil() {
-				err = dc.ReadNil()
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				z.AllTierStats = nil
-			} else {
-				if z.AllTierStats == nil {
-					z.AllTierStats = new(allTierStats)
-				}
-				var zb0004 uint32
-				zb0004, err = dc.ReadMapHeader()
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "AllTierStats")
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "ts":
-						var zb0005 uint32
-						zb0005, err = dc.ReadMapHeader()
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats", "Tiers")
-							return
-						}
-						if z.AllTierStats.Tiers == nil {
-							z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
-						} else if len(z.AllTierStats.Tiers) > 0 {
-							clear(z.AllTierStats.Tiers)
-						}
-						for zb0005 > 0 {
-							zb0005--
-							var za0003 string
-							za0003, err = dc.ReadString()
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers")
-								return
-							}
-							var za0004 tierStats
-							var zb0006 uint32
-							zb0006, err = dc.ReadMapHeader()
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-								return
-							}
-							for zb0006 > 0 {
-								zb0006--
-								field, err = dc.ReadMapKeyPtr()
-								if err != nil {
-									err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-									return
-								}
-								switch msgp.UnsafeString(field) {
-								case "ts":
-									za0004.TotalSize, err = dc.ReadUint64()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
-										return
-									}
-								case "nv":
-									za0004.NumVersions, err = dc.ReadInt()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
-										return
-									}
-								case "no":
-									za0004.NumObjects, err = dc.ReadInt()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
-										return
-									}
-								default:
-									err = dc.Skip()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-										return
-									}
-								}
-							}
-							z.AllTierStats.Tiers[za0003] = za0004
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats")
-							return
-						}
-					}
-				}
-			}
-			zb0001Mask |= 0x1
-		case "c":
-			z.Compacted, err = dc.ReadBool()
-			if err != nil {
-				err = msgp.WrapError(err, "Compacted")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	// Clear omitted fields.
-	if (zb0001Mask & 0x1) == 0 {
-		z.AllTierStats = nil
-	}
-
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageEntryV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var zb0001Mask uint8 /* 1 bits */
-	_ = zb0001Mask
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "ch":
-			bts, err = z.Children.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Children")
-				return
-			}
-		case "sz":
-			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Size")
-				return
-			}
-		case "os":
-			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Objects")
-				return
-			}
-		case "vs":
-			z.Versions, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Versions")
-				return
-			}
-		case "dms":
-			z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "DeleteMarkers")
-				return
-			}
-		case "szs":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ObjSizes")
-				return
-			}
-			if zb0002 != uint32(dataUsageBucketLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
-				return
-			}
-			for za0001 := range z.ObjSizes {
-				z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "ObjSizes", za0001)
-					return
-				}
-			}
-		case "vh":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ObjVersions")
-				return
-			}
-			if zb0003 != uint32(dataUsageVersionLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
-				return
-			}
-			for za0002 := range z.ObjVersions {
-				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "ObjVersions", za0002)
-					return
-				}
-			}
-		case "ats":
-			if msgp.IsNil(bts) {
-				bts, err = msgp.ReadNilBytes(bts)
-				if err != nil {
-					return
-				}
-				z.AllTierStats = nil
-			} else {
-				if z.AllTierStats == nil {
-					z.AllTierStats = new(allTierStats)
-				}
-				var zb0004 uint32
-				zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "AllTierStats")
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "ts":
-						var zb0005 uint32
-						zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats", "Tiers")
-							return
-						}
-						if z.AllTierStats.Tiers == nil {
-							z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
-						} else if len(z.AllTierStats.Tiers) > 0 {
-							clear(z.AllTierStats.Tiers)
-						}
-						for zb0005 > 0 {
-							var za0004 tierStats
-							zb0005--
-							var za0003 string
-							za0003, bts, err = msgp.ReadStringBytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers")
-								return
-							}
-							var zb0006 uint32
-							zb0006, bts, err = msgp.ReadMapHeaderBytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-								return
-							}
-							for zb0006 > 0 {
-								zb0006--
-								field, bts, err = msgp.ReadMapKeyZC(bts)
-								if err != nil {
-									err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-									return
-								}
-								switch msgp.UnsafeString(field) {
-								case "ts":
-									za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
-										return
-									}
-								case "nv":
-									za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
-										return
-									}
-								case "no":
-									za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
-										return
-									}
-								default:
-									bts, err = msgp.Skip(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-										return
-									}
-								}
-							}
-							z.AllTierStats.Tiers[za0003] = za0004
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats")
-							return
-						}
-					}
-				}
-			}
-			zb0001Mask |= 0x1
-		case "c":
-			z.Compacted, bts, err = msgp.ReadBoolBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Compacted")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	// Clear omitted fields.
-	if (zb0001Mask & 0x1) == 0 {
-		z.AllTierStats = nil
-	}
-
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageEntryV8) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
-	if z.AllTierStats == nil {
-		s += msgp.NilSize
-	} else {
-		s += 1 + 3 + msgp.MapHeaderSize
-		if z.AllTierStats.Tiers != nil {
-			for za0003, za0004 := range z.AllTierStats.Tiers {
-				_ = za0004
-				s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
-			}
-		}
-	}
-	s += 2 + msgp.BoolSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{

cmd/data-usage-utils.go

@@ -46,8 +46,7 @@ type BucketTargetUsageInfo struct {
 // - total objects in a bucket
 // - object size histogram per bucket
 type BucketUsageInfo struct {
-	Size        uint64 `json:"size"`
-	HotTierSize uint64 `json:"hotTierSize,omitempty"`
+	Size uint64 `json:"size"`
 	// Following five fields suffixed with V1 are here for backward compatibility
 	// Total Size for objects that have not yet been replicated
 	ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
@@ -79,10 +78,6 @@ type DataUsageInfo struct {
 	// LastUpdate is the timestamp of when the data usage info was last updated.
 	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
-	// ScannerCycle changes only after a complete scanner pass. Background
-	// consumers use it to distinguish a partial cache update from a baseline
-	// that has visited every bucket and server pool.
-	ScannerCycle uint32 `json:"scannerCycle,omitempty"`
 
 	// Objects total count across all buckets
 	ObjectsTotalCount uint64 `json:"objectsCount"`

cmd/erasure-multipart.go

@@ -1179,7 +1179,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		switch {
 		case gerr == nil:
 			reconcileStoredObjectLock(fi.Metadata, storedObjectLockState(curr.UserDefined))
-			reconcileStoredObjectTags(fi.Metadata, curr.UserDefined)
+			reconcileStoredObjectTags(fi.Metadata, curr.UserTags, curr.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
 		case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
 			// No existing version to order against: keep the upload's own accepted
 			// lock, including a pre-upgrade upload that persisted values without

cmd/erasure-object.go

@@ -135,7 +135,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 
 	if dstOpts.ReplicaLockReconcile {
 		reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(fi.Metadata))
-		reconcileStoredObjectTags(srcInfo.UserDefined, fi.Metadata)
+		reconcileStoredObjectTags(srcInfo.UserDefined, fi.Metadata[xhttp.AmzObjectTagging], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
 	}
 
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
@@ -1311,7 +1311,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		// existing version contributes independently ordered lock and tags.
 		if opts.ReplicaLockReconcile && err == nil {
 			reconcileStoredObjectLock(opts.UserDefined, storedObjectLockState(obj.UserDefined))
-			reconcileStoredObjectTags(opts.UserDefined, obj.UserDefined)
+			reconcileStoredObjectTags(opts.UserDefined, obj.UserTags, obj.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
 		}
 	}
 
@@ -2327,7 +2327,11 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	fi.Metadata[xhttp.AmzObjectTagging] = tags
 	fi.ReplicationState = opts.PutReplicationState()
+	stamp := monotonicTaggingTimestamp(opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
 	maps.Copy(fi.Metadata, opts.UserDefined)
+	if stamp != "" {
+		fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
+	}
 
 	if err = er.updateObjectMeta(ctx, bucket, object, fi, onlineDisks); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)

cmd/erasure-server-pool-consistency.go

@@ -126,10 +126,9 @@ func mergedPoolObjectInfo(copies []PoolObjInfo) ObjectInfo {
 		stamp, _ := time.Parse(time.RFC3339Nano, ts)
 		if olderThan(oi.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], stamp) {
 			oi.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = ts
-			oi.UserDefined[xhttp.AmzObjectTagging] = copy.ObjInfo.UserDefined[xhttp.AmzObjectTagging]
+			oi.UserTags = copy.ObjInfo.UserTags
 		}
 	}
-	oi.UserTags = oi.UserDefined[xhttp.AmzObjectTagging]
 	return oi
 }
 
@@ -190,6 +189,12 @@ func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, obj
 			changes[key] = ""
 		}
 	}
+	// Metadata callbacks may explicitly replace tags in the raw write map.
+	// Otherwise retain the merged value from the ObjectInfo read model.
+	tags, ok := updated.UserDefined[xhttp.AmzObjectTagging]
+	if !ok {
+		tags = updated.UserTags
+	}
 	state := storedObjectLockState(updated.UserDefined)
 	opts.VersionID = updated.VersionID
 	if opts.VersionID == "" {
@@ -199,11 +204,13 @@ func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, obj
 	opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
 		maps.Copy(oi.UserDefined, changes)
 		replaceObjectLockMetadata(oi.UserDefined, state)
-		for _, key := range []string{xhttp.AmzObjectTagging, ReservedMetadataPrefixLower + TaggingTimestamp} {
-			value, exists := updated.UserDefined[key]
-			if exists || oi.UserDefined[key] != "" {
-				oi.UserDefined[key] = value
-			}
+		// Reassemble the tag value and its ordering timestamp for storage.
+		if tags != "" || oi.UserTags != "" {
+			oi.UserDefined[xhttp.AmzObjectTagging] = tags
+		}
+		key := ReservedMetadataPrefixLower + TaggingTimestamp
+		if value, exists := updated.UserDefined[key]; exists || oi.UserDefined[key] != "" {
+			oi.UserDefined[key] = value
 		}
 		return ReplicateDecision{}, nil
 	}
@@ -220,19 +227,36 @@ func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, obj
 	return primary, nil
 }
 
-func reconcileStoredObjectTags(metadata, stored map[string]string) {
+// Pass the stored tag value explicitly: ObjectInfo.UserDefined excludes it,
+// whereas FileInfo.Metadata retains the raw storage key.
+func reconcileStoredObjectTags(metadata map[string]string, storedTags, storedTimestamp string) {
 	key := ReservedMetadataPrefixLower + TaggingTimestamp
-	stamp, err := time.Parse(time.RFC3339Nano, stored[key])
+	stamp, err := time.Parse(time.RFC3339Nano, storedTimestamp)
 	if err != nil {
 		return
 	}
 	incoming, err := time.Parse(time.RFC3339Nano, metadata[key])
 	if err != nil || !stamp.Before(incoming) {
-		metadata[key] = stored[key]
-		metadata[xhttp.AmzObjectTagging] = stored[xhttp.AmzObjectTagging]
+		metadata[key] = storedTimestamp
+		metadata[xhttp.AmzObjectTagging] = storedTags
 	}
 }
 
+// Local tagging mutations must advance the revision they overwrite, even when
+// a request's clock or lock acquisition order is behind the stored revision.
+// Replica writes use reconcileStoredObjectTags instead of minting a revision.
+func monotonicTaggingTimestamp(incoming, stored string) string {
+	requested, err := time.Parse(time.RFC3339Nano, incoming)
+	if err != nil {
+		return incoming
+	}
+	current, err := time.Parse(time.RFC3339Nano, stored)
+	if err != nil || requested.After(current) {
+		return incoming
+	}
+	return current.Add(time.Nanosecond).UTC().Format(time.RFC3339Nano)
+}
+
 // A restored version still owns its tier reference even while IsRemote is
 // false. Only the last copy of a reference may schedule its contents for GC.
 func sharesTierObject(oi ObjectInfo, copies []PoolObjInfo) bool {
@@ -287,33 +311,51 @@ func (z *erasureServerPools) retireReplicaCopies(ctx context.Context, bucket, ob
 	return nil
 }
 
-// deleteObjectConditional evaluates the condition once against the logical
+// deleteObjectReconciled evaluates any condition once against the logical
 // version, then removes all its copies under the same lock as pooled writers.
-func (z *erasureServerPools) deleteObjectConditional(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	primary := copies[0]
-	if opts.CheckPrecondFn(primary.ObjInfo) {
+	if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(primary.ObjInfo) {
 		return ObjectInfo{}, PreConditionFailed{}
 	}
 	opts.CheckPrecondFn = nil
 	opts.NoLock = true
 	if opts.EvalRetentionBypassFn != nil || opts.EvalMetadataFn != nil {
-		versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
-		if err != nil {
-			return ObjectInfo{}, err
+		logical := primary.ObjInfo
+		var gerr error
+		switch {
+		case logical.DeleteMarker:
+			// Markers can be deleted by version ID. Match the set layer's
+			// callback inputs instead of rejecting them as metadata updates.
+			gerr = toObjectErr(errMethodNotAllowed, bucket, object)
+			if opts.VersionID == "" || opts.DeleteMarker {
+				gerr = toObjectErr(errFileNotFound, bucket, object)
+			}
+		case opts.VersionID != "":
+			// An addressed version already resolved every copy above.
+			logical = mergedPoolObjectInfo(copies)
+		default:
+			versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
+			if err != nil {
+				return ObjectInfo{}, err
+			}
+			logical = mergedPoolObjectInfo(versions)
 		}
-		logical := mergedPoolObjectInfo(versions)
+		// Keep the retention gate first. These callbacks independently evaluate
+		// the logical version and run once before any deletion; the handler only
+		// sweeps metadata's transition state after a successful delete.
 		if opts.EvalRetentionBypassFn != nil {
-			if err := opts.EvalRetentionBypassFn(logical, nil); err != nil {
+			if err := opts.EvalRetentionBypassFn(logical, gerr); err != nil {
 				return ObjectInfo{}, err
 			}
 			opts.EvalRetentionBypassFn = nil
 		}
 		if opts.EvalMetadataFn != nil {
-			decision, err := opts.EvalMetadataFn(&logical, nil)
+			decision, err := opts.EvalMetadataFn(&logical, gerr)
 			if err != nil {
 				return ObjectInfo{}, err
 			}

cmd/erasure-server-pool-consistency_test.go

@@ -24,12 +24,16 @@ import (
 	"fmt"
 	"io"
 	"maps"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	madmin "github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
 
@@ -62,6 +66,490 @@ func putConsistencyObject(t *testing.T, z *erasureServerPools, bucket, object st
 	return oi
 }
 
+func consistencyRequest(t *testing.T, router http.Handler, method, bucket, object, version string) *httptest.ResponseRecorder {
+	t.Helper()
+	url := getGetObjectURL("", bucket, object)
+	if version != "" {
+		url += "?versionId=" + version
+	}
+	req, err := newTestSignedRequestV4(method, url, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	return rec
+}
+
+// Exercise the handler's real replication/retention callbacks, including the
+// MethodNotAllowed metadata returned for an explicitly addressed delete marker.
+func TestPoolsDeleteVersionAPI(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, kind := range []string{"uuid", "null", "marker"} {
+		for primary := range 2 {
+			t.Run(fmt.Sprintf("%s/primary=%d", kind, primary), func(t *testing.T) {
+				object := fmt.Sprintf("%s-%d", kind, primary)
+				opts := ObjectOptions{Versioned: kind != "null", MTime: UTCNow().Add(-time.Hour)}
+				var addressed ObjectInfo
+				if kind == "marker" {
+					opts.VersionID, opts.DeleteMarker = mustGetUUID(), true
+					addressed, err = z.serverPools[primary].DeleteObject(ctx, bucket, object, opts)
+					if err != nil {
+						t.Fatal(err)
+					}
+				} else {
+					addressed = putConsistencyObject(t, z, bucket, object, primary, "payload", opts)
+				}
+				version := addressed.VersionID
+				if version == "" {
+					version = nullVersionID
+				}
+				opts.VersionID = version
+				opts.MTime = addressed.ModTime.Add(-time.Minute)
+				if kind == "marker" {
+					opts.DeleteMarker = true
+					if _, err := z.serverPools[1-primary].DeleteObject(ctx, bucket, object, opts); err != nil {
+						t.Fatal(err)
+					}
+				} else {
+					putConsistencyObject(t, z, bucket, object, 1-primary, "payload", opts)
+				}
+				latest := putConsistencyObject(t, z, bucket, object, 1-primary, "keep-latest", ObjectOptions{Versioned: true})
+				rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, version)
+				if rec.Code != http.StatusNoContent {
+					t.Fatalf("DELETE: %d %s", rec.Code, rec.Body.String())
+				}
+				if kind == "marker" && (strings.Join(rec.Header()[xhttp.AmzDeleteMarker], "") != "true" || strings.Join(rec.Header()[xhttp.AmzVersionID], "") != version) {
+					t.Errorf("lost deleted marker response headers: %v", rec.Header())
+				}
+				for i, pool := range z.serverPools {
+					if _, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) {
+						t.Errorf("pool %d retained addressed version: %v", i, err)
+					}
+				}
+				for _, method := range []string{http.MethodGet, http.MethodHead} {
+					if rec := consistencyRequest(t, router, method, bucket, object, version); rec.Code != http.StatusNotFound {
+						t.Errorf("%s after DELETE: %d %s", method, rec.Code, rec.Body.String())
+					}
+				}
+				if got, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err != nil || got.VersionID != latest.VersionID {
+					t.Errorf("DELETE changed another version: %+v, %v", got, err)
+				}
+				if rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, version); rec.Code != http.StatusNoContent {
+					t.Errorf("idempotent retry: %d %s", rec.Code, rec.Body.String())
+				}
+			})
+		}
+	}
+	if rec := consistencyRequest(t, router, http.MethodDelete, bucket, "absent-key", mustGetUUID()); rec.Code != http.StatusNoContent {
+		t.Errorf("absent key: %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+type consistencyReadFaultDisk struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d consistencyReadFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+	if volume == d.bucket && path == d.object {
+		return FileInfo{}, errDiskNotFound
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d consistencyReadFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) {
+	if volume == d.bucket && path == d.object {
+		return RawFileInfo{}, errDiskNotFound
+	}
+	return d.StorageAPI.ReadXL(ctx, volume, path, readData)
+}
+
+func TestPoolsDeleteVersionUnreadablePool(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, holding := range []bool{false, true} {
+		for failed := range 2 {
+			t.Run(fmt.Sprintf("holding=%t/pool=%d", holding, failed), func(t *testing.T) {
+				object := fmt.Sprintf("unreadable-%t-%d", holding, failed)
+				oi := putConsistencyObject(t, z, bucket, object, 1-failed, "payload", ObjectOptions{Versioned: true})
+				if holding {
+					putConsistencyObject(t, z, bucket, object, failed, "payload", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime})
+				}
+				set := z.serverPools[failed].getHashedSet(object)
+				getDisks := set.getDisks
+				faulty := append([]StorageAPI(nil), getDisks()...)
+				for i := range faulty {
+					faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+				}
+				set.getDisks = func() []StorageAPI { return faulty }
+				defer func() { set.getDisks = getDisks }()
+				if rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, oi.VersionID); rec.Code != http.StatusServiceUnavailable {
+					t.Errorf("unreadable pool DELETE: %d %s", rec.Code, rec.Body.String())
+				}
+				if _, err := z.serverPools[1-failed].GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: oi.VersionID}); err != nil {
+					t.Errorf("DELETE lost the readable copy: %v", err)
+				}
+				set.getDisks = getDisks
+				if rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, oi.VersionID); rec.Code != http.StatusNoContent {
+					t.Errorf("recovered pool retry: %d %s", rec.Code, rec.Body.String())
+				}
+				for i, pool := range z.serverPools {
+					if _, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) {
+						t.Errorf("pool %d retained version after retry: %v", i, err)
+					}
+				}
+			})
+		}
+	}
+}
+
+type consistencyDeleteCountDisk struct {
+	StorageAPI
+	bucket, object         string
+	metadataReads, deletes *atomic.Int32
+}
+
+func (d consistencyDeleteCountDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+	if volume == d.bucket && path == d.object {
+		d.metadataReads.Add(1)
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d consistencyDeleteCountDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
+	if volume == d.bucket && path == d.object {
+		d.deletes.Add(1)
+	}
+	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, force, opts)
+}
+
+func TestPoolsDeleteVersionSingleCopy(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for primary := range 2 {
+		t.Run(fmt.Sprintf("pool=%d", primary), func(t *testing.T) {
+			object := fmt.Sprintf("single-copy-%d", primary)
+			oi := putConsistencyObject(t, z, bucket, object, primary, "payload", ObjectOptions{Versioned: true})
+			var metadataReads, deletes [2]atomic.Int32
+			for i, pool := range z.serverPools {
+				set := pool.getHashedSet(object)
+				getDisks := set.getDisks
+				disks := append([]StorageAPI(nil), getDisks()...)
+				for j := range disks {
+					disks[j] = consistencyDeleteCountDisk{StorageAPI: disks[j], bucket: bucket, object: object, metadataReads: &metadataReads[i], deletes: &deletes[i]}
+				}
+				set.getDisks = func() []StorageAPI { return disks }
+				defer func() { set.getDisks = getDisks }()
+			}
+			metadata, retention := 0, 0
+			_, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{
+				Versioned: true, VersionID: oi.VersionID,
+				EvalMetadataFn: func(current *ObjectInfo, err error) (ReplicateDecision, error) {
+					metadata++
+					if err != nil || current.VersionID != oi.VersionID {
+						t.Errorf("metadata callback: %+v, %v", current, err)
+					}
+					// Count preflight reads before the physical delete reads its own metadata.
+					for i := range metadataReads {
+						if got := metadataReads[i].Load(); got != 16 {
+							t.Errorf("pool %d metadata reads before callbacks = %d; want once per disk (16)", i, got)
+						}
+					}
+					return ReplicateDecision{}, nil
+				},
+				EvalRetentionBypassFn: func(current ObjectInfo, err error) error {
+					retention++
+					return err
+				},
+			})
+			if err != nil || metadata != 1 || retention != 1 {
+				t.Fatalf("DELETE: %v, metadata=%d retention=%d", err, metadata, retention)
+			}
+			if deletes[primary].Load() != 16 || deletes[1-primary].Load() != 0 {
+				t.Errorf("physical deletes per pool = %d, %d; want once per holding disk only", deletes[0].Load(), deletes[1].Load())
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionReplicationPurge(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for _, marker := range []bool{false, true} {
+		t.Run(fmt.Sprintf("marker=%t", marker), func(t *testing.T) {
+			object := fmt.Sprintf("local-replication-purge-%t", marker)
+			version := mustGetUUID()
+			for _, pool := range z.serverPools {
+				opts := ObjectOptions{Versioned: true, VersionID: version, DeleteMarker: marker}
+				var err error
+				if marker {
+					_, err = pool.DeleteObject(t.Context(), bucket, object, opts)
+				} else {
+					_, err = pool.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("payload"), 7, "", ""), opts)
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			dsc := ReplicateDecision{}
+			dsc.Set(newReplicateTargetDecision("arn1", true, false))
+			opts := ObjectOptions{Versioned: true, VersionID: version}
+			for attempt := range 2 {
+				calls := 0
+				opts.EvalMetadataFn = func(current *ObjectInfo, gerr error) (ReplicateDecision, error) {
+					calls++
+					wantMethodNotAllowed := marker || attempt > 0
+					if (wantMethodNotAllowed && !isErrMethodNotAllowed(gerr)) || (!wantMethodNotAllowed && gerr != nil) {
+						t.Errorf("pending purge callback lost set-layer read error: %v", gerr)
+					}
+					return dsc, nil
+				}
+				got, err := z.DeleteObject(t.Context(), bucket, object, opts)
+				if err != nil || calls != 1 || got.VersionPurgeStatus != replication.VersionPurgePending || got.replicationDecision != dsc.String() {
+					t.Fatalf("replication scheduling result: %+v, %v, calls=%d", got, err, calls)
+				}
+				for i, pool := range z.serverPools {
+					got, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version})
+					if (err != nil && (!got.DeleteMarker || !isErrMethodNotAllowed(err))) || got.VersionPurgeStatus != replication.VersionPurgePending {
+						t.Errorf("pool %d lost pending purge: %+v, %v", i, got, err)
+					}
+				}
+			}
+			// The local worker completes the purge without ReplicationRequest.
+			// Both pending copies must be removed, including a pending marker purge.
+			opts.EvalMetadataFn = nil
+			opts.DeleteReplication = ReplicationState{
+				VersionPurgeStatusInternal: "arn1=COMPLETED;",
+				PurgeTargets:               map[string]VersionPurgeStatusType{"arn1": replication.VersionPurgeComplete},
+			}
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) {
+					t.Errorf("pool %d retained completed replication purge: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionCleanupFailure(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for primary := range 2 {
+		t.Run(fmt.Sprintf("primary=%d", primary), func(t *testing.T) {
+			object := fmt.Sprintf("cleanup-failure-%d", primary)
+			oi := putConsistencyObject(t, z, bucket, object, primary, "payload", ObjectOptions{Versioned: true})
+			putConsistencyObject(t, z, bucket, object, 1-primary, "payload", ObjectOptions{
+				Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime.Add(-time.Minute),
+			})
+			set := z.serverPools[1-primary].getHashedSet(object)
+			getDisks := set.getDisks
+			faulty := append([]StorageAPI(nil), getDisks()...)
+			for i := range faulty {
+				faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
+			}
+			set.getDisks = func() []StorageAPI { return faulty }
+			defer func() { set.getDisks = getDisks }()
+			opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID}
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err == nil {
+				t.Error("acknowledged DELETE despite failed secondary cleanup")
+			}
+			if got, err := z.serverPools[primary].GetObjectInfo(t.Context(), bucket, object, opts); err != nil || got.ETag != oi.ETag {
+				t.Errorf("lost authoritative copy after secondary failure: %+v, %v", got, err)
+			}
+			set.getDisks = getDisks
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(t.Context(), bucket, object, opts); !isErrVersionNotFound(err) {
+					t.Errorf("pool %d retained version after retry: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionCallbacks(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for _, marker := range []bool{false, true} {
+		t.Run(fmt.Sprintf("marker=%t", marker), func(t *testing.T) {
+			object := fmt.Sprintf("callbacks-%t", marker)
+			old, recent := "2026-09-09T09:00:00Z", "2026-09-09T10:00:00Z"
+			oi := putConsistencyObject(t, z, bucket, object, 0, "payload", ObjectOptions{
+				Versioned: true, UserDefined: poolLockMetadata("GOVERNANCE", "OFF", recent, old),
+			})
+			putConsistencyObject(t, z, bucket, object, 1, "payload", ObjectOptions{
+				Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime,
+				UserDefined: poolLockMetadata("", "ON", old, recent),
+			})
+			if marker {
+				markerOpts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow()}
+				for _, pool := range z.serverPools {
+					var err error
+					oi, err = pool.DeleteObject(t.Context(), bucket, object, markerOpts)
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			check := func(current ObjectInfo, err error) {
+				t.Helper()
+				if current.VersionID != oi.VersionID || current.DeleteMarker != marker || (marker && !isErrMethodNotAllowed(err)) || (!marker && err != nil) {
+					t.Errorf("wrong callback version or read error: %+v, %v", current, err)
+				}
+				if !marker {
+					state := storedObjectLockState(current.UserDefined)
+					if state.mode != "GOVERNANCE" || state.legalHold != "ON" {
+						t.Errorf("callback did not reconcile independently ordered lock state: %+v", state)
+					}
+				}
+			}
+			for _, reject := range []string{"retention", "metadata", "none"} {
+				metadata, retention := 0, 0
+				denied := errors.New("callback denied deletion")
+				opts := ObjectOptions{
+					Versioned: true, VersionID: oi.VersionID,
+					EvalRetentionBypassFn: func(current ObjectInfo, err error) error {
+						retention++
+						check(current, err)
+						if reject == "retention" {
+							return denied
+						}
+						return nil
+					},
+					EvalMetadataFn: func(current *ObjectInfo, err error) (ReplicateDecision, error) {
+						metadata++
+						check(*current, err)
+						if reject == "metadata" {
+							return ReplicateDecision{}, denied
+						}
+						return ReplicateDecision{}, nil
+					},
+				}
+				_, err := z.DeleteObject(t.Context(), bucket, object, opts)
+				if reject == "none" {
+					if err != nil || metadata != 1 || retention != 1 {
+						t.Fatalf("DELETE: %v, metadata=%d retention=%d", err, metadata, retention)
+					}
+				} else if !errors.Is(err, denied) {
+					t.Fatalf("%s rejection was lost: %v", reject, err)
+				}
+				for i, pool := range z.serverPools {
+					current, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
+					if reject == "none" {
+						if !isErrVersionNotFound(err) {
+							t.Errorf("pool %d retained deleted version: %v", i, err)
+						}
+					} else if current.VersionID != oi.VersionID || (err != nil && (!marker || !isErrMethodNotAllowed(err))) {
+						t.Errorf("pool %d changed despite %s rejection: %+v, %v", i, reject, current, err)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionSpecialCalls(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	t.Run("incoming-marker-replication", func(t *testing.T) {
+		const object = "incoming-marker"
+		opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, ReplicationRequest: true}
+		opts.SetReplicaStatus(replication.Replica)
+		if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+			t.Fatalf("replication could not create an absent marker: %v", err)
+		}
+		if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: opts.VersionID}); !isErrMethodNotAllowed(err) || !got.DeleteMarker {
+			t.Fatalf("replicated marker missing: %+v, %v", got, err)
+		}
+	})
+	for _, rebalance := range []bool{false, true} {
+		t.Run(fmt.Sprintf("movement/rebalance=%t", rebalance), func(t *testing.T) {
+			object := fmt.Sprintf("movement-%t", rebalance)
+			putConsistencyObject(t, z, bucket, object, 1, "destination", ObjectOptions{Versioned: true})
+			opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow()}
+			if _, err := z.serverPools[0].DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			if rebalance {
+				z.rebalMu.Lock()
+				z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}, {}}}
+				z.rebalMu.Unlock()
+				defer func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() }()
+			} else {
+				z.poolMetaMutex.Lock()
+				z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+				z.poolMetaMutex.Unlock()
+				defer func() { z.poolMetaMutex.Lock(); z.poolMeta.Pools[0].Decommission = nil; z.poolMetaMutex.Unlock() }()
+			}
+			// These are the marker-copy options used by rebalance/decommission;
+			// Source cleanup belongs to the mover.
+			opts.DataMovement, opts.SrcPoolIdx = true, 0
+			opts.SkipRebalancing, opts.SkipDecommissioned = rebalance, !rebalance
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			for i, pool := range z.serverPools {
+				if got, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: opts.VersionID}); !isErrMethodNotAllowed(err) || !got.DeleteMarker {
+					t.Errorf("movement lost marker in pool %d: %+v, %v", i, got, err)
+				}
+			}
+		})
+	}
+	for _, expiration := range []bool{false, true} {
+		t.Run(fmt.Sprintf("scanner/expiration=%t", expiration), func(t *testing.T) {
+			object := fmt.Sprintf("scanner-%t", expiration)
+			oi := putConsistencyObject(t, z, bucket, object, 0, "payload", ObjectOptions{Versioned: true})
+			set := z.serverPools[1].getHashedSet(object)
+			getDisks := set.getDisks
+			faulty := append([]StorageAPI(nil), getDisks()...)
+			for i := range faulty {
+				faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+			}
+			set.getDisks = func() []StorageAPI { return faulty }
+			defer func() { set.getDisks = getDisks }()
+			opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID, InclFreeVersions: !expiration, Expiration: ExpirationOptions{Expire: expiration}}
+			_, err := z.DeleteObject(t.Context(), bucket, object, opts)
+			if expiration {
+				// No lifecycle rule authorizes expiration. Preserve its existing
+				// version-not-found result, even with an unrelated unreadable pool.
+				if !isErrVersionNotFound(err) {
+					t.Errorf("expiration changed its scanner contract: %v", err)
+				}
+			} else if err != nil {
+				t.Errorf("free-version cleanup was forced through all-pool resolution: %v", err)
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteUnversionedFanout(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const object = "unversioned-fanout"
+	for i := range z.serverPools {
+		putConsistencyObject(t, z, bucket, object, i, "payload", ObjectOptions{})
+	}
+	if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	for i, pool := range z.serverPools {
+		if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
+			t.Errorf("unversioned fanout left pool %d readable: %v", i, err)
+		}
+	}
+}
+
 func TestPoolsConditionalDeleteVersionSelection(t *testing.T) {
 	z, bucket := consistencyPools(t)
 	const object = "split-versions"
@@ -108,7 +596,7 @@ func TestPoolsConditionalDeleteReportsOtherPoolFailure(t *testing.T) {
 	getDisks := set.getDisks
 	faulty := append([]StorageAPI(nil), getDisks()...)
 	for i := range faulty {
-		faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+		faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
 	}
 	set.getDisks = func() []StorageAPI { return faulty }
 	defer func() { set.getDisks = getDisks }()
@@ -135,9 +623,8 @@ func testPoolsConditionalDeleteWriter(t *testing.T, multipart bool) {
 	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
 	defer cancel()
 	reader := mustGetPutObjReader(t, bytes.NewBufferString("after"), 5, "", "")
-	destination := 1
 	write := func() error {
-		_, err := z.PutObject(ctx, bucket, object, reader, ObjectOptions{DataMovement: true, SrcPoolIdx: 0, DstPoolIdx: &destination})
+		_, err := z.PutObject(ctx, bucket, object, reader, ObjectOptions{})
 		return err
 	}
 	if multipart {
@@ -548,7 +1035,7 @@ func TestPoolsReplicaCleanupFailureCanRetry(t *testing.T) {
 	getDisks := set.getDisks
 	faulty := append([]StorageAPI(nil), getDisks()...)
 	for i := range faulty {
-		faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
+		faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
 	}
 	set.getDisks = func() []StorageAPI { return faulty }
 	defer func() { set.getDisks = getDisks }()
@@ -582,6 +1069,8 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		failPrimaryDelete bool
 		differentRemote   bool
 		restored          bool
+		unconditional     bool
+		skipFreeVersion   bool
 	}{
 		{name: "metadata-copy"},
 		{name: "restored-metadata-copy", restored: true},
@@ -589,6 +1078,9 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		{name: "failed-primary-delete", deleting: true, failPrimaryDelete: true},
 		{name: "failed-primary-delete-distinct-reference", deleting: true, failPrimaryDelete: true, differentRemote: true},
 		{name: "successful-delete", deleting: true},
+		{name: "ordinary-failed-primary-delete", deleting: true, unconditional: true, failPrimaryDelete: true},
+		{name: "ordinary-successful-delete", deleting: true, unconditional: true},
+		{name: "ordinary-skip-free-version", deleting: true, unconditional: true, skipFreeVersion: true},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			z, bucket := consistencyPools(t)
@@ -622,16 +1114,20 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 				getDisks := set.getDisks
 				faulty := append([]StorageAPI(nil), getDisks()...)
 				for i := range faulty {
-					faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
+					faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
 				}
 				set.getDisks = func() []StorageAPI { return faulty }
 				defer func() { set.getDisks = getDisks }()
 			}
 			if test.deleting {
-				_, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{
-					Versioned: true, VersionID: oi.VersionID,
+				opts := ObjectOptions{
+					Versioned: true, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
 					CheckPrecondFn: func(info ObjectInfo) bool { return info.ETag != current.ETag },
-				})
+				}
+				if test.unconditional {
+					opts.CheckPrecondFn = nil
+				}
+				_, err := z.DeleteObject(t.Context(), bucket, object, opts)
 				if (err != nil) != test.failPrimaryDelete {
 					t.Fatalf("unexpected authoritative delete result: %v", err)
 				}
@@ -656,6 +1152,7 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 				t.Fatalf("lost retained authoritative copy: %v", err)
 			}
 			for pool, wantFree := range []bool{primaryDeleted, test.differentRemote} {
+				wantFree = wantFree && !test.skipFreeVersion
 				for _, disk := range z.serverPools[pool].getHashedSet(object).getDisks() {
 					data, err := disk.ReadAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile))
 					if errors.Is(err, errFileNotFound) && !wantFree {
@@ -680,3 +1177,132 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		})
 	}
 }
+
+// Fault injection shared by general multi-pool regressions.
+type consistencyDeleteFaultDisk struct {
+	StorageAPI
+	bucket, object, version string
+}
+
+func (d consistencyDeleteFaultDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
+	if volume == d.bucket && path == d.object && fi.VersionID == d.version {
+		return errDiskFull
+	}
+	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
+}
+
+// Exercise the surviving production rebalance copy, then interrupt the workflow
+// before its later source cleanup. Repeated versions are reachable without the
+// retired access-tier mover, and an API delete must remove both copies.
+func TestPoolsDeleteVersionAfterInterruptedRebalance(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for source := range 2 {
+		t.Run(fmt.Sprintf("source=%d", source), func(t *testing.T) {
+			object := fmt.Sprintf("interrupted-rebalance-%d", source)
+			original := putConsistencyObject(t, z, bucket, object, source, "original", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+			stats := []*rebalanceStats{{}, {}}
+			stats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
+			z.rebalMu.Lock()
+			z.rebalMeta = &rebalanceMeta{PoolStats: stats}
+			z.rebalMu.Unlock()
+			defer func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() }()
+			gr, err := z.serverPools[source].GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{VersionID: original.VersionID, NoLock: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := z.rebalanceObject(ctx, source, bucket, gr); err != nil {
+				t.Fatalf("rebalance copy: %v", err)
+			}
+			// Simulate interruption after rebalanceObject, before the enclosing loop
+			// removes the source version stack. Stop rebalance before the API request.
+			z.rebalMu.Lock()
+			z.rebalMeta = nil
+			z.rebalMu.Unlock()
+			for i, pool := range z.serverPools {
+				got, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: original.VersionID})
+				if err != nil || got.VersionID != original.VersionID || !got.ModTime.Equal(original.ModTime) {
+					t.Fatalf("copy missing/changed in pool %d: %+v, %v", i, got, err)
+				}
+			}
+			later, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewBufferString("later"), 5, "", ""), ObjectOptions{Versioned: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, original.VersionID)
+			if rec.Code != http.StatusNoContent {
+				t.Fatalf("DELETE: %d %s", rec.Code, rec.Body.String())
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: original.VersionID}); !isErrVersionNotFound(err) {
+					t.Errorf("pool %d retained old version: %v", i, err)
+				}
+			}
+			if got, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err != nil || got.VersionID != later.VersionID {
+				t.Fatalf("other version changed: %+v, %v", got, err)
+			}
+			for _, method := range []string{http.MethodGet, http.MethodHead} {
+				if rec := consistencyRequest(t, router, method, bucket, object, original.VersionID); rec.Code != http.StatusNotFound {
+					t.Errorf("%s returned %d", method, rec.Code)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteDirectoryMarker(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for holding := range 2 {
+		for _, unreadable := range []bool{false, true} {
+			t.Run(fmt.Sprintf("holding=%d/unreadable=%t", holding, unreadable), func(t *testing.T) {
+				object := fmt.Sprintf("directory-%d-%t/", holding, unreadable)
+				encoded := encodeDirObject(object)
+				putConsistencyObject(t, z, bucket, encoded, holding, "", ObjectOptions{})
+				set := z.serverPools[1-holding].getHashedSet(encoded)
+				getDisks := set.getDisks
+				defer func() { set.getDisks = getDisks }()
+				if unreadable {
+					disks := append([]StorageAPI(nil), getDisks()...)
+					for i := range disks {
+						disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: encoded}
+					}
+					set.getDisks = func() []StorageAPI { return disks }
+				}
+				// No explicit versionId: delOpts permanently deletes the null version.
+				rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, "")
+				if unreadable {
+					if rec.Code != http.StatusServiceUnavailable {
+						t.Fatalf("unreadable pool: %d %s", rec.Code, rec.Body.String())
+					}
+					if _, err := z.serverPools[holding].GetObjectInfo(ctx, bucket, encoded, ObjectOptions{VersionID: nullVersionID}); err != nil {
+						t.Fatalf("readable directory marker lost: %v", err)
+					}
+					set.getDisks = getDisks
+					rec = consistencyRequest(t, router, http.MethodDelete, bucket, object, "")
+				}
+				if rec.Code != http.StatusNoContent {
+					t.Fatalf("DELETE: %d %s", rec.Code, rec.Body.String())
+				}
+				for i, pool := range z.serverPools {
+					if _, err := pool.GetObjectInfo(ctx, bucket, encoded, ObjectOptions{VersionID: nullVersionID}); !isErrVersionNotFound(err) {
+						t.Errorf("pool %d retained null version: %v", i, err)
+					}
+				}
+				if rec := consistencyRequest(t, router, http.MethodHead, bucket, object, ""); rec.Code != http.StatusNotFound {
+					t.Errorf("directory still visible: %d", rec.Code)
+				}
+			})
+		}
+	}
+}

cmd/erasure-server-pool-multipart-cond_test.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+	"time"
+)
+
+// A multipart completion's If-Match must be evaluated against the logical
+// latest object across pools, not against the copy local to a pool.
+//
+// Multi-pool write placement is not sticky (getPoolIdx picks by available
+// space even for existing objects), so an upload and a newer overwrite of
+// the same name routinely end up in different pools. Uploads are pinned to
+// their pools directly: routing through z.NewMultipartUpload would make the
+// placement depend on the space-weighted random choice.
+func TestPoolsMultipartConditionalUsesLogicalLatest(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	ctx := t.Context()
+
+	ifMatch := func(etag string) (opts ObjectOptions) {
+		return ObjectOptions{
+			HasIfMatch: true,
+			CheckPrecondFn: func(oi ObjectInfo) bool {
+				return oi.ETag != etag
+			},
+		}
+	}
+
+	uploadPart := func(t *testing.T, bucket, object, uploadID string) []CompletePart {
+		t.Helper()
+		pi, err := z.PutObjectPart(ctx, bucket, object, uploadID, 1,
+			mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return []CompletePart{{PartNumber: 1, ETag: pi.ETag}}
+	}
+
+	// Scenario A: the uploaded If-Match carries the stale ETag of the pool-0
+	// copy while the logical latest object lives in pool 1. The completion
+	// must fail with 412 instead of shadowing the newer logical state.
+	objectA := "cond-mp-stale-etag"
+	base := time.Now()
+	oldA := putConsistencyObject(t, z, bucket, objectA, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpA, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectA, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerA := putConsistencyObject(t, z, bucket, objectA, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+
+	latest, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerA.ETag {
+		t.Fatalf("logical latest should be the pool-1 copy: got %s want %s", latest.ETag, newerA.ETag)
+	}
+
+	if _, err = z.CompleteMultipartUpload(ctx, bucket, objectA, mpA.UploadID,
+		uploadPart(t, bucket, objectA, mpA.UploadID), ifMatch(oldA.ETag)); err == nil {
+		t.Fatal("If-Match with the stale pool-0 ETag must not complete over the newer pool-1 object")
+	} else if _, ok := err.(PreConditionFailed); !ok {
+		t.Fatalf("expected PreconditionFailed, got %v", err)
+	}
+
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerA.ETag {
+		t.Fatalf("the newer pool-1 object must remain the logical latest, got %s", latest.ETag)
+	}
+
+	// Scenario B: the uploaded If-Match carries the logical latest ETag (the
+	// pool-1 copy) while the upload sits next to the stale pool-0 copy. The
+	// precondition is satisfied, so completion must succeed and its result
+	// must become the logical latest.
+	objectB := "cond-mp-latest-etag"
+	putConsistencyObject(t, z, bucket, objectB, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpB, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectB, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerB := putConsistencyObject(t, z, bucket, objectB, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	oiB, err := z.CompleteMultipartUpload(ctx, bucket, objectB, mpB.UploadID,
+		uploadPart(t, bucket, objectB, mpB.UploadID), ifMatch(newerB.ETag))
+	if err != nil {
+		t.Fatalf("If-Match with the logical latest ETag must complete, got %v", err)
+	}
+	if oiB.ETag == "" {
+		t.Fatal("completion returned an empty ETag")
+	}
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectB, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != oiB.ETag {
+		t.Fatalf("the completed object must be the logical latest: got %s want %s", latest.ETag, oiB.ETag)
+	}
+
+	// Scenario C: the upload lives in pool 1 with the newer copy while pool 0
+	// holds the stale one. A set-local evaluation order would let pool 0's
+	// stale copy fail the request before pool 1 is reached; the logical
+	// latest ETag must complete.
+	objectC := "cond-mp-upload-other-pool"
+	putConsistencyObject(t, z, bucket, objectC, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpC, err := z.serverPools[1].NewMultipartUpload(ctx, bucket, objectC, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerC := putConsistencyObject(t, z, bucket, objectC, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	if _, err = z.CompleteMultipartUpload(ctx, bucket, objectC, mpC.UploadID,
+		uploadPart(t, bucket, objectC, mpC.UploadID), ifMatch(newerC.ETag)); err != nil {
+		t.Fatalf("If-Match with the logical latest ETag must complete regardless of upload pool, got %v", err)
+	}
+
+	// Scenario D: pool 1 holds the newer copy but cannot be read. An
+	// unreadable pool may contain the newest state, so the unverifiable
+	// condition must fail the request rather than pass it against pool 0's
+	// stale ETag.
+	objectD := "cond-mp-unreadable-pool"
+	oldD := putConsistencyObject(t, z, bucket, objectD, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpD, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectD, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerD := putConsistencyObject(t, z, bucket, objectD, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectD, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerD.ETag {
+		t.Fatalf("logical latest before faulting pool 1 should be its copy: got %s want %s", latest.ETag, newerD.ETag)
+	}
+	set := z.serverPools[1].getHashedSet(objectD)
+	getDisks := set.getDisks
+	faulty := append([]StorageAPI(nil), getDisks()...)
+	for i := range faulty {
+		faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: objectD}
+	}
+	set.getDisks = func() []StorageAPI { return faulty }
+	defer func() { set.getDisks = getDisks }()
+
+	_, err = z.CompleteMultipartUpload(ctx, bucket, objectD, mpD.UploadID,
+		uploadPart(t, bucket, objectD, mpD.UploadID), ifMatch(oldD.ETag))
+	if !isErrReadQuorum(err) {
+		t.Fatalf("expected an insufficient read quorum error, got %v", err)
+	}
+}

cmd/erasure-server-pool-multipart-http_test.go

@@ -0,0 +1,341 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func multipartConditionRequest(t *testing.T, router http.Handler, method, target, body string, headers map[string]string) *httptest.ResponseRecorder {
+	t.Helper()
+	req, err := newTestSignedRequestV4(method, target, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	return rec
+}
+
+// The server writes the wire spelling ETag directly into Header; unlike a
+// network response, httptest's map has not canonicalized it to Etag.
+func multipartConditionResponseETag(rec *httptest.ResponseRecorder) string {
+	for key, values := range rec.Header() {
+		if strings.EqualFold(key, xhttp.ETag) && len(values) > 0 {
+			return strings.Trim(values[0], "\"")
+		}
+	}
+	return ""
+}
+
+func multipartConditionUpload(t *testing.T, z *erasureServerPools, bucket, object string, owner int, opts ObjectOptions) (string, []CompletePart) {
+	t.Helper()
+	mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, object, opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	part, err := z.serverPools[owner].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}
+}
+
+func multipartConditionCompleteBody(parts []CompletePart) string {
+	data, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
+	if err != nil {
+		panic(err)
+	}
+	return string(data)
+}
+
+func multipartConditionError(t *testing.T, rec *httptest.ResponseRecorder, code string) {
+	t.Helper()
+	decoder := xml.NewDecoder(strings.NewReader(rec.Body.String()))
+	var response APIErrorResponse
+	if err := decoder.Decode(&response); err != nil || response.Code != code {
+		t.Fatalf("expected %s error, got %q: %v", code, rec.Body.String(), err)
+	}
+	if err := decoder.Decode(&response); err != io.EOF {
+		t.Fatalf("expected exactly one error response, got %q: %v", rec.Body.String(), err)
+	}
+}
+
+// State is deliberately placed per pool; the final request uses signed HTTP
+// and the real handler, precondition callback, erasure metadata and rename.
+func TestPoolsMultipartConditionalHTTPMatrix(t *testing.T) {
+	z, _ := consistencyPools(t)
+	bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for owner := range 2 {
+		for _, localOld := range []bool{false, true} {
+			for _, condition := range []string{"match-old", "match-current", "none-match"} {
+				t.Run(fmt.Sprintf("owner=%d/old=%t/%s", owner, localOld, condition), func(t *testing.T) {
+					object := fmt.Sprintf("http-%d-%t-%s", owner, localOld, condition)
+					oldETag := "old-does-not-exist"
+					if localOld {
+						oldETag = putConsistencyObject(t, z, bucket, object, owner, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)}).ETag
+					}
+					id, parts := multipartConditionUpload(t, z, bucket, object, owner, ObjectOptions{})
+					current := putConsistencyObject(t, z, bucket, object, 1-owner, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
+					head := multipartConditionRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object), "", nil)
+					if head.Code != 200 || multipartConditionResponseETag(head) != current.ETag {
+						t.Fatalf("bad HEAD: %d %v", head.Code, head.Header())
+					}
+					h := map[string]string{xhttp.IfMatch: "\"" + oldETag + "\""}
+					want := 412
+					if condition == "match-current" {
+						h[xhttp.IfMatch] = "\"" + current.ETag + "\""
+						want = 200
+					}
+					if condition == "none-match" {
+						h = map[string]string{xhttp.IfNoneMatch: "*"}
+					}
+					rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, id), multipartConditionCompleteBody(parts), h)
+					t.Logf("HEAD current=%s, requested=%v, complete HTTP=%d", current.ETag, h, rec.Code)
+					if rec.Code != want {
+						t.Errorf("want HTTP %d, got %d: %s", want, rec.Code, rec.Body.String())
+					}
+					if want == 412 {
+						multipartConditionError(t, rec, "PreconditionFailed")
+						got := multipartConditionRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object), "", nil)
+						if got.Code != 200 || got.Body.String() != "current" {
+							t.Errorf("rejected request must preserve current data: %d %q", got.Code, got.Body.String())
+						}
+						if _, err := z.serverPools[owner].ListObjectParts(t.Context(), bucket, object, id, 0, 10, ObjectOptions{}); err != nil {
+							t.Errorf("rejected request consumed upload: %v", err)
+						}
+					}
+				})
+			}
+		}
+	}
+}
+
+func TestPoolsMultipartConditionalHTTPAbsentObject(t *testing.T) {
+	for _, deleted := range []bool{false, true} {
+		for _, match := range []bool{false, true} {
+			t.Run(fmt.Sprintf("delete-marker=%t/if-match=%t", deleted, match), func(t *testing.T) {
+				z, _ := consistencyPools(t)
+				bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: deleted})
+				if err != nil {
+					t.Fatal(err)
+				}
+				object := "http-absent"
+				if deleted {
+					putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+					_, err = z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)})
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+				id, parts := multipartConditionUpload(t, z, bucket, object, 0, ObjectOptions{Versioned: deleted})
+				headers := map[string]string{xhttp.IfNoneMatch: "*"}
+				want := http.StatusOK
+				if match {
+					headers = map[string]string{xhttp.IfMatch: "\"missing\""}
+					want = http.StatusNotFound
+				}
+				rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, id), multipartConditionCompleteBody(parts), headers)
+				if rec.Code != want {
+					t.Fatalf("expected HTTP %d, got %d: %s", want, rec.Code, rec.Body.String())
+				}
+				if match {
+					multipartConditionError(t, rec, "NoSuchKey")
+					if _, err := z.serverPools[0].ListObjectParts(t.Context(), bucket, object, id, 0, 10, ObjectOptions{}); err != nil {
+						t.Errorf("rejected request consumed upload: %v", err)
+					}
+				}
+			})
+		}
+	}
+}
+
+// All writes below use ordinary signed S3 requests. The result must be correct
+// for every placement; the matrix above deterministically covers split pools.
+func TestPoolsMultipartConditionalHTTPNormalRouting(t *testing.T) {
+	z, _ := consistencyPools(t)
+	bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	object := "normal-routing"
+	url := getPutObjectURL("", bucket, object)
+	old := multipartConditionRequest(t, router, http.MethodPut, url, "old-data", nil)
+	if old.Code != http.StatusOK {
+		t.Fatalf("initial PUT %d: %s", old.Code, old.Body.String())
+	}
+	init := multipartConditionRequest(t, router, http.MethodPost, url+"?uploads", "", nil)
+	if init.Code != http.StatusOK {
+		t.Fatalf("init %d: %s", init.Code, init.Body.String())
+	}
+	var mp InitiateMultipartUploadResponse
+	if err := xml.Unmarshal(init.Body.Bytes(), &mp); err != nil {
+		t.Fatal(err)
+	}
+	part := multipartConditionRequest(t, router, http.MethodPut, getPutObjectPartURL("", bucket, object, mp.UploadID, "1"), "replacement", nil)
+	if part.Code != http.StatusOK {
+		t.Fatalf("part %d: %s", part.Code, part.Body.String())
+	}
+	parts := []CompletePart{{PartNumber: 1, ETag: multipartConditionResponseETag(part)}}
+	newer := multipartConditionRequest(t, router, http.MethodPut, url, "newer-data", nil)
+	if newer.Code != http.StatusOK {
+		t.Fatalf("new PUT %d: %s", newer.Code, newer.Body.String())
+	}
+	head := multipartConditionRequest(t, router, http.MethodHead, url, "", nil)
+	if head.Code != http.StatusOK || multipartConditionResponseETag(head) != multipartConditionResponseETag(newer) {
+		t.Fatalf("HEAD did not pick new object: %d %v", head.Code, head.Header())
+	}
+	rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, mp.UploadID), multipartConditionCompleteBody(parts), map[string]string{xhttp.IfMatch: "\"" + multipartConditionResponseETag(old) + "\""})
+	if rec.Code != http.StatusPreconditionFailed {
+		t.Errorf("stale If-Match should be HTTP 412, got %d: %s", rec.Code, rec.Body.String())
+	}
+	get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
+	if get.Code != http.StatusOK || get.Body.String() != "newer-data" {
+		t.Errorf("conditional completion changed newer data: %d %q", get.Code, get.Body.String())
+	}
+}
+
+func TestPoolsMultipartConditionalUnreadablePool(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	object := "quorum-with-readable-copy"
+	old := putConsistencyObject(t, z, bucket, object, 0, "readable", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
+	putConsistencyObject(t, z, bucket, object, 1, "hidden-newer", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
+	id, parts := multipartConditionUpload(t, z, bucket, object, 0, ObjectOptions{})
+	set := z.serverPools[1].getHashedSet(object)
+	original := set.getDisks
+	disks := append([]StorageAPI(nil), original()...)
+	for i := range disks {
+		disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
+	}
+	set.getDisks = func() []StorageAPI { return disks }
+	defer func() { set.getDisks = original }()
+	read, readErr := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+	t.Logf("ordinary GET lookup: etag=%s err=%v", read.ETag, readErr)
+	called := 0
+	_, err := z.CompleteMultipartUpload(t.Context(), bucket, object, id, parts, ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { called++; return oi.ETag != old.ETag }})
+	if !isErrReadQuorum(err) {
+		t.Errorf("conditional write must fail on unreadable pool, got %v", err)
+	}
+	if called != 0 {
+		t.Errorf("callback evaluated without complete state: %d calls", called)
+	}
+}
+
+func TestPoolsMultipartConditionalLatestVersionAndCallbackOnce(t *testing.T) {
+	for _, explicit := range []bool{false, true} {
+		t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			object := "tie-version"
+			opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour), Versioned: explicit}
+			if explicit {
+				opts.VersionID = mustGetUUID()
+			}
+			current := putConsistencyObject(t, z, bucket, object, 0, "first", opts)
+			putConsistencyObject(t, z, bucket, object, 1, "second", opts)
+			if explicit {
+				current = putConsistencyObject(t, z, bucket, object, 1, "latest-other-version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
+			}
+			id, parts := multipartConditionUpload(t, z, bucket, object, 1, opts)
+			called := 0
+			opts.CheckPrecondFn = func(oi ObjectInfo) bool { called++; return oi.ETag != current.ETag }
+			opts.MTime = time.Time{}
+			opts.HasIfMatch = true
+			_, err := z.CompleteMultipartUpload(t.Context(), bucket, object, id, parts, opts)
+			if err != nil {
+				t.Errorf("logical current object should match: %v", err)
+			}
+			if called != 1 {
+				t.Errorf("condition evaluated %d times; want exactly once", called)
+			}
+		})
+	}
+}
+
+func TestPoolsMultipartConditionalConcurrentCompletes(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	object := "concurrent-completes"
+	old := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
+	putConsistencyObject(t, z, bucket, object, 1, "old", ObjectOptions{MTime: old.ModTime})
+	ids := make([]string, 2)
+	parts := make([][]CompletePart, 2)
+	for i := range 2 {
+		ids[i], parts[i] = multipartConditionUpload(t, z, bucket, object, i, ObjectOptions{})
+	}
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var gate, releaseOnce sync.Once
+	defer releaseOnce.Do(func() { close(release) })
+	errs := make([]error, 2)
+	var wg sync.WaitGroup
+	// Let pool 1's completion hold the object lock before pool 0's starts.
+	// Once pool 1 commits, a set-local read in pool 0 would still see the
+	// old ETag and incorrectly accept the second completion.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, errs[1] = z.CompleteMultipartUpload(t.Context(), bucket, object, ids[1], parts[1], ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool {
+			gate.Do(func() { close(entered); <-release })
+			return oi.ETag != old.ETag
+		}})
+	}()
+	select {
+	case <-entered:
+	case <-time.After(10 * time.Second):
+		t.Fatal("first completion did not enter its condition callback")
+	}
+	started := make(chan struct{})
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		close(started)
+		_, errs[0] = z.CompleteMultipartUpload(t.Context(), bucket, object, ids[0], parts[0], ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != old.ETag }})
+	}()
+	<-started
+	releaseOnce.Do(func() { close(release) })
+	wg.Wait()
+	success, failed := 0, 0
+	for _, err := range errs {
+		var p PreConditionFailed
+		switch {
+		case err == nil:
+			success++
+		case errors.As(err, &p):
+			failed++
+		default:
+			t.Errorf("unexpected completion error %v", err)
+		}
+	}
+	if success != 1 || failed != 1 {
+		t.Errorf("CAS writers: success=%d conditional failures=%d errors=%v", success, failed, errs)
+	}
+}

cmd/erasure-server-pool-multipart-preconditions_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestPoolsMultipartConditionMatrix(t *testing.T) {
+	for owner := range 2 {
+		for _, withOld := range []bool{false, true} {
+			for _, condition := range []string{"match-current", "match-old", "none-match-any"} {
+				t.Run(fmt.Sprintf("upload-pool=%d/old-copy=%t/%s", owner, withOld, condition), func(t *testing.T) {
+					z, bucket := consistencyPools(t)
+					object := "conditional-multipart"
+					oldETag := "arbitrary-old"
+					if withOld {
+						old := putConsistencyObject(t, z, bucket, object, owner, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
+						oldETag = old.ETag
+					}
+					mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					part, err := z.serverPools[owner].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					current := putConsistencyObject(t, z, bucket, object, 1-owner, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
+					visible, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil || visible.ETag != current.ETag {
+						t.Fatalf("invalid current state: %v", err)
+					}
+					opts := ObjectOptions{HasIfMatch: condition != "none-match-any", CheckPrecondFn: func(oi ObjectInfo) bool {
+						switch condition {
+						case "match-current":
+							return oi.ETag != current.ETag
+						case "match-old":
+							return oi.ETag != oldETag
+						default:
+							return true
+						}
+					}}
+					_, err = z.CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, opts)
+					if condition == "match-current" {
+						if err != nil {
+							t.Errorf("correct logical If-Match rejected: %v", err)
+						}
+					} else {
+						var expected PreConditionFailed
+						if !errors.As(err, &expected) {
+							t.Errorf("logical condition must fail, got %v", err)
+						}
+					}
+				})
+			}
+		}
+	}
+}
+
+func TestPoolsMultipartConditionBoundaries(t *testing.T) {
+	for _, state := range []string{"missing", "latest-delete-marker", "unreadable-other-pool"} {
+		for _, match := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/if-match=%t", state, match), func(t *testing.T) {
+				z, bucket := consistencyPools(t)
+				object := "conditional-boundary"
+				versioned := state == "latest-delete-marker"
+				if versioned {
+					putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+					if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)}); err != nil {
+						t.Fatal(err)
+					}
+				}
+				mp, err := z.serverPools[0].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{Versioned: versioned})
+				if err != nil {
+					t.Fatal(err)
+				}
+				part, err := z.serverPools[0].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("new"), 3, "", ""), ObjectOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+				if state == "unreadable-other-pool" {
+					set := z.serverPools[1].getHashedSet(object)
+					getDisks := set.getDisks
+					faulty := append([]StorageAPI(nil), getDisks()...)
+					for i := range faulty {
+						faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+					}
+					set.getDisks = func() []StorageAPI { return faulty }
+					defer func() { set.getDisks = getDisks }()
+				}
+				opts := ObjectOptions{Versioned: versioned, HasIfMatch: match, CheckPrecondFn: func(ObjectInfo) bool { return true }}
+				_, err = z.CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, opts)
+				switch {
+				case state == "unreadable-other-pool":
+					if !isErrReadQuorum(err) {
+						t.Errorf("unreadable pool must not mean absence: %v", err)
+					}
+				case match:
+					if !isErrObjectNotFound(err) {
+						t.Errorf("If-Match against logical absence should report absence: %v", err)
+					}
+				default:
+					if err != nil {
+						t.Errorf("If-None-Match against logical absence must succeed: %v", err)
+					}
+				}
+				if err != nil {
+					if _, lerr := z.serverPools[0].ListObjectParts(t.Context(), bucket, object, mp.UploadID, 0, 10, ObjectOptions{}); lerr != nil {
+						t.Errorf("failed condition consumed upload: %v", lerr)
+					}
+				}
+			})
+		}
+	}
+}

cmd/erasure-server-pool-tags_test.go

@@ -0,0 +1,459 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"strings"
+	"testing"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// The fixture places copies directly in real erasure pools. It models a
+// duplicated version; it does not claim to exercise a rebalance workflow.
+func TestPoolsMetadataUpdatePreservesTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, test := range []struct {
+		name         string
+		tags, stamps [2]string
+		winner       int
+		single       bool
+	}{
+		{name: "newer-secondary", tags: [2]string{"key=old", "key=new"}, stamps: [2]string{old, recent}, winner: 1},
+		{name: "newer-primary", tags: [2]string{"key=new", "key=old"}, stamps: [2]string{recent, old}},
+		{name: "empty-secondary", tags: [2]string{"key=old", ""}, stamps: [2]string{old, recent}, winner: 1},
+		{name: "empty-primary", tags: [2]string{"", "key=old"}, stamps: [2]string{recent, old}},
+		{name: "single-copy-primary", tags: [2]string{"key=only", ""}, stamps: [2]string{recent, ""}, single: true},
+		{name: "single-copy-secondary", tags: [2]string{"", "key=only"}, stamps: [2]string{"", recent}, winner: 1, single: true},
+		{name: "legacy-single-copy", tags: [2]string{"key=legacy", ""}, single: true},
+		{name: "legacy-duplicates", tags: [2]string{"key=legacy", "key=legacy"}},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			object := test.name
+			var original ObjectInfo
+			for pool := range 2 {
+				if test.single && pool != test.winner {
+					continue
+				}
+				metadata := map[string]string{
+					xhttp.AmzObjectTagging: test.tags[pool],
+					"copy-local":           fmt.Sprint(pool),
+				}
+				if test.stamps[pool] != "" {
+					metadata[timestamp] = test.stamps[pool]
+				}
+				original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
+				})
+				got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
+				if err != nil || got.UserTags != test.tags[pool] || got.UserDefined[timestamp] != test.stamps[pool] {
+					t.Fatalf("pool %d fixture: tags=%q timestamp=%q err=%v", pool, got.UserTags, got.UserDefined[timestamp], err)
+				}
+				if _, exists := got.UserDefined[xhttp.AmzObjectTagging]; exists {
+					t.Fatal("fixture must use the cleaned ObjectInfo representation")
+				}
+			}
+			wantTags, wantStamp := test.tags[test.winner], test.stamps[test.winner]
+			called := 0
+			got, err := z.PutObjectMetadata(t.Context(), bucket, object, ObjectOptions{
+				VersionID: original.VersionID, MTime: original.ModTime,
+				EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
+					called++
+					if current.UserTags != wantTags || current.UserDefined[timestamp] != wantStamp {
+						t.Errorf("callback tags=%q timestamp=%q; want %q %q", current.UserTags, current.UserDefined[timestamp], wantTags, wantStamp)
+					}
+					current.UserDefined["unrelated-update"] = "preserved"
+					return ReplicateDecision{}, nil
+				},
+			})
+			if err != nil || called != 1 {
+				t.Fatalf("metadata update: %v, callbacks=%d", err, called)
+			}
+			if got.UserTags != wantTags || got.UserDefined[timestamp] != wantStamp {
+				t.Errorf("response tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], wantTags, wantStamp)
+			}
+			for pool := range 2 {
+				got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
+				if test.single && pool != test.winner {
+					if !isErrVersionNotFound(err) {
+						t.Errorf("metadata update created another copy: %v", err)
+					}
+					continue
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Logf("pool %d persisted tags=%q timestamp=%q", pool, got.UserTags, got.UserDefined[timestamp])
+				if got.UserTags != wantTags || got.UserDefined[timestamp] != wantStamp {
+					t.Errorf("pool %d persisted tags=%q timestamp=%q; want %q %q", pool, got.UserTags, got.UserDefined[timestamp], wantTags, wantStamp)
+				}
+				if got.UserDefined["unrelated-update"] != "preserved" || got.UserDefined["copy-local"] != fmt.Sprint(pool) {
+					t.Errorf("pool %d lost unrelated metadata: %v", pool, got.UserDefined)
+				}
+			}
+		})
+	}
+}
+
+func TestReplicaWritesPreserveTagOrdering(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	// Pool allocation checks the host's used-space percentage. Present only
+	// its free space as fixture capacity; all reads and writes still use the
+	// real disks. This keeps unrelated host disk usage out of the tag test.
+	for _, pool := range z.serverPools {
+		for _, set := range pool.sets {
+			getDisks := set.getDisks
+			disks := append([]StorageAPI(nil), getDisks()...)
+			for i := range disks {
+				disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			t.Cleanup(func() { set.getDisks = getDisks })
+		}
+	}
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, path := range []struct {
+		name, operation string
+		direct          bool
+		winner          int
+	}{
+		{name: "set-put", operation: "put", direct: true},
+		{name: "set-multipart", operation: "multipart", direct: true},
+		{name: "set-copy", operation: "copy", direct: true},
+		{name: "pools-put", operation: "put", winner: 1},
+		{name: "pools-multipart", operation: "multipart", winner: 1},
+		{name: "pools-copy-primary", operation: "copy"},
+		{name: "pools-copy-secondary", operation: "copy", winner: 1},
+	} {
+		for _, test := range []struct {
+			name, storedTags, incomingTags, storedStamp, incomingStamp, wantTags string
+		}{
+			{"stored-newer", "key=stored", "key=incoming", recent, old, "key=stored"},
+			{"stored-deleted", "", "key=incoming", recent, old, ""},
+			{"incoming-newer", "key=stored", "key=incoming", old, recent, "key=incoming"},
+			{"incoming-deleted", "key=stored", "", old, recent, ""},
+		} {
+			t.Run(path.name+"/"+test.name, func(t *testing.T) {
+				object := path.name + "-" + test.name
+				var original ObjectInfo
+				for pool := range 2 {
+					if path.direct && pool != 0 {
+						continue
+					}
+					metadata := map[string]string{
+						xhttp.AmzObjectTagging: "key=older-copy",
+						timestamp:              "2026-09-09T08:00:00Z",
+					}
+					if pool == path.winner {
+						metadata[xhttp.AmzObjectTagging] = test.storedTags
+						metadata[timestamp] = test.storedStamp
+					}
+					original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
+						Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
+					})
+				}
+				opts := ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, ReplicaLockReconcile: true,
+					UserDefined: map[string]string{
+						xhttp.AmzObjectTagging: test.incomingTags,
+						timestamp:              test.incomingStamp,
+					},
+				}
+				var got ObjectInfo
+				var err error
+				switch path.operation {
+				case "put":
+					put := z.PutObject
+					if path.direct {
+						put = z.serverPools[0].PutObject
+					}
+					got, err = put(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), opts)
+				case "multipart":
+					// Persist the incoming tags with the upload, before completion
+					// reconciles the destination version through its real resolver.
+					mp, err := z.serverPools[0].NewMultipartUpload(t.Context(), bucket, object, opts)
+					if err != nil {
+						t.Fatal(err)
+					}
+					part, err := z.serverPools[0].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1,
+						mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					complete := z.CompleteMultipartUpload
+					if path.direct {
+						complete = z.serverPools[0].CompleteMultipartUpload
+					}
+					got, err = complete(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{
+						Versioned: true, MTime: original.ModTime, ReplicaLockReconcile: true,
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+				case "copy":
+					src := original
+					src.metadataOnly = true
+					src.UserDefined = maps.Clone(opts.UserDefined)
+					copyObject := z.CopyObject
+					if path.direct {
+						copyObject = z.serverPools[0].CopyObject
+					}
+					got, err = copyObject(t.Context(), bucket, object, bucket, object, src, ObjectOptions{VersionID: original.VersionID}, opts)
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+				if got.UserTags != test.wantTags || got.UserDefined[timestamp] != recent {
+					t.Errorf("response tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], test.wantTags, recent)
+				}
+				copies := 0
+				for pool := range 2 {
+					got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
+					if isErrVersionNotFound(err) {
+						continue
+					}
+					if err != nil {
+						t.Fatal(err)
+					}
+					copies++
+					if got.UserTags != test.wantTags || got.UserDefined[timestamp] != recent {
+						t.Errorf("pool %d persisted tags=%q timestamp=%q; want %q %q", pool, got.UserTags, got.UserDefined[timestamp], test.wantTags, recent)
+					}
+				}
+				if copies != 1 {
+					t.Errorf("replacement left %d copies; want 1", copies)
+				}
+			})
+		}
+	}
+}
+
+type tagTestCapacityDisk struct{ StorageAPI }
+
+func (d tagTestCapacityDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (DiskInfo, error) {
+	info, err := d.StorageAPI.DiskInfo(ctx, opts)
+	info.Total, info.Used = info.Free, 0
+	return info, err
+}
+
+func TestMergedPoolObjectInfoTagOrdering(t *testing.T) {
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, test := range []struct {
+		name, firstStamp, secondStamp, secondTags string
+		winner                                    int
+	}{
+		{"newer", old, recent, "key=second", 1},
+		{"newer-removal", old, recent, "", 1},
+		{"equal", recent, recent, "key=second", 0},
+		{"unordered", "", "", "key=second", 0},
+		{"missing-first", "", recent, "key=second", 1},
+		{"missing-second", recent, "", "key=second", 0},
+		{"invalid-first", "invalid", recent, "key=second", 1},
+		{"invalid-second", recent, "invalid", "key=second", 0},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			tagValues := []string{"key=first", test.secondTags}
+			stamps := []string{test.firstStamp, test.secondStamp}
+			copies := make([]PoolObjInfo, 2)
+			before := make([]map[string]string, 2)
+			for i := range copies {
+				fi := FileInfo{Metadata: map[string]string{xhttp.AmzObjectTagging: tagValues[i]}}
+				if stamps[i] != "" {
+					fi.Metadata[timestamp] = stamps[i]
+				}
+				copies[i] = PoolObjInfo{Index: i, ObjInfo: fi.ToObjectInfo("bucket", "object", true)}
+				before[i] = maps.Clone(copies[i].ObjInfo.UserDefined)
+			}
+			got := mergedPoolObjectInfo(copies)
+			if got.UserTags != tagValues[test.winner] || got.UserDefined[timestamp] != stamps[test.winner] {
+				t.Errorf("merged tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], tagValues[test.winner], stamps[test.winner])
+			}
+			if _, exists := got.UserDefined[xhttp.AmzObjectTagging]; exists {
+				t.Error("merged ObjectInfo leaked the raw tagging key into UserDefined")
+			}
+			for i := range copies {
+				if !maps.Equal(copies[i].ObjInfo.UserDefined, before[i]) || copies[i].ObjInfo.UserTags != tagValues[i] {
+					t.Errorf("merge mutated input copy %d", i)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsMetadataCallbackReplacesTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	for _, test := range []struct{ name, tags string }{
+		{"replace", "key=callback"},
+		{"remove", ""},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var original ObjectInfo
+			for pool := range 2 {
+				original = putConsistencyObject(t, z, bucket, test.name, pool, "data", ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime,
+					UserDefined: map[string]string{
+						xhttp.AmzObjectTagging: []string{"key=old", "key=new"}[pool],
+						timestamp:              []string{"2026-09-09T09:00:00Z", "2026-09-09T10:00:00Z"}[pool],
+					},
+				})
+			}
+			const updatedStamp = "2026-09-09T11:00:00Z"
+			got, err := z.PutObjectMetadata(t.Context(), bucket, test.name, ObjectOptions{
+				VersionID: original.VersionID, MTime: original.ModTime,
+				EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
+					if current.UserTags != "key=new" {
+						t.Errorf("callback read tags=%q; want key=new", current.UserTags)
+					}
+					if _, exists := current.UserDefined[xhttp.AmzObjectTagging]; exists {
+						t.Error("callback received the raw tagging key")
+					}
+					current.UserDefined[xhttp.AmzObjectTagging] = test.tags
+					current.UserDefined[timestamp] = updatedStamp
+					return ReplicateDecision{}, nil
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.UserTags != test.tags || got.UserDefined[timestamp] != updatedStamp {
+				t.Errorf("callback update response tags=%q timestamp=%q", got.UserTags, got.UserDefined[timestamp])
+			}
+			for pool := range 2 {
+				got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, test.name, ObjectOptions{VersionID: original.VersionID})
+				if err != nil || got.UserTags != test.tags || got.UserDefined[timestamp] != updatedStamp {
+					t.Errorf("pool %d did not persist callback tags: tags=%q timestamp=%q err=%v", pool, got.UserTags, got.UserDefined[timestamp], err)
+				}
+			}
+		})
+	}
+}
+
+func TestReconcileStoredObjectTagOrdering(t *testing.T) {
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, test := range []struct {
+		name, storedStamp, incomingStamp, storedTags string
+		wantStored                                   bool
+	}{
+		{"stored-newer", recent, old, "key=stored", true},
+		{"incoming-newer", old, recent, "key=stored", false},
+		{"equal", recent, recent, "key=stored", true},
+		{"equal-removal", recent, recent, "", true},
+		{"missing-stored", "", recent, "key=stored", false},
+		{"missing-incoming", recent, "", "key=stored", true},
+		{"invalid-stored", "invalid", recent, "key=stored", false},
+		{"invalid-incoming", recent, "invalid", "key=stored", true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			metadata := map[string]string{
+				xhttp.AmzObjectTagging: "key=incoming",
+				timestamp:              test.incomingStamp,
+				"unrelated":            "preserved",
+			}
+			reconcileStoredObjectTags(metadata, test.storedTags, test.storedStamp)
+			wantTags, wantStamp := "key=incoming", test.incomingStamp
+			if test.wantStored {
+				wantTags, wantStamp = test.storedTags, test.storedStamp
+			}
+			if metadata[xhttp.AmzObjectTagging] != wantTags || metadata[timestamp] != wantStamp || metadata["unrelated"] != "preserved" {
+				t.Errorf("reconciled metadata=%v; want tags=%q timestamp=%q and unrelated field preserved", metadata, wantTags, wantStamp)
+			}
+		})
+	}
+}
+
+func TestPoolsMetadataUpdatePreservesAbsentTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, removal := range []bool{false, true} {
+		t.Run(fmt.Sprintf("timestamp-only-removal=%t", removal), func(t *testing.T) {
+			object := fmt.Sprintf("absent-tags-%t", removal)
+			var original ObjectInfo
+			checkStored := func(pool int, wantKey bool, wantTags, wantStamp string) {
+				t.Helper()
+				infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(object).getDisks(), "", bucket, object, original.VersionID, false, false)
+				for disk, info := range infos {
+					if errs[disk] != nil {
+						t.Fatal(errs[disk])
+					}
+					tags, exists := info.Metadata[xhttp.AmzObjectTagging]
+					if exists != wantKey || tags != wantTags || info.Metadata[timestamp] != wantStamp {
+						t.Errorf("pool %d disk %d raw tagging key=%t value=%q stamp=%q; want %t %q %q", pool, disk, exists, tags, info.Metadata[timestamp], wantKey, wantTags, wantStamp)
+					}
+				}
+			}
+			for pool := range 2 {
+				metadata := map[string]string{}
+				if removal {
+					metadata[timestamp] = recent
+					if pool == 1 {
+						metadata[xhttp.AmzObjectTagging] = "key=old"
+						metadata[timestamp] = old
+					}
+				}
+				original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
+				})
+				checkStored(pool, removal && pool == 1, metadata[xhttp.AmzObjectTagging], metadata[timestamp])
+			}
+			_, err := z.PutObjectMetadata(t.Context(), bucket, object, ObjectOptions{
+				VersionID: original.VersionID, MTime: original.ModTime,
+				EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
+					current.UserDefined["unrelated-update"] = "preserved"
+					return ReplicateDecision{}, nil
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			wantStamp := ""
+			if removal {
+				wantStamp = recent
+			}
+			for pool := range 2 {
+				// A previously non-empty key needs an explicit empty value to
+				// propagate deletion; an absent key should remain absent.
+				checkStored(pool, removal && pool == 1, "", wantStamp)
+			}
+		})
+	}
+}

cmd/erasure-server-pool.go

@@ -615,17 +615,7 @@ func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucke
 	})
 }
 
-func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) {
-	if dstPoolIdx != nil {
-		if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
-			return -1, errInvalidArgument
-		}
-		if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
-			return -1, toObjectErr(errDiskFull)
-		}
-		return *dstPoolIdx, nil
-	}
-
+func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
 	idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object)
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
@@ -644,23 +634,13 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 // getPoolIdx returns the found previous object and its corresponding pool idx,
 // if none are found falls back to most available space pool, this function is
 // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
-func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) {
-	return z.getWritePoolIdx(ctx, bucket, object, size, dstPoolIdx, false)
+func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
+	return z.getWritePoolIdx(ctx, bucket, object, size, false)
 }
 
 // getWritePoolIdx keeps the write-allocation policy when the caller already
 // holds the pools-layer object lock and must not reacquire a set read lock.
-func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int, noLock bool) (idx int, err error) {
-	if dstPoolIdx != nil {
-		if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
-			return -1, errInvalidArgument
-		}
-		if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
-			return -1, toObjectErr(errDiskFull)
-		}
-		return *dstPoolIdx, nil
-	}
-
+func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock bool) (idx int, err error) {
 	pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{
 		NoLock:             noLock,
 		SkipDecommissioned: true,
@@ -683,13 +663,6 @@ func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object
 	return idx, nil
 }
 
-func dataMovementDstPool(opts ObjectOptions) *int {
-	if !opts.DataMovement {
-		return nil
-	}
-	return opts.DstPoolIdx
-}
-
 func (z *erasureServerPools) Shutdown(ctx context.Context) error {
 	g := errgroup.WithNErrs(len(z.serverPools))
 
@@ -1158,16 +1131,10 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 
 	object = encodeDirObject(object)
 	if z.SinglePool() {
-		idx, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
+		_, err := z.getPoolIdx(ctx, bucket, object, data.Size())
 		if err != nil {
 			return ObjectInfo{}, err
 		}
-		if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx {
-			return ObjectInfo{}, DataMovementOverwriteErr{
-				Bucket: bucket, Object: object, VersionID: opts.VersionID,
-				Err: errDataMovementSrcDstPoolSame,
-			}
-		}
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
 
@@ -1182,7 +1149,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 	}
 	opts.NoLock = true
 
-	idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts), true)
+	idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -1238,30 +1205,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
 	}
 
-	// Access-tier moves must recreate delete markers on the explicitly
-	// selected destination. The regular data-movement path discovers a pool
-	// from existing object state, which is ambiguous while both source and
-	// destination temporarily contain the version stack.
-	if dstPoolIdx := dataMovementDstPool(opts); dstPoolIdx != nil {
-		if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
-			return ObjectInfo{}, errInvalidArgument
-		}
-		if *dstPoolIdx == opts.SrcPoolIdx {
-			return ObjectInfo{}, DataMovementOverwriteErr{
-				Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID,
-				Err: errDataMovementSrcDstPoolSame,
-			}
-		}
-		if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
-			return ObjectInfo{}, toObjectErr(errDiskFull)
-		}
-		objInfo, err = z.serverPools[*dstPoolIdx].DeleteObject(ctx, bucket, object, opts)
-		objInfo.Name = decodeDirObject(object)
-		return objInfo, err
-	}
-
-	if !z.SinglePool() && opts.CheckPrecondFn != nil {
-		return z.deleteObjectConditional(ctx, bucket, object, opts)
+	// Reconcile ordinary addressed-version deletes independently of pool movement.
+	reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
+		!opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
+	if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
+		return z.deleteObjectReconciled(ctx, bucket, object, opts)
 	}
 
 	gopts := opts
@@ -1499,7 +1447,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		}
 		stored := mergedPoolObjectInfo(copies)
 		reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(stored.UserDefined))
-		reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserDefined)
+		reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserTags, stored.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
 		idx := copies[0].Index
 		oi, err := z.serverPools[idx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		if err == nil {
@@ -1508,16 +1456,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		return oi, err
 	}
 
-	poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size, dataMovementDstPool(dstOpts))
+	poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
-	if dataMovementDstPool(dstOpts) != nil && poolIdx == dstOpts.SrcPoolIdx {
-		return ObjectInfo{}, DataMovementOverwriteErr{
-			Bucket: dstBucket, Object: dstObject, VersionID: dstOpts.VersionID,
-			Err: errDataMovementSrcDstPoolSame,
-		}
-	}
 
 	if !z.SinglePool() && dstOpts.ReplicaLockReconcile {
 		dstOpts.replicaObjectInfo = z.replicaObjectInfo
@@ -1977,41 +1919,29 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}()
 
 	if z.SinglePool() {
-		idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts))
-		if err != nil {
-			return nil, err
-		}
-		if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx {
-			return nil, DataMovementOverwriteErr{
-				Bucket: bucket, Object: object, VersionID: opts.VersionID,
-				Err: errDataMovementSrcDstPoolSame,
-			}
-		}
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
 
-	if dataMovementDstPool(opts) == nil {
-		for idx, pool := range z.serverPools {
-			if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
-				continue
-			}
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
+			continue
+		}
 
-			result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
-			if err != nil {
-				return nil, err
-			}
-			// If there is a multipart upload with the same bucket/object name,
-			// create the new multipart in the same pool, this will avoid
-			// creating two multiparts uploads in two different pools.
-			if len(result.Uploads) != 0 {
-				return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
-			}
+		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
+		if err != nil {
+			return nil, err
+		}
+		// If there is a multipart upload with the same bucket/object name,
+		// create the new multipart in the same pool, this will avoid
+		// creating two multiparts uploads in two different pools
+		if len(result.Uploads) != 0 {
+			return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
 		}
 	}
 
 	// any parallel writes on the object will block for this poolIdx
 	// to return since this holds a read lock on the namespace.
-	idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts))
+	idx, err := z.getPoolIdx(ctx, bucket, object, -1)
 	if err != nil {
 		return nil, err
 	}
@@ -2045,7 +1975,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 	}
 
 	if z.SinglePool() {
-		_, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
+		_, err := z.getPoolIdx(ctx, bucket, object, data.Size())
 		if err != nil {
 			return PartInfo{}, err
 		}
@@ -2224,6 +2154,47 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 			defer lk.Unlock(lkctx)
 		}
 		opts.NoLock = true
+
+		// A conditional completion must be evaluated against the logical
+		// latest object across pools, under the object write lock held for
+		// this operation. The pool hosting the upload may only hold a stale
+		// duplicate, so its set-local check would both accept an outdated
+		// ETag and reject the current one. An unreadable pool is not
+		// absence: it may hold the newest copy, so a read that cannot be
+		// verified fails the request instead of passing the condition.
+		// Once satisfied, the callback is cleared so the set layer does not
+		// re-evaluate it against its local copy.
+		if opts.CheckPrecondFn != nil {
+			copies, lerr := z.objectPoolInfos(ctx, bucket, encodeDirObject(object), ObjectOptions{
+				// Conditions always compare the logical current object,
+				// independently of the completion's destination version.
+				VersionID:        "",
+				Versioned:        opts.Versioned,
+				VersionSuspended: opts.VersionSuspended,
+				NoAuditLog:       true,
+			})
+			var latest ObjectInfo
+			if lerr == nil {
+				latest = copies[0].ObjInfo
+				if latest.DeleteMarker {
+					// A delete-marker latest reads as an absent key, matching
+					// the set layer's getObjectInfo.
+					lerr = toObjectErr(errFileNotFound, bucket, object)
+				}
+			}
+			if lerr == nil && opts.CheckPrecondFn(latest) {
+				return ObjectInfo{}, PreConditionFailed{}
+			}
+			if lerr != nil && !isErrVersionNotFound(lerr) && !isErrObjectNotFound(lerr) {
+				return ObjectInfo{}, lerr
+			}
+			// if object doesn't exist return error for If-Match conditional requests
+			// If-None-Match should be allowed to proceed for non-existent objects
+			if lerr != nil && opts.HasIfMatch && (isErrObjectNotFound(lerr) || isErrVersionNotFound(lerr)) {
+				return ObjectInfo{}, lerr
+			}
+			opts.CheckPrecondFn = nil
+		}
 	}
 
 	// Hold write locks to verify uploaded parts, also disallows any
@@ -2276,6 +2247,11 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, o
 	if err != nil {
 		return bucketInfo, toObjectErr(err, bucket)
 	}
+	// Physical existence/creation probes must not be overwritten by cached
+	// metadata, which can legitimately lack Created on an unmigrated bucket.
+	if opts.NoMetadata {
+		return bucketInfo, nil
+	}
 
 	meta, err := globalBucketMetadataSys.Get(bucket)
 	if err == nil {
@@ -3075,6 +3051,15 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 	if err != nil {
 		return ObjectInfo{}, err
 	}
+	// Ordinary reads and replication can return any owning pool. Persist one
+	// revision beyond all copies, so the returned value and every pool agree.
+	if stamp := opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; stamp != "" {
+		for _, copy := range copies {
+			stamp = monotonicTaggingTimestamp(stamp, copy.ObjInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
+		}
+		opts.UserDefined = cloneMSS(opts.UserDefined)
+		opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = stamp
+	}
 	opts.NoLock = true
 	opts.VersionID = copies[0].ObjInfo.VersionID
 	if opts.VersionID == "" {
@@ -3214,7 +3199,7 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
 		defer ns.Unlock(lkctx)
 		opts.NoLock = true
 	}
-	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size, dataMovementDstPool(opts))
+	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
 	if err != nil {
 		return err
 	}

cmd/handler-utils.go

@@ -246,16 +246,6 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
 
 // extractMetadata extracts metadata from map values.
 func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
-	return extractMetadataFromMimeWithReplication(ctx, v, m, false)
-}
-
-// extractReplicationMetadataFromMime restores replication-only metadata after the
-// caller has validated that the request is a trusted replication write.
-func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
-	return extractMetadataFromMimeWithReplication(ctx, v, m, true)
-}
-
-func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIMEHeader, m map[string]string, allowReplication bool) error {
 	if v == nil {
 		bugLogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
@@ -267,18 +257,14 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
 		nv[http.CanonicalHeaderKey(k)] = kv
 	}
 
-	// Save all supported headers.
+	// Save ordinary object metadata. Replication-only headers are restored only
+	// after the request has been validated as a trusted replication write.
 	for _, supportedHeader := range supportedHeaders {
-		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
-		if ok {
-			if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
-				if !allowReplication {
-					continue
-				}
-				m[v] = strings.Join(value, ",")
-			} else {
-				m[supportedHeader] = strings.Join(value, ",")
-			}
+		if _, ok := replicationToInternalHeaders[supportedHeader]; ok {
+			continue
+		}
+		if value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]; ok {
+			m[supportedHeader] = strings.Join(value, ",")
 		}
 	}
 
@@ -287,8 +273,7 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
 			if !stringsHasPrefixFold(key, prefix) {
 				continue
 			}
-			value, ok := nv[http.CanonicalHeaderKey(key)]
-			if ok {
+			if value, ok := nv[http.CanonicalHeaderKey(key)]; ok {
 				m[key] = strings.Join(value, ",")
 				break
 			}
@@ -297,6 +282,31 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
 	return nil
 }
 
+// extractReplicationMetadataFromMime restores replication-only metadata after the
+// caller has validated that the request is a trusted replication write.
+func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
+	if v == nil {
+		bugLogIf(ctx, errInvalidArgument)
+		return errInvalidArgument
+	}
+
+	nv := make(textproto.MIMEHeader, len(v))
+	for k, kv := range v {
+		// Canonicalize all headers, to remove any duplicates.
+		nv[http.CanonicalHeaderKey(k)] = kv
+	}
+
+	// Ordinary object metadata belongs to the caller. Re-extracting it would
+	// undo normalization (such as removing aws-chunked) or copy an outer
+	// Snowball archive's metadata onto its individual entries.
+	for header, internalHeader := range replicationToInternalHeaders {
+		if value, ok := nv[http.CanonicalHeaderKey(header)]; ok {
+			m[internalHeader] = strings.Join(value, ",")
+		}
+	}
+	return nil
+}
+
 // Returns access credentials in the request Authorization header.
 func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
 	cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)

cmd/handler-utils_test.go

@@ -254,6 +254,9 @@ func TestExtractMetadataFromRequestKeepsQueryCompatibility(t *testing.T) {
 
 func TestExtractReplicationMetadataHeaders(t *testing.T) {
 	header := http.Header{
+		"Content-Type":      []string{"application/wasm"},
+		"Content-Encoding":  []string{"aws-chunked"},
+		"X-Amz-Meta-Source": []string{"client"},
 		"X-Minio-Replication-Server-Side-Encryption-Sealed-Key":     []string{"sealed-key"},
 		"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
 		"X-Minio-Replication-Server-Side-Encryption-Iv":             []string{"iv"},
@@ -262,12 +265,17 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
 		ReplicationSsecChecksumHeader:                               []string{"checksum"},
 	}
 
-	metadata := make(map[string]string)
+	metadata := map[string]string{
+		"content-type":      "application/wasm",
+		"x-amz-meta-source": "client",
+	}
 	if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
 		t.Fatalf("failed to extract replication metadata: %v", err)
 	}
 
 	expected := map[string]string{
+		"content-type":      "application/wasm",
+		"x-amz-meta-source": "client",
 		"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "sealed-key",
 		"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
 		"X-Minio-Internal-Server-Side-Encryption-Iv":             "iv",
@@ -279,6 +287,9 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
 	if !reflect.DeepEqual(metadata, expected) {
 		t.Fatalf("unexpected replication metadata: expected %#v, got %#v", expected, metadata)
 	}
+	if _, ok := metadata["content-encoding"]; ok {
+		t.Fatalf("replication metadata restored transport content-encoding: %#v", metadata)
+	}
 }
 
 func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {

cmd/iam-credential-retention_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+func TestIAMCredentialRetention(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			secret, err := getTokenSigningKey()
+			mustIAM(t, err)
+			parent := "external-idp-parent"
+			credential := func(exp time.Time) auth.Credentials {
+				cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": exp.Unix(), parentClaim: parent}, secret)
+				mustIAM(t, err)
+				cred.ParentUser = parent
+				return cred
+			}
+			// Disablement of an external identity must include cached STS,
+			// which are kept separately from regular and service accounts.
+			cred := credential(UTCNow().Add(time.Hour))
+			_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
+			mustIAM(t, err)
+			mustIAM(t, sys.store.DeleteUsers(ctx, []string{parent}))
+			r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
+			mustIAM(t, err)
+			if !r.Deleted || !r.ExpiresAt.Equal(cred.Expiration.Add(globalMaxSkewTime)) || r.Credentials.SessionToken != "" || r.Credentials.SecretKey != "" {
+				t.Fatal("early STS revocation lost its retention boundary or retained a secret")
+			}
+			if _, ok := sys.store.GetUser(cred.AccessKey); ok {
+				t.Fatal("external disablement left the STS cache live")
+			}
+			_, err = sys.SetTempUser(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), cred.AccessKey, cred, "")
+			if !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("same revoked token was reissued by replay: %v", err)
+			}
+			var mp MappedPolicy
+			err = sys.store.loadIAMConfig(ctx, &mp, getMappedPolicyPath(cred.AccessKey, stsUser, false))
+			if !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("random STS key produced a permanent mapping: %v", err)
+			}
+
+			// Seed genuinely expired immutable tokens, as an ordinary startup
+			// loader sees them. Natural expiry leaves no permanent tombstone.
+			expired := credential(UTCNow().Add(-time.Hour))
+			path := getUserIdentityPath(expired.AccessKey, stsUser)
+			mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: expired, UpdatedAt: UTCNow().Add(-2 * time.Hour)}, path))
+			_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
+			var u UserIdentity
+			if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("natural expiration retained a random key: %v", err)
+			}
+			// A retained early-revocation record is collectable only after the
+			// immutable token's expiration plus the skew allowance.
+			tomb := UserIdentity{Version: 1, Deleted: true, UpdatedAt: UTCNow().Add(-2 * time.Hour), ExpiresAt: expired.Expiration.Add(globalMaxSkewTime)}
+			mustIAM(t, sys.store.saveIAMConfig(ctx, &tomb, path))
+			_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
+			if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("expired STS revocation not collected: %v", err)
+			}
+			if _, ok := sys.store.revisionIndex().snapshot()[path]; ok {
+				t.Fatal("expired STS retained an index entry")
+			}
+		})
+	}
+}
+
+func TestIAMPolicyDeletionRemainsExplicit(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			mustIAM(t, sys.DeletePolicy(ctx, "misspelled-policy", true))
+			r, err := loadIAMRevision(ctx, sys.store, getPolicyDocPath("misspelled-policy"))
+			mustIAM(t, err)
+			if r.Deleted {
+				t.Fatal("local nonexistent policy created a tombstone")
+			}
+			p, err := sys.store.GetPolicy("readwrite")
+			mustIAM(t, err)
+			if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil {
+				t.Fatal("local pristine builtin policy became deletable")
+			}
+			_, err = sys.SetPolicy(ctx, "readwrite", p)
+			mustIAM(t, err)
+			mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true))
+			mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+			if _, err := sys.store.GetPolicy("readwrite"); !errors.Is(err, errNoSuchPolicy) {
+				t.Fatalf("reload restored an explicitly deleted override: %v", err)
+			}
+			_, err = sys.SetPolicy(ctx, "readwrite", p)
+			mustIAM(t, err)
+			if _, err := sys.store.GetPolicy("readwrite"); err != nil {
+				t.Fatal("explicit policy recreation failed", err)
+			}
+			mustIAM(t, globalSiteReplicationSys.PeerAddPolicyHandler(ctx, "remote-unknown-policy", nil, UTCNow()))
+			r, err = loadIAMRevision(ctx, sys.store, getPolicyDocPath("remote-unknown-policy"))
+			mustIAM(t, err)
+			if !r.Deleted {
+				t.Fatal("replicated unknown deletion lost its version")
+			}
+		})
+	}
+}

cmd/iam-etcd-store.go

@@ -26,7 +26,6 @@ import (
 	"sync"
 	"time"
 
-	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
@@ -62,6 +61,7 @@ type IAMEtcdStore struct {
 	sync.RWMutex
 
 	*iamCache
+	index iamRevisionIndex
 
 	usersSysType UsersSysType
 
@@ -69,13 +69,17 @@ type IAMEtcdStore struct {
 }
 
 func newIAMEtcdStore(client *etcd.Client, usersSysType UsersSysType) *IAMEtcdStore {
-	return &IAMEtcdStore{
+	store := &IAMEtcdStore{
 		iamCache:     newIamCache(),
 		client:       client,
 		usersSysType: usersSysType,
 	}
+	store.revisions = &store.index
+	return store
 }
 
+func (ies *IAMEtcdStore) revisionIndex() *iamRevisionIndex { return &ies.index }
+
 func (ies *IAMEtcdStore) rlock() *iamCache {
 	ies.RLock()
 	return ies.iamCache
@@ -103,6 +107,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
 	if err != nil {
 		return err
 	}
+	plain := data
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, itemPath),
@@ -111,24 +116,28 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
 			return err
 		}
 	}
-	return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
+	if err := saveKeyEtcd(ctx, ies.client, itemPath, data, opts...); err != nil {
+		return err
+	}
+	ies.index.observe(itemPath, plain)
+	return nil
 }
 
-func getIAMConfig(item any, data []byte, itemPath string) error {
-	data, err := decryptData(data, itemPath)
+func (ies *IAMEtcdStore) decodeIAMConfig(item any, data []byte, path string) error {
+	data, err := decryptData(data, path)
 	if err != nil {
 		return err
 	}
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	ies.index.observe(path, data)
 	return json.Unmarshal(data, item)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item any, path string) error {
-	data, err := readKeyEtcd(ctx, ies.client, path)
+	data, err := ies.loadIAMConfigBytes(ctx, path)
 	if err != nil {
 		return err
 	}
-	return getIAMConfig(item, data, path)
+	return json.Unmarshal(data, item)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) {
@@ -136,11 +145,19 @@ func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([
 	if err != nil {
 		return nil, err
 	}
-	return decryptData(data, path)
+	data, err = decryptData(data, path)
+	if err == nil {
+		ies.index.observe(path, data)
+	}
+	return data, err
 }
 
 func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error {
-	return deleteKeyEtcd(ctx, ies.client, path)
+	if err := deleteKeyEtcd(ctx, ies.client, path); err != nil {
+		return err
+	}
+	ies.index.forget(path)
+	return nil
 }
 
 func (ies *IAMEtcdStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, _ int) error {
@@ -162,6 +179,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
 		return err
 	}
 
+	if p.Deleted {
+		return errNoSuchPolicy
+	}
 	m[policy] = p
 	return nil
 }
@@ -181,7 +201,11 @@ func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValu
 		return err
 	}
 
+	ies.index.observe(string(kvs.Key), data)
 	policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key)))
+	if p.Deleted {
+		return errNoSuchPolicy
+	}
 	m[policy] = p
 	return nil
 }
@@ -207,7 +231,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]Policy
 
 func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
 	var u UserIdentity
-	err := getIAMConfig(&u, userkv.Value, string(userkv.Key))
+	err := ies.decodeIAMConfig(&u, userkv.Value, string(userkv.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -219,10 +243,14 @@ func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue,
 }
 
 func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error {
+	if u.Deleted {
+		if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
+			bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
+		}
+		return errNoSuchUser
+	}
 	if u.Credentials.IsExpired() {
-		// Delete expired identity.
-		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
+		bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
 		return nil
 	}
 	if u.Credentials.AccessKey == "" {
@@ -231,16 +259,17 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
-			if u.Credentials.IsTemp() {
-				// We should delete such that the client can re-request
-				// for the expiring credentials.
-				deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-				deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
-			}
+			// A temporarily unavailable signing key is not proof of expiration.
 			return nil
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
+	if err := checkIAMParentRevision(ctx, ies, u.Credentials); err != nil {
+		if errors.Is(err, errIAMStaleUpdate) {
+			return errNoSuchUser
+		}
+		return err
+	}
 	if u.Credentials.Description == "" {
 		u.Credentials.Description = u.Credentials.Comment
 	}
@@ -258,6 +287,9 @@ func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userTyp
 		}
 		return "", err
 	}
+	if u.Deleted {
+		return "", errNoSuchUser
+	}
 	return u.Credentials.SecretKey, nil
 }
 
@@ -274,6 +306,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 }
 
 func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
+	ctx = withIAMExpirationCleanup(ctx)
 	var basePrefix string
 	switch userType {
 	case svcUser:
@@ -312,6 +345,9 @@ func (ies *IAMEtcdStore) loadGroup(ctx context.Context, group string, m map[stri
 		}
 		return err
 	}
+	if gi.Deleted {
+		return errNoSuchGroup
+	}
 	m[group] = gi
 	return nil
 }
@@ -349,13 +385,16 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
 		}
 		return err
 	}
+	if !ies.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+		return errNoSuchPolicy
+	}
 	m.Store(name, p)
 	return nil
 }
 
-func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
+func (ies *IAMEtcdStore) getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
 	var p MappedPolicy
-	err := getIAMConfig(&p, kv.Value, string(kv.Key))
+	err := ies.decodeIAMConfig(&p, kv.Value, string(kv.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -363,6 +402,9 @@ func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy],
 		return err
 	}
 	name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json")
+	if !ies.index.mappingAllowed(string(kv.Key), p) {
+		return errNoSuchPolicy
+	}
 	m.Store(name, p)
 	return nil
 }
@@ -392,61 +434,13 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Parse all policies mapping to create the proper data model
 	for _, kv := range r.Kvs {
-		if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if err = ies.getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
 			return err
 		}
 	}
 	return nil
 }
 
-func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
-	return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
-}
-
-func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
-	return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
-}
-
-func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
-	return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
-}
-
-func (ies *IAMEtcdStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
-	return ies.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
-}
-
-func (ies *IAMEtcdStore) deletePolicyDoc(ctx context.Context, name string) error {
-	err := ies.deleteIAMConfig(ctx, getPolicyDocPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
-	err := ies.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
-	err := ies.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
-	if err == errConfigNotFound {
-		err = errNoSuchUser
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteGroupInfo(ctx context.Context, name string) error {
-	err := ies.deleteIAMConfig(ctx, getGroupInfoPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchGroup
-	}
-	return err
-}
-
 func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWatchEvent {
 	ch := make(chan iamWatchEvent)
 

cmd/iam-group-revision.go

@@ -0,0 +1,164 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"maps"
+	"slices"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+type (
+	iamGroupGrantsKey   struct{}
+	iamGroupMutationKey struct{}
+	iamGroupMutation    struct {
+		Members    []string
+		Remove     bool
+		StatusOnly bool
+	}
+)
+
+// Merge the intended mutation with the record read under the distributed
+// revision lock, not the older cache used to prepare the request.
+func mergeIAMGroupMutation(ctx context.Context, previous GroupInfo, next *GroupInfo) {
+	op, ok := ctx.Value(iamGroupMutationKey{}).(iamGroupMutation)
+	if !ok || previous.Deleted || previous.Version == 0 {
+		return
+	}
+	members := set.CreateStringSet(previous.Members...)
+	grants := maps.Clone(previous.MemberGrants)
+	if grants == nil {
+		grants = make(map[string]time.Time)
+	}
+	switch {
+	case op.StatusOnly:
+		// Only the status changes.
+	case op.Remove:
+		for _, member := range op.Members {
+			members.Remove(member)
+			delete(grants, member)
+		}
+		next.Status = previous.Status
+	default:
+		requested := set.CreateStringSet(next.Members...)
+		for _, member := range op.Members {
+			if !requested.Contains(member) {
+				continue
+			}
+			at := next.MemberGrants[member]
+			if at.Before(grants[member]) {
+				continue
+			}
+			members.Add(member)
+			grants[member] = at
+		}
+		next.Status = previous.Status
+	}
+	next.Members, next.MemberGrants = members.ToSlice(), grants
+	slices.Sort(next.Members)
+}
+
+// A non-nil map is supplied by the versioned peer envelope, including for
+// snapshots. Missing times are unknown, never the snapshot's newer timestamp.
+func withIAMGroupGrants(ctx context.Context, grants map[string]time.Time) context.Context {
+	return context.WithValue(ctx, iamGroupGrantsKey{}, grants)
+}
+
+func (c *iamCache) effectiveGroupMembers(gi GroupInfo) []string {
+	var members []string
+	for _, member := range gi.Members {
+		if c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
+			members = append(members, member)
+		}
+	}
+	return members
+}
+
+func (c *iamCache) effectiveUserGroups(user string) []string {
+	var groups []string
+	for group := range c.iamUserGroupMemberships[user] {
+		gi, ok := c.iamGroupsMap[group]
+		r := c.revisions.get(getGroupInfoPath(group))
+		if r.RevokedBefore.After(gi.RevokedBefore) {
+			gi.RevokedBefore = r.RevokedBefore
+		}
+		if ok && !r.Deleted && c.groupMemberAllowed(user, gi.MemberGrants[user], gi.RevokedBefore) {
+			groups = append(groups, group)
+		}
+	}
+	return groups
+}
+
+func (c *iamCache) addGroupMembers(ctx context.Context, gi GroupInfo, members []string) (GroupInfo, error) {
+	grants, versioned := ctx.Value(iamGroupGrantsKey{}).(map[string]time.Time)
+	if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(gi.RevokedBefore) {
+		gi.RevokedBefore = boundary
+	}
+	origin, replicated := iamReplicationTime(ctx)
+	gi.Members = slices.Clone(gi.Members)
+	gi.MemberGrants = maps.Clone(gi.MemberGrants)
+	if gi.MemberGrants == nil {
+		gi.MemberGrants = make(map[string]time.Time)
+	}
+	current := set.CreateStringSet(gi.Members...)
+	gi.UpdatedAt = UTCNow()
+	if replicated {
+		gi.UpdatedAt = origin
+	}
+	for _, member := range members {
+		at := gi.UpdatedAt
+		r := c.userRevocation(member)
+		if replicated {
+			switch {
+			case versioned:
+				at = grants[member]
+				if at.After(origin) {
+					return gi, errInvalidArgument
+				}
+			case !r.RevokedBefore.IsZero() || r.Deleted || !gi.RevokedBefore.IsZero():
+				// Legacy snapshots cannot prove a post-revocation grant.
+				continue
+			case current.Contains(member):
+				continue
+			}
+			if !c.groupMemberAllowed(member, at, gi.RevokedBefore) {
+				continue
+			}
+		} else {
+			if current.Contains(member) && c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
+				continue // Editing the group is not reissuing every grant.
+			}
+			if !at.After(gi.RevokedBefore) {
+				at = gi.RevokedBefore.Add(time.Nanosecond)
+			}
+			if !at.After(r.RevokedBefore) {
+				at = r.RevokedBefore.Add(time.Nanosecond)
+			}
+			if !at.After(gi.MemberGrants[member]) {
+				at = gi.MemberGrants[member].Add(time.Nanosecond)
+			}
+			if at.After(gi.UpdatedAt) {
+				gi.UpdatedAt = at
+			}
+		}
+		u, ok := c.iamUsersMap[member]
+		if !ok {
+			return gi, errNoSuchUser
+		}
+		if u.Credentials.IsTemp() || u.Credentials.IsServiceAccount() {
+			return gi, errIAMActionNotAllowed
+		}
+		if previous := gi.MemberGrants[member]; previous.After(at) {
+			continue
+		}
+		current.Add(member)
+		gi.MemberGrants[member] = at
+	}
+	gi.Members = current.ToSlice()
+	slices.Sort(gi.Members)
+	return gi, nil
+}

cmd/iam-healing-leadership_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestIAMHealingResumesAfterLeadershipLoss(t *testing.T) {
+	previous := globalLeaderLock
+	locks := make(chan LockContext)
+	globalLeaderLock = &sharedLock{lockContext: locks}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	c := &SiteReplicationSys{}
+	go func() { c.startHealRoutine(ctx, nil); close(done) }()
+	t.Cleanup(func() {
+		cancel()
+		select {
+		case <-done:
+		case locks <- LockContext{ctx: ctx}:
+		}
+		<-done
+		globalLeaderLock = previous
+	})
+	first, loseFirst := context.WithCancel(ctx)
+	defer loseFirst()
+	select {
+	case locks <- LockContext{ctx: first}:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not acquire its first leader context")
+	}
+	loseFirst() // A transient quorum loss cancels the distributed lease.
+	select {
+	case <-done:
+		t.Fatal("healer permanently exited after temporary leadership loss")
+	case locks <- LockContext{ctx: ctx}:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not wait for reacquired leadership")
+	}
+	// Shutdown must also interrupt the wait for leadership after lease loss.
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not stop with its owning context")
+	}
+}
+
+func TestIAMHealingLeadershipWaitCancels(t *testing.T) {
+	previous := globalLeaderLock
+	locks := make(chan LockContext)
+	globalLeaderLock = &sharedLock{lockContext: locks}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	c := &SiteReplicationSys{}
+	go func() { c.startHealRoutine(ctx, nil); close(done) }()
+	cancel()
+	t.Cleanup(func() {
+		select {
+		case <-done:
+		case locks <- LockContext{ctx: ctx}:
+		}
+		<-done
+		globalLeaderLock = previous
+	})
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("healer ignored shutdown while waiting for leadership")
+	}
+}

cmd/iam-healing-performance_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+// Measures steady-state index traversal, sorting and the capability request.
+// The network peer acknowledges real batches but performs no disk I/O; this
+// benchmark deliberately does not claim durable catch-up throughput.
+func BenchmarkIAMRevisionConvergedHealing(b *testing.B) {
+	for _, n := range []int{1000, 10000} {
+		b.Run(fmt.Sprint(n), func(b *testing.B) {
+			ctx, sys, _ := prepareIAMRevisionFixture(b)
+			_, err := sys.CreateUser(ctx, "benchmark-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
+			mustIAM(b, err)
+			for i := range n {
+				at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
+				data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
+				mustIAM(b, err)
+				sys.store.revisionIndex().observe(getUserIdentityPath(fmt.Sprintf("deleted-%06d", i), regUser), data)
+			}
+			var puts atomic.Int64
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.Path == "/minio/health/live" {
+					w.WriteHeader(http.StatusOK)
+					return
+				}
+				if r.Method == http.MethodPut {
+					puts.Add(1)
+				}
+				_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: "benchmark-peer", Digest: "constant"}})
+			}))
+			defer server.Close()
+			c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "benchmark-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
+			mustIAM(b, c.healIAMDeletions(ctx))
+			before := puts.Load()
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				mustIAM(b, c.healIAMDeletions(ctx))
+			}
+			b.StopTimer()
+			b.ReportMetric(float64(puts.Load()-before)/float64(b.N), "PUT/op")
+			if puts.Load() != before {
+				b.Fatal("steady-state healing replayed acknowledged records")
+			}
+		})
+	}
+}

cmd/iam-object-store.go

@@ -45,6 +45,7 @@ type IAMObjectStore struct {
 	sync.RWMutex
 
 	*iamCache
+	index iamRevisionIndex
 
 	usersSysType UsersSysType
 
@@ -52,13 +53,17 @@ type IAMObjectStore struct {
 }
 
 func newIAMObjectStore(objAPI ObjectLayer, usersSysType UsersSysType) *IAMObjectStore {
-	return &IAMObjectStore{
+	store := &IAMObjectStore{
 		iamCache:     newIamCache(),
 		objAPI:       objAPI,
 		usersSysType: usersSysType,
 	}
+	store.revisions = &store.index
+	return store
 }
 
+func (iamOS *IAMObjectStore) revisionIndex() *iamRevisionIndex { return &iamOS.index }
+
 func (iamOS *IAMObjectStore) rlock() *iamCache {
 	iamOS.RLock()
 	return iamOS.iamCache
@@ -87,6 +92,7 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
 	if err != nil {
 		return err
 	}
+	plain := data
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, objPath),
@@ -95,7 +101,11 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
 			return err
 		}
 	}
-	return saveConfig(ctx, iamOS.objAPI, objPath, data)
+	if err := saveConfig(ctx, iamOS.objAPI, objPath, data); err != nil {
+		return err
+	}
+	iamOS.index.observe(objPath, plain)
+	return nil
 }
 
 func decryptData(data []byte, objPath string) ([]byte, error) {
@@ -133,6 +143,7 @@ func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context,
 	if err != nil {
 		return nil, meta, err
 	}
+	iamOS.index.observe(objPath, data)
 	return data, meta, nil
 }
 
@@ -146,7 +157,11 @@ func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item any, objPat
 }
 
 func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) error {
-	return deleteConfig(ctx, iamOS.objAPI, path)
+	if err := deleteConfig(ctx, iamOS.objAPI, path); err != nil {
+		return err
+	}
+	iamOS.index.forget(path)
+	return nil
 }
 
 func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error {
@@ -171,6 +186,10 @@ func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy
 			return err
 		}
 
+		if p.Deleted {
+			return errNoSuchPolicy
+		}
+
 		if p.Version == 0 {
 			// This means that policy was in the old version (without any
 			// timestamp info). We fetch the mod time of the file and save
@@ -200,6 +219,10 @@ func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (Pol
 		return p, err
 	}
 
+	if p.Deleted {
+		return PolicyDoc{}, errNoSuchPolicy
+	}
+
 	if p.Version == 0 {
 		// This means that policy was in the old version (without any
 		// timestamp info). We fetch the mod time of the file and save
@@ -245,6 +268,9 @@ func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, use
 		}
 		return "", err
 	}
+	if u.Deleted {
+		return "", errNoSuchUser
+	}
 	return u.Credentials.SecretKey, nil
 }
 
@@ -258,10 +284,15 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
 		return u, err
 	}
 
+	if u.Deleted {
+		if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
+			bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
+		}
+		return UserIdentity{}, errNoSuchUser
+	}
+
 	if u.Credentials.IsExpired() {
-		// Delete expired identity - ignoring errors here.
-		iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
-		iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
+		bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
 		return u, errNoSuchUser
 	}
 
@@ -272,16 +303,18 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
-			if u.Credentials.IsTemp() {
-				// We should delete such that the client can re-request
-				// for the expiring credentials.
-				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
-				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
-			}
-			return u, errNoSuchUser
+			// During startup the site signing key may not be available yet.
+			// Reject this load without deleting a credential that has not expired.
+			return UserIdentity{}, errNoSuchUser
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
+	if err := checkIAMParentRevision(ctx, iamOS, u.Credentials); err != nil {
+		if errors.Is(err, errIAMStaleUpdate) {
+			return UserIdentity{}, errNoSuchUser
+		}
+		return UserIdentity{}, err
+	}
 
 	if u.Credentials.Description == "" {
 		u.Credentials.Description = u.Credentials.Comment
@@ -320,6 +353,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 }
 
 func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
+	ctx = withIAMExpirationCleanup(ctx)
 	var basePrefix string
 	switch userType {
 	case svcUser:
@@ -354,6 +388,9 @@ func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[
 		}
 		return err
 	}
+	if g.Deleted {
+		return errNoSuchGroup
+	}
 	m[group] = g
 	return nil
 }
@@ -391,6 +428,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
 			goto retry
 		}
 
+		if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+			return errNoSuchPolicy
+		}
 		m.Store(name, p)
 		return nil
 	}
@@ -405,6 +445,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name
 		}
 		return p, err
 	}
+	if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+		return MappedPolicy{}, errNoSuchPolicy
+	}
 	return p, nil
 }
 
@@ -824,54 +867,6 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	return nil
 }
 
-func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
-	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
-}
-
-func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
-	return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
-}
-
-func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
-	return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
-}
-
-func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
-	return iamOS.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
-}
-
-func (iamOS *IAMObjectStore) deletePolicyDoc(ctx context.Context, name string) error {
-	err := iamOS.deleteIAMConfig(ctx, getPolicyDocPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
-	err := iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
-	err := iamOS.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
-	if err == errConfigNotFound {
-		err = errNoSuchUser
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteGroupInfo(ctx context.Context, name string) error {
-	err := iamOS.deleteIAMConfig(ctx, getGroupInfoPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchGroup
-	}
-	return err
-}
-
 // Lists objects in the minioMetaBucket at the given path prefix. All returned
 // items have the pathPrefix removed from their names.
 func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr[string] {

cmd/iam-peer-reload_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/grid"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// Two independent IAM caches share the same real object backend, as sibling
+// nodes do. Deliver the actual peer handler only after the source committed.
+func TestIAMPeerDeleteNotificationReloadsCommittedState(t *testing.T) {
+	for _, name := range []string{"deleted", "recreated", "recreated_without_grant"} {
+		recreate := name != "deleted"
+		t.Run(name, func(t *testing.T) {
+			resetTestGlobals()
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			obj, disk, err := prepareFS(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.RemoveAll(disk)
+			defer obj.Shutdown(ctx)
+			defer resetTestGlobals()
+			globalObjLayerMutex.Lock()
+			globalObjectAPI = obj
+			globalObjLayerMutex.Unlock()
+			must := func(err error) {
+				t.Helper()
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			source := globalIAMSys
+			const user = "peer-reload-user"
+			req := madmin.AddOrUpdateUserReq{SecretKey: "original-test-password", Status: madmin.AccountEnabled}
+			_, err = source.CreateUser(ctx, user, req)
+			must(err)
+			_, err = source.PolicyDBSet(ctx, user, "readwrite", regUser, false)
+			must(err)
+			_, err = source.AddUsersToGroup(ctx, "peer-reload-group", []string{user})
+			must(err)
+			_, err = source.PolicyDBSet(ctx, "peer-reload-group", "readwrite", regUser, true)
+			must(err)
+			svc, _, err := source.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{accessKey: "peer-reload-service", secretKey: "service-test-password"})
+			must(err)
+			signingKey, err := getTokenSigningKey()
+			must(err)
+			sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: user}, signingKey)
+			must(err)
+			sts.ParentUser = user
+			_, err = source.SetTempUser(ctx, sts.AccessKey, sts, "")
+			must(err)
+
+			siblingStore := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			must(siblingStore.LoadIAMCache(ctx, true))
+			must(siblingStore.UserNotificationHandler(ctx, sts.AccessKey, stsUser))
+			for _, key := range []string{user, svc.AccessKey, sts.AccessKey} {
+				if _, ok := siblingStore.GetUser(key); !ok {
+					t.Fatalf("fixture did not load %s", key)
+				}
+			}
+			must(source.DeleteUser(ctx, user, false))
+			if recreate {
+				req.SecretKey = "recreated-test-password"
+				_, err = source.CreateUser(ctx, user, req)
+				must(err)
+				if name == "recreated" {
+					_, err = source.PolicyDBSet(ctx, user, "readonly", regUser, false)
+					must(err)
+				}
+			}
+
+			sibling := &IAMSys{store: siblingStore, usersSysType: MinIOUsersSysType}
+			globalIAMSys = sibling
+			defer func() { globalIAMSys = source }()
+			server := &peerRESTServer{}
+			for range 2 {
+				_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+				if remoteErr != nil {
+					t.Fatal(remoteErr)
+				}
+			}
+			if recreate {
+				u, ok := siblingStore.GetUser(user)
+				if !ok || u.Credentials.SecretKey != req.SecretKey {
+					t.Fatal("delayed deletion notification removed the recreated user")
+				}
+				loaded := make(map[string]UserIdentity)
+				must(source.store.loadUser(ctx, user, regUser, loaded))
+				if loaded[user].Credentials.SecretKey != req.SecretKey {
+					t.Fatal("notification changed the persisted recreated identity")
+				}
+				if allowed := sibling.IsAllowed(policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}); allowed != (name == "recreated") {
+					t.Fatal("notification did not load the recreated user's current grant")
+				}
+			}
+			if sibling.IsAllowed(policy.Args{AccountName: user, Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) {
+				t.Fatal("notification retained an old direct or group grant")
+			}
+			for _, key := range []string{svc.AccessKey, sts.AccessKey} {
+				if _, ok := siblingStore.GetUser(key); ok {
+					t.Fatalf("notification retained a revoked child: %s", key)
+				}
+			}
+			if !recreate {
+				for _, key := range []string{user} {
+					if _, ok := siblingStore.GetUser(key); ok {
+						t.Fatalf("notification retained a revoked cached identity: %s", key)
+					}
+				}
+				if sibling.IsAllowed(policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}) {
+					t.Fatal("notification retained the user's old grant")
+				}
+				cache := siblingStore.rlock()
+				member := cache.iamUserGroupMemberships[user].Contains("peer-reload-group")
+				siblingStore.runlock()
+				if member {
+					t.Fatal("notification retained the deleted user's group membership")
+				}
+			}
+		})
+	}
+}

cmd/iam-performance_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// Uses APIs shared with the pre-revision tree so the same benchmark can be
+// overlaid on that tree for a comparable local baseline.
+func prepareIAMPerformanceFixture(b *testing.B) (context.Context, *IAMSys) {
+	b.Helper()
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		b.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err != nil {
+		b.Fatal(err)
+	}
+	initAllSubsystems(ctx)
+	globalIAMSys.initStore(obj, nil)
+	if err := globalIAMSys.Load(ctx, true); err != nil {
+		b.Fatal(err)
+	}
+	b.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
+	return ctx, globalIAMSys
+}
+
+func BenchmarkIAMCachedCredential(b *testing.B) {
+	for _, kind := range []string{"user", "service", "sts"} {
+		b.Run(kind, func(b *testing.B) {
+			ctx, sys := prepareIAMPerformanceFixture(b)
+			const parent = "benchmark-parent"
+			_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
+			if err != nil {
+				b.Fatal(err)
+			}
+			key := parent
+			if kind == "service" {
+				c, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "benchmark-service", secretKey: "benchmark-service-password"})
+				if err != nil {
+					b.Fatal(err)
+				}
+				key = c.AccessKey
+			}
+			if kind == "sts" {
+				secret, err := getTokenSigningKey()
+				if err != nil {
+					b.Fatal(err)
+				}
+				c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+				if err != nil {
+					b.Fatal(err)
+				}
+				c.ParentUser = parent
+				if _, err := sys.SetTempUser(ctx, c.AccessKey, c, ""); err != nil {
+					b.Fatal(err)
+				}
+				key = c.AccessKey
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				if _, ok := sys.store.GetUser(key); !ok {
+					b.Fatal("credential missing")
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkIAMSetTempUser(b *testing.B) {
+	ctx, sys := prepareIAMPerformanceFixture(b)
+	const parent = "benchmark-sts-parent"
+	_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
+	if err != nil {
+		b.Fatal(err)
+	}
+	secret, err := getTokenSigningKey()
+	if err != nil {
+		b.Fatal(err)
+	}
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cred.ParentUser = parent
+	b.ReportAllocs()
+	b.ResetTimer()
+	for b.Loop() {
+		if _, err := sys.SetTempUser(ctx, cred.AccessKey, cred, "readwrite"); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// Run with -benchtime=1x. Preparation is outside the timer; each measured load
+// sees a fresh set of expired reusable service-account records.
+func BenchmarkIAMColdLoadExpiredServices(b *testing.B) {
+	for _, count := range []int{100, 1000} {
+		b.Run(fmt.Sprint(count), func(b *testing.B) {
+			ctx, sys := prepareIAMPerformanceFixture(b)
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				b.StopTimer()
+				for j := 0; j < count; j++ {
+					key := fmt.Sprintf("expired-benchmark-%d-%d", i, j)
+					u := UserIdentity{Version: 1, UpdatedAt: UTCNow().Add(-2 * time.Hour), Credentials: auth.Credentials{AccessKey: key, SecretKey: "expired-benchmark-password", ParentUser: "absent-idp-parent", Expiration: UTCNow().Add(-time.Hour), Status: auth.AccountOn}}
+					if err := sys.store.saveIAMConfig(ctx, &u, getUserIdentityPath(key, svcUser)); err != nil {
+						b.Fatal(err)
+					}
+				}
+				b.StartTimer()
+				if err := sys.store.LoadIAMCache(ctx, true); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

cmd/iam-replay_test.go

@@ -0,0 +1,168 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func TestReviewIAMRevokedUserReplay(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "review-revoked-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
+	created, err := globalIAMSys.CreateUser(ctx, user, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	policyAt, err := globalIAMSys.PolicyDBSet(ctx, user, "readwrite", regUser, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "review-bucket", ObjectName: "review-object"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("seed must allow object read")
+	}
+	if err := globalIAMSys.DeleteUser(ctx, user, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("deletion did not remove initial permission")
+	}
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Errorf("revoked user restored by an older replicated create, GetUserInfo error = %v", err)
+	}
+	if err := globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: user, UserType: int(regUser), Policy: "readwrite"}, policyAt); err != nil {
+		t.Fatal(err)
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Error("older replicated identity and policy events restored revoked S3 read permission")
+	}
+}
+
+func TestReviewIAMSourceTimestampOrder(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "review-ordered-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour)
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Fatalf("newer source deletion skipped after delayed creation, GetUserInfo error = %v", err)
+	}
+}
+
+// A user's old group grant must not return after deletion and deliberate recreation.
+func TestR3CandidateOldGroupReplayAfterRecreation(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user, group := "r3-group-member", "r3-granting-group"
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
+	peer := &globalSiteReplicationSys
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
+	add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
+	must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
+	must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("fixture must grant through group")
+	}
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("recreation must start without deleted group membership")
+	}
+	must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("old group event restored the deleted user's read grant after recreation and durable reload")
+	}
+}
+
+// The user delete is also a revocation of its earlier group memberships.
+func TestR3CandidateLateDeleteRetainsOldGroupGrant(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user, group := "r3-late-member", "r3-late-group"
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
+	peer := &globalSiteReplicationSys
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
+	must(peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin.Add(time.Minute)))
+	must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("fixture must grant through group")
+	}
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if _, ok := globalIAMSys.GetUser(ctx, user); !ok {
+		t.Fatal("newer identity must survive")
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("late user deletion retained the older group grant on the recreated identity")
+	}
+}

cmd/iam-replication-protocol.go

@@ -0,0 +1,321 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+const (
+	iamRevisionProtocol  = 1
+	iamRevisionPeerPath  = "/v3/site-replication/peer/iam-revisions"
+	iamUserBoundaryType  = "silo-user-revocation"
+	iamGroupBoundaryType = "silo-group-revocation"
+	maxIAMRevisionBatch  = 128
+)
+
+var iamRevisionInstance = mustGetUUID()
+
+type iamUserBoundary struct {
+	User   string    `json:"user"`
+	Before time.Time `json:"before"`
+}
+
+type iamGroupBoundary struct {
+	Group  string    `json:"group"`
+	Before time.Time `json:"before"`
+}
+
+// The server owns this additive protocol, without changing the client SDK or
+// overloading a policy/document field. Old servers reject the dedicated route
+// before applying any change that would lose revocation or member metadata.
+type iamReplicationItem struct {
+	madmin.SRIAMItem
+	GroupGrants     map[string]time.Time `json:"groupGrants,omitempty"`
+	GroupSnapshot   bool                 `json:"groupSnapshot,omitempty"`
+	UserRevocation  *iamUserBoundary     `json:"userRevocation,omitempty"`
+	GroupRevocation *iamGroupBoundary    `json:"groupRevocation,omitempty"`
+	RevokedBefore   time.Time            `json:"revokedBefore,omitempty"`
+}
+
+type iamRevisionBatch struct {
+	Version int                  `json:"version"`
+	Items   []iamReplicationItem `json:"items"`
+}
+
+type iamRevisionStatus struct {
+	Version  int    `json:"version"`
+	Node     string `json:"node"`
+	Instance string `json:"instance"`
+	Digest   string `json:"digest"`
+}
+
+type iamRevisionResponse struct {
+	iamRevisionStatus
+	Errors []string `json:"errors,omitempty"`
+}
+
+type iamRevisionBatchError struct{ failures []string }
+
+func (e *iamRevisionBatchError) Error() string {
+	return "IAM revision batch: " + strings.Join(e.failures, "; ")
+}
+
+type iamRevisionProgress struct {
+	Instances    map[string]string
+	Acknowledged map[string]string
+}
+
+type iamRevisionMetrics struct {
+	healFailures       atomic.Uint64
+	healLastSuccess    atomic.Int64
+	healDurationMillis atomic.Int64
+}
+
+func iamRevisionDigest(items map[string]iamRevision) string {
+	paths := make([]string, 0, len(items))
+	for path := range items {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	h := sha256.New()
+	for _, path := range paths {
+		r := items[path]
+		fmt.Fprintf(h, "%q %s %t %s\n", path, r.timestamp().UTC().Format(time.RFC3339Nano), r.Deleted, r.RevokedBefore.UTC().Format(time.RFC3339Nano))
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func (store *IAMStoreSys) iamRevisionStatus() iamRevisionStatus {
+	node := globalLocalNodeName
+	if node == "" {
+		node = "local"
+	}
+	return iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: iamRevisionInstance, Digest: store.revisionIndex().digest()}
+}
+
+func executeIAMRevisionRequest(ctx context.Context, client *madmin.AdminClient, method string, batch *iamRevisionBatch) (status iamRevisionStatus, err error) {
+	var content []byte
+	if batch != nil {
+		content, err = json.Marshal(batch)
+		if err != nil {
+			return status, err
+		}
+	}
+	resp, err := client.ExecuteMethod(ctx, method, madmin.RequestData{RelPath: iamRevisionPeerPath, QueryValues: url.Values{"api-version": {madmin.SiteReplAPIVersion}}, Content: content})
+	if resp != nil {
+		defer xhttp.DrainBody(resp.Body)
+	}
+	if err != nil {
+		return status, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		var remote madmin.ErrorResponse
+		if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&remote) == nil && remote.Code != "" {
+			return status, remote
+		}
+		return status, fmt.Errorf("IAM revision protocol requires upgraded peers: %s", resp.Status)
+	}
+	var response iamRevisionResponse
+	if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&response); err != nil {
+		return status, err
+	}
+	status = response.iamRevisionStatus
+	if status.Version != iamRevisionProtocol || status.Node == "" || status.Instance == "" || status.Digest == "" {
+		return status, errors.New("peer did not acknowledge the IAM revision protocol")
+	}
+	if len(response.Errors) != 0 {
+		return status, &iamRevisionBatchError{failures: response.Errors}
+	}
+	return status, nil
+}
+
+type (
+	iamRecordBoundaryKey struct{}
+	iamGroupSnapshotKey  struct{}
+)
+
+func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) {
+	out := iamReplicationItem{SRIAMItem: item}
+	if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil {
+		var key string
+		if item.SvcAccChange.Create != nil {
+			key = item.SvcAccChange.Create.AccessKey
+		} else if item.SvcAccChange.Update != nil {
+			key = item.SvcAccChange.Update.AccessKey
+		}
+		if key != "" {
+			r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(key, svcUser))
+			if err != nil {
+				return out, err
+			}
+			if r.Deleted || r.timestamp().After(item.UpdatedAt) {
+				return out, errIAMStaleUpdate
+			}
+			out.RevokedBefore = r.RevokedBefore
+		}
+	}
+	if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && !item.GroupInfo.UpdateReq.IsRemove {
+		out.GroupSnapshot = true
+		var gi GroupInfo
+		if err := globalIAMSys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath(item.GroupInfo.UpdateReq.Group)); err != nil {
+			return out, err
+		}
+		// The matching persisted snapshot carries member grant times. If a
+		// later write won before sending, propagate that whole newer state.
+		if gi.Deleted {
+			return out, errIAMStaleUpdate
+		}
+		out.UpdatedAt = gi.UpdatedAt
+		out.RevokedBefore = gi.RevokedBefore
+		out.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: item.GroupInfo.UpdateReq.Group, Status: madmin.GroupStatus(gi.Status)}}
+		cache := globalIAMSys.store.rlock()
+		out.GroupInfo.UpdateReq.Members = cache.effectiveGroupMembers(gi)
+		globalIAMSys.store.runlock()
+		out.GroupGrants = make(map[string]time.Time, len(out.GroupInfo.UpdateReq.Members))
+		for _, member := range out.GroupInfo.UpdateReq.Members {
+			out.GroupGrants[member] = gi.MemberGrants[member]
+		}
+	}
+	if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, getGroupInfoPath(item.GroupInfo.UpdateReq.Group))
+		if err != nil {
+			return out, err
+		}
+		if !r.Deleted && !r.RevokedBefore.IsZero() {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+			out.UpdatedAt = r.RevokedBefore
+		}
+	}
+	if item.Type == madmin.SRIAMItemIAMUser && item.IAMUser != nil {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(item.IAMUser.AccessKey, regUser))
+		if err != nil {
+			return out, err
+		}
+		if item.IAMUser.IsDeleteReq && !r.Deleted && !r.RevokedBefore.IsZero() {
+			out.Type = iamUserBoundaryType
+			out.IAMUser = nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+			out.UpdatedAt = r.RevokedBefore
+		} else if !item.IAMUser.IsDeleteReq {
+			if r.Deleted || r.timestamp().After(item.UpdatedAt) {
+				return out, errIAMStaleUpdate
+			}
+			out.RevokedBefore = r.RevokedBefore
+		}
+	}
+	return out, nil
+}
+
+func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error {
+	if item.GroupInfo != nil {
+		if item.GroupSnapshot {
+			ctx = context.WithValue(ctx, iamGroupSnapshotKey{}, true)
+		}
+		// A nil map also explicitly denotes unknown legacy grants. Do not
+		// turn an unrelated group edit into a new grant after a revocation.
+		ctx = withIAMGroupGrants(ctx, item.GroupGrants)
+	}
+	if !item.RevokedBefore.IsZero() {
+		if item.RevokedBefore.After(item.UpdatedAt) {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore)
+	}
+	switch item.Type {
+	case iamUserBoundaryType:
+		if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		return iamReplicationError(globalIAMSys.DeleteUser(withIAMReplicationTime(ctx, item.UserRevocation.Before), item.UserRevocation.User, true))
+	case iamGroupBoundaryType:
+		if item.GroupRevocation == nil || item.GroupRevocation.Group == "" || item.GroupRevocation.Before.IsZero() {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		_, err := globalIAMSys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, item.GroupRevocation.Before), item.GroupRevocation.Group, nil)
+		return iamReplicationError(err)
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
+		}
+		p, err := policy.ParseConfig(bytes.NewReader(item.Policy))
+		if err != nil {
+			return err
+		}
+		if p.IsEmpty() {
+			p = nil
+		}
+		return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, p, item.UpdatedAt)
+	case madmin.SRIAMItemSvcAcc:
+		return globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
+	case madmin.SRIAMItemPolicyMapping:
+		return globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
+	case madmin.SRIAMItemSTSAcc:
+		return globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
+	case madmin.SRIAMItemIAMUser:
+		return globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
+	case madmin.SRIAMItemGroupInfo:
+		return globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
+	default:
+		return errSRInvalidRequest(errInvalidArgument)
+	}
+}
+
+func (a adminAPIHandlers) SRPeerIAMRevisions(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	if obj, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction); obj == nil {
+		return
+	}
+	var failures []string
+	if r.Method == http.MethodPut {
+		var batch iamRevisionBatch
+		if err := parseJSONBody(ctx, r.Body, &batch, ""); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if batch.Version != iamRevisionProtocol || len(batch.Items) == 0 || len(batch.Items) > maxIAMRevisionBatch {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
+			return
+		}
+		for i, item := range batch.Items {
+			if err := applyIAMReplicationItem(ctx, item); err != nil {
+				failures = append(failures, fmt.Sprintf("item %d (%s): %v", i, item.Type, err))
+			}
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: globalIAMSys.store.iamRevisionStatus(), Errors: failures})
+}
+
+// A site endpoint can balance requests across nodes sharing durable IAM state.
+// Switching between known node incarnations preserves ACKs; a new incarnation
+// conservatively invalidates them so restoring an old backend cannot inherit
+// acknowledgements from before the restore.
+func (p *iamRevisionProgress) observePeer(status iamRevisionStatus) {
+	if p.Instances == nil {
+		p.Instances = make(map[string]string)
+	}
+	if p.Instances[status.Node] != status.Instance || p.Acknowledged == nil {
+		p.Instances[status.Node] = status.Instance
+		p.Acknowledged = make(map[string]string)
+	}
+}

cmd/iam-replication-protocol_test.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+func TestIAMRevisionProtocolDoesNotFallBackToLegacy(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
+			t.Errorf("unsafe fallback path: %s", r.URL.Path)
+		}
+		w.WriteHeader(http.StatusNotFound)
+		_, _ = w.Write([]byte(`{"Code":"NotImplemented","Message":"old server"}`))
+	}))
+	defer server.Close()
+	client, err := madmin.New(strings.TrimPrefix(server.URL, "http://"), "test-access", "valid-test-secret", false)
+	mustIAM(t, err)
+	_, err = executeIAMRevisionRequest(context.Background(), client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{{SRIAMItem: madmin.SRIAMItem{Type: iamUserBoundaryType}, UserRevocation: &iamUserBoundary{User: "recreated", Before: UTCNow()}}}})
+	if err == nil || requests.Load() != 1 {
+		t.Fatalf("old peer must reject without fallback, err=%v requests=%d", err, requests.Load())
+	}
+}
+
+type iamNoHealingScanStore struct{ IAMStorageAPI }
+
+func (s *iamNoHealingScanStore) listIAMConfigPaths(context.Context) ([]string, error) {
+	panic("healing must use the loaded revision index")
+}
+
+func TestIAMRevisionHealingAcknowledgements(t *testing.T) {
+	for _, balanced := range []bool{false, true} {
+		t.Run(fmt.Sprintf("load_balanced_%t", balanced), func(t *testing.T) { testIAMRevisionHealingAcknowledgements(t, balanced) })
+	}
+}
+
+func testIAMRevisionHealingAcknowledgements(t *testing.T, balanced bool) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "ack-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	for i := range maxIAMRevisionBatch*2 + 1 {
+		at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
+		mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath(fmt.Sprintf("ack-%04d", i), regUser)))
+	}
+	sys.store.IAMStorageAPI = &iamNoHealingScanStore{IAMStorageAPI: sys.store.IAMStorageAPI}
+	var mu sync.Mutex
+	var puts, gets int
+	var applied int
+	instance := "boot-1"
+	failSecondBatch := true
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		mu.Lock()
+		defer mu.Unlock()
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
+			t.Errorf("unexpected request: %s", r.URL.Path)
+			w.WriteHeader(404)
+			return
+		}
+		var failures []string
+		if r.Method == http.MethodGet {
+			gets++
+		} else {
+			puts++
+			var batch iamRevisionBatch
+			if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+				t.Error(err)
+				w.WriteHeader(400)
+				return
+			}
+			if len(batch.Items) > maxIAMRevisionBatch {
+				t.Error("batch exceeds limit")
+			}
+			if failSecondBatch && puts == 2 {
+				failures = []string{"injected item error"}
+			} else {
+				applied += len(batch.Items)
+			}
+		}
+		node := "node-1"
+		if balanced {
+			node = fmt.Sprintf("node-%d", (gets+puts)%2+1)
+		}
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: node + instance, Digest: fmt.Sprintf("%d", applied)}, Errors: failures})
+	}))
+	defer server.Close()
+	c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "ack-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
+	if err := c.healIAMDeletions(ctx); err == nil {
+		t.Fatal("item failure was hidden")
+	}
+	mu.Lock()
+	if puts != 3 || applied != maxIAMRevisionBatch+1 {
+		t.Errorf("failed middle batch blocked later revocations: puts=%d applied=%d", puts, applied)
+	}
+	failSecondBatch = false
+	applied++ // Unrelated remote mutation changes its digest.
+	mu.Unlock()
+	at := UTCNow()
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath("ack-new-local", regUser)))
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	if puts != 5 {
+		t.Errorf("did not resume at unacknowledged batch: puts=%d", puts)
+	}
+	mu.Unlock()
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	if puts != 5 || gets != 3 {
+		t.Errorf("converged records were replayed: puts=%d gets=%d", puts, gets)
+	}
+	instance = "boot-2"
+	mu.Unlock()
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	defer mu.Unlock()
+	if puts != 8 {
+		t.Fatalf("peer restart reused an old acknowledgement: puts=%d", puts)
+	}
+}
+
+func TestIAMRevisionIndexRebuildsFromStorage(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	const user = "index-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-parent-password", Status: madmin.AccountEnabled}
+	_, err := sys.CreateUser(ctx, user, req)
+	mustIAM(t, err)
+	mustIAM(t, sys.DeleteUser(ctx, user, false))
+	before := sys.store.revisionIndex().snapshot()
+	store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+	mustIAM(t, store.LoadIAMCache(ctx, true))
+	if iamRevisionDigest(before) != iamRevisionDigest(store.revisionIndex().snapshot()) {
+		t.Fatal("ordinary IAM loading did not restore the deletion index")
+	}
+	_, err = store.AddUser(ctx, user, req)
+	mustIAM(t, err)
+	r := store.revisionIndex().get(getUserIdentityPath(user, regUser))
+	if r.Deleted || r.RevokedBefore.IsZero() {
+		t.Fatal("recreation discarded the retained boundary")
+	}
+	if r.Credentials.SecretKey != "" || r.Credentials.SessionToken != "" {
+		t.Fatal("index retained credentials")
+	}
+}

cmd/iam-revision-boundary_test.go

@@ -0,0 +1,393 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+func prepareIAMRevisionFixture(t testing.TB, backend ...string) (context.Context, *IAMSys, ObjectLayer) {
+	t.Helper()
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	disks, err := getRandomDisks(1)
+	mustIAM(t, err)
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	mustIAM(t, err)
+	initAllSubsystems(ctx)
+	// Deliberately omit the periodic refresh goroutine. Fault injection can
+	// replace this fixture's storage interface without racing initialization.
+	var client *etcd.Client
+	if len(backend) != 0 && backend[0] == "etcd" {
+		endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+		if endpoint == "" {
+			cancel()
+			obj.Shutdown(context.Background())
+			os.RemoveAll(disks[0])
+			t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+		}
+		client, err = etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
+		mustIAM(t, err)
+		prefix := fmt.Sprintf("/silo-boundary-test/%d/", time.Now().UnixNano())
+		client.KV = namespace.NewKV(client.KV, prefix)
+		client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
+		t.Cleanup(func() { client.Delete(context.Background(), "", etcd.WithPrefix()); client.Close() })
+	}
+	globalIAMSys.initStore(obj, client)
+	mustIAM(t, globalIAMSys.Load(ctx, true))
+	t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
+	return ctx, globalIAMSys, obj
+}
+
+func mustIAM(t testing.TB, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+var errIAMInjectedWrite = errors.New("injected IAM persistence failure")
+
+type iamFailingCleanupStore struct {
+	IAMStorageAPI
+	parentPath   string
+	beforeCommit bool
+}
+
+func (s *iamFailingCleanupStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
+	if s.beforeCommit || path != s.parentPath {
+		return errIAMInjectedWrite
+	}
+	return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
+}
+
+func TestIAMRevocationCommitBoundary(t *testing.T) {
+	for _, before := range []bool{true, false} {
+		name := "after_identity_commit"
+		if before {
+			name = "before_identity_commit"
+		}
+		t.Run(name, func(t *testing.T) {
+			ctx, sys, obj := prepareIAMRevisionFixture(t)
+			const user = "commit-boundary-user"
+			origin := UTCNow().Add(-time.Hour)
+			req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+			_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
+			mustIAM(t, err)
+			_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(time.Minute)), user, "readwrite", regUser, false)
+			mustIAM(t, err)
+			_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(time.Minute)), "commit-group", []string{user})
+			mustIAM(t, err)
+			_, err = sys.PolicyDBSet(ctx, "commit-group", "readwrite", regUser, true)
+			mustIAM(t, err)
+			child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), user, nil, newServiceAccountOpts{accessKey: "commit-child", secretKey: "valid-child-password"})
+			mustIAM(t, err)
+			args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+			if !sys.IsAllowed(args) {
+				t.Fatal("fixture has no grant")
+			}
+			siblingStore := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			mustIAM(t, siblingStore.LoadIAMCache(ctx, true))
+			sibling := &IAMSys{store: siblingStore, usersSysType: MinIOUsersSysType}
+			tg, err := grid.SetupTestGrid(2)
+			mustIAM(t, err)
+			defer tg.Cleanup()
+			var notifications atomic.Int32
+			mustIAM(t, deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+				notifications.Add(1)
+				if err := sibling.LoadUserAfterDelete(ctx, r.Get(peerRESTUser)); err != nil {
+					return grid.NoPayload{}, grid.NewRemoteErr(err)
+				}
+				return grid.NoPayload{}, nil
+			}))
+			host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+			mustIAM(t, err)
+			globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{host: host, gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}}}
+			original := sys.store.IAMStorageAPI
+			sys.store.IAMStorageAPI = &iamFailingCleanupStore{IAMStorageAPI: original, parentPath: getUserIdentityPath(user, regUser), beforeCommit: before}
+			boundary := origin.Add(2 * time.Minute)
+			err = sys.DeleteUser(withIAMReplicationTime(ctx, boundary), user, true)
+			if !errors.Is(err, errIAMInjectedWrite) {
+				t.Fatalf("expected write failure, got %v", err)
+			}
+			sys.store.IAMStorageAPI = original
+			r, err := loadIAMRevision(ctx, original, getUserIdentityPath(user, regUser))
+			mustIAM(t, err)
+			if before {
+				if r.Deleted || !sys.IsAllowed(args) || !sibling.IsAllowed(args) || notifications.Load() != 0 {
+					t.Fatal("failure before commit changed the identity or grant")
+				}
+				return
+			}
+			if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
+				t.Fatal("cleanup failure lost durable revocation")
+			}
+			if sys.IsAllowed(args) || sibling.IsAllowed(args) || notifications.Load() != 1 {
+				t.Fatal("cleanup failure retained old permission")
+			}
+			// Subsequent fixture writes need no additional RPC handlers.
+			globalNotificationSys = &NotificationSys{}
+			// Recreate after the partial cleanup. The old mapping, group member
+			// and child still exist in storage; none may authorize this identity.
+			_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), user, req)
+			mustIAM(t, err)
+			reloaded := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			mustIAM(t, reloaded.LoadIAMCache(ctx, true))
+			fresh := &IAMSys{store: reloaded, usersSysType: MinIOUsersSysType}
+			if fresh.IsAllowed(args) {
+				t.Fatal("cold reload restored partially cleaned-up grants")
+			}
+			if _, ok := reloaded.GetUser(child.AccessKey); ok {
+				t.Fatal("cold reload restored the old child")
+			}
+			gd, err := reloaded.GetGroupDescription("commit-group")
+			mustIAM(t, err)
+			if len(gd.Members) != 0 {
+				t.Fatalf("listing exposed a revoked group relation: %v", gd.Members)
+			}
+			_, err = sys.AddUsersToGroup(ctx, "commit-group", []string{user})
+			mustIAM(t, err)
+			if !sys.IsAllowed(args) {
+				t.Fatal("explicit new group grant was not accepted")
+			}
+		})
+	}
+}
+
+func TestIAMGroupGrantVersionsSurviveSnapshotsAndRecreation(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	for _, user := range []string{"grant-alice", "grant-bob"} {
+		_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
+		mustIAM(t, err)
+	}
+	grant := origin.Add(time.Minute)
+	_, err := sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), "grant-group", []string{"grant-alice"})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(ctx, "grant-group", "readwrite", regUser, true)
+	mustIAM(t, err)
+	boundary := origin.Add(2 * time.Minute)
+	mustIAM(t, sys.DeleteUser(withIAMReplicationTime(ctx, boundary), "grant-alice", false))
+	_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), "grant-alice", req)
+	mustIAM(t, err)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), "grant-group", []string{"grant-bob"})
+	mustIAM(t, err)
+	_, err = sys.SetGroupStatus(withIAMReplicationTime(ctx, origin.Add(5*time.Minute)), "grant-group", true)
+	mustIAM(t, err)
+	var gi GroupInfo
+	mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
+	if !gi.MemberGrants["grant-alice"].Equal(grant) {
+		t.Fatal("unrelated group edits refreshed an old grant")
+	}
+	args := policy.Args{AccountName: "grant-alice", Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	for _, stale := range []time.Time{grant, boundary, {}} {
+		item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(6 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: "grant-group", Members: []string{"grant-alice", "grant-bob"}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{"grant-alice": stale, "grant-bob": origin.Add(4 * time.Minute)}}
+		mustIAM(t, applyIAMReplicationItem(ctx, item))
+		mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+		if sys.IsAllowed(args) {
+			t.Fatalf("snapshot restored revoked grant %s", stale)
+		}
+		gd, err := sys.GetGroupDescription("grant-group")
+		mustIAM(t, err)
+		if len(gd.Members) != 1 || gd.Members[0] != "grant-bob" {
+			t.Fatalf("inconsistent effective members: %v", gd.Members)
+		}
+	}
+	// Only an explicit post-revocation grant restores access.
+	freshAt, err := sys.AddUsersToGroup(ctx, "grant-group", []string{"grant-alice"})
+	mustIAM(t, err)
+	if !sys.IsAllowed(args) {
+		t.Fatal("explicit regrant rejected")
+	}
+	mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+	mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
+	if !gi.MemberGrants["grant-alice"].Equal(freshAt) {
+		t.Fatal("new grant version was not persisted")
+	}
+	if !gi.MemberGrants["grant-bob"].Equal(origin.Add(4 * time.Minute)) {
+		t.Fatal("regranting Alice changed Bob's grant")
+	}
+}
+
+func TestIAMGroupRevocationCommitAndRecreation(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) { testIAMGroupRevocationCommitAndRecreation(t, backend) })
+	}
+}
+
+func testIAMGroupRevocationCommitAndRecreation(t *testing.T, backend string) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
+	origin := UTCNow().Add(-time.Hour)
+	user, group := "group-boundary-user", "group-boundary"
+	_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	grant, boundary := origin.Add(time.Minute), origin.Add(2*time.Minute)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
+	mustIAM(t, err)
+	// A newer mapping must not veto the authoritative group deletion.
+	_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, boundary), group, nil)
+	mustIAM(t, err)
+	r, err := loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
+	mustIAM(t, err)
+	if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
+		t.Fatal("newer mapping swallowed group deletion")
+	}
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), group, nil)
+	mustIAM(t, err)
+	for _, at := range []time.Time{grant, boundary, {}} {
+		item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(5 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{user: at}}
+		mustIAM(t, applyIAMReplicationItem(ctx, item))
+		gd, err := sys.GetGroupDescription(group)
+		mustIAM(t, err)
+		if len(gd.Members) != 0 {
+			t.Fatalf("group recreation restored grant %s", at)
+		}
+	}
+	_, err = sys.AddUsersToGroup(ctx, group, []string{user})
+	mustIAM(t, err)
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	if !sys.IsAllowed(args) {
+		t.Fatal("explicit group regrant was rejected")
+	}
+	// The newer live snapshot may arrive before an older group deletion.
+	lateBoundary := origin.Add(6 * time.Minute)
+	_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, lateBoundary), group, nil)
+	mustIAM(t, err)
+	r, err = loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
+	mustIAM(t, err)
+	if r.Deleted || !r.RevokedBefore.Equal(lateBoundary) {
+		t.Fatal("late deletion lost the live group's revocation boundary")
+	}
+	// The old mapping is now revoked; a new explicit mapping restores access.
+	if sys.IsAllowed(args) {
+		t.Fatal("late group boundary retained an old mapping")
+	}
+	_, err = sys.PolicyDBSet(ctx, group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+	if es, ok := sys.store.IAMStorageAPI.(*IAMEtcdStore); ok {
+		store.IAMStorageAPI = newIAMEtcdStore(es.client, MinIOUsersSysType)
+	}
+	mustIAM(t, store.LoadIAMCache(ctx, true))
+	fresh := &IAMSys{store: store, usersSysType: MinIOUsersSysType}
+	if !fresh.IsAllowed(args) {
+		t.Fatal("reload lost explicit grants after a retained group boundary")
+	}
+	item, err := globalSiteReplicationSys.replicationItem(ctx, madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group}}, UpdatedAt: r.timestamp()})
+	mustIAM(t, err)
+	if !item.RevokedBefore.Equal(lateBoundary) || !item.GroupGrants[user].After(lateBoundary) {
+		t.Fatal("group snapshot lost revision metadata")
+	}
+}
+
+// A committed revision is observable before all cached dependents have been
+// cleaned up. Every authorization read must apply that boundary in this window.
+func TestIAMCachedMappingHonorsCommittedRevision(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	parent := "cached-external-parent"
+	_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), parent, "readwrite", stsUser, false)
+	mustIAM(t, err)
+	policies, err := sys.PolicyDBGet(parent)
+	mustIAM(t, err)
+	if len(policies) == 0 {
+		t.Fatal("fixture has no STS-parent mapping")
+	}
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Deleted: true, UpdatedAt: origin.Add(time.Minute)}, getMappedPolicyPath(parent, stsUser, false)))
+	policies, err = sys.PolicyDBGet(parent)
+	mustIAM(t, err)
+	if len(policies) != 0 {
+		t.Fatal("cached STS mapping ignored its own namespace tombstone")
+	}
+
+	user, group := "cached-group-user", "cached-group"
+	_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	grant := origin.Add(5 * time.Minute)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	if !sys.IsAllowed(args) {
+		t.Fatal("fixture has no group grant")
+	}
+	// A late deletion preserves the newer member grant but revokes the older
+	// policy mapping. Simulate the interval before mapping cleanup completes.
+	gi := GroupInfo{Version: 1, Status: statusEnabled, Members: []string{user}, MemberGrants: map[string]time.Time{user: grant}, UpdatedAt: grant, RevokedBefore: origin.Add(2 * time.Minute)}
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &gi, getGroupInfoPath(group)))
+	if sys.IsAllowed(args) {
+		t.Fatal("cached group mapping ignored the committed group boundary")
+	}
+	gd, err := sys.GetGroupDescription(group)
+	mustIAM(t, err)
+	if gd.Policy != "" {
+		t.Fatal("group listing exposed a revoked mapping")
+	}
+}
+
+type (
+	iamExpiryLockFailure struct {
+		ObjectLayer
+		path string
+	}
+	iamFailedExpiryLock struct{ RWLocker }
+)
+
+func (o *iamExpiryLockFailure) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path+".revision-lock" {
+		return &iamFailedExpiryLock{RWLocker: lock}
+	}
+	return lock
+}
+
+func (l *iamFailedExpiryLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) {
+	return LockContext{}, errIAMInjectedWrite
+}
+
+func TestIAMExpiredCredentialCleanupDoesNotBlockLoading(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "healthy-user", madmin.AddOrUpdateUserReq{SecretKey: "healthy-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(ctx, "healthy-user", "readwrite", regUser, false)
+	mustIAM(t, err)
+	c, _, err := sys.NewServiceAccount(ctx, "healthy-user", nil, newServiceAccountOpts{accessKey: "expired-service", secretKey: "expired-service-password"})
+	mustIAM(t, err)
+	c.Expiration = UTCNow().Add(-time.Hour)
+	path := getUserIdentityPath(c.AccessKey, svcUser)
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: c, UpdatedAt: UTCNow()}, path))
+	// A cold loader sees the existing version but cannot acquire the cleanup
+	// write lock. Healthy users must still load; the expired one stays denied.
+	fresh := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(&iamExpiryLockFailure{ObjectLayer: obj, path: path}, MinIOUsersSysType)}
+	mustIAM(t, fresh.LoadIAMCache(ctx, true))
+	if _, ok := fresh.GetUser("healthy-user"); !ok {
+		t.Fatal("cleanup failure prevented healthy IAM state from loading")
+	}
+	if _, ok := fresh.GetUser(c.AccessKey); ok {
+		t.Fatal("cleanup failure admitted an expired service account")
+	}
+	r, err := loadIAMRevision(ctx, fresh, path)
+	mustIAM(t, err)
+	if r.Deleted || !r.Credentials.IsExpired() {
+		t.Fatal("failed cleanup lost the existing expired revision")
+	}
+}

cmd/iam-revision-index.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+// This index is rebuilt by the existing IAM loaders and updated by successful
+// storage operations. It avoids a second full IAM walk during every heal pass.
+// It is an optimization of the durable records, never a reason to delete them.
+// The index contains no secrets or grants.
+type iamParentRevision struct {
+	deleted bool
+	before  time.Time
+}
+
+type iamRevisionIndex struct {
+	mu         sync.RWMutex
+	items      map[string]iamRevision
+	parents    map[string]iamParentRevision
+	floors     map[string]time.Time
+	generation uint64
+}
+
+func (idx *iamRevisionIndex) observe(path string, data []byte) {
+	if !strings.HasPrefix(path, iamConfigPrefix+"/") {
+		return
+	}
+	var r iamRevision
+	if json.Unmarshal(data, &r) != nil {
+		return // The caller reports malformed data using its normal decoder.
+	}
+	r.Credentials = auth.Credentials{ParentUser: r.Credentials.ParentUser, Expiration: r.Credentials.Expiration}
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+	if strings.HasPrefix(path, iamConfigUsersPrefix) {
+		// Keep a compact name-keyed view for the authentication hot path;
+		// constructing a config path on every S3 request allocates needlessly.
+		defer func() {
+			name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+			if current, ok := idx.items[path]; ok {
+				if idx.parents == nil {
+					idx.parents = make(map[string]iamParentRevision)
+				}
+				idx.parents[name] = iamParentRevision{deleted: current.Deleted, before: current.RevokedBefore}
+			} else {
+				delete(idx.parents, name)
+			}
+		}()
+	}
+	if floor, ok := idx.floors[path]; ok && r.timestamp().Before(floor) {
+		return
+	}
+	if previous, ok := idx.items[path]; ok {
+		// A concurrent read that began before a write must not roll it back.
+		if previous.timestamp().After(r.timestamp()) || (previous.Deleted && !r.Deleted && !r.timestamp().After(previous.timestamp())) {
+			return
+		}
+		if previous.RevokedBefore.After(r.RevokedBefore) {
+			r.RevokedBefore = previous.RevokedBefore
+		}
+		if previous.timestamp().Equal(r.timestamp()) && previous.Deleted == r.Deleted && previous.RevokedBefore.Equal(r.RevokedBefore) {
+			return
+		}
+	}
+	if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
+		if _, tracked := idx.items[path]; tracked {
+			delete(idx.items, path)
+			idx.generation++
+		}
+		delete(idx.floors, path)
+		return
+	}
+	if !r.Deleted && r.RevokedBefore.IsZero() {
+		_, tracked := idx.items[path]
+		_, hasFloor := idx.floors[path]
+		if tracked || hasFloor {
+			if idx.floors == nil {
+				idx.floors = make(map[string]time.Time)
+			}
+			idx.floors[path] = r.timestamp()
+		}
+		if tracked {
+			delete(idx.items, path)
+			idx.generation++
+		}
+		return
+	}
+	if idx.items == nil {
+		idx.items = make(map[string]iamRevision)
+	}
+	idx.items[path] = r
+	delete(idx.floors, path)
+	idx.generation++
+}
+
+func (idx *iamRevisionIndex) get(path string) iamRevision {
+	if idx == nil {
+		return iamRevision{}
+	}
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return idx.items[path]
+}
+
+func (idx *iamRevisionIndex) snapshot() map[string]iamRevision {
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+	for path, r := range idx.items {
+		if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
+			delete(idx.items, path)
+			delete(idx.floors, path)
+			idx.generation++
+		}
+	}
+	return maps.Clone(idx.items)
+}
+
+func (idx *iamRevisionIndex) count() int {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return len(idx.items)
+}
+
+// A process-local generation plus the protocol's instance ID is sufficient
+// for acknowledgements. Avoid hashing the entire index on every IAM write.
+func (idx *iamRevisionIndex) digest() string {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return fmt.Sprintf("%x:%x", idx.generation, len(idx.items))
+}
+
+func (idx *iamRevisionIndex) forget(path string) {
+	idx.mu.Lock()
+	if _, ok := idx.items[path]; ok {
+		delete(idx.items, path)
+		idx.generation++
+	}
+	delete(idx.floors, path)
+	if strings.HasPrefix(path, iamConfigUsersPrefix) {
+		delete(idx.parents, strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile))
+	}
+	idx.mu.Unlock()
+}
+
+func (c *iamCache) userRevocation(user string) iamRevision {
+	r := c.revisions.parentRevision(user)
+	if u, ok := c.iamUsersMap[user]; ok && u.RevokedBefore.After(r.RevokedBefore) {
+		r.RevokedBefore = u.RevokedBefore
+	}
+	return r
+}
+
+func (c *iamCache) groupMemberAllowed(member string, grantedAt, groupBoundary time.Time) bool {
+	r := c.userRevocation(member)
+	return !r.Deleted && (r.RevokedBefore.IsZero() || grantedAt.After(r.RevokedBefore)) && (groupBoundary.IsZero() || grantedAt.After(groupBoundary))
+}
+
+func iamMappingParentPath(path string) string {
+	kind, name, ok := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+	if !ok {
+		return ""
+	}
+	name = strings.TrimSuffix(name, ".json")
+	switch kind {
+	case "users", "sts-users":
+		return getUserIdentityPath(name, regUser)
+	case "service-accounts":
+		return getUserIdentityPath(name, svcUser)
+	case "groups":
+		return getGroupInfoPath(name)
+	}
+	return ""
+}
+
+func (idx *iamRevisionIndex) mappingAllowed(path string, mp MappedPolicy) bool {
+	if mp.Deleted || idx.get(path).Deleted {
+		return false
+	}
+	r := idx.get(iamMappingParentPath(path))
+	return !r.Deleted && (r.RevokedBefore.IsZero() || mp.UpdatedAt.After(r.RevokedBefore))
+}
+
+// Apply the persisted commit boundary even before dependent cache cleanup has
+// completed. The map namespace is part of the authorization record's identity.
+func (c *iamCache) cachedMappedPolicy(name string, userType IAMUserType, isGroup bool) (MappedPolicy, bool) {
+	var mp MappedPolicy
+	var ok bool
+	switch {
+	case isGroup:
+		mp, ok = c.iamGroupPolicyMap.Load(name)
+	case userType == stsUser:
+		mp, ok = c.iamSTSPolicyMap.Load(name)
+	default:
+		mp, ok = c.iamUserPolicyMap.Load(name)
+	}
+	if !ok || !c.revisions.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), mp) {
+		return MappedPolicy{}, false
+	}
+	return mp, true
+}
+
+func (idx *iamRevisionIndex) parentRevision(user string) iamRevision {
+	if idx == nil {
+		return iamRevision{}
+	}
+	idx.mu.RLock()
+	p := idx.parents[user]
+	idx.mu.RUnlock()
+	return iamRevision{Deleted: p.deleted, RevokedBefore: p.before}
+}

cmd/iam-revision-lock_test.go

@@ -0,0 +1,305 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+type iamRevisionLockObserver struct {
+	ObjectLayer
+	path    string
+	waiting chan struct{}
+	once    sync.Once
+}
+
+func (o *iamRevisionLockObserver) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path {
+		return &iamRevisionObservedLock{RWLocker: lock, observe: func() { o.once.Do(func() { close(o.waiting) }) }}
+	}
+	return lock
+}
+
+type iamRevisionObservedLock struct {
+	RWLocker
+	observe func()
+}
+
+func (l *iamRevisionObservedLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	l.observe()
+	return l.RWLocker.GetLock(ctx, timeout)
+}
+
+type iamRevisionWatchObserver struct {
+	etcd.Watcher
+	waiting chan struct{}
+	once    sync.Once
+}
+
+func (w *iamRevisionWatchObserver) Watch(ctx context.Context, key string, opts ...etcd.OpOption) etcd.WatchChan {
+	w.once.Do(func() { close(w.waiting) })
+	return w.Watcher.Watch(ctx, key, opts...)
+}
+
+// Simulate an unavailable cleanup RPC. Mutex.Lock calls Delete after its wait
+// is canceled; that RPC must inherit a deadline too, not Client.Ctx() forever.
+type iamRevisionCleanupBlocker struct {
+	etcd.KV
+	release chan struct{}
+}
+
+func (b *iamRevisionCleanupBlocker) Delete(ctx context.Context, key string, opts ...etcd.OpOption) (*etcd.DeleteResponse, error) {
+	if strings.Contains(key, "/iam-revision-locks/") {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-b.release:
+		}
+	}
+	return b.KV.Delete(ctx, key, opts...)
+}
+
+type iamRevisionReadBlocker struct {
+	IAMStorageAPI
+	path    string
+	after   int
+	waiting chan struct{}
+}
+
+func (b *iamRevisionReadBlocker) loadIAMConfig(ctx context.Context, item any, path string) error {
+	if path == b.path {
+		b.after--
+		if b.after == 0 {
+			close(b.waiting)
+			<-ctx.Done()
+			return ctx.Err()
+		}
+	}
+	return b.IAMStorageAPI.loadIAMConfig(ctx, item, path)
+}
+
+func TestIAMRevisionReadDoesNotBlockAuthentication(t *testing.T) {
+	for _, stage := range []struct {
+		name   string
+		offset time.Duration
+	}{{"deletion", time.Minute}, {"retained_revocation", -time.Minute}} {
+		t.Run(stage.name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			disks, err := getRandomDisks(1)
+			if err != nil {
+				t.Fatal(err)
+			}
+			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() {
+				obj.Shutdown(context.Background())
+				os.RemoveAll(disks[0])
+			})
+			store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			const user = "read-blocked-parent"
+			created, err := store.AddUser(ctx, user, madmin.AddOrUpdateUserReq{SecretKey: "original-password", Status: madmin.AccountEnabled})
+			if err != nil {
+				t.Fatal(err)
+			}
+			blocked := &iamRevisionReadBlocker{IAMStorageAPI: store.IAMStorageAPI, path: getUserIdentityPath(user, regUser), after: 1, waiting: make(chan struct{})}
+			store.IAMStorageAPI = blocked
+			done := make(chan error, 1)
+			go func() {
+				done <- store.DeleteUser(withIAMReplicationTime(ctx, created.Add(stage.offset)), user, regUser)
+			}()
+			defer func() { cancel(); <-done }()
+			select {
+			case <-blocked.waiting:
+			case <-time.After(5 * time.Second):
+				t.Fatal("revision read was not attempted")
+			}
+			read := make(chan bool, 1)
+			go func() {
+				u, ok := store.GetUser(user)
+				read <- ok && u.Credentials.SecretKey == "original-password"
+			}()
+			select {
+			case ok := <-read:
+				if !ok {
+					t.Fatal("pending revision read changed the cached identity")
+				}
+			case <-time.After(time.Second):
+				t.Fatal("revision read blocked cached authentication")
+			}
+		})
+	}
+}
+
+func TestIAMRevisionLockContention(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+			if backend == "etcd" && endpoint == "" {
+				t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+			}
+			for _, outcome := range []string{"release", "cancel", "default_timeout"} {
+				t.Run(outcome, func(t *testing.T) {
+					resetTestGlobals()
+					t.Cleanup(resetTestGlobals)
+					ctx, cancel := context.WithCancel(context.Background())
+					defer cancel()
+					oldTimeout := defaultContextTimeout
+					defaultContextTimeout = 2 * time.Second
+					t.Cleanup(func() { defaultContextTimeout = oldTimeout })
+					must := func(err error) {
+						t.Helper()
+						if err != nil {
+							t.Fatal(err)
+						}
+					}
+					const user = "contended-user"
+					path := getUserIdentityPath(user, regUser)
+					waiting := make(chan struct{})
+					var store *IAMStoreSys
+					var hold func() func()
+					unblockCleanup := func() {}
+					if backend == "object" {
+						disks, err := getRandomDisks(1)
+						must(err)
+						obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+						must(err)
+						t.Cleanup(func() {
+							obj.Shutdown(context.Background())
+							os.RemoveAll(disks[0])
+						})
+						observed := &iamRevisionLockObserver{ObjectLayer: obj, path: path + ".revision-lock", waiting: waiting}
+						store = &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+						hold = func() func() {
+							lock := obj.NewNSLock(minioMetaBucket, observed.path)
+							lc, err := lock.GetLock(ctx, newDynamicTimeout(time.Second, time.Second))
+							must(err)
+							store.IAMStorageAPI.(*IAMObjectStore).objAPI = observed
+							return func() { lock.Unlock(lc) }
+						}
+					} else {
+						client, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: time.Second})
+						must(err)
+						t.Cleanup(func() { client.Close() })
+						prefix := fmt.Sprintf("/silo-lock-test/%d/", time.Now().UnixNano())
+						client.KV = namespace.NewKV(client.KV, prefix)
+						client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
+						store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(client, MinIOUsersSysType)}
+						hold = func() func() {
+							session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+							must(err)
+							lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+							must(lock.Lock(ctx))
+							client.Watcher = &iamRevisionWatchObserver{Watcher: client.Watcher, waiting: waiting}
+							blocker := &iamRevisionCleanupBlocker{KV: client.KV, release: make(chan struct{})}
+							client.KV = blocker
+							unblockCleanup = sync.OnceFunc(func() { close(blocker.release) })
+							t.Cleanup(unblockCleanup)
+							return func() { session.Close() }
+						}
+					}
+					request := func(secret string) madmin.AddOrUpdateUserReq {
+						return madmin.AddOrUpdateUserReq{SecretKey: secret, Status: madmin.AccountEnabled}
+					}
+					_, err := store.AddUser(ctx, user, request("original-password"))
+					must(err)
+					release := sync.OnceFunc(hold())
+					t.Cleanup(release)
+					writeCtx, cancelWrite := context.WithCancel(ctx)
+					defer cancelWrite()
+					first, second := make(chan error, 1), make(chan error, 1)
+					var writers sync.WaitGroup
+					t.Cleanup(func() {
+						cancelWrite()
+						unblockCleanup()
+						release()
+						writers.Wait()
+					})
+					writers.Go(func() {
+						_, err := store.AddUser(writeCtx, user, request("first-password"))
+						first <- err
+					})
+					select {
+					case <-waiting:
+					case <-time.After(5 * time.Second):
+						t.Fatal("writer did not attempt the held revision lock")
+					}
+					// A second writer must queue without taking the cache's RWMutex:
+					// Go's writer preference would otherwise block every new reader.
+					writers.Go(func() {
+						_, err := store.AddUser(ctx, user, request("second-password"))
+						second <- err
+					})
+					select {
+					case err := <-second:
+						t.Fatalf("second writer bypassed the first: %v", err)
+					case <-time.After(50 * time.Millisecond):
+					}
+					read := make(chan UserIdentity, 1)
+					go func() {
+						u, _ := store.GetUser(user)
+						read <- u
+					}()
+					select {
+					case u := <-read:
+						if u.Credentials.SecretKey != "original-password" {
+							t.Fatal("pending write changed the cached credential")
+						}
+					case <-time.After(time.Second):
+						t.Fatal("distributed lock contention blocked cached authentication")
+					}
+					switch outcome {
+					case "release":
+						release()
+					case "cancel":
+						cancelWrite()
+					}
+					select {
+					case err := <-first:
+						if outcome == "release" {
+							must(err)
+						} else if err == nil {
+							t.Fatal("canceled or timed-out write succeeded")
+						}
+					case <-time.After(5 * time.Second):
+						t.Fatal("lock wait or cancellation cleanup exceeded its deadline")
+					}
+					release()
+					select {
+					case err := <-second:
+						must(err)
+					case <-time.After(5 * time.Second):
+						t.Fatal("queued writer did not recover after the first completed")
+					}
+					cached, ok := store.GetUser(user)
+					if !ok || cached.Credentials.SecretKey != "second-password" {
+						t.Fatal("cached write order was lost")
+					}
+					var persisted UserIdentity
+					must(store.loadIAMConfig(ctx, &persisted, path))
+					if persisted.Credentials.SecretKey != cached.Credentials.SecretKey || !persisted.UpdatedAt.Equal(cached.UpdatedAt) {
+						t.Fatal("persistent and cached revisions differ")
+					}
+				})
+			}
+		})
+	}
+}

cmd/iam-revision.go

@@ -0,0 +1,749 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+var errIAMStaleUpdate = errors.New("IAM update predates a stored revision or revocation")
+
+// The parent is still live; callers must not broadcast a user deletion when
+// only its revocation boundary was retained.
+var errIAMRevocationRetained = errors.New("IAM revocation recorded without deleting the record")
+
+// A revocation advances the boundary even when a newer identity already
+// exists. Keep this operation distinct from replacing/deleting that identity.
+type iamUserRevocation struct {
+	UserIdentity
+	retained bool
+}
+
+type iamGroupRevocation struct {
+	GroupInfo
+	retained     bool
+	requireEmpty bool
+}
+
+// Natural expiration is distinct from revoking a live credential. An expired
+// immutable STS token can be removed; a reusable service-account key retains
+// its revision so an older non-expiring credential cannot return.
+type iamExpireIdentity struct{}
+
+// The authoritative revocation is durable even if dependent cleanup fails.
+// Callers must publish it to sibling caches before returning the error.
+type iamCommittedCleanupError struct {
+	err      error
+	retained bool
+}
+
+func (e *iamCommittedCleanupError) Error() string {
+	return "IAM revocation committed; cleanup failed: " + e.err.Error()
+}
+func (e *iamCommittedCleanupError) Unwrap() error { return e.err }
+
+type iamReplicationTimeKey struct{}
+
+func withIAMReplicationTime(ctx context.Context, at time.Time) context.Context {
+	return context.WithValue(ctx, iamReplicationTimeKey{}, at)
+}
+
+func iamReplicationTime(ctx context.Context) (time.Time, bool) {
+	at, ok := ctx.Value(iamReplicationTimeKey{}).(time.Time)
+	return at, ok
+}
+
+func iamReplicationError(err error) error {
+	if errors.Is(err, errIAMStaleUpdate) {
+		// Retrying an obsolete event cannot change the result.
+		return nil
+	}
+	return wrapSRErr(err)
+}
+
+// Deletions occupy the original IAM config path. They contain no secret or
+// grant and are hidden by the normal loaders, but remain available to heal
+// and to timestamp comparisons after a restart. Do not age them out: a peer
+// can be offline indefinitely.
+type iamRevision struct {
+	UpdatedAt     time.Time        `json:"updatedAt"`
+	UpdateDate    time.Time        `json:"UpdateDate"`
+	Deleted       bool             `json:"deleted"`
+	RevokedBefore time.Time        `json:"revokedBefore"`
+	ExpiresAt     time.Time        `json:"expiresAt,omitempty"`
+	Credentials   auth.Credentials `json:"credentials"`
+}
+
+func (r iamRevision) timestamp() time.Time {
+	if r.UpdateDate.After(r.UpdatedAt) {
+		return r.UpdateDate
+	}
+	return r.UpdatedAt
+}
+
+func loadIAMRevision(ctx context.Context, store IAMStorageAPI, path string) (iamRevision, error) {
+	var r iamRevision
+	err := store.loadIAMConfig(ctx, &r, path)
+	if errors.Is(err, errConfigNotFound) {
+		err = nil
+	}
+	return r, err
+}
+
+func (store *IAMStoreSys) checkIAMRevision(ctx context.Context, path string, deleting bool) error {
+	at, replicated := iamReplicationTime(ctx)
+	if !replicated {
+		return nil
+	}
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		r, err := loadIAMRevision(ctx, store.IAMStorageAPI, path)
+		if err != nil {
+			return err
+		}
+		if r.timestamp().After(at) || (r.Deleted && !deleting && !at.After(r.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		return nil
+	})
+}
+
+// This signed claim records the parent's revocation boundary at issuance.
+// Unlike UpdatedAt, it cannot advance when an offline site edits an old child.
+// It travels in the existing service-account Claims and STS SessionToken fields.
+const iamParentRevocationClaim = "siloParentRevocation"
+
+func setIAMParentRevocationClaim(ctx context.Context, store IAMStorageAPI, parent string, claims map[string]any) error {
+	delete(claims, iamParentRevocationClaim)
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted {
+		return errIAMStaleUpdate
+	}
+	if !r.RevokedBefore.IsZero() {
+		claims[iamParentRevocationClaim] = r.RevokedBefore.Format(time.RFC3339Nano)
+	}
+	return nil
+}
+
+func iamCredentialSurvivesRevocation(cred auth.Credentials, at time.Time) bool {
+	if at.IsZero() {
+		return true
+	}
+	s, _ := cred.Claims[iamParentRevocationClaim].(string)
+	issuedAfter, err := time.Parse(time.RFC3339Nano, s)
+	return err == nil && !issuedAfter.Before(at)
+}
+
+// Parent revocations delete old children even if an offline peer has edited
+// them later. Preserve children that prove issuance after this revocation.
+func iamChildDeletionContext(ctx context.Context, child UserIdentity) (context.Context, bool) {
+	if at, replicated := iamReplicationTime(ctx); replicated {
+		if !at.IsZero() && iamCredentialSurvivesRevocation(child.Credentials, at) {
+			return ctx, false
+		}
+		if child.UpdatedAt.After(at) {
+			ctx = withIAMReplicationTime(ctx, child.UpdatedAt)
+		}
+	}
+	return ctx, true
+}
+
+// A delayed service account or STS event must not outlive deletion of its
+// built-in parent. The caller must populate Claims from the verified token.
+func checkIAMParentRevision(ctx context.Context, store IAMStorageAPI, cred auth.Credentials) error {
+	parent := cred.ParentUser
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted || !iamCredentialSurvivesRevocation(cred, r.RevokedBefore) {
+		return errIAMStaleUpdate
+	}
+	return nil
+}
+
+// Called with the IAM writer mutex and cache lock held. Persistence only
+// touches the caller's record, not the cache. Keep writers serialized while
+// allowing cached authentication reads throughout storage and lock waits.
+func (store *IAMStoreSys) withIAMStorage(ctx context.Context, fn func(context.Context) error) error {
+	store.IAMStorageAPI.unlock()
+	defer store.IAMStorageAPI.lock()
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	return fn(ctx)
+}
+
+func (store *IAMStoreSys) saveIAMRevision(ctx context.Context, path string, item any, opts ...options) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return saveIAMRevision(ctx, store.IAMStorageAPI, path, item, opts...)
+	})
+}
+
+func (store *IAMStoreSys) checkIAMParentRevision(ctx context.Context, cred auth.Credentials) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return checkIAMParentRevision(ctx, store.IAMStorageAPI, cred)
+	})
+}
+
+// Update the caller's record with the persisted revision before it is cached.
+func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item any, opts ...options) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	// Serialize compare-and-write across nodes, as well as goroutines. Use a
+	// separate lock name so saving the config does not reacquire this lock.
+	switch s := store.(type) {
+	case *IAMObjectStore:
+		lock := s.objAPI.NewNSLock(minioMetaBucket, path+".revision-lock")
+		lc, err := lock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		defer lock.Unlock(lc)
+		ctx = lc.Context()
+	case *IAMEtcdStore:
+		// Mutex.Lock also uses Client.Ctx() for cleanup after cancellation.
+		// Borrow the existing services with the operation's bounded context;
+		// never close this facade, which does not own those services.
+		client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(s.client.GetLogger()))
+		client.KV, client.Lease, client.Watcher = s.client.KV, s.client.Lease, s.client.Watcher
+		session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+		if err != nil {
+			return err
+		}
+		defer func() {
+			session.Orphan()
+			// A canceled operation must still release its lease when etcd is
+			// reachable. If it is unavailable, stop waiting and let it expire.
+			cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultContextTimeout)
+			defer cancel()
+			_, _ = s.client.Revoke(cleanupCtx, session.Lease())
+		}()
+		lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+		if err = lock.Lock(ctx); err != nil {
+			return err
+		}
+		// Revoking the session lease releases the lock, including on cancellation.
+	}
+	previous, err := loadIAMRevision(ctx, store, path)
+	if err != nil {
+		return err
+	}
+	if _, expiring := item.(*iamExpireIdentity); expiring {
+		sts := strings.HasPrefix(path, iamConfigSTSPrefix)
+		if previous.Deleted {
+			if sts && !previous.ExpiresAt.IsZero() && UTCNow().After(previous.ExpiresAt) {
+				return expireIAMSTSConfig(ctx, store, path)
+			}
+			return nil
+		}
+		if previous.timestamp().IsZero() || !previous.Credentials.IsExpired() {
+			return nil
+		}
+		if sts {
+			return expireIAMSTSConfig(ctx, store, path)
+		}
+		item = &UserIdentity{Version: 1, Deleted: true}
+		ctx = withIAMReplicationTime(ctx, previous.timestamp())
+	}
+	var revocation *iamUserRevocation
+	if op, ok := item.(*iamUserRevocation); ok {
+		revocation = op
+		op.UserIdentity = UserIdentity{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.UserIdentity = UserIdentity{Version: 1, Credentials: previous.Credentials, UpdatedAt: previous.timestamp(), RevokedBefore: origin}
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.UserIdentity
+	}
+	var groupRevocation *iamGroupRevocation
+	if op, ok := item.(*iamGroupRevocation); ok {
+		groupRevocation = op
+		var group GroupInfo
+		if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+			return err
+		}
+		if op.requireEmpty && !group.Deleted {
+			for _, member := range group.Members {
+				r := store.revisionIndex().get(getUserIdentityPath(member, regUser))
+				at := group.MemberGrants[member]
+				if !r.Deleted && (r.RevokedBefore.IsZero() || at.After(r.RevokedBefore)) && (group.RevokedBefore.IsZero() || at.After(group.RevokedBefore)) {
+					return errGroupNotEmpty
+				}
+			}
+		}
+		op.GroupInfo = GroupInfo{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.GroupInfo = group
+			op.RevokedBefore = origin
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.GroupInfo
+	}
+	var at *time.Time
+	var deleted bool
+	switch v := item.(type) {
+	case *UserIdentity:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *GroupInfo:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if !deleted {
+			var group GroupInfo
+			if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+				return err
+			}
+			mergeIAMGroupMutation(ctx, group, v)
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *MappedPolicy:
+		at, deleted = &v.UpdatedAt, v.Deleted
+	case *PolicyDoc:
+		at, deleted = &v.UpdateDate, v.Deleted
+	default:
+		return errInvalidArgument
+	}
+	if strings.HasPrefix(path, iamConfigSTSPrefix) && previous.Deleted && !deleted {
+		// STS access keys identify immutable tokens, not reusable user names.
+		return errIAMStaleUpdate
+	}
+	if origin, replicated := iamReplicationTime(ctx); replicated {
+		*at = origin
+		if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		if strings.HasPrefix(path, iamConfigServiceAccountsPrefix) && !deleted && previous.Credentials.AccessKey != "" && previous.timestamp().Equal(origin) {
+			// Duplicate service snapshots are acknowledgements, not new creates
+			// or edits. Reload the winner without writing, so even a stale
+			// sibling cache is refreshed by the retry before acknowledging it.
+			return store.loadIAMConfig(ctx, item, path)
+		}
+		if previous.Deleted && deleted && !origin.After(previous.timestamp()) {
+			// An already-applied tombstone needs no further persistent write.
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+	} else {
+		if previous.Deleted && deleted {
+			// A peer notification without an originating revision must not
+			// advance a tombstone past a subsequent deliberate recreation.
+			*at = previous.timestamp()
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+		if at.IsZero() {
+			*at = UTCNow()
+		}
+		if !at.After(previous.timestamp()) {
+			*at = previous.timestamp().Add(time.Nanosecond)
+		}
+	}
+	if v, ok := item.(*UserIdentity); ok {
+		if deleted {
+			// Retain only the parent name for root-account exclusion during heal.
+			v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
+			v.RevokedBefore = *at
+			if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) {
+				// The signed STS token cannot authorize beyond this time, even
+				// if an offline site replays it with a newer event timestamp.
+				v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime)
+				opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
+			}
+		} else {
+			if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
+				claims, err := extractJWTClaims(*v)
+				if err != nil {
+					return err
+				}
+				v.Credentials.Claims = claims.Map()
+			}
+			if err = checkIAMParentRevision(ctx, store, v.Credentials); err != nil {
+				return err
+			}
+		}
+	}
+	if v, ok := item.(*GroupInfo); ok && deleted {
+		v.RevokedBefore = *at
+		v.Members, v.MemberGrants = nil, nil
+	}
+	if _, ok := item.(*MappedPolicy); ok && !deleted {
+		if parentPath := iamMappingParentPath(path); parentPath != "" {
+			parent, err := loadIAMRevision(ctx, store, parentPath)
+			if err != nil {
+				return err
+			}
+			if parent.Deleted {
+				return errIAMStaleUpdate
+			}
+			if !parent.RevokedBefore.IsZero() && !at.After(parent.RevokedBefore) {
+				if _, replicated := iamReplicationTime(ctx); replicated {
+					return errIAMStaleUpdate
+				}
+				*at = parent.RevokedBefore.Add(time.Nanosecond)
+			}
+		}
+	}
+	if err := store.saveIAMConfig(ctx, item, path, opts...); err != nil {
+		return err
+	}
+	if revocation != nil && revocation.retained {
+		return errIAMRevocationRetained
+	}
+	if groupRevocation != nil && groupRevocation.retained {
+		return errIAMRevocationRetained
+	}
+	return nil
+}
+
+func (iamOS *IAMObjectStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var paths []string
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+"/") {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+		paths = append(paths, iamConfigPrefix+"/"+item.Item)
+	}
+	return paths, nil
+}
+
+func (ies *IAMEtcdStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	r, err := ies.client.Get(ctx, iamConfigPrefix+"/", etcd.WithPrefix(), etcd.WithKeysOnly())
+	if err != nil {
+		return nil, err
+	}
+	paths := make([]string, 0, len(r.Kvs))
+	for _, kv := range r.Kvs {
+		paths = append(paths, string(kv.Key))
+	}
+	return paths, nil
+}
+
+func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool) {
+	if (strings.HasPrefix(path, iamConfigUsersPrefix) || strings.HasPrefix(path, iamConfigGroupsPrefix)) && !r.RevokedBefore.IsZero() {
+		// Recreating a parent does not cancel its older revocation of derived
+		// credentials. Replay this boundary even after the parent is live again.
+		r.Deleted = true
+		r.UpdatedAt, r.UpdateDate = r.RevokedBefore, time.Time{}
+	}
+	if !r.Deleted {
+		return item, false
+	}
+	item.UpdatedAt = r.timestamp()
+	switch {
+	case strings.HasPrefix(path, iamConfigUsersPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+		item.Type = madmin.SRIAMItemIAMUser
+		item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
+	case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
+		if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
+			return item, false
+		}
+		item.Type = madmin.SRIAMItemSvcAcc
+		item.SvcAccChange = &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: name}}
+	case strings.HasPrefix(path, iamConfigGroupsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigGroupsPrefix), "/"+iamGroupMembersFile)
+		item.Type = madmin.SRIAMItemGroupInfo
+		item.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: name, IsRemove: true}}
+	case strings.HasPrefix(path, iamConfigPoliciesPrefix):
+		item.Type = madmin.SRIAMItemPolicy
+		item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigPoliciesPrefix), "/"+iamPolicyFile)
+	case strings.HasPrefix(path, iamConfigPolicyDBPrefix):
+		prefix, name, found := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+		if !found {
+			return item, false
+		}
+		typ := regUser
+		switch prefix {
+		case "sts-users":
+			typ = stsUser
+		case "service-accounts":
+			typ = svcUser
+		}
+		item.Type = madmin.SRIAMItemPolicyMapping
+		item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
+	default:
+		// Expired STS credentials are not replayed. Parent revocations and
+		// their retained timestamp reject delayed copies of derived tokens.
+		return item, false
+	}
+	return item, true
+}
+
+func iamDeletionPath(item madmin.SRIAMItem) string {
+	switch item.Type {
+	case madmin.SRIAMItemIAMUser:
+		if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
+			return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
+		}
+	case madmin.SRIAMItemSvcAcc:
+		if item.SvcAccChange != nil && item.SvcAccChange.Delete != nil {
+			return getUserIdentityPath(item.SvcAccChange.Delete.AccessKey, svcUser)
+		}
+	case madmin.SRIAMItemGroupInfo:
+		if item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+			return getGroupInfoPath(item.GroupInfo.UpdateReq.Group)
+		}
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return getPolicyDocPath(item.Name)
+		}
+	case madmin.SRIAMItemPolicyMapping:
+		if p := item.PolicyMapping; p != nil && p.Policy == "" {
+			return getMappedPolicyPath(p.UserOrGroup, IAMUserType(p.UserType), p.IsGroup)
+		}
+	}
+	return ""
+}
+
+func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
+	started := time.Now()
+	defer func() {
+		c.iamRevisionMetrics.healDurationMillis.Store(time.Since(started).Milliseconds())
+		if err != nil {
+			c.iamRevisionMetrics.healFailures.Add(1)
+		} else {
+			c.iamRevisionMetrics.healLastSuccess.Store(time.Now().Unix())
+		}
+	}()
+	c.iamHealMu.Lock()
+	defer c.iamHealMu.Unlock()
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	snapshot := globalIAMSys.store.revisionIndex().snapshot()
+	paths := make([]string, 0, len(snapshot))
+	for path := range snapshot {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	byType := make(map[string][]iamReplicationItem)
+	for _, path := range paths {
+		r := snapshot[path]
+		item, ok := iamDeletionItem(path, r)
+		if !ok {
+			continue
+		}
+		out := iamReplicationItem{SRIAMItem: item}
+		if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
+			out.Type, out.IAMUser = iamUserBoundaryType, nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+		}
+		if item.Type == madmin.SRIAMItemGroupInfo && !r.Deleted {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+		}
+		byType[item.Type] = append(byType[item.Type], out)
+	}
+	var items []iamReplicationItem
+	for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
+		items = append(items, byType[typ]...)
+	}
+	if len(items) == 0 {
+		return nil
+	}
+	if c.iamRevisionProgress == nil {
+		c.iamRevisionProgress = make(map[string]iamRevisionProgress)
+	}
+	for id := range c.iamRevisionProgress {
+		if _, present := c.state.Peers[id]; !present {
+			delete(c.iamRevisionProgress, id)
+		}
+	}
+	var progressMu sync.Mutex
+	cerr := c.concDo(nil, func(id string, p madmin.PeerInfo) error {
+		// Bound each pass, but retain acknowledgements independently of the
+		// pass deadline or unrelated changes at either site.
+		peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+		defer cancel()
+		client, err := c.getAdminClient(peerCtx, id)
+		if err != nil {
+			return err
+		}
+		remote, err := executeIAMRevisionRequest(peerCtx, client, http.MethodGet, nil)
+		if err != nil {
+			return err
+		}
+		progressMu.Lock()
+		progress := c.iamRevisionProgress[id]
+		progressMu.Unlock()
+		progress.observePeer(remote)
+		defer func() {
+			progressMu.Lock()
+			c.iamRevisionProgress[id] = progress
+			progressMu.Unlock()
+		}()
+		var pending []iamReplicationItem
+		for _, item := range items {
+			path, version := iamReplicationMarker(item)
+			if progress.Acknowledged[path] != version {
+				pending = append(pending, item)
+			}
+		}
+		// Acknowledgements are only a replay optimization, never GC proof.
+		for path := range progress.Acknowledged {
+			if _, retained := snapshot[path]; !retained {
+				delete(progress.Acknowledged, path)
+			}
+		}
+		var failures []error
+		for next := 0; next < len(pending); {
+			end := min(next+maxIAMRevisionBatch, len(pending))
+			batch := pending[next:end]
+			remote, err = executeIAMRevisionRequest(peerCtx, client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: batch})
+			if err != nil {
+				var batchErr *iamRevisionBatchError
+				if !errors.As(err, &batchErr) {
+					return errors.Join(append(failures, err)...)
+				}
+				failures = append(failures, err)
+			} else {
+				progress.observePeer(remote)
+				for _, item := range batch {
+					path, version := iamReplicationMarker(item)
+					progress.Acknowledged[path] = version
+				}
+			}
+			next = end
+		}
+		return errors.Join(failures...)
+	}, "IAM revision convergence")
+	return errors.Unwrap(cerr)
+}
+
+func iamReplicationMarker(item iamReplicationItem) (path, version string) {
+	path = iamDeletionPath(item.SRIAMItem)
+	if item.UserRevocation != nil {
+		path = getUserIdentityPath(item.UserRevocation.User, regUser)
+	}
+	if item.GroupRevocation != nil {
+		path = getGroupInfoPath(item.GroupRevocation.Group)
+	}
+	return path, item.Type + ":" + item.UpdatedAt.UTC().Format(time.RFC3339Nano)
+}
+
+func (store *IAMStoreSys) savePolicyDoc(ctx context.Context, policyName string, p *PolicyDoc) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(policyName), p)
+}
+
+func (store *IAMStoreSys) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp *MappedPolicy, opts ...options) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), mp, opts...)
+}
+
+func (store *IAMStoreSys) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u *UserIdentity, opts ...options) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), u, opts...)
+}
+
+func (store *IAMStoreSys) saveGroupInfo(ctx context.Context, name string, gi *GroupInfo) error {
+	return store.saveIAMRevision(ctx, getGroupInfoPath(name), gi)
+}
+
+func (store *IAMStoreSys) deletePolicyDoc(ctx context.Context, name string) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(name), &PolicyDoc{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), &MappedPolicy{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), &UserIdentity{Version: 1, Deleted: true})
+}
+
+// Called under the identity's distributed revision lock, after verifying that
+// its immutable STS token (or early-revocation retention) has expired. Only the
+// old token-key mapping is removed; the reusable parent mapping is unaffected.
+func expireIAMSTSConfig(ctx context.Context, store IAMStorageAPI, path string) error {
+	key := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
+	if err := store.deleteIAMConfig(ctx, getMappedPolicyPath(key, stsUser, false)); err != nil && !errors.Is(err, errConfigNotFound) {
+		return err
+	}
+	return store.deleteIAMConfig(ctx, path)
+}
+
+type (
+	iamExpirationCleanupKey   struct{}
+	iamExpirationCleanupState struct{ failed atomic.Bool }
+)
+
+func withIAMExpirationCleanup(ctx context.Context) context.Context {
+	if _, ok := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState); ok {
+		return ctx
+	}
+	return context.WithValue(ctx, iamExpirationCleanupKey{}, &iamExpirationCleanupState{})
+}
+
+func bestEffortIAMExpiration(ctx context.Context, store IAMStorageAPI, path string) {
+	state, _ := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState)
+	if ctx.Err() != nil || (state != nil && state.failed.Load()) {
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	// Failure leaves the expired record and its existing version intact.
+	// Stop optional reclamation for this load, while still loading healthy
+	// users. Healthy cleanup has no per-scan quota that could build a backlog.
+	if err := saveIAMRevision(ctx, store, path, &iamExpireIdentity{}); err != nil {
+		if state != nil {
+			state.failed.Store(true)
+		}
+		iamLogIf(ctx, err)
+	}
+}

cmd/iam-revision_test.go

@@ -0,0 +1,330 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+)
+
+// Count physical saves: comparing timestamps alone would miss identical
+// tombstones being rewritten on every heal pass.
+type iamRevisionWriteCounter struct {
+	IAMStorageAPI
+	data   []byte
+	writes int
+}
+
+func (s *iamRevisionWriteCounter) loadIAMConfig(_ context.Context, item any, _ string) error {
+	return json.Unmarshal(s.data, item)
+}
+
+func (s *iamRevisionWriteCounter) saveIAMConfig(_ context.Context, item any, _ string, _ ...options) error {
+	data, err := json.Marshal(item)
+	if err == nil {
+		s.data = data
+		s.writes++
+	}
+	return err
+}
+
+func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
+	at := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC)
+	for _, record := range []struct {
+		name string
+		new  func(bool) any
+	}{
+		{"user", func(deleted bool) any { return &UserIdentity{Version: 1, Deleted: deleted} }},
+		{"group", func(deleted bool) any { return &GroupInfo{Version: 1, Deleted: deleted} }},
+		{"policy", func(deleted bool) any { return &PolicyDoc{Version: 1, Deleted: deleted} }},
+		{"mapping", func(deleted bool) any { return &MappedPolicy{Version: 1, Deleted: deleted} }},
+	} {
+		t.Run(record.name, func(t *testing.T) {
+			data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
+			if err != nil {
+				t.Fatal(err)
+			}
+			store := &iamRevisionWriteCounter{data: data}
+			ctx := context.Background()
+			for range 3 {
+				// Site heal carries the original timestamp. Sibling notifications
+				// have no timestamp; both must leave an applied deletion untouched.
+				for _, replay := range []context.Context{withIAMReplicationTime(ctx, at), ctx} {
+					if err := saveIAMRevision(replay, store, record.name, record.new(true)); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			if store.writes != 0 || string(store.data) != string(data) {
+				t.Fatalf("replayed tombstone changed storage: writes=%d, record=%s", store.writes, store.data)
+			}
+			for _, deleted := range []bool{false, true} {
+				err := saveIAMRevision(withIAMReplicationTime(ctx, at.Add(-time.Second)), store, record.name, record.new(deleted))
+				if !errors.Is(err, errIAMStaleUpdate) {
+					t.Fatalf("older event accepted, deleted=%t: %v", deleted, err)
+				}
+			}
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, at), store, record.name, record.new(false)); !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("equal-time recreation accepted: %v", err)
+			}
+			newer := at.Add(time.Minute)
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer), store, record.name, record.new(true)); err != nil {
+				t.Fatal(err)
+			}
+			r, err := loadIAMRevision(ctx, store, record.name)
+			if err != nil || store.writes != 1 || !r.timestamp().Equal(newer) || !r.Deleted {
+				t.Fatalf("newer deletion did not advance storage: writes=%d, revision=%+v, error=%v", store.writes, r, err)
+			}
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer.Add(time.Minute)), store, record.name, record.new(false)); err != nil {
+				t.Fatalf("newer recreation rejected: %v", err)
+			}
+		})
+	}
+}
+
+// Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
+// The object-store case uses the real peer RPC and deletion handler, so a
+// spurious notification actually destroys the parent instead of only counting it.
+func testIAMRevocationReplayAfterRecreation(ctx context.Context, t *testing.T, sys *IAMSys) {
+	t.Helper()
+	peer := &globalSiteReplicationSys
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user := "heal-recreated-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
+	create := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
+	}
+	revoke := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
+	}
+	create(origin)
+	revoke(deleted)
+	create(recreated)
+	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "heal-recreated-child", secretKey: "valid-service-password",
+	})
+	must(err)
+
+	tg, err := grid.SetupTestGrid(2)
+	must(err)
+	t.Cleanup(tg.Cleanup)
+	var deletes atomic.Int32
+	server := &peerRESTServer{}
+	must(deleteUserRPC.Register(tg.Managers[1], func(req *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		deletes.Add(1)
+		return server.DeleteUserHandler(req)
+	}))
+	// Future user updates still use the normal peer reload notification.
+	must(loadUserRPC.Register(tg.Managers[1], server.LoadUserHandler))
+	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+	must(err)
+	previousNotifications := globalNotificationSys
+	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
+		host: host,
+		gridConn: func() *grid.Connection {
+			return tg.Managers[0].Connection(tg.Hosts[1])
+		},
+	}}}
+	t.Cleanup(func() { globalNotificationSys = previousNotifications })
+	assertLive := func(key string) {
+		t.Helper()
+		if _, ok := sys.GetUser(ctx, key); !ok {
+			t.Fatalf("live credential %s lost during deletion replay", key)
+		}
+	}
+	assertNoDelete := func() {
+		t.Helper()
+		if n := deletes.Load(); n != 0 {
+			t.Fatalf("retained revocation sent %d destructive sibling notifications", n)
+		}
+	}
+	for range 3 {
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+		must(err)
+		item, ok := iamDeletionItem(getUserIdentityPath(user, regUser), r)
+		if !ok || item.IAMUser == nil || !item.UpdatedAt.Equal(deleted) {
+			t.Fatal("recreated user lost its durable revocation replay")
+		}
+		must(peer.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt))
+		must(sys.store.LoadIAMCache(ctx, false))
+		assertLive(user)
+		assertLive(child.AccessKey)
+		assertNoDelete()
+	}
+
+	// A divergent site sends a previously unseen revocation between our old
+	// boundary and recreation. Retain it and revoke old children, but never
+	// turn it into an unversioned delete of the recreated parent.
+	delayed := deleted.Add(time.Minute)
+	revoke(delayed)
+	assertNoDelete()
+	must(sys.store.LoadIAMCache(ctx, false))
+	assertLive(user)
+	if _, ok := sys.GetUser(ctx, child.AccessKey); ok {
+		t.Fatal("child from before the delayed revocation remains usable")
+	}
+	r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+	must(err)
+	if r.Deleted || !r.RevokedBefore.Equal(delayed) || !r.timestamp().Equal(recreated) {
+		t.Fatalf("retained revocation damaged the recreated identity: %+v", r)
+	}
+
+	// A genuinely newer deletion must still reach siblings and remove the
+	// parent plus credentials issued under its latest revocation boundary.
+	fresh, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "heal-fresh-child", secretKey: "valid-service-password",
+	})
+	must(err)
+	latest := recreated.Add(time.Minute)
+	revoke(latest)
+	wantDeletes := int32(1)
+	if sys.HasWatcher() {
+		wantDeletes = 0
+	}
+	if n := deletes.Load(); n != wantDeletes {
+		t.Fatalf("new deletion notifications=%d, want %d", n, wantDeletes)
+	}
+	for _, key := range []string{user, fresh.AccessKey} {
+		if _, ok := sys.GetUser(ctx, key); ok {
+			t.Fatalf("newer deletion left credential %s usable", key)
+		}
+	}
+	// Exercise the actual sibling handler again against the already persisted
+	// tombstone. Its context has no revision; it must not re-stamp the record.
+	_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+	if remoteErr != nil {
+		t.Fatal(remoteErr)
+	}
+	r, err = loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+	must(err)
+	if !r.Deleted || !r.timestamp().Equal(latest) {
+		t.Fatalf("sibling re-stamped the tombstone: got %s, want %s", r.timestamp(), latest)
+	}
+	create(latest.Add(time.Minute))
+	_, remoteErr = server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+	if remoteErr != nil {
+		t.Fatal(remoteErr)
+	}
+	assertLive(user)
+	revoke(latest)
+	must(sys.store.LoadIAMCache(ctx, false))
+	assertLive(user)
+}
+
+// Counts what a retained revocation actually sends to sibling nodes.
+func TestIAMRevocationRetainedReloadsSibling(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	initAllSubsystems(ctx)
+	globalIAMSys.Init(ctx, obj, nil, 2*time.Second)
+	defer os.RemoveAll(disks[0])
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+
+	sys, peer := globalIAMSys, &globalSiteReplicationSys
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user := "retained-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
+	create := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
+	}
+	revoke := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
+	}
+	create(origin)
+	revoke(deleted)
+	create(recreated)
+	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "retained-child", secretKey: "valid-service-password",
+	})
+	must(err)
+
+	// A sibling shares persistent state but has an independent IAM cache.
+	sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, sys.usersSysType)}
+	must(sibling.LoadIAMCache(ctx, false))
+	if _, ok := sibling.GetUser(child.AccessKey); !ok {
+		t.Fatal("sibling fixture did not load child")
+	}
+	tg, err := grid.SetupTestGrid(2)
+	must(err)
+	t.Cleanup(tg.Cleanup)
+	var deletes, loads atomic.Int32
+	server := &peerRESTServer{}
+	must(deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		deletes.Add(1)
+		return server.DeleteUserHandler(r)
+	}))
+	must(loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		loads.Add(1)
+		// LoadUserHandler delegates to this same cache reload method.
+		if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), regUser); err != nil {
+			return grid.NoPayload{}, grid.NewRemoteErr(err)
+		}
+		return grid.NoPayload{}, nil
+	}))
+	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+	must(err)
+	prev := globalNotificationSys
+	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
+		host:     host,
+		gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
+	}}}
+	t.Cleanup(func() { globalNotificationSys = prev })
+
+	delayed := deleted.Add(time.Minute)
+	revoke(delayed)
+
+	t.Logf("sibling notifications after a retained revocation: destructive=%d reload=%d", deletes.Load(), loads.Load())
+	if deletes.Load() != 0 {
+		t.Errorf("destructive sibling delete sent: %d", deletes.Load())
+	}
+	if loads.Load() == 0 {
+		t.Errorf("retained revocation did not notify the sibling")
+	}
+	if _, ok := sibling.GetUser(child.AccessKey); ok {
+		t.Error("sibling still resolves revoked child")
+	}
+	if _, ok := sibling.GetUser(user); !ok {
+		t.Error("sibling lost live parent")
+	}
+	if _, ok := sys.store.GetUser(user); !ok {
+		t.Error("live parent lost")
+	}
+	if _, ok := sys.store.GetUser(child.AccessKey); ok {
+		t.Error("revoked child still resolves on the receiving node")
+	}
+}

cmd/iam-revocation_test.go

@@ -0,0 +1,535 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+// Exercise the persisted IAM store and the same peer handler used by site heal.
+// A delete must survive a cache reload and an older create arriving afterwards.
+func TestIAMRevocationRejectsOfflineUser(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "offline-revoked-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "test-password-valid", Status: madmin.AccountEnabled}
+	created, err := globalIAMSys.CreateUser(ctx, user, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = globalIAMSys.DeleteUser(ctx, user, false); err != nil {
+		t.Fatal(err)
+	}
+	if err = globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Fatalf("revoked user restored by old peer event: %v", err)
+	}
+}
+
+func TestIAMRevocationHealingContinuesAfterPeerRejectsDelete(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	if _, err := globalIAMSys.CreateUser(ctx, "heal-sync", req); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.CreateUser(ctx, "heal-deleted", req); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalIAMSys.DeleteUser(ctx, "heal-deleted", false); err != nil {
+		t.Fatal(err)
+	}
+	p, err := globalIAMSys.store.GetPolicy("readwrite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.SetPolicy(ctx, "heal-new-policy", p); err != nil {
+		t.Fatal(err)
+	}
+	var liveUpdates atomic.Int32
+	peer := func(id string, rejectDelete bool) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			switch {
+			case strings.HasSuffix(r.URL.Path, "/metainfo"):
+				_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: id})
+			case r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions":
+				if r.Method == http.MethodGet {
+					_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
+					return
+				}
+				var batch iamRevisionBatch
+				if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+					t.Error(err)
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				for _, item := range batch.Items {
+					if rejectDelete && iamDeletionPath(item.SRIAMItem) != "" {
+						w.WriteHeader(http.StatusForbidden)
+						_, _ = w.Write([]byte(`{"Code":"AccessDenied","Message":"delete rejected"}`))
+						return
+					}
+					if id == "healthy" && item.Type == madmin.SRIAMItemPolicy && item.Name == "heal-new-policy" && len(item.Policy) > 0 {
+						liveUpdates.Add(1)
+					}
+				}
+				_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
+			default:
+				t.Errorf("unexpected peer request %s", r.URL.Path)
+				w.WriteHeader(http.StatusNotFound)
+			}
+		}))
+	}
+	healthy, rejected := peer("healthy", false), peer("rejected", true)
+	defer healthy.Close()
+	defer rejected.Close()
+	c := &SiteReplicationSys{enabled: true, state: srState{
+		ServiceAccountAccessKey: "heal-sync",
+		Peers: map[string]madmin.PeerInfo{
+			globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
+			"healthy":            {Name: "healthy", DeploymentID: "healthy", Endpoint: healthy.URL},
+			"rejected":           {Name: "rejected", DeploymentID: "rejected", Endpoint: rejected.URL},
+		},
+	}}
+	if err := c.healIAMSystem(ctx, obj); err == nil {
+		t.Fatal("deletion failure was not reported")
+	}
+	if liveUpdates.Load() == 0 {
+		t.Fatal("one peer rejecting a deletion blocked unrelated live IAM healing to a healthy peer")
+	}
+}
+
+func TestIAMRevocationLifecycle(t *testing.T) {
+	testIAMRevocationLifecycle(t, nil)
+}
+
+func TestIAMRevocationEtcdLifecycle(t *testing.T) {
+	endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+	if endpoint == "" {
+		t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+	}
+	connection, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer connection.Close()
+	// The facade borrows the connection's services. Close the owning client,
+	// not namespace.Watcher while IAM's canceled watch loop is winding down.
+	ctx, cancel := context.WithCancel(connection.Ctx())
+	defer cancel()
+	client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(connection.GetLogger()))
+	prefix := fmt.Sprintf("/silo-revocation-test/%d/", time.Now().UnixNano())
+	client.KV = namespace.NewKV(connection.KV, prefix)
+	client.Watcher = namespace.NewWatcher(connection.Watcher, prefix)
+	client.Lease = connection.Lease
+	testIAMRevocationLifecycle(t, client)
+}
+
+func testIAMRevocationLifecycle(t *testing.T, client *etcd.Client) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	disk := disks[0]
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err == nil {
+		initAllSubsystems(ctx)
+		globalIAMSys.Init(ctx, obj, client, 2*time.Second)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	sys, peer := globalIAMSys, &globalSiteReplicationSys
+	must := func(t *testing.T, err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	reload := func(t *testing.T) { t.Helper(); must(t, sys.store.LoadIAMCache(ctx, false)) }
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	createUser := func(t *testing.T, name string) {
+		t.Helper()
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, origin))
+	}
+	assertAbsent := func(t *testing.T, name string) {
+		t.Helper()
+		if _, ok := sys.GetUser(ctx, name); ok {
+			t.Fatalf("revoked credential %s is usable", name)
+		}
+	}
+
+	t.Run("replay after recreation", func(t *testing.T) {
+		testIAMRevocationReplayAfterRecreation(ctx, t, sys)
+	})
+
+	t.Run("origin timestamp and recreation", func(t *testing.T) {
+		name := "revocation-recreate"
+		createUser(t, name)
+		ui, ok := sys.store.GetUser(name)
+		if !ok || !ui.UpdatedAt.Equal(origin) {
+			t.Fatalf("origin time changed: %v", ui.UpdatedAt)
+		}
+		must(t, sys.DeleteUser(ctx, name, false))
+		reload(t)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, time.Time{}))
+		assertAbsent(t, name)
+		newTime := UTCNow().Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, newTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, origin.Add(time.Second)))
+		reload(t)
+		ui, ok = sys.store.GetUser(name)
+		if !ok || !ui.UpdatedAt.Equal(newTime) || ui.RevokedBefore.IsZero() {
+			t.Fatalf("newer recreation lost, or deletion boundary missing: present=%v", ok)
+		}
+	})
+
+	t.Run("groups policies and mappings", func(t *testing.T) {
+		user, group, name := "revocation-member", "revocation-group", "revocation-policy"
+		createUser(t, user)
+		p, err := sys.store.GetPolicy("readwrite")
+		must(t, err)
+		must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
+		add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
+		for _, isGroup := range []bool{false, true} {
+			entity := user
+			if isGroup {
+				entity = group
+			}
+			mp := &madmin.SRPolicyMapping{UserOrGroup: entity, Policy: name, UserType: int(regUser), IsGroup: isGroup}
+			must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
+			_, err = sys.PolicyDBSet(ctx, entity, "", regUser, isGroup)
+			must(t, err)
+			must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
+			if _, ok := sys.store.GetMappedPolicy(entity, isGroup); ok {
+				t.Fatal("old grant restored")
+			}
+		}
+		// This receiver never saw the member-removal event preceding deletion.
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, UTCNow()))
+		must(t, sys.DeletePolicy(ctx, name, true))
+		reload(t)
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
+		must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
+		if _, err = sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
+			t.Fatalf("group restored: %v", err)
+		}
+		if _, err = sys.store.GetPolicyDoc(name); !errors.Is(err, errNoSuchPolicy) {
+			t.Fatalf("policy restored: %v", err)
+		}
+		paths, err := sys.store.listIAMConfigPaths(ctx)
+		must(t, err)
+		found := make(map[string]bool)
+		for _, path := range paths {
+			r, err := loadIAMRevision(ctx, sys.store, path)
+			must(t, err)
+			if item, ok := iamDeletionItem(path, r); ok {
+				found[iamDeletionPath(item)] = true
+				if item.UpdatedAt.IsZero() {
+					t.Fatal("undated delete replay")
+				}
+			}
+		}
+		for _, path := range []string{getGroupInfoPath(group), getPolicyDocPath(name), getMappedPolicyPath(user, regUser, false), getMappedPolicyPath(group, regUser, true)} {
+			if !found[path] {
+				t.Errorf("deletion missing from heal: %s", path)
+			}
+		}
+	})
+
+	t.Run("parent revokes service accounts and STS", func(t *testing.T) {
+		parent := "revocation-parent"
+		createUser(t, parent)
+		svc, svcAt, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "revocation-service", secretKey: "valid-service-password"})
+		must(t, err)
+		secret, err := getTokenSigningKey()
+		must(t, err)
+		sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		sts.ParentUser = parent
+		_, err = sys.SetTempUser(withIAMReplicationTime(ctx, origin), sts.AccessKey, sts, "readwrite")
+		must(t, err)
+		must(t, sys.DeleteUser(ctx, parent, false))
+		reload(t)
+		assertAbsent(t, parent)
+		assertAbsent(t, svc.AccessKey)
+		assertAbsent(t, sts.AccessKey)
+		// Recreate the parent, then deliver old child events from the offline site.
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, UTCNow()))
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: svc.AccessKey, SecretKey: svc.SecretKey}}, svcAt))
+		must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken, ParentPolicyMapping: "readwrite"}, origin))
+		reload(t)
+		assertAbsent(t, svc.AccessKey)
+		assertAbsent(t, sts.AccessKey)
+		// A freshly issued credential is still supported after deliberate recreation.
+		_, _, err = sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-service", secretKey: "valid-service-password"})
+		must(t, err)
+		if _, ok := sys.GetUser(ctx, "new-service"); !ok {
+			t.Fatal("fresh service account rejected")
+		}
+	})
+
+	t.Run("delete before first create", func(t *testing.T) {
+		name := "revocation-unseen"
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, UTCNow()))
+		createUser(t, name)
+		assertAbsent(t, name)
+		svc := "unseen-service"
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: svc}}, UTCNow()))
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "revocation-recreate", AccessKey: svc, SecretKey: "valid-service-password"}}, origin))
+		assertAbsent(t, svc)
+	})
+
+	t.Run("recreation arrives before revocation", func(t *testing.T) {
+		parent := "reordered-parent"
+		createUser(t, parent)
+		child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "reordered-child", secretKey: "valid-service-password"})
+		must(t, err)
+		newTime, deleteTime := UTCNow(), origin.Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, newTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+		assertAbsent(t, child.AccessKey)
+		reload(t)
+		assertAbsent(t, child.AccessKey)
+		u, ok := sys.GetUser(ctx, parent)
+		if !ok || !u.UpdatedAt.Equal(newTime) || !u.RevokedBefore.Equal(deleteTime) {
+			t.Fatal("reordered revocation damaged the new parent or lost its boundary")
+		}
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(parent, regUser))
+		must(t, err)
+		item, ok := iamDeletionItem(getUserIdentityPath(parent, regUser), r)
+		if !ok || !item.UpdatedAt.Equal(deleteTime) {
+			t.Fatal("recreation erased deletion replay")
+		}
+	})
+
+	t.Run("user cleanup does not supersede group deletion", func(t *testing.T) {
+		user, group := "cascade-user", "cascade-group"
+		createUser(t, user)
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin))
+		// On the origin site the group was removed before the user, but the
+		// recovering receiver processes those independent events in reverse.
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, origin.Add(time.Minute)))
+		reload(t)
+		if _, err := sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
+			t.Fatalf("deleted group survived reordered cleanup: %v", err)
+		}
+		groups, err := sys.ListGroups(ctx)
+		must(t, err)
+		for _, name := range groups {
+			if name == group {
+				t.Fatal("deleted group listed")
+			}
+		}
+	})
+
+	t.Run("parent revocation covers later updates to existing children", func(t *testing.T) {
+		parent, key := "late-update-parent", "late-update-child"
+		createUser(t, parent)
+		_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
+		must(t, err)
+		// This site missed the deletion and subsequently edited an old child.
+		_, err = sys.UpdateServiceAccount(withIAMReplicationTime(ctx, origin.Add(2*time.Minute)), key, updateServiceAccountOpts{description: "edited while the peer was offline"})
+		must(t, err)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, origin.Add(time.Minute)))
+		assertAbsent(t, key)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(3*time.Minute)))
+		reload(t)
+		assertAbsent(t, key)
+	})
+
+	t.Run("old generation cannot return with a newer event timestamp", func(t *testing.T) {
+		parent := "generation-parent"
+		createUser(t, parent)
+		deleteTime := origin.Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(2*time.Minute)))
+		// Another offline site issued this child under the original parent,
+		// after this site's delete/recreate. Wall-clock ordering cannot identify it.
+		late := origin.Add(3 * time.Minute)
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: "old-gen-service", SecretKey: "valid-service-password"}}, late))
+		secret, err := getTokenSigningKey()
+		must(t, err)
+		sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken}, late))
+		reload(t)
+		assertAbsent(t, "old-gen-service")
+		assertAbsent(t, sts.AccessKey)
+		// A local issuer knows the new boundary and signs it into both kinds
+		// of child. Untrusted inherited claims cannot select that boundary.
+		child, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-gen-service", secretKey: "valid-service-password", claims: map[string]any{iamParentRevocationClaim: "forged"}})
+		must(t, err)
+		newClaims := map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}
+		must(t, setIAMParentRevocationClaim(ctx, sys.store, parent, newClaims))
+		fresh, err := auth.GetNewCredentialsWithMetadata(newClaims, secret)
+		must(t, err)
+		fresh.ParentUser = parent
+		_, err = sys.SetTempUser(ctx, fresh.AccessKey, fresh, "")
+		must(t, err)
+		reload(t)
+		// A periodic reload retains the STS cache. Explicitly clear it to
+		// exercise the cold credential load performed after process restart.
+		cache := sys.store.lock()
+		cache.iamSTSAccountsMap = make(map[string]UserIdentity)
+		sys.store.unlock()
+		for _, key := range []string{child.AccessKey, fresh.AccessKey} {
+			u, ok := sys.GetUser(ctx, key)
+			if !ok || !iamCredentialSurvivesRevocation(u.Credentials, deleteTime) {
+				t.Fatalf("new-generation credential %s rejected", key)
+			}
+		}
+	})
+
+	t.Run("late revocation preserves proven new-generation children", func(t *testing.T) {
+		for _, recreateFirst := range []bool{false, true} {
+			parent := fmt.Sprintf("gen-parent-%t", recreateFirst)
+			createUser(t, parent)
+			deleteTime, createTime := origin.Add(time.Minute), origin.Add(2*time.Minute)
+			if recreateFirst {
+				must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
+			}
+			key := fmt.Sprintf("gen-child-%t", recreateFirst)
+			must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: key, SecretKey: "valid-service-password", Claims: map[string]any{iamParentRevocationClaim: deleteTime.Format(time.RFC3339Nano)}}}, createTime.Add(time.Second)))
+			must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+			if !recreateFirst {
+				assertAbsent(t, key)
+				must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
+			}
+			reload(t)
+			if _, ok := sys.GetUser(ctx, key); !ok {
+				t.Fatal("late revocation deleted a child issued by the recreated parent")
+			}
+		}
+	})
+
+	t.Run("cold loading preserves site-signed STS", func(t *testing.T) {
+		parent := "cold-sts-parent"
+		createUser(t, parent)
+		secret := "site-signing-key-valid"
+		_, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{
+			accessKey: siteReplicatorSvcAcc, secretKey: secret, allowSiteReplicatorAccount: true,
+		})
+		must(t, err)
+		setReplication := func(enabled bool) {
+			globalSiteReplicationSys.Lock()
+			globalSiteReplicationSys.enabled = enabled
+			globalSiteReplicationSys.Unlock()
+			globalSiteReplicatorCred.Set("")
+		}
+		setReplication(true)
+		defer setReplication(false)
+		cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		cred.ParentUser = parent
+		_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
+		must(t, err)
+		// IAM can load before the site replication manager during startup.
+		// A signing key that is not available yet must not delete live tokens.
+		setReplication(false)
+		for range 3 {
+			unverified := make(map[string]UserIdentity)
+			_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, unverified)
+			if _, ok := unverified[cred.AccessKey]; ok {
+				t.Fatal("accepted STS before the signing key became available")
+			}
+		}
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
+		must(t, err)
+		if r.Credentials.SessionToken == "" {
+			t.Fatal("cold IAM load physically deleted a non-expired site-signed STS credential")
+		}
+		setReplication(true)
+		loaded := make(map[string]UserIdentity)
+		must(t, sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded))
+		if _, ok := loaded[cred.AccessKey]; !ok {
+			t.Fatal("STS credential did not recover when the signing key became available")
+		}
+	})
+
+	t.Run("unverifiable STS stay denied and expired STS are removed", func(t *testing.T) {
+		parent := "invalid-sts-parent"
+		createUser(t, parent)
+		// Keep the etcd watcher from cleaning half of the fixture before the
+		// second record is seeded; this subtest exercises the loader directly.
+		sys.store.lock()
+		defer sys.store.unlock()
+		for _, expired := range []bool{false, true} {
+			cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, "unavailable-test-signing-key")
+			must(t, err)
+			cred.ParentUser = parent
+			if expired {
+				cred.Expiration = UTCNow().Add(-time.Minute)
+			}
+			identityPath := getUserIdentityPath(cred.AccessKey, stsUser)
+			mappingPath := getMappedPolicyPath(cred.AccessKey, stsUser, false)
+			// Seed disk directly to exercise loading, including existing records
+			// whose key is unknown. The write API should not accept such tokens.
+			must(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()}, identityPath))
+			must(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Policies: "readwrite"}, mappingPath))
+			loaded := make(map[string]UserIdentity)
+			_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded)
+			if _, ok := loaded[cred.AccessKey]; ok {
+				t.Fatalf("invalid STS accepted, expired=%t", expired)
+			}
+			for _, path := range []string{identityPath, mappingPath} {
+				var record map[string]any
+				err := sys.store.loadIAMConfig(ctx, &record, path)
+				if expired {
+					if !errors.Is(err, errConfigNotFound) {
+						t.Fatalf("expired STS data not cleaned up at %s: %v", path, err)
+					}
+				} else {
+					must(t, err)
+				}
+			}
+		}
+	})
+}

cmd/iam-service-recreation_test.go

@@ -0,0 +1,239 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// The receiver missed a deletion and still has the previous service key.
+// A later full snapshot must replace it, including when the owner changed.
+func TestIAMServiceAccountRecreation(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			origin := UTCNow().Add(-time.Hour)
+			for _, parent := range []string{"old-owner", "new-owner"} {
+				_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), parent, madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+				mustIAM(t, err)
+			}
+			const key = "reusable-service"
+			old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "old-owner", AccessKey: key, SecretKey: "old-service-password"}}
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+			oldIdentity, _ := sys.store.GetUser(key)
+			_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
+			mustIAM(t, err)
+			boundary, newer := origin.Add(time.Minute), origin.Add(2*time.Minute)
+			fresh := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemSvcAcc, UpdatedAt: newer, SvcAccChange: &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "new-owner", AccessKey: key, SecretKey: "new-service-password", Status: auth.AccountOff}}}, RevokedBefore: boundary}
+			mustIAM(t, applyIAMReplicationItem(ctx, fresh))
+			// A sibling may have missed the notification of the committed
+			// replacement. An equal-version retry must refresh that cache too.
+			staleCache := sys.store.lock()
+			staleCache.iamUsersMap[key] = oldIdentity
+			sys.store.unlock()
+			mustIAM(t, applyIAMReplicationItem(ctx, fresh)) // duplicate delivery is acknowledged
+			if current, _ := sys.store.GetUser(key); current.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey {
+				t.Fatal("duplicate snapshot acknowledged without refreshing the stale cache")
+			}
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: key}}, boundary))
+			mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+			u, ok := sys.store.GetUser(key)
+			if !ok || u.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey || u.Credentials.ParentUser != "new-owner" || u.Credentials.Status != auth.AccountOff || !u.UpdatedAt.Equal(newer) || !u.RevokedBefore.Equal(boundary) {
+				t.Fatal("recreation did not retain the new identity, disabled status, source version and revocation")
+			}
+			if _, ok := sys.GetUser(ctx, key); ok {
+				t.Fatal("replicated disabled service can authenticate")
+			}
+			cache := sys.store.rlock()
+			_, mapped := cache.cachedMappedPolicy(key, svcUser, false)
+			sys.store.runlock()
+			if mapped {
+				t.Fatal("recreated service inherited an older mapping")
+			}
+			_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
+			if !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("old service mapping replay was accepted: %v", err)
+			}
+			_, _, err = sys.NewServiceAccount(ctx, "new-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "local-service-password"})
+			if !errors.Is(err, errIAMServiceAccountNotAllowed) {
+				t.Fatalf("local duplicate creation must remain rejected: %v", err)
+			}
+			// Outbound snapshots must carry the retained service boundary too.
+			out, err := globalSiteReplicationSys.replicationItem(ctx, fresh.SRIAMItem)
+			mustIAM(t, err)
+			if !out.RevokedBefore.Equal(boundary) {
+				t.Fatal("outbound service snapshot lost its revocation")
+			}
+		})
+	}
+}
+
+func TestIAMServiceAccountReplicationRejectsOtherCredentialKinds(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "builtin-collision", madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	secret, err := getTokenSigningKey()
+	mustIAM(t, err)
+	token, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "builtin-collision"}, secret)
+	mustIAM(t, err)
+	token.ParentUser = "builtin-collision"
+	_, err = sys.SetTempUser(ctx, token.AccessKey, token, "")
+	mustIAM(t, err)
+	for _, key := range []string{"builtin-collision", token.AccessKey} {
+		_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), "another-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
+		if !errors.Is(err, errIAMServiceAccountNotAllowed) {
+			t.Fatalf("service replication replaced another credential kind: %v", err)
+		}
+	}
+}
+
+// SR configuration can be temporarily unreadable even though a service token
+// is signed with its own valid secret. Do not acknowledge a failed cache load.
+func TestIAMServiceAccountRetryReportsClaimLoadFailure(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	globalSiteReplicatorCred.RLock()
+	previousSigningKey := globalSiteReplicatorCred.secretKey
+	globalSiteReplicatorCred.RUnlock()
+	globalSiteReplicatorCred.Set("")
+	t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSigningKey) })
+	_, err := sys.CreateUser(ctx, "retry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	opts := newServiceAccountOpts{accessKey: "retry-service", secretKey: "valid-service-password"}
+	_, _, err = sys.NewServiceAccount(ctx, "retry-owner", nil, opts)
+	mustIAM(t, err)
+	old, _ := sys.store.GetUser(opts.accessKey)
+	opts.secretKey = "replacement-service-password"
+	at, err := sys.UpdateServiceAccount(ctx, opts.accessKey, updateServiceAccountOpts{secretKey: opts.secretKey})
+	mustIAM(t, err)
+	cache := sys.store.lock()
+	cache.iamUsersMap[opts.accessKey] = old // Missed sibling notification.
+	sys.store.unlock()
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = true // No site-replicator credential is installed.
+	globalSiteReplicationSys.Unlock()
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
+	if err == nil {
+		t.Fatal("acknowledged service retry despite failed claims loading")
+	}
+	if _, ok := sys.store.GetUser(opts.accessKey); ok {
+		t.Fatal("failed cache refresh retained the superseded service secret")
+	}
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = false
+	globalSiteReplicationSys.Unlock()
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
+	mustIAM(t, err)
+}
+
+// A delayed snapshot still has its original absolute expiration. Reapplying
+// the local minimum issuance lifetime would leave the old unexpired key alive.
+func TestIAMServiceAccountReplicationPreservesExpiration(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		for _, action := range []string{"create", "update"} {
+			for _, expired := range []bool{false, true} {
+				name := backend + "/" + action + "/near_expiry"
+				if expired {
+					name = backend + "/" + action + "/expired"
+				}
+				t.Run(name, func(t *testing.T) {
+					ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+					origin := UTCNow().Add(-time.Hour)
+					_, err := sys.CreateUser(ctx, "expiry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+					mustIAM(t, err)
+					old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "old-service-password"}}
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+					expires := UTCNow().Add(time.Minute)
+					if expired {
+						expires = UTCNow().Add(-time.Minute)
+					}
+					change := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
+					if action == "update" {
+						change = &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
+					}
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, change, origin.Add(2*time.Minute)))
+					u, ok := sys.store.GetUser("expiry-service")
+					if !ok || u.Credentials.SecretKey != "new-service-password" || !u.Credentials.Expiration.Equal(expires) {
+						t.Fatal("delayed snapshot lost its new secret or absolute expiration")
+					}
+					_, allowed := sys.GetUser(ctx, "expiry-service")
+					if allowed == expired {
+						t.Fatal("credential validity disagrees with its absolute expiration")
+					}
+					mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+					r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath("expiry-service", svcUser))
+					mustIAM(t, err)
+					if r.Credentials.SecretKey == old.Create.SecretKey || (expired && !r.Deleted) {
+						t.Fatal("old non-expiring credential returned after reload")
+					}
+					_, _, err = sys.NewServiceAccount(ctx, "expiry-owner", nil, newServiceAccountOpts{accessKey: "local-expiry", secretKey: "valid-service-password", expiration: &expires})
+					if !errors.Is(err, errInvalidSvcAcctExpiration) {
+						t.Fatalf("local issuance lifetime check changed: %v", err)
+					}
+				})
+			}
+		}
+	}
+}
+
+// Status-only summaries intentionally omit secrets. Different revisions must
+// still trigger live healing, and disabled identities must be eligible sources.
+func TestIAMServiceAccountHealingNewerSnapshot(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	_, err := sys.CreateUser(ctx, "heal-svc-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), "heal-svc-owner", nil, newServiceAccountOpts{accessKey: "heal-service", secretKey: "valid-service-password"})
+	mustIAM(t, err)
+	at, err := sys.UpdateServiceAccount(ctx, "heal-service", updateServiceAccountOpts{status: auth.AccountOff})
+	mustIAM(t, err)
+	var sent atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			return
+		}
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" || r.Method != http.MethodPut {
+			t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		var batch iamRevisionBatch
+		if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+			t.Error(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		for _, item := range batch.Items {
+			if item.SvcAccChange != nil && item.SvcAccChange.Create != nil && item.SvcAccChange.Create.AccessKey == "heal-service" && item.SvcAccChange.Create.Status == auth.AccountOff && item.UpdatedAt.Equal(at) {
+				sent.Add(1)
+			}
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node", Instance: "boot", Digest: "fixture"}})
+	}))
+	defer server.Close()
+	peers := map[string]madmin.PeerInfo{globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()}, "remote": {Name: "remote", DeploymentID: "remote", Endpoint: server.URL}}
+	c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "heal-svc-owner", Peers: peers}}
+	local := madmin.UserInfo{Status: madmin.AccountStatus(auth.AccountOff), UpdatedAt: at}
+	remote := local
+	remote.UpdatedAt = origin
+	if isUserInfoReplicated(2, 2, []madmin.UserInfo{local, remote}) {
+		t.Fatal("status-only summaries concealed different service revisions")
+	}
+	info := srStatusInfo{Sites: peers, UserStats: map[string]map[string]srUserStatsSummary{"heal-service": {globalDeploymentID(): {userInfo: srUserInfo{UserInfo: local}}, "remote": {SRUserStatsSummary: madmin.SRUserStatsSummary{UserInfoMismatch: true}, userInfo: srUserInfo{UserInfo: remote}}}}}
+	mustIAM(t, c.healUsers(ctx, obj, "heal-service", info))
+	if sent.Load() == 0 {
+		t.Fatal("disabled newer service snapshot was not healed")
+	}
+}

cmd/iam.go

@@ -162,6 +162,15 @@ func (sys *IAMSys) LoadUser(ctx context.Context, objAPI ObjectLayer, accessKey s
 	return sys.store.UserNotificationHandler(ctx, accessKey, userType)
 }
 
+// LoadUserAfterDelete reloads a parent's identity and cached dependents after a
+// sibling committed a deletion. Each record may already have been recreated.
+func (sys *IAMSys) LoadUserAfterDelete(ctx context.Context, accessKey string) error {
+	if !sys.Initialized() {
+		return errServerNotInitialized
+	}
+	return sys.store.UserDeletionNotificationHandler(ctx, accessKey)
+}
+
 // LoadServiceAccount - reloads a specific service account from backend disks or etcd.
 func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) error {
 	if !sys.Initialized() {
@@ -596,10 +605,22 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 		return errServerNotInitialized
 	}
 
-	for _, v := range policy.DefaultPolicies {
-		if v.Name == policyName {
-			if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound {
-				return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+	if _, replicated := iamReplicationTime(ctx); !replicated && notifyPeers {
+		for _, v := range policy.DefaultPolicies {
+			if v.Name == policyName {
+				var err error
+				if objectStore, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
+					err = checkConfig(ctx, objectStore.objAPI, getPolicyDocPath(policyName))
+				} else {
+					var r iamRevision
+					err = sys.store.loadIAMConfig(ctx, &r, getPolicyDocPath(policyName))
+				}
+				if errors.Is(err, errConfigNotFound) {
+					return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+				}
+				if err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -705,20 +726,30 @@ func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers
 		return errServerNotInitialized
 	}
 
-	if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil {
+	err := sys.store.DeleteUser(ctx, accessKey, regUser)
+	var cleanupErr *iamCommittedCleanupError
+	retained := errors.Is(err, errIAMRevocationRetained)
+	if errors.As(err, &cleanupErr) {
+		retained = cleanupErr.retained
+	} else if err != nil && !retained {
 		return err
 	}
-
-	// Notify all other MinIO peers to delete user.
+	// Publish the committed state even when dependent cleanup must be retried.
 	if notifyPeers && !sys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				iamLogIf(ctx, nerr.Err)
+		if retained {
+			sys.notifyForUser(ctx, accessKey, false)
+		} else {
+			for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
+				if nerr.Err != nil {
+					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+					iamLogIf(ctx, nerr.Err)
+				}
 			}
 		}
 	}
-
+	if cleanupErr != nil {
+		return cleanupErr
+	}
 	return nil
 }
 
@@ -1052,6 +1083,7 @@ type newServiceAccountOpts struct {
 	sessionPolicy              *policy.Policy
 	accessKey                  string
 	secretKey                  string
+	status                     string // Used by replication snapshots; local creates default to enabled.
 	name, description          string
 	expiration                 *time.Time
 	allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
@@ -1116,6 +1148,11 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 			m[k] = v
 		}
 	}
+	if _, replicated := iamReplicationTime(ctx); !replicated {
+		if err := setIAMParentRevocationClaim(ctx, sys.store, parentUser, m); err != nil {
+			return auth.Credentials{}, time.Time{}, err
+		}
+	}
 
 	var accessKey, secretKey string
 	var err error
@@ -1134,12 +1171,19 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	cred.ParentUser = parentUser
 	cred.Groups = groups
 	cred.Status = string(auth.AccountOn)
+	switch opts.status {
+	case "", auth.AccountOn, string(madmin.AccountEnabled):
+	case auth.AccountOff, string(madmin.AccountDisabled):
+		cred.Status = auth.AccountOff
+	default:
+		return auth.Credentials{}, time.Time{}, errInvalidArgument
+	}
 	cred.Name = opts.name
 	cred.Description = opts.description
 
 	if opts.expiration != nil {
 		expirationInUTC := opts.expiration.UTC()
-		if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
+		if err := validateSvcExpirationInUTC(ctx, expirationInUTC); err != nil {
 			return auth.Credentials{}, time.Time{}, err
 		}
 		cred.Expiration = expirationInUTC
@@ -1370,7 +1414,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.Credentials.IsServiceAccount() {
+	if _, replicated := iamReplicationTime(ctx); (!ok || !sa.Credentials.IsServiceAccount()) && !replicated {
 		return nil
 	}
 
@@ -1474,8 +1518,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 		}
 	}
 
-	// We ignore any errors
-	_ = sys.store.DeleteUsers(ctx, expiredUsers)
+	// Keep failed revocations visible so the next purge can retry.
+	iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
 }
 
 // purgeExpiredCredentialsForLDAP - validates if local credentials are still
@@ -1503,8 +1547,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 		return
 	}
 
-	// We ignore any errors
-	_ = sys.store.DeleteUsers(ctx, expiredUsers)
+	// Keep failed revocations visible so the next purge can retry.
+	iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
 }
 
 // updateGroupMembershipsForLDAP - updates the list of groups associated with the credential.
@@ -1925,12 +1969,12 @@ func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, membe
 	}
 
 	updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members)
-	if err != nil {
+	var cleanupErr *iamCommittedCleanupError
+	if err != nil && !errors.As(err, &cleanupErr) {
 		return updatedAt, err
 	}
-
 	sys.notifyForGroup(ctx, group)
-	return updatedAt, nil
+	return updatedAt, err
 }
 
 // SetGroupStatus - enable/disabled a group

cmd/ilm-access-move_test.go

@@ -1,529 +0,0 @@
-// Copyright 2026 PGSTY contributors.
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"crypto/md5"
-	"encoding/base64"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io"
-	"maps"
-	"net/http"
-	"net/http/httptest"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/minio/minio/internal/dsync"
-	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
-)
-
-func accessMovePools(t *testing.T) (*erasureServerPools, string) {
-	t.Helper()
-	ctx, cancel := context.WithCancel(t.Context())
-	dirs, err := getRandomDisks(32)
-	if err != nil {
-		t.Fatal(err)
-	}
-	endpoints := mustGetPoolEndpoints(0, dirs[:16]...)
-	endpoints = append(endpoints, mustGetPoolEndpoints(1, dirs[16:]...)...)
-	obj, _, err := initObjectLayer(ctx, endpoints)
-	if err != nil {
-		cancel()
-		removeRoots(dirs)
-		t.Fatal(err)
-	}
-	z := obj.(*erasureServerPools)
-	previous := newObjectLayerFn()
-	setObjectLayer(z)
-	t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
-	bucket := "access-move-test"
-	if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
-		t.Fatal(err)
-	}
-	return z, bucket
-}
-
-func putAccessMoveVersion(t *testing.T, z *erasureServerPools, bucket, object string, pool int, body string, moved bool) ObjectInfo {
-	t.Helper()
-	metadata := map[string]string{"test-value": body}
-	if moved {
-		metadata[accessTierMetadataKey] = accessTierStamp(pool, time.Now().UnixNano())
-	}
-	cs := hash.NewChecksumFromData(hash.ChecksumCRC32C, []byte(body))
-	oi, err := z.serverPools[pool].PutObject(t.Context(), bucket, object,
-		mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""),
-		ObjectOptions{Versioned: true, UserDefined: metadata, WantChecksum: cs})
-	if err != nil {
-		t.Fatal(err)
-	}
-	return oi
-}
-
-func assertAccessMoveVersion(t *testing.T, z *erasureServerPools, bucket, object string, pool int, want ObjectInfo, body string) {
-	t.Helper()
-	gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, http.Header{}, ObjectOptions{VersionID: want.VersionID})
-	if err != nil {
-		t.Errorf("pool %d version %s: %v", pool, want.VersionID, err)
-		return
-	}
-	got, err := io.ReadAll(gr)
-	gr.Close()
-	if err != nil || string(got) != body {
-		t.Errorf("pool %d payload = %q, err = %v, want %q", pool, got, err, body)
-	}
-	if gr.ObjInfo.ETag != want.ETag || !gr.ObjInfo.ModTime.Equal(want.ModTime) {
-		t.Error("move changed ETag or modification time")
-	}
-	if len(want.Checksum) == 0 {
-		t.Fatal("fixture has no checksum")
-	}
-	if !bytes.Equal(gr.ObjInfo.Checksum, want.Checksum) {
-		t.Error("move changed or dropped the stored checksum")
-	}
-	if gr.ObjInfo.UserDefined["test-value"] != body {
-		t.Error("move changed user metadata")
-	}
-}
-
-func TestAccessMoveVersionStack(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "versions"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	dm, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	for _, pair := range [][2]int{{1, 0}, {0, 1}} {
-		if _, err := moveObjectPool(t.Context(), z, bucket, object, pair[0], pair[1], nil); err != nil {
-			t.Fatal(err)
-		}
-		assertAccessMoveVersion(t, z, bucket, object, pair[1], old, "old")
-		assertAccessMoveVersion(t, z, bucket, object, pair[1], latest, "new")
-		versions, err := accessObjectVersions(t.Context(), z, pair[1], bucket, object)
-		if err != nil {
-			t.Fatal(err)
-		}
-		found := false
-		for _, version := range versions {
-			if version.VersionID == dm.VersionID && version.Deleted {
-				found = true
-			}
-		}
-		if !found || len(versions) != 3 {
-			t.Errorf("move lost a version or delete marker: %+v", versions)
-		}
-		if _, err := z.serverPools[pair[0]].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
-			t.Errorf("source remains after completed move: %v", err)
-		}
-	}
-}
-
-func TestAccessMovePreservesNestedObject(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	parent := putAccessMoveVersion(t, z, bucket, "parent", 1, "parent", false)
-	child := putAccessMoveVersion(t, z, bucket, "parent/child", 1, "child", false)
-	if _, err := moveObjectPool(t.Context(), z, bucket, "parent", 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, "parent", 0, parent, "parent")
-	assertAccessMoveVersion(t, z, bucket, "parent/child", 1, child, "child")
-}
-
-func TestAccessMoveNullVersion(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object, body = "null-version", "unversioned"
-	// A null version can remain in a bucket after versioning is enabled.
-	oi, err := z.serverPools[1].PutObject(t.Context(), bucket, object,
-		mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""), ObjectOptions{
-			UserDefined:  map[string]string{"test-value": body},
-			WantChecksum: hash.NewChecksumFromData(hash.ChecksumCRC32C, []byte(body)),
-		})
-	if err != nil || oi.VersionID != "" {
-		t.Fatalf("null version fixture failed: %v %q", err, oi.VersionID)
-	}
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, oi, body)
-	if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
-		t.Fatalf("null version source was not removed: %v", err)
-	}
-}
-
-// Source removal can partially commit before a process or disk fails. The
-// destination can therefore hold the only remaining copy of an older version.
-func TestAccessMoveRetryPreservesDestinationVersions(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "retry"
-	onlyAtDestination := putAccessMoveVersion(t, z, bucket, object, 0, "unique-old", true)
-	source := putAccessMoveVersion(t, z, bucket, object, 1, "source-new", false)
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, onlyAtDestination, "unique-old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, source, "source-new")
-}
-
-type accessMoveFaultDisk struct {
-	StorageAPI
-	failVersion string
-}
-
-func (d accessMoveFaultDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
-	if fi.VersionID == d.failVersion {
-		return RenameDataResp{}, errDiskFull
-	}
-	return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
-}
-
-func TestAccessMoveWriteFailureCanResume(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "write-failure"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	existing := putAccessMoveVersion(t, z, bucket, object, 0, "unique-destination", true)
-	set := z.serverPools[0].getHashedSet(object)
-	getDisks := set.getDisks
-	disks := getDisks()
-	faulty := make([]StorageAPI, len(disks))
-	for i, disk := range disks {
-		faulty[i] = accessMoveFaultDisk{StorageAPI: disk, failVersion: latest.VersionID}
-	}
-	set.getDisks = func() []StorageAPI { return faulty }
-	_, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil)
-	set.getDisks = getDisks
-	if err == nil {
-		t.Fatal("injected destination write failure was ignored")
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 1, latest, "new")
-	assertAccessMoveVersion(t, z, bucket, object, 0, existing, "unique-destination")
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-	assertAccessMoveVersion(t, z, bucket, object, 0, existing, "unique-destination")
-}
-
-func TestAccessMoveExcludesSourceWriter(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "concurrent"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	_, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, func(ObjectInfo, uint64) error {
-		ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
-		defer cancel()
-		_, err := z.serverPools[1].PutObject(ctx, bucket, object,
-			mustGetPutObjReader(t, bytes.NewBufferString("racing-write"), 12, "", ""), ObjectOptions{Versioned: true})
-		if err == nil {
-			t.Error("source writer committed while move was in progress")
-		}
-		if ctx.Err() == nil {
-			return errors.New("writer did not wait for the move lock")
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-}
-
-func TestAccessMoveResumesPartialCopy(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "partial-copy"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	gr, err := z.serverPools[1].GetObjectNInfo(t.Context(), bucket, object, nil, http.Header{}, ObjectOptions{VersionID: old.VersionID, NoDecryption: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Model a process stopping after the first copy commits, before cleanup.
-	if err := moveAccessTierVersion(t.Context(), z, 1, 0, bucket, gr, time.Now().UnixNano()); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-}
-
-func TestAccessMoveRefusesConflictingVersion(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "conflicting-version"
-	source := putAccessMoveVersion(t, z, bucket, object, 1, "source", false)
-	destination, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
-		mustGetPutObjReader(t, bytes.NewBufferString("target"), 6, "", ""), ObjectOptions{
-			VersionID: source.VersionID, MTime: source.ModTime,
-			UserDefined:  map[string]string{"test-value": "target"},
-			WantChecksum: hash.NewChecksumFromData(hash.ChecksumCRC32C, []byte("target")),
-		})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); !errors.Is(err, errAccessTierNotEligible) {
-		t.Fatalf("conflicting version should be left intact: %v", err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, source, "source")
-	assertAccessMoveVersion(t, z, bucket, object, 0, destination, "target")
-}
-
-type accessMoveDeleteFaultDisk struct {
-	StorageAPI
-	bucket, object, version string
-}
-
-func (d accessMoveDeleteFaultDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
-	if volume == d.bucket && path == d.object && fi.VersionID == d.version {
-		return errDiskFull
-	}
-	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
-}
-
-func TestAccessMoveSourceDeleteFailureCanResume(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "source-delete-failure"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	set := z.serverPools[1].getHashedSet(object)
-	getDisks := set.getDisks
-	faulty := append([]StorageAPI(nil), getDisks()...)
-	// Commit removal of the old version, then reject the latest version on
-	// every disk. This fixes the retry boundary without relying on how a
-	// partially deleted erasure quorum is subsequently resolved or healed.
-	for i := range faulty {
-		faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: latest.VersionID}
-	}
-	set.getDisks = func() []StorageAPI { return faulty }
-	_, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil)
-	set.getDisks = getDisks
-	if err == nil {
-		t.Fatal("injected partial source purge was ignored")
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-	if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: old.VersionID}); !isErrVersionNotFound(err) {
-		t.Fatalf("old source version should already be removed: %v", err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, latest, "new")
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-}
-
-// Exercise the public multipart API and compare whole-object and part reads
-// across both directions of a real pool move, including raw SSE-C ciphertext.
-func TestAccessMoveMultipartChecksums(t *testing.T) {
-	z, _ := accessMovePools(t)
-	ctx, cancel := context.WithCancel(t.Context())
-	defer cancel()
-	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	previousTLS := globalIsTLS
-	globalIsTLS = true
-	defer func() { globalIsTLS = previousTLS }()
-	partData, full := multipartChecksumTestData()
-	for _, tc := range []struct {
-		typ  hash.ChecksumType
-		ssec bool
-	}{{hash.ChecksumCRC32, false}, {hash.ChecksumCRC32C, false}, {hash.ChecksumCRC64NVME, false}, {hash.ChecksumCRC32C, true}} {
-		t.Run(fmt.Sprintf("%s/ssec=%t", tc.typ, tc.ssec), func(t *testing.T) {
-			object := getRandomObjectName()
-			headers := map[string]string{}
-			if tc.ssec {
-				key := bytes.Repeat([]byte{0x42}, 32)
-				keyMD5 := md5.Sum(key)
-				headers[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
-				headers[xhttp.AmzServerSideEncryptionCustomerKey] = base64.StdEncoding.EncodeToString(key)
-				headers[xhttp.AmzServerSideEncryptionCustomerKeyMD5] = base64.StdEncoding.EncodeToString(keyMD5[:])
-			}
-			do := func(method, url string, body []byte, extra map[string]string) *httptest.ResponseRecorder {
-				t.Helper()
-				h := maps.Clone(headers)
-				maps.Copy(h, extra)
-				req, err := newTestSignedRequestV4(method, url, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, h)
-				if err != nil {
-					t.Fatal(err)
-				}
-				rec := httptest.NewRecorder()
-				router.ServeHTTP(rec, req)
-				if rec.Code != http.StatusOK && rec.Code != http.StatusPartialContent {
-					t.Fatalf("%s %s: %d %s", method, url, rec.Code, rec.Body.String())
-				}
-				return rec
-			}
-			init := do(http.MethodPost, getNewMultipartURL("", bucket, object), nil, map[string]string{
-				xhttp.AmzChecksumAlgo: tc.typ.String(), xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
-			})
-			var upload InitiateMultipartUploadResponse
-			if err := xml.Unmarshal(init.Body.Bytes(), &upload); err != nil {
-				t.Fatal(err)
-			}
-			parts := make([]CompletePart, len(partData))
-			for i, data := range partData {
-				rec := do(http.MethodPut, getPutObjectPartURL("", bucket, object, upload.UploadID, fmt.Sprint(i+1)), data,
-					map[string]string{tc.typ.Key(): mustChecksum(t, tc.typ, data)})
-				parts[i] = CompletePart{PartNumber: i + 1, ETag: canonicalizeETag(rec.Header()[xhttp.ETag][0])}
-			}
-			body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
-			if err != nil {
-				t.Fatal(err)
-			}
-			do(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, upload.UploadID), body,
-				map[string]string{tc.typ.Key(): mustChecksum(t, tc.typ, full), xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject})
-			source, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
-			if err != nil || len(source.Checksum) == 0 {
-				t.Fatalf("source checksum missing: %v", err)
-			}
-			src := 0
-			if _, err := z.serverPools[src].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); isErrObjectNotFound(err) {
-				src = 1
-			}
-			for i := 0; i < 2; i++ {
-				dst := 1 - src
-				if _, err := moveObjectPool(t.Context(), z, bucket, object, src, dst, nil); err != nil {
-					t.Fatal(err)
-				}
-				got, err := z.serverPools[dst].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
-				if err != nil || !bytes.Equal(got.Checksum, source.Checksum) || got.ETag != source.ETag || got.VersionID != source.VersionID || !got.ModTime.Equal(source.ModTime) {
-					t.Fatalf("move changed version metadata: %v checksumEqual=%t ETag=%q/%q version=%q/%q modTimeEqual=%t", err, bytes.Equal(got.Checksum, source.Checksum), got.ETag, source.ETag, got.VersionID, source.VersionID, got.ModTime.Equal(source.ModTime))
-				}
-				rec := do(http.MethodGet, getGetObjectURL("", bucket, object), nil, map[string]string{xhttp.AmzChecksumMode: "ENABLED"})
-				if !bytes.Equal(rec.Body.Bytes(), full) || rec.Header().Get(tc.typ.Key()) != mustChecksum(t, tc.typ, full) {
-					t.Fatal("GET payload or checksum changed after move")
-				}
-				for j, data := range partData {
-					rec := do(http.MethodGet, getGetObjectURL("", bucket, object)+fmt.Sprintf("?partNumber=%d", j+1), nil, nil)
-					if !bytes.Equal(rec.Body.Bytes(), data) {
-						t.Fatalf("part %d changed after move", j+1)
-					}
-				}
-				src = dst
-			}
-		})
-	}
-}
-
-type accessMoveNamedLocker struct {
-	*localLocker
-	address string
-}
-
-func (l accessMoveNamedLocker) String() string { return l.address }
-
-func TestAccessMoveSharedDistributedLockers(t *testing.T) {
-	// Three overlapping sets of real dsync lock servers. Taking independent
-	// quorum locks for the same object would contend with our own earlier lock.
-	peers := make([]dsync.NetLocker, 5)
-	for i := range peers {
-		peers[i] = accessMoveNamedLocker{localLocker: newLocker(), address: fmt.Sprint(i)}
-	}
-	z := &erasureServerPools{}
-	for i := 0; i < 3; i++ {
-		setPeers := peers[i : i+3]
-		set := &erasureObjects{nsMutex: &nsLockMap{isDistErasure: true}, getLockers: func() ([]dsync.NetLocker, string) {
-			return setPeers, "access-move-fixture"
-		}}
-		z.serverPools = append(z.serverPools, &erasureSets{sets: []*erasureObjects{set}, distributionAlgo: formatErasureVersionV3DistributionAlgoV3})
-	}
-	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
-	defer cancel()
-	_, unlock, err := lockAccessTierObject(ctx, z, "bucket", "object", 1, 2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	release := sync.OnceFunc(unlock)
-	defer release()
-	for i, pool := range z.serverPools {
-		ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
-		lock := pool.NewNSLock("bucket", "object")
-		lc, err := lock.GetLock(ctx, globalOperationTimeout)
-		cancel()
-		if err == nil {
-			lock.Unlock(lc)
-			t.Fatalf("pool %d writer acquired its quorum during the move", i)
-		}
-	}
-	release()
-	// Every acquired peer lock is released, so ordinary writes can resume.
-	for _, peer := range peers {
-		// Distributed Unlock sends releases asynchronously.
-		lock := (&nsLockMap{isDistErasure: true}).NewNSLock(func() ([]dsync.NetLocker, string) {
-			return []dsync.NetLocker{peer}, "access-move-fixture"
-		}, "bucket", "object")
-		ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
-		lc, err := lock.GetLock(ctx, globalOperationTimeout)
-		cancel()
-		if err != nil {
-			t.Fatalf("peer %s leaked a lock: %v", peer.String(), err)
-		}
-		lock.Unlock(lc)
-	}
-}
-
-func TestAccessMoveConfiguredPromotionAndDemotion(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "configured-move"
-	version := putAccessMoveVersion(t, z, bucket, object, 1, "configured", false)
-	oldCfg, oldTracker := globalILMConfig.accessCfg(), globalAccessTracker
-	defer func() { globalILMConfig.update(oldCfg); globalAccessTracker = oldTracker }()
-	cfg := oldCfg
-	cfg.AccessTiering, cfg.AccessPools = true, []int{0, 1}
-	cfg.AccessMinResidency, cfg.AccessPromoteWatermark = 0, 99
-	globalILMConfig.update(cfg)
-	globalAccessTracker = newAccessTracker()
-	lc := accessLifecycleForTest(t)
-	data, err := xml.Marshal(lc)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, data); err != nil {
-		t.Fatal(err)
-	}
-	state := newAccessTierState(t.Context())
-	state.z, state.objAPI = z, z
-	task := accessTierTask{ctx: t.Context(), bucket: bucket, object: object, src: 1, dst: 0, direction: accessTierPromote, bytes: uint64(version.Size)}
-	// A queued move is rechecked when configuration is disabled dynamically.
-	cfg.AccessTiering = false
-	globalILMConfig.update(cfg)
-	if err := state.processTask(task); !errors.Is(err, errAccessTierNotEligible) {
-		t.Fatalf("disabled mover = %v", err)
-	}
-	cfg.AccessTiering = true
-	globalILMConfig.update(cfg)
-	globalAccessTracker.merged.Store(&mergedAccess{binWidth: 60, entries: map[string]accessEntry{
-		accessKey(bucket, object): {Bins: []uint32{100}, LastAt: time.Now().Unix()},
-	}})
-	if reason, ok := state.reservePromotion(task, cfg, 0); !ok {
-		t.Fatalf("promotion reservation failed: %s", reason)
-	}
-	if err := state.processTask(task); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, version, "configured")
-	// Advance the counter view beyond the rule's idle period.
-	globalAccessTracker.merged.Store(&mergedAccess{binWidth: 60, entries: map[string]accessEntry{
-		accessKey(bucket, object): {Bins: []uint32{0}, LastAt: time.Now().Add(-2 * time.Hour).Unix()},
-	}})
-	task.src, task.dst, task.direction, task.bytes = 0, 1, accessTierDemote, 0
-	if err := state.processTask(task); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, version, "configured")
-	if state.promotions.Load() != 1 || state.demotions.Load() != 1 || state.bytesMoved.Load() != 2*uint64(version.Size) || len(state.pending) != 0 || len(state.reserved) != 0 {
-		t.Fatal("promotion/demotion metrics or reservation cleanup are incorrect")
-	}
-}

cmd/ilm-access-tier_test.go

@@ -1,263 +0,0 @@
-// Copyright (c) 2015-2026 MinIO, Inc.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/klauspost/compress/zstd"
-	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/config/ilm"
-	"github.com/tinylib/msgp/msgp"
-)
-
-func accessLifecycleForTest(t *testing.T) *lifecycle.Lifecycle {
-	t.Helper()
-	xml := "<LifecycleConfiguration>" +
-		"<Rule><ID>access</ID><Status>Enabled</Status>" +
-		"<AccessTransition><Window>10m</Window><PromoteAfterAccesses>100</PromoteAfterAccesses>" +
-		"<DemoteAfterAccesses>5</DemoteAfterAccesses><DemoteAfterIdle>1h</DemoteAfterIdle></AccessTransition>" +
-		"</Rule></LifecycleConfiguration>"
-	lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml))
-	if err != nil {
-		t.Fatal(err)
-	}
-	return lc
-}
-
-func TestAccessTierStampAndDemoteEligibility(t *testing.T) {
-	oldTracker := globalAccessTracker
-	globalAccessTracker = newAccessTracker()
-	t.Cleanup(func() { globalAccessTracker = oldTracker })
-
-	now := time.Now()
-	cfg := ilm.Config{
-		AccessTiering: true, AccessPools: []int{0, 1},
-		AccessBinWidth: time.Minute, AccessBins: 12,
-		AccessMinResidency: 30 * time.Minute,
-	}
-	oi := ObjectInfo{
-		Bucket: "bucket", Name: "object", Size: 10, IsLatest: true,
-		UserDefined: map[string]string{
-			accessTierMetadataKey: "0:" + strconv.FormatInt(now.Add(-2*time.Hour).UnixNano(), 10),
-		},
-	}
-	if !accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
-		t.Fatal("cold, resident object should be demotion eligible")
-	}
-	oi.UserDefined[accessTierMetadataKey] = "0:" + strconv.FormatInt(now.Add(-time.Minute).UnixNano(), 10)
-	if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
-		t.Fatal("object inside minimum residency was eligible")
-	}
-	oi.UserDefined[accessTierMetadataKey] = "broken"
-	if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
-		t.Fatal("object with malformed marker was eligible")
-	}
-}
-
-func TestAccessTierReservationsEnforceCaps(t *testing.T) {
-	state := newAccessTierState(t.Context())
-	state.usageReady = true
-	state.baseUsage["a"] = 80
-	state.baseTotal = 180
-
-	cfg := ilm.Config{AccessMaxSize: 200}
-	task := accessTierTask{ctx: t.Context(), bucket: "a", object: "one", bytes: 30}
-	if reason, ok := state.reservePromotion(task, cfg, 0); ok || reason != "max-size" {
-		t.Fatalf("max-size reserve = %q/%v", reason, ok)
-	}
-
-	cfg.AccessMaxSize = 0
-	if reason, ok := state.reservePromotion(task, cfg, 100); ok || reason != "quota" {
-		t.Fatalf("quota reserve = %q/%v", reason, ok)
-	}
-
-	task.bytes = 20
-	if reason, ok := state.reservePromotion(task, cfg, 100); !ok || reason != "" {
-		t.Fatalf("valid reserve = %q/%v", reason, ok)
-	}
-	if got := state.bucketUsageLocked("a"); got != 100 {
-		t.Fatalf("reserved bucket usage = %d, want 100", got)
-	}
-	state.releasePending(task, true)
-}
-
-func TestDataMovementDestinationIsGated(t *testing.T) {
-	dst := 0
-	if got := dataMovementDstPool(ObjectOptions{DstPoolIdx: &dst}); got != nil {
-		t.Fatal("destination honored without DataMovement")
-	}
-	if got := dataMovementDstPool(ObjectOptions{DataMovement: true, DstPoolIdx: &dst}); got == nil || *got != 0 {
-		t.Fatalf("destination = %v", got)
-	}
-}
-
-func TestForcedDestinationPoolSelection(t *testing.T) {
-	z := &erasureServerPools{serverPools: make([]*erasureSets, 2)}
-	dst := 1
-	if got, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err != nil || got != dst {
-		t.Fatalf("destination = %d, err = %v", got, err)
-	}
-	bad := 2
-	if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &bad); !errors.Is(err, errInvalidArgument) {
-		t.Fatalf("out-of-range error = %v", err)
-	}
-	z.poolMeta.Pools = make([]PoolStatus, 2)
-	z.poolMeta.Pools[1].Decommission = &PoolDecommissionInfo{}
-	if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err == nil {
-		t.Fatal("suspended destination was accepted")
-	}
-}
-
-func TestHotTierAccounting(t *testing.T) {
-	entry := dataUsageEntry{}
-	entry.addSizes(sizeSummary{totalSize: 100, hotTierSize: 60, versions: 1})
-	entry.merge(dataUsageEntry{Size: 50, HotTierSize: 25})
-	if entry.Size != 150 || entry.HotTierSize != 85 {
-		t.Fatalf("usage = size:%d hot:%d", entry.Size, entry.HotTierSize)
-	}
-}
-
-func TestScannerCyclesApart(t *testing.T) {
-	tests := []struct {
-		current, previous uint32
-		want              uint32
-	}{
-		{current: 12, previous: 10, want: 2},
-		{current: 0, previous: ^uint32(0), want: 1},
-		// A cycle counter reset is not evidence that a complete pass covered
-		// recent moves, so it must not release conservative deltas.
-		{current: 1, previous: 100, want: 0},
-	}
-	for _, tt := range tests {
-		if got := scannerCyclesApart(tt.current, tt.previous); got != tt.want {
-			t.Fatalf("scannerCyclesApart(%d, %d) = %d, want %d", tt.current, tt.previous, got, tt.want)
-		}
-	}
-}
-
-func TestDataUsageCacheV8Migration(t *testing.T) {
-	var encoded bytes.Buffer
-	if err := encoded.WriteByte(dataUsageCacheVerV8); err != nil {
-		t.Fatal(err)
-	}
-	zw, err := zstd.NewWriter(&encoded)
-	if err != nil {
-		t.Fatal(err)
-	}
-	mw := msgp.NewWriter(zw)
-	if err = mw.WriteMapHeader(2); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("Info"); err != nil {
-		t.Fatal(err)
-	}
-	info := dataUsageCacheInfo{Name: dataUsageRoot, NextCycle: 17, LastUpdate: time.Now().UTC()}
-	if err = info.EncodeMsg(mw); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("Cache"); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteMapHeader(1); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("entry"); err != nil {
-		t.Fatal(err)
-	}
-	// A v8 entry has no hts field. Encoding only populated fields also
-	// verifies that its map decoder retains normal msgpack compatibility.
-	if err = mw.WriteMapHeader(2); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("sz"); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteInt64(123); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("os"); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteUint64(2); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if err = zw.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	var migrated dataUsageCache
-	if err = migrated.deserialize(bytes.NewReader(encoded.Bytes())); err != nil {
-		t.Fatal(err)
-	}
-	entry := migrated.Cache["entry"]
-	if entry.Size != 123 || entry.Objects != 2 || entry.HotTierSize != 0 {
-		t.Fatalf("migrated entry = size:%d objects:%d hot:%d", entry.Size, entry.Objects, entry.HotTierSize)
-	}
-	if migrated.Info.NextCycle != 17 || !migrated.Info.LastUpdate.Equal(info.LastUpdate) {
-		t.Fatalf("migrated cache info = %+v", migrated.Info)
-	}
-}
-
-// The stamp written on every moved version and the stamp the rollback path
-// matches against must agree, otherwise rollback silently skips its own work.
-func TestAccessTierStampRoundTrip(t *testing.T) {
-	movedAt := time.Now().UnixNano()
-	stamp := accessTierStamp(3, movedAt)
-
-	pool, at, ok := parseAccessTierStamp(map[string]string{accessTierMetadataKey: stamp})
-	if !ok {
-		t.Fatalf("stamp %q did not parse", stamp)
-	}
-	if pool != 3 {
-		t.Fatalf("pool = %d, want 3", pool)
-	}
-	if at.UnixNano() != movedAt {
-		t.Fatalf("movedAt = %d, want %d", at.UnixNano(), movedAt)
-	}
-	// A different move of the same object must not match, so a concurrent
-	// client overwrite is never mistaken for our own copy.
-	if stamp == accessTierStamp(3, movedAt+1) {
-		t.Fatal("stamps from distinct moves collided")
-	}
-	if stamp == accessTierStamp(4, movedAt) {
-		t.Fatal("stamps from distinct pools collided")
-	}
-}
-
-func TestAccessHitsMightPromote(t *testing.T) {
-	lc := accessLifecycleForTest(t)
-	hot := accessEntry{Bins: []uint32{100}, HeadAt: 0}
-	cold := accessEntry{Bins: []uint32{1}, HeadAt: 0}
-	if !accessHitsMightPromote(lc, "object", hot, 60) {
-		t.Fatal("object above promote threshold was rejected")
-	}
-	if accessHitsMightPromote(lc, "object", cold, 60) {
-		t.Fatal("object below promote threshold was accepted")
-	}
-
-	xml := "<LifecycleConfiguration><Rule><ID>logs</ID><Status>Enabled</Status>" +
-		"<Filter><Prefix>logs/</Prefix></Filter>" +
-		"<AccessTransition><Window>10m</Window><PromoteAfterAccesses>100</PromoteAfterAccesses>" +
-		"<DemoteAfterAccesses>5</DemoteAfterAccesses><DemoteAfterIdle>1h</DemoteAfterIdle></AccessTransition>" +
-		"</Rule></LifecycleConfiguration>"
-	prefixed, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if accessHitsMightPromote(prefixed, "data/object", hot, 60) {
-		t.Fatal("object outside the rule prefix was accepted")
-	}
-	if !accessHitsMightPromote(prefixed, "logs/object", hot, 60) {
-		t.Fatal("object inside the rule prefix was rejected")
-	}
-}

cmd/ilm-access-tracker.go

@@ -1,598 +0,0 @@
-// Copyright (c) 2015-2026 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"cmp"
-	"context"
-	"fmt"
-	"slices"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/minio/minio/internal/config/ilm"
-	"github.com/zeebo/xxh3"
-)
-
-//go:generate msgp -file=$GOFILE -unexported
-//msgp:ignore accessTracker mergedAccess
-
-const (
-	// accessTrackerPrefix is where each node publishes its own counters.
-	// One object per node, merged by every node on a timer.
-	accessTrackerPrefix = minioConfigPrefix + "/ilm/access"
-
-	// accessQueueSize bounds the GET -> tracker handoff. Overflow drops
-	// samples rather than slowing down reads.
-	accessQueueSize = 100000
-
-	// accessMaxDemoteCandidates bounds what the scanner may hand over in a
-	// single flush interval.
-	accessMaxDemoteCandidates = 100000
-
-	// accessShardStaleFactor multiplies the flush interval to decide when a
-	// peer's counters are too old to trust, e.g. after a node is removed.
-	accessShardStaleFactor = 5
-)
-
-func accessShardFresh(now int64, shard accessShard, stale int64) bool {
-	if shard.UpdatedAt <= 0 {
-		return false
-	}
-	if stale <= 0 {
-		return true
-	}
-	age := now - shard.UpdatedAt
-	return age >= -stale && age <= stale
-}
-
-// accessEntry is a rolling hit counter for one object. Bins[0] is the current
-// bin and each subsequent bin is one bin-width older, so a rule asking for
-// "100 hits in 10 minutes" sums the newest ceil(10m/binWidth) bins.
-//
-// A fixed window is used rather than an exponentially decayed score because
-// the rule is stated to operators in exactly those terms.
-type accessEntry struct {
-	Bins   []uint32 `msg:"b"`
-	HeadAt int64    `msg:"h"` // unix seconds at the start of Bins[0]
-	LastAt int64    `msg:"l"` // unix seconds of the most recent hit
-}
-
-// demoteCandidate is an object the scanner found sitting on a fast pool with
-// no recent reads. Candidates ride the node's own counter shard so they reach
-// the leader without a new peer RPC.
-type demoteCandidate struct {
-	Bucket string `msg:"b"`
-	Object string `msg:"o"`
-	Pool   int    `msg:"p"`
-}
-
-// accessShard is what one node publishes. BinWidth is carried so a peer that
-// has not yet picked up a configuration change is ignored rather than merged
-// with mismatched bins.
-type accessShard struct {
-	UpdatedAt int64                  `msg:"u"`
-	BinWidth  int64                  `msg:"bw"`
-	Entries   map[string]accessEntry `msg:"e"`
-	Demote    []demoteCandidate      `msg:"d"`
-}
-
-// binStart truncates a unix timestamp to the start of its bin.
-func binStart(now, binWidth int64) int64 {
-	if binWidth <= 0 {
-		return now
-	}
-	return now - now%binWidth
-}
-
-// rollTo advances the counter to now, zeroing the bins that elapsed since the
-// last update and resizing if the configured bin count changed.
-func (e *accessEntry) rollTo(now, binWidth int64, nbins int) {
-	if nbins <= 0 || binWidth <= 0 {
-		return
-	}
-	if len(e.Bins) != nbins {
-		resized := make([]uint32, nbins)
-		copy(resized, e.Bins)
-		e.Bins = resized
-	}
-	head := binStart(now, binWidth)
-	if e.HeadAt == 0 {
-		e.HeadAt = head
-		return
-	}
-	steps := (head - e.HeadAt) / binWidth
-	if steps <= 0 {
-		return
-	}
-	if steps >= int64(nbins) {
-		clear(e.Bins)
-	} else {
-		copy(e.Bins[steps:], e.Bins[:nbins-int(steps)])
-		clear(e.Bins[:steps])
-	}
-	e.HeadAt = head
-}
-
-// hits returns the number of accesses recorded over the newest bins covering
-// window. A window longer than the configured history is clamped to it.
-//
-// The newest bin is partial, so the covered span is between window-binWidth
-// and window. Operators tune resolution with ilm access_bin_width.
-func (e accessEntry) hits(window time.Duration, binWidth int64) uint64 {
-	if binWidth <= 0 || len(e.Bins) == 0 {
-		return 0
-	}
-	n := int((int64(window/time.Second) + binWidth - 1) / binWidth)
-	if n < 1 {
-		n = 1
-	}
-	if n > len(e.Bins) {
-		n = len(e.Bins)
-	}
-	var total uint64
-	for _, v := range e.Bins[:n] {
-		total += uint64(v)
-	}
-	return total
-}
-
-// total is the whole retained history, used to decide what to evict.
-func (e accessEntry) total() uint64 {
-	var t uint64
-	for _, v := range e.Bins {
-		t += uint64(v)
-	}
-	return t
-}
-
-// mergeFrom adds another node's counters for the same object. Both sides must
-// already be rolled to the same head.
-func (e *accessEntry) mergeFrom(o accessEntry) {
-	for i := range e.Bins {
-		if i < len(o.Bins) {
-			total := uint64(e.Bins[i]) + uint64(o.Bins[i])
-			if total > uint64(^uint32(0)) {
-				total = uint64(^uint32(0))
-			}
-			e.Bins[i] = uint32(total)
-		}
-	}
-	if o.LastAt > e.LastAt {
-		e.LastAt = o.LastAt
-	}
-}
-
-// mergedAccess is an immutable cluster-wide snapshot. Readers take it from an
-// atomic pointer, so the hot scanner and sweep paths never take a lock.
-type mergedAccess struct {
-	entries  map[string]accessEntry
-	binWidth int64
-	at       int64
-}
-
-func (m *mergedAccess) hits(key string, window time.Duration) uint64 {
-	if m == nil {
-		return 0
-	}
-	e, ok := m.entries[key]
-	if !ok {
-		return 0
-	}
-	return e.hits(window, m.binWidth)
-}
-
-func (m *mergedAccess) lastAccess(key string) int64 {
-	if m == nil {
-		return 0
-	}
-	return m.entries[key].LastAt
-}
-
-// accessTracker records how often each object is read.
-//
-// Ownership is deliberately narrow: the live counter map is touched only by
-// run()'s goroutine, so it needs no lock. Everything read from elsewhere goes
-// through the immutable merged snapshot.
-type accessTracker struct {
-	ch      chan string
-	enabled atomic.Bool
-	merged  atomic.Pointer[mergedAccess]
-
-	// Demote candidates arrive from scanner goroutines, so this one does
-	// need a lock. It is small: only objects we previously promoted.
-	demoteMu sync.Mutex
-	demote   map[string]demoteCandidate
-
-	dropped atomic.Uint64
-}
-
-var globalAccessTracker = newAccessTracker()
-
-func newAccessTracker() *accessTracker {
-	return &accessTracker{
-		ch:     make(chan string, accessQueueSize),
-		demote: make(map[string]demoteCandidate),
-	}
-}
-
-// accessKey is the tracker's map key. Bucket names cannot contain '/', so the
-// join is unambiguous.
-func accessKey(bucket, object string) string {
-	return bucket + "/" + object
-}
-
-func splitAccessKey(key string) (bucket, object string, ok bool) {
-	bucket, object, ok = strings.Cut(key, "/")
-	if !ok || bucket == "" || object == "" {
-		return "", "", false
-	}
-	return bucket, object, true
-}
-
-// note records one read. It is called from the GET path and must never block
-// or allocate meaningfully: on a full queue the sample is dropped.
-func (t *accessTracker) note(bucket, object string) {
-	if t == nil || !t.enabled.Load() {
-		return
-	}
-	select {
-	case t.ch <- accessKey(bucket, object):
-	default:
-		t.dropped.Add(1)
-	}
-}
-
-// noteDemoteCandidate is called by the scanner for an object it found on a
-// fast pool that has gone quiet. The leader picks these up on the next merge.
-func (t *accessTracker) noteDemoteCandidate(bucket, object string, pool int) {
-	if t == nil || !t.enabled.Load() {
-		return
-	}
-	t.demoteMu.Lock()
-	defer t.demoteMu.Unlock()
-	if len(t.demote) >= accessMaxDemoteCandidates {
-		return
-	}
-	t.demote[accessKey(bucket, object)] = demoteCandidate{Bucket: bucket, Object: object, Pool: pool}
-}
-
-// hits reports cluster-wide accesses to an object over window.
-func (t *accessTracker) hits(bucket, object string, window time.Duration) uint64 {
-	if t == nil {
-		return 0
-	}
-	return t.merged.Load().hits(accessKey(bucket, object), window)
-}
-
-// lastAccess reports the cluster-wide time an object was last read. A zero
-// time means "no read on record", which for demotion purposes is idle.
-func (t *accessTracker) lastAccess(bucket, object string) time.Time {
-	if t == nil {
-		return time.Time{}
-	}
-	sec := t.merged.Load().lastAccess(accessKey(bucket, object))
-	if sec == 0 {
-		return time.Time{}
-	}
-	return time.Unix(sec, 0)
-}
-
-// snapshot returns the current merged view, or nil if none has been published.
-func (t *accessTracker) snapshot() *mergedAccess {
-	if t == nil {
-		return nil
-	}
-	return t.merged.Load()
-}
-
-// takeDemoteCandidates drains and returns the pending candidates.
-func (t *accessTracker) takeDemoteCandidates() []demoteCandidate {
-	t.demoteMu.Lock()
-	defer t.demoteMu.Unlock()
-	if len(t.demote) == 0 {
-		return nil
-	}
-	out := make([]demoteCandidate, 0, len(t.demote))
-	for _, c := range t.demote {
-		out = append(out, c)
-	}
-	t.demote = make(map[string]demoteCandidate)
-	return out
-}
-
-// restoreDemoteCandidates puts candidates back when the publisher is still
-// busy. Dropping access samples is acceptable; dropping the only scanner
-// discovery of an idle promoted object would delay demotion by a full scan.
-func (t *accessTracker) restoreDemoteCandidates(candidates []demoteCandidate) {
-	if len(candidates) == 0 {
-		return
-	}
-	t.demoteMu.Lock()
-	defer t.demoteMu.Unlock()
-	for _, c := range candidates {
-		if len(t.demote) >= accessMaxDemoteCandidates {
-			return
-		}
-		t.demote[accessKey(c.Bucket, c.Object)] = c
-	}
-}
-
-// shardName is this node's counter object. The node name is hashed so that
-// host:port never has to be escaped into an object key.
-func (t *accessTracker) shardName() string {
-	return fmt.Sprintf("%s/%016x.bin", accessTrackerPrefix, xxh3.HashString(globalLocalNodeName))
-}
-
-// run owns the live counter map. It drains reads, and on every flush interval
-// hands a marshaled shard to a background publisher.
-//
-// Everything here is best effort: this is accounting for a background data
-// movement decision, not a durability path.
-func (t *accessTracker) run(ctx context.Context, objAPI ObjectLayer) {
-	cfg := globalILMConfig.accessCfg()
-	t.enabled.Store(cfg.AccessTiering)
-
-	live := make(map[string]accessEntry)
-	if cfg.AccessTiering {
-		if buf, err := readConfig(ctx, objAPI, t.shardName()); err == nil {
-			var previous accessShard
-			if _, err = previous.UnmarshalMsg(buf); err == nil && previous.BinWidth == int64(cfg.AccessBinWidth/time.Second) {
-				now := time.Now().Unix()
-				for key, entry := range previous.Entries {
-					entry.rollTo(now, previous.BinWidth, cfg.AccessBins)
-					if entry.total() != 0 {
-						live[key] = entry
-					}
-				}
-			}
-		}
-	}
-	ticker := time.NewTicker(cfg.AccessFlush)
-	defer ticker.Stop()
-
-	// One publisher goroutine keeps object-layer I/O off the drain loop.
-	type pub struct {
-		shard accessShard
-		cfg   ilm.Config
-	}
-	pubCh := make(chan pub, 1)
-	go func() {
-		for p := range pubCh {
-			t.publish(ctx, objAPI, p.shard, p.cfg)
-		}
-	}()
-	defer close(pubCh)
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-
-		case key := <-t.ch:
-			now := time.Now().Unix()
-			e := live[key]
-			e.rollTo(now, int64(cfg.AccessBinWidth/time.Second), cfg.AccessBins)
-			if len(e.Bins) > 0 {
-				if e.Bins[0] != ^uint32(0) {
-					e.Bins[0]++
-				}
-			}
-			e.LastAt = now
-			live[key] = e
-
-		case <-ticker.C:
-			newCfg := globalILMConfig.accessCfg()
-			t.enabled.Store(newCfg.AccessTiering)
-			if newCfg.AccessFlush != cfg.AccessFlush && newCfg.AccessFlush > 0 {
-				ticker.Reset(newCfg.AccessFlush)
-			}
-			cfg = newCfg
-			if !cfg.AccessTiering {
-				// Feature turned off: release the counters rather than
-				// holding a stale working set for the process lifetime.
-				clear(live)
-				t.merged.Store(nil)
-				continue
-			}
-
-			now := time.Now().Unix()
-			binWidth := int64(cfg.AccessBinWidth / time.Second)
-			t.evict(live, now, binWidth, cfg)
-
-			shard := accessShard{
-				UpdatedAt: now,
-				BinWidth:  binWidth,
-				Entries:   make(map[string]accessEntry, len(live)),
-				Demote:    t.takeDemoteCandidates(),
-			}
-			for k, e := range live {
-				e.Bins = slices.Clone(e.Bins)
-				shard.Entries[k] = e
-			}
-			select {
-			case pubCh <- pub{shard: shard, cfg: cfg}:
-			default:
-				// Previous publish still running; skip this round
-				// rather than queueing work we cannot keep up with.
-				t.restoreDemoteCandidates(shard.Demote)
-			}
-		}
-	}
-}
-
-// evict rolls every counter forward and drops the ones with no hits left in
-// the retained history, then enforces the tracked-object cap.
-//
-// ponytail: amortized O(n) sweep on the flush tick; a heap would only pay off
-// past ~10M tracked keys.
-func (t *accessTracker) evict(live map[string]accessEntry, now, binWidth int64, cfg ilm.Config) {
-	for k, e := range live {
-		e.rollTo(now, binWidth, cfg.AccessBins)
-		if e.total() == 0 {
-			delete(live, k)
-			continue
-		}
-		live[k] = e
-	}
-	if len(live) <= cfg.AccessMaxTracked {
-		return
-	}
-	// Over the cap: keep the hottest and drop the rest. Objects that fall
-	// out are cold by construction, which is exactly what the demotion path
-	// already assumes about anything missing from the map.
-	type kt struct {
-		key   string
-		total uint64
-	}
-	all := make([]kt, 0, len(live))
-	for k, e := range live {
-		all = append(all, kt{k, e.total()})
-	}
-	slices.SortFunc(all, func(a, b kt) int { return cmp.Compare(b.total, a.total) })
-	for _, x := range all[cfg.AccessMaxTracked:] {
-		delete(live, x.key)
-	}
-}
-
-// publish writes this node's shard and rebuilds the merged snapshot from every
-// node's shard. Both halves are best effort.
-func (t *accessTracker) publish(ctx context.Context, objAPI ObjectLayer, shard accessShard, cfg ilm.Config) {
-	buf, err := shard.MarshalMsg(nil)
-	if err != nil {
-		ilmLogIf(ctx, err)
-		return
-	}
-	if err := saveConfig(ctx, objAPI, t.shardName(), buf); err != nil {
-		ilmLogIf(ctx, err)
-		// Still rebuild the snapshot below: our own counters are already
-		// in hand and a stale peer view beats no view.
-	}
-	t.merged.Store(t.mergeShards(ctx, objAPI, shard, cfg))
-}
-
-// mergeShards sums every live node's counters, including our own in-memory
-// shard so this node's most recent reads are never a flush behind.
-func (t *accessTracker) mergeShards(ctx context.Context, objAPI ObjectLayer, own accessShard, cfg ilm.Config) *mergedAccess {
-	now := time.Now().Unix()
-	binWidth := int64(cfg.AccessBinWidth / time.Second)
-	out := &mergedAccess{
-		entries:  make(map[string]accessEntry, len(own.Entries)),
-		binWidth: binWidth,
-		at:       now,
-	}
-
-	add := func(s accessShard) {
-		if s.BinWidth != binWidth {
-			// A peer has not yet picked up a bin-width change; merging
-			// its bins would silently mis-scale the counts.
-			return
-		}
-		for k, e := range s.Entries {
-			e.rollTo(now, binWidth, cfg.AccessBins)
-			cur, ok := out.entries[k]
-			if !ok {
-				cur = accessEntry{Bins: make([]uint32, cfg.AccessBins)}
-			}
-			cur.mergeFrom(e)
-			out.entries[k] = cur
-		}
-	}
-
-	add(own)
-
-	ownName := t.shardName()
-	stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor
-	for _, name := range t.listShards(ctx, objAPI) {
-		if name == ownName {
-			continue
-		}
-		buf, err := readConfig(ctx, objAPI, name)
-		if err != nil {
-			continue
-		}
-		var s accessShard
-		if _, err := s.UnmarshalMsg(buf); err != nil {
-			continue
-		}
-		if !accessShardFresh(now, s, stale) {
-			continue // node is gone or wedged
-		}
-		add(s)
-	}
-	return out
-}
-
-func (t *accessTracker) listShards(ctx context.Context, objAPI ObjectLayer) []string {
-	res, err := objAPI.ListObjects(ctx, minioMetaBucket, accessTrackerPrefix+"/", "", "", maxObjectList)
-	if err != nil {
-		return nil
-	}
-	names := make([]string, 0, len(res.Objects))
-	for _, o := range res.Objects {
-		if strings.HasSuffix(o.Name, ".bin") {
-			names = append(names, o.Name)
-		}
-	}
-	return names
-}
-
-// collectDemoteCandidates returns every node's pending demote candidates. Only
-// the leader calls this, right before running a demotion pass.
-//
-// Our own shard is read back rather than skipped: run() drains the local map
-// into the published shard, so anything found since the last leader pass lives
-// there, not in memory. The local map is still drained here to pick up
-// candidates recorded since that publish.
-func (t *accessTracker) collectDemoteCandidates(ctx context.Context, objAPI ObjectLayer, cfg ilm.Config) []demoteCandidate {
-	seen := make(map[string]struct{})
-	var out []demoteCandidate
-
-	for _, c := range t.takeDemoteCandidates() {
-		key := accessKey(c.Bucket, c.Object)
-		seen[key] = struct{}{}
-		out = append(out, c)
-	}
-
-	now := time.Now().Unix()
-	stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor
-	for _, name := range t.listShards(ctx, objAPI) {
-		buf, err := readConfig(ctx, objAPI, name)
-		if err != nil {
-			continue
-		}
-		var s accessShard
-		if _, err := s.UnmarshalMsg(buf); err != nil {
-			continue
-		}
-		if !accessShardFresh(now, s, stale) {
-			continue
-		}
-		for _, c := range s.Demote {
-			key := accessKey(c.Bucket, c.Object)
-			if _, dup := seen[key]; dup {
-				continue
-			}
-			seen[key] = struct{}{}
-			out = append(out, c)
-		}
-	}
-	return out
-}

cmd/ilm-access-tracker_gen.go

@@ -1,742 +0,0 @@
-// Code generated by github.com/tinylib/msgp DO NOT EDIT.
-
-package cmd
-
-import (
-	"github.com/tinylib/msgp/msgp"
-)
-
-// DecodeMsg implements msgp.Decodable
-func (z *accessEntry) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Bins")
-				return
-			}
-			if cap(z.Bins) >= int(zb0002) {
-				z.Bins = (z.Bins)[:zb0002]
-			} else {
-				z.Bins = make([]uint32, zb0002)
-			}
-			for za0001 := range z.Bins {
-				z.Bins[za0001], err = dc.ReadUint32()
-				if err != nil {
-					err = msgp.WrapError(err, "Bins", za0001)
-					return
-				}
-			}
-		case "h":
-			z.HeadAt, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "HeadAt")
-				return
-			}
-		case "l":
-			z.LastAt, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "LastAt")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *accessEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
-	// write "b"
-	err = en.Append(0x83, 0xa1, 0x62)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.Bins)))
-	if err != nil {
-		err = msgp.WrapError(err, "Bins")
-		return
-	}
-	for za0001 := range z.Bins {
-		err = en.WriteUint32(z.Bins[za0001])
-		if err != nil {
-			err = msgp.WrapError(err, "Bins", za0001)
-			return
-		}
-	}
-	// write "h"
-	err = en.Append(0xa1, 0x68)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.HeadAt)
-	if err != nil {
-		err = msgp.WrapError(err, "HeadAt")
-		return
-	}
-	// write "l"
-	err = en.Append(0xa1, 0x6c)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.LastAt)
-	if err != nil {
-		err = msgp.WrapError(err, "LastAt")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *accessEntry) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
-	// string "b"
-	o = append(o, 0x83, 0xa1, 0x62)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.Bins)))
-	for za0001 := range z.Bins {
-		o = msgp.AppendUint32(o, z.Bins[za0001])
-	}
-	// string "h"
-	o = append(o, 0xa1, 0x68)
-	o = msgp.AppendInt64(o, z.HeadAt)
-	// string "l"
-	o = append(o, 0xa1, 0x6c)
-	o = msgp.AppendInt64(o, z.LastAt)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *accessEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Bins")
-				return
-			}
-			if cap(z.Bins) >= int(zb0002) {
-				z.Bins = (z.Bins)[:zb0002]
-			} else {
-				z.Bins = make([]uint32, zb0002)
-			}
-			for za0001 := range z.Bins {
-				z.Bins[za0001], bts, err = msgp.ReadUint32Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Bins", za0001)
-					return
-				}
-			}
-		case "h":
-			z.HeadAt, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "HeadAt")
-				return
-			}
-		case "l":
-			z.LastAt, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "LastAt")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *accessEntry) Msgsize() (s int) {
-	s = 1 + 2 + msgp.ArrayHeaderSize + (len(z.Bins) * (msgp.Uint32Size)) + 2 + msgp.Int64Size + 2 + msgp.Int64Size
-	return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *accessShard) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "u":
-			z.UpdatedAt, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "UpdatedAt")
-				return
-			}
-		case "bw":
-			z.BinWidth, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "BinWidth")
-				return
-			}
-		case "e":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Entries")
-				return
-			}
-			if z.Entries == nil {
-				z.Entries = make(map[string]accessEntry, zb0002)
-			} else if len(z.Entries) > 0 {
-				clear(z.Entries)
-			}
-			for zb0002 > 0 {
-				zb0002--
-				var za0001 string
-				za0001, err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Entries")
-					return
-				}
-				var za0002 accessEntry
-				err = za0002.DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "Entries", za0001)
-					return
-				}
-				z.Entries[za0001] = za0002
-			}
-		case "d":
-			var zb0003 uint32
-			zb0003, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Demote")
-				return
-			}
-			if cap(z.Demote) >= int(zb0003) {
-				z.Demote = (z.Demote)[:zb0003]
-			} else {
-				z.Demote = make([]demoteCandidate, zb0003)
-			}
-			for za0003 := range z.Demote {
-				var zb0004 uint32
-				zb0004, err = dc.ReadMapHeader()
-				if err != nil {
-					err = msgp.WrapError(err, "Demote", za0003)
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "Demote", za0003)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "b":
-						z.Demote[za0003].Bucket, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Bucket")
-							return
-						}
-					case "o":
-						z.Demote[za0003].Object, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Object")
-							return
-						}
-					case "p":
-						z.Demote[za0003].Pool, err = dc.ReadInt()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Pool")
-							return
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003)
-							return
-						}
-					}
-				}
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *accessShard) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 4
-	// write "u"
-	err = en.Append(0x84, 0xa1, 0x75)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.UpdatedAt)
-	if err != nil {
-		err = msgp.WrapError(err, "UpdatedAt")
-		return
-	}
-	// write "bw"
-	err = en.Append(0xa2, 0x62, 0x77)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.BinWidth)
-	if err != nil {
-		err = msgp.WrapError(err, "BinWidth")
-		return
-	}
-	// write "e"
-	err = en.Append(0xa1, 0x65)
-	if err != nil {
-		return
-	}
-	err = en.WriteMapHeader(uint32(len(z.Entries)))
-	if err != nil {
-		err = msgp.WrapError(err, "Entries")
-		return
-	}
-	for za0001, za0002 := range z.Entries {
-		err = en.WriteString(za0001)
-		if err != nil {
-			err = msgp.WrapError(err, "Entries")
-			return
-		}
-		err = za0002.EncodeMsg(en)
-		if err != nil {
-			err = msgp.WrapError(err, "Entries", za0001)
-			return
-		}
-	}
-	// write "d"
-	err = en.Append(0xa1, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.Demote)))
-	if err != nil {
-		err = msgp.WrapError(err, "Demote")
-		return
-	}
-	for za0003 := range z.Demote {
-		// map header, size 3
-		// write "b"
-		err = en.Append(0x83, 0xa1, 0x62)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Demote[za0003].Bucket)
-		if err != nil {
-			err = msgp.WrapError(err, "Demote", za0003, "Bucket")
-			return
-		}
-		// write "o"
-		err = en.Append(0xa1, 0x6f)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Demote[za0003].Object)
-		if err != nil {
-			err = msgp.WrapError(err, "Demote", za0003, "Object")
-			return
-		}
-		// write "p"
-		err = en.Append(0xa1, 0x70)
-		if err != nil {
-			return
-		}
-		err = en.WriteInt(z.Demote[za0003].Pool)
-		if err != nil {
-			err = msgp.WrapError(err, "Demote", za0003, "Pool")
-			return
-		}
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *accessShard) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
-	// string "u"
-	o = append(o, 0x84, 0xa1, 0x75)
-	o = msgp.AppendInt64(o, z.UpdatedAt)
-	// string "bw"
-	o = append(o, 0xa2, 0x62, 0x77)
-	o = msgp.AppendInt64(o, z.BinWidth)
-	// string "e"
-	o = append(o, 0xa1, 0x65)
-	o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
-	for za0001, za0002 := range z.Entries {
-		o = msgp.AppendString(o, za0001)
-		o, err = za0002.MarshalMsg(o)
-		if err != nil {
-			err = msgp.WrapError(err, "Entries", za0001)
-			return
-		}
-	}
-	// string "d"
-	o = append(o, 0xa1, 0x64)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.Demote)))
-	for za0003 := range z.Demote {
-		// map header, size 3
-		// string "b"
-		o = append(o, 0x83, 0xa1, 0x62)
-		o = msgp.AppendString(o, z.Demote[za0003].Bucket)
-		// string "o"
-		o = append(o, 0xa1, 0x6f)
-		o = msgp.AppendString(o, z.Demote[za0003].Object)
-		// string "p"
-		o = append(o, 0xa1, 0x70)
-		o = msgp.AppendInt(o, z.Demote[za0003].Pool)
-	}
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *accessShard) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "u":
-			z.UpdatedAt, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "UpdatedAt")
-				return
-			}
-		case "bw":
-			z.BinWidth, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "BinWidth")
-				return
-			}
-		case "e":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Entries")
-				return
-			}
-			if z.Entries == nil {
-				z.Entries = make(map[string]accessEntry, zb0002)
-			} else if len(z.Entries) > 0 {
-				clear(z.Entries)
-			}
-			for zb0002 > 0 {
-				var za0002 accessEntry
-				zb0002--
-				var za0001 string
-				za0001, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Entries")
-					return
-				}
-				bts, err = za0002.UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Entries", za0001)
-					return
-				}
-				z.Entries[za0001] = za0002
-			}
-		case "d":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Demote")
-				return
-			}
-			if cap(z.Demote) >= int(zb0003) {
-				z.Demote = (z.Demote)[:zb0003]
-			} else {
-				z.Demote = make([]demoteCandidate, zb0003)
-			}
-			for za0003 := range z.Demote {
-				var zb0004 uint32
-				zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Demote", za0003)
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Demote", za0003)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "b":
-						z.Demote[za0003].Bucket, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Bucket")
-							return
-						}
-					case "o":
-						z.Demote[za0003].Object, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Object")
-							return
-						}
-					case "p":
-						z.Demote[za0003].Pool, bts, err = msgp.ReadIntBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Pool")
-							return
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003)
-							return
-						}
-					}
-				}
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *accessShard) Msgsize() (s int) {
-	s = 1 + 2 + msgp.Int64Size + 3 + msgp.Int64Size + 2 + msgp.MapHeaderSize
-	if z.Entries != nil {
-		for za0001, za0002 := range z.Entries {
-			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
-		}
-	}
-	s += 2 + msgp.ArrayHeaderSize
-	for za0003 := range z.Demote {
-		s += 1 + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Bucket) + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Object) + 2 + msgp.IntSize
-	}
-	return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *demoteCandidate) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			z.Bucket, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Bucket")
-				return
-			}
-		case "o":
-			z.Object, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Object")
-				return
-			}
-		case "p":
-			z.Pool, err = dc.ReadInt()
-			if err != nil {
-				err = msgp.WrapError(err, "Pool")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z demoteCandidate) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
-	// write "b"
-	err = en.Append(0x83, 0xa1, 0x62)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Bucket)
-	if err != nil {
-		err = msgp.WrapError(err, "Bucket")
-		return
-	}
-	// write "o"
-	err = en.Append(0xa1, 0x6f)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Object)
-	if err != nil {
-		err = msgp.WrapError(err, "Object")
-		return
-	}
-	// write "p"
-	err = en.Append(0xa1, 0x70)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.Pool)
-	if err != nil {
-		err = msgp.WrapError(err, "Pool")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z demoteCandidate) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
-	// string "b"
-	o = append(o, 0x83, 0xa1, 0x62)
-	o = msgp.AppendString(o, z.Bucket)
-	// string "o"
-	o = append(o, 0xa1, 0x6f)
-	o = msgp.AppendString(o, z.Object)
-	// string "p"
-	o = append(o, 0xa1, 0x70)
-	o = msgp.AppendInt(o, z.Pool)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *demoteCandidate) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Bucket")
-				return
-			}
-		case "o":
-			z.Object, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Object")
-				return
-			}
-		case "p":
-			z.Pool, bts, err = msgp.ReadIntBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Pool")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z demoteCandidate) Msgsize() (s int) {
-	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 2 + msgp.IntSize
-	return
-}

cmd/ilm-access-tracker_gen_test.go

@@ -1,349 +0,0 @@
-// Code generated by github.com/tinylib/msgp DO NOT EDIT.
-
-package cmd
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/tinylib/msgp/msgp"
-)
-
-func TestMarshalUnmarshalaccessEntry(t *testing.T) {
-	v := accessEntry{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeaccessEntry(t *testing.T) {
-	v := accessEntry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeaccessEntry Msgsize() is inaccurate")
-	}
-
-	vn := accessEntry{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalaccessShard(t *testing.T) {
-	v := accessShard{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgaccessShard(b *testing.B) {
-	v := accessShard{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgaccessShard(b *testing.B) {
-	v := accessShard{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalaccessShard(b *testing.B) {
-	v := accessShard{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeaccessShard(t *testing.T) {
-	v := accessShard{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeaccessShard Msgsize() is inaccurate")
-	}
-
-	vn := accessShard{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeaccessShard(b *testing.B) {
-	v := accessShard{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeaccessShard(b *testing.B) {
-	v := accessShard{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshaldemoteCandidate(t *testing.T) {
-	v := demoteCandidate{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgdemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgdemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshaldemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodedemoteCandidate(t *testing.T) {
-	v := demoteCandidate{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodedemoteCandidate Msgsize() is inaccurate")
-	}
-
-	vn := demoteCandidate{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodedemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodedemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

cmd/ilm-access-tracker_test.go

@@ -1,95 +0,0 @@
-// Copyright (c) 2015-2026 MinIO, Inc.
-
-package cmd
-
-import (
-	"math"
-	"testing"
-	"time"
-
-	"github.com/minio/minio/internal/config/ilm"
-)
-
-func TestAccessEntryRollAndHits(t *testing.T) {
-	entry := accessEntry{Bins: []uint32{3, 2, 1}, HeadAt: 100, LastAt: 107}
-	entry.rollTo(120, 10, 3)
-	want := []uint32{0, 0, 3}
-	for i := range want {
-		if entry.Bins[i] != want[i] {
-			t.Fatalf("bins = %v, want %v", entry.Bins, want)
-		}
-	}
-	if entry.HeadAt != 120 || entry.LastAt != 107 {
-		t.Fatalf("head/last = %d/%d", entry.HeadAt, entry.LastAt)
-	}
-
-	entry = accessEntry{Bins: []uint32{10, 20, 30}, HeadAt: 120}
-	if got := entry.hits(20*time.Second, 10); got != 30 {
-		t.Fatalf("20s hits = %d, want 30", got)
-	}
-	if got := entry.hits(21*time.Second, 10); got != 60 {
-		t.Fatalf("21s hits = %d, want 60", got)
-	}
-	entry.rollTo(200, 10, 3)
-	if got := entry.total(); got != 0 {
-		t.Fatalf("expired total = %d, want 0", got)
-	}
-}
-
-func TestAccessEntryMergeSaturates(t *testing.T) {
-	entry := accessEntry{Bins: []uint32{math.MaxUint32 - 1}}
-	entry.mergeFrom(accessEntry{Bins: []uint32{10}, LastAt: 50})
-	if entry.Bins[0] != math.MaxUint32 || entry.LastAt != 50 {
-		t.Fatalf("merged entry = %+v", entry)
-	}
-}
-
-func TestAccessTrackerEvictsColdest(t *testing.T) {
-	tracker := newAccessTracker()
-	live := map[string]accessEntry{
-		"a": {Bins: []uint32{1}, HeadAt: 100},
-		"b": {Bins: []uint32{5}, HeadAt: 100},
-		"c": {Bins: []uint32{3}, HeadAt: 100},
-	}
-	tracker.evict(live, 100, 10, ilm.Config{AccessBins: 1, AccessMaxTracked: 2})
-	if len(live) != 2 {
-		t.Fatalf("len = %d, want 2", len(live))
-	}
-	if _, ok := live["a"]; ok {
-		t.Fatal("coldest entry was retained")
-	}
-}
-
-func TestAccessKeyRoundTrip(t *testing.T) {
-	key := accessKey("bucket", "a/b/c")
-	bucket, object, ok := splitAccessKey(key)
-	if !ok || bucket != "bucket" || object != "a/b/c" {
-		t.Fatalf("split %q = %q/%q/%v", key, bucket, object, ok)
-	}
-}
-
-func TestAccessShardFresh(t *testing.T) {
-	const now = int64(1000)
-	if !accessShardFresh(now, accessShard{UpdatedAt: 950}, 100) {
-		t.Fatal("fresh shard rejected")
-	}
-	if accessShardFresh(now, accessShard{UpdatedAt: 899}, 100) {
-		t.Fatal("stale shard accepted")
-	}
-	if accessShardFresh(now, accessShard{UpdatedAt: 1101}, 100) {
-		t.Fatal("far-future shard accepted")
-	}
-	if accessShardFresh(now, accessShard{}, 100) {
-		t.Fatal("zero timestamp accepted")
-	}
-}
-
-func TestAccessTrackerRestoresDemoteCandidates(t *testing.T) {
-	tracker := newAccessTracker()
-	candidate := demoteCandidate{Bucket: "bucket", Object: "object", Pool: 1}
-	tracker.restoreDemoteCandidates([]demoteCandidate{candidate})
-	got := tracker.takeDemoteCandidates()
-	if len(got) != 1 || got[0] != candidate {
-		t.Fatalf("restored candidates = %+v, want %+v", got, candidate)
-	}
-}

cmd/ilm-config.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"sync"
-	"time"
 
 	"github.com/minio/minio/internal/config/ilm"
 )
@@ -28,16 +27,6 @@ var globalILMConfig = ilmConfig{
 	cfg: ilm.Config{
 		ExpirationWorkers: 100,
 		TransitionWorkers: 100,
-		// Access tiering stays off until configured, but the counter
-		// geometry must be sane from the start: the tracker divides by
-		// AccessBinWidth before any config is loaded.
-		AccessPromoteWatermark: 85,
-		AccessBinWidth:         time.Minute,
-		AccessBins:             12,
-		AccessFlush:            time.Minute,
-		AccessMinResidency:     24 * time.Hour,
-		AccessWorkers:          10,
-		AccessMaxTracked:       1000000,
 	},
 }
 
@@ -60,25 +49,6 @@ func (c *ilmConfig) getTransitionWorkers() int {
 	return c.cfg.TransitionWorkers
 }
 
-// accessCfg returns a copy of the access tiering settings. Callers take the
-// whole struct rather than one getter per field because the promotion and
-// demotion paths need a consistent view of several knobs at once.
-func (c *ilmConfig) accessCfg() ilm.Config {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	return c.cfg
-}
-
-// accessTieringEnabled is the cheap gate used on the scanner path.
-func (c *ilmConfig) accessTieringEnabled() bool {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	_, ok := c.cfg.HotPool()
-	return ok
-}
-
 func (c *ilmConfig) update(cfg ilm.Config) {
 	c.mu.Lock()
 	defer c.mu.Unlock()

cmd/lceventsrc_string.go

@@ -19,12 +19,11 @@ func _() {
 	_ = x[lcEventSrc_s3PutObject-8]
 	_ = x[lcEventSrc_s3CopyObject-9]
 	_ = x[lcEventSrc_s3CompleteMultipartUpload-10]
-	_ = x[lcEventSrc_AccessTier-11]
 }
 
-const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUploadAccessTier"
+const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUpload"
 
-var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109, 119}
+var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109}
 
 func (i lcEventSrc) String() string {
 	idx := int(i) - 0

cmd/metrics-v3-cluster-iam.go

@@ -34,6 +34,10 @@ const (
 	sinceLastSyncMillis                    = "since_last_sync_millis"
 	syncFailures                           = "sync_failures"
 	syncSuccesses                          = "sync_successes"
+	revocationRecords                      = "revocation_records"
+	revocationHealFailures                 = "revocation_heal_failures"
+	revocationHealDurationMillis           = "revocation_heal_duration_millis"
+	revocationHealLastSuccess              = "revocation_heal_last_success_timestamp_seconds"
 )
 
 var (
@@ -47,10 +51,20 @@ var (
 	sinceLastSyncMillisMD                    = NewCounterMD(sinceLastSyncMillis, "Time (in milliseconds) since last successful IAM data sync.")
 	syncFailuresMD                           = NewCounterMD(syncFailures, "Number of failed IAM data syncs since server start.")
 	syncSuccessesMD                          = NewCounterMD(syncSuccesses, "Number of successful IAM data syncs since server start.")
+	revocationRecordsMD                      = NewGaugeMD(revocationRecords, "Retained IAM deletion records and revocation boundaries in this node's index.")
+	revocationHealFailuresMD                 = NewCounterMD(revocationHealFailures, "Failed IAM revocation convergence passes since server start.")
+	revocationHealDurationMillisMD           = NewGaugeMD(revocationHealDurationMillis, "Duration of the last IAM revocation convergence pass in milliseconds.")
+	revocationHealLastSuccessMD              = NewGaugeMD(revocationHealLastSuccess, "Unix timestamp of the last successful IAM revocation convergence pass.")
 )
 
 // loadClusterIAMMetrics - `MetricsLoaderFn` for cluster IAM metrics.
 func loadClusterIAMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
+	if globalIAMSys.Initialized() {
+		m.Set(revocationRecords, float64(globalIAMSys.store.revisionIndex().count()))
+	}
+	m.Set(revocationHealFailures, float64(globalSiteReplicationSys.iamRevisionMetrics.healFailures.Load()))
+	m.Set(revocationHealDurationMillis, float64(globalSiteReplicationSys.iamRevisionMetrics.healDurationMillis.Load()))
+	m.Set(revocationHealLastSuccess, float64(globalSiteReplicationSys.iamRevisionMetrics.healLastSuccess.Load()))
 	m.Set(lastSyncDurationMillis, float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)))
 	pluginAuthNMetrics := globalAuthNPlugin.Metrics()
 	m.Set(pluginAuthnServiceFailedRequestsMinute, float64(pluginAuthNMetrics.FailedRequests))

cmd/metrics-v3-ilm.go

@@ -26,17 +26,6 @@ const (
 	transitionActiveTasks          = "transition_active_tasks"
 	transitionPendingTasks         = "transition_pending_tasks"
 	transitionMissedImmediateTasks = "transition_missed_immediate_tasks"
-	accessTierActiveTasks          = "access_tier_active_tasks"
-	accessTierPendingTasks         = "access_tier_pending_tasks"
-	accessTierPromotionsTotal      = "access_tier_promotions_total"
-	accessTierDemotionsTotal       = "access_tier_demotions_total"
-	accessTierBytesMovedTotal      = "access_tier_bytes_moved_total"
-	accessTierFailuresTotal        = "access_tier_failures_total"
-	accessTierSkippedWatermark     = "access_tier_skipped_watermark_total"
-	accessTierSkippedMaxSize       = "access_tier_skipped_max_size_total"
-	accessTierSkippedQuota         = "access_tier_skipped_bucket_quota_total"
-	accessTierHotBytes             = "access_tier_hot_bytes"
-	accessTierSamplesDropped       = "access_tier_samples_dropped_total"
 	versionsScanned                = "versions_scanned"
 )
 
@@ -45,17 +34,6 @@ var (
 	ilmTransitionActiveTasksMD          = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks")
 	ilmTransitionPendingTasksMD         = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue")
 	ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks")
-	ilmAccessTierActiveTasksMD          = NewGaugeMD(accessTierActiveTasks, "Number of active access-tier pool moves")
-	ilmAccessTierPendingTasksMD         = NewGaugeMD(accessTierPendingTasks, "Number of pending access-tier pool moves")
-	ilmAccessTierPromotionsTotalMD      = NewCounterMD(accessTierPromotionsTotal, "Total objects promoted by access-tier ILM")
-	ilmAccessTierDemotionsTotalMD       = NewCounterMD(accessTierDemotionsTotal, "Total objects demoted by access-tier ILM")
-	ilmAccessTierBytesMovedTotalMD      = NewCounterMD(accessTierBytesMovedTotal, "Total logical bytes moved by access-tier ILM")
-	ilmAccessTierFailuresTotalMD        = NewCounterMD(accessTierFailuresTotal, "Total failed access-tier ILM moves")
-	ilmAccessTierSkippedWatermarkMD     = NewCounterMD(accessTierSkippedWatermark, "Promotions skipped because the hot pool reached its watermark")
-	ilmAccessTierSkippedMaxSizeMD       = NewCounterMD(accessTierSkippedMaxSize, "Promotions skipped because the cluster hot-tier size cap was reached")
-	ilmAccessTierSkippedQuotaMD         = NewCounterMD(accessTierSkippedQuota, "Promotions skipped because the bucket hot-tier quota was reached")
-	ilmAccessTierHotBytesMD             = NewGaugeMD(accessTierHotBytes, "Logical bytes currently accounted to the hot tier", "bucket")
-	ilmAccessTierSamplesDroppedMD       = NewCounterMD(accessTierSamplesDropped, "GET samples dropped because the access tracker queue was full")
 	ilmVersionsScannedMD                = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start")
 )
 
@@ -69,21 +47,6 @@ func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
 		m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks()))
 		m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks()))
 	}
-	if globalAccessTierState != nil {
-		m.Set(accessTierActiveTasks, float64(globalAccessTierState.ActiveTasks()))
-		m.Set(accessTierPendingTasks, float64(globalAccessTierState.PendingTasks()))
-		m.Set(accessTierPromotionsTotal, float64(globalAccessTierState.promotions.Load()))
-		m.Set(accessTierDemotionsTotal, float64(globalAccessTierState.demotions.Load()))
-		m.Set(accessTierBytesMovedTotal, float64(globalAccessTierState.bytesMoved.Load()))
-		m.Set(accessTierFailuresTotal, float64(globalAccessTierState.failures.Load()))
-		m.Set(accessTierSkippedWatermark, float64(globalAccessTierState.skippedWatermark.Load()))
-		m.Set(accessTierSkippedMaxSize, float64(globalAccessTierState.skippedMaxSize.Load()))
-		m.Set(accessTierSkippedQuota, float64(globalAccessTierState.skippedQuota.Load()))
-		for bucket, bytes := range globalAccessTierState.hotUsageSnapshot() {
-			m.Set(accessTierHotBytes, float64(bytes), "bucket", bucket)
-		}
-	}
-	m.Set(accessTierSamplesDropped, float64(globalAccessTracker.dropped.Load()))
 	m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM)))
 
 	return nil

cmd/metrics-v3.go

@@ -323,6 +323,10 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			sinceLastSyncMillisMD,
 			syncFailuresMD,
 			syncSuccessesMD,
+			revocationRecordsMD,
+			revocationHealFailuresMD,
+			revocationHealDurationMillisMD,
+			revocationHealLastSuccessMD,
 		},
 		loadClusterIAMMetrics,
 	)
@@ -392,17 +396,6 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			ilmTransitionActiveTasksMD,
 			ilmTransitionPendingTasksMD,
 			ilmTransitionMissedImmediateTasksMD,
-			ilmAccessTierActiveTasksMD,
-			ilmAccessTierPendingTasksMD,
-			ilmAccessTierPromotionsTotalMD,
-			ilmAccessTierDemotionsTotalMD,
-			ilmAccessTierBytesMovedTotalMD,
-			ilmAccessTierFailuresTotalMD,
-			ilmAccessTierSkippedWatermarkMD,
-			ilmAccessTierSkippedMaxSizeMD,
-			ilmAccessTierSkippedQuotaMD,
-			ilmAccessTierHotBytesMD,
-			ilmAccessTierSamplesDroppedMD,
 			ilmVersionsScannedMD,
 		},
 		loadILMMetrics,

cmd/object-api-interface.go

@@ -122,10 +122,6 @@ type ObjectOptions struct {
 	SkipRebalancing bool
 
 	SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same
-	// DstPoolIdx forces a data-movement write onto a specific server pool.
-	// It is ignored unless DataMovement is true; a pointer keeps pool zero
-	// distinguishable from the unset value.
-	DstPoolIdx *int
 
 	DataMovement bool // indicates an going decommisionning or rebalacing
 

cmd/object-api-options-replication_test.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/base64"
+	"net/http"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func TestPutOptsFromHeadersReplicationTimestamps(t *testing.T) {
+	stamp := time.Date(2026, 9, 15, 1, 2, 3, 123456789, time.UTC)
+	context := base64.StdEncoding.EncodeToString([]byte(`{"purpose":"tag-replication"}`))
+	for _, encryption := range []struct {
+		name    string
+		headers map[string]string
+	}{
+		{name: "none"},
+		{name: "SSE-S3", headers: map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}},
+		{name: "SSE-KMS", headers: map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS}},
+		{name: "SSE-KMS-context", headers: map[string]string{
+			xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS, xhttp.AmzServerSideEncryptionKmsID: "tag-replication-key",
+			xhttp.AmzServerSideEncryptionKmsContext: context,
+		}},
+		{name: "SSE-C", headers: ssecKeyHeaders([]byte("01234567890123456789012345678901"), false)},
+	} {
+		t.Run(encryption.name, func(t *testing.T) {
+			for _, trusted := range []bool{false, true} {
+				t.Run("trusted="+strconv.FormatBool(trusted), func(t *testing.T) {
+					for _, tagging := range []struct {
+						name, header string
+						want         time.Time
+						invalid      bool
+					}{
+						{name: "absent"},
+						{name: "nanoseconds", header: stamp.Format(time.RFC3339Nano), want: stamp},
+						{name: "offset-whitespace", header: " " + stamp.In(time.FixedZone("UTC+8", 8*60*60)).Format(time.RFC3339Nano) + " ", want: stamp},
+						{name: "invalid", header: "not-a-timestamp", invalid: true},
+					} {
+						t.Run(tagging.name, func(t *testing.T) {
+							for _, metadata := range []map[string]string{nil, {"x-amz-meta-test": "kept"}} {
+								hdr := make(http.Header)
+								wantEncryption := make(http.Header)
+								for key, value := range encryption.headers {
+									hdr.Set(key, value)
+									wantEncryption.Set(key, value)
+								}
+								hdr.Set(xhttp.MinIOSourceTaggingTimestamp, tagging.header)
+								hdr.Set(xhttp.MinIOSourceMTime, stamp.Add(-time.Hour).Format(time.RFC3339Nano))
+								hdr.Set(xhttp.MinIOSourceObjectRetentionTimestamp, stamp.Add(-time.Minute).Format(time.RFC3339Nano))
+								hdr.Set(xhttp.MinIOSourceObjectLegalHoldTimestamp, stamp.Add(-time.Second).Format(time.RFC3339Nano))
+								hdr.Set(xhttp.MinIOSourceETag, "source-etag")
+								opts, err := putOptsFromHeaders(t.Context(), hdr, metadata, trusted)
+								if trusted && tagging.invalid {
+									if err == nil || !strings.Contains(err.Error(), xhttp.MinIOSourceTaggingTimestamp) {
+										t.Fatalf("malformed trusted timestamp: got %v", err)
+									}
+									continue
+								}
+								if err != nil {
+									t.Fatal(err)
+								}
+								wantTag, wantMTime, wantRetention, wantLegalhold, wantETag := time.Time{}, time.Time{}, time.Time{}, time.Time{}, ""
+								if trusted {
+									wantTag, wantMTime = tagging.want, stamp.Add(-time.Hour)
+									wantRetention, wantLegalhold, wantETag = stamp.Add(-time.Minute), stamp.Add(-time.Second), "source-etag"
+								}
+								if !opts.ReplicationSourceTaggingTimestamp.Equal(wantTag) {
+									t.Errorf("tag timestamp=%s, want %s", opts.ReplicationSourceTaggingTimestamp, wantTag)
+								}
+								if !opts.MTime.Equal(wantMTime) || !opts.ReplicationSourceRetentionTimestamp.Equal(wantRetention) ||
+									!opts.ReplicationSourceLegalholdTimestamp.Equal(wantLegalhold) || opts.PreserveETag != wantETag || opts.ReplicationRequest != trusted {
+									t.Error("other source fields did not preserve the replication trust boundary")
+								}
+								if opts.UserDefined == nil || (metadata != nil && !reflect.DeepEqual(opts.UserDefined, metadata)) {
+									t.Errorf("metadata=%v, want nonnil map preserving %v", opts.UserDefined, metadata)
+								}
+								gotEncryption := make(http.Header)
+								if opts.ServerSideEncryption != nil {
+									opts.ServerSideEncryption.Marshal(gotEncryption)
+								}
+								if !reflect.DeepEqual(gotEncryption, wantEncryption) {
+									t.Errorf("SSE headers=%v, want %v", gotEncryption, wantEncryption)
+								}
+							}
+						})
+					}
+				})
+			}
+		})
+	}
+}

cmd/object-api-options.go

@@ -452,11 +452,11 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
 			MTime:                mtime,
 			PreserveETag:         etag,
 			ReplicationRequest:   trustedReplication,
-			// The Object Lock timestamps order replicated retention and legal
-			// hold updates. Dropping them here would leave every update on an
-			// SSE-KMS destination unordered.
+			// These timestamps order replicated retention, legal hold and tagging
+			// updates on an SSE-KMS destination.
 			ReplicationSourceLegalholdTimestamp: lholdtimestmp,
 			ReplicationSourceRetentionTimestamp: retaintimestmp,
+			ReplicationSourceTaggingTimestamp:   taggingtimestmp,
 		}
 		return op, nil
 	}

cmd/object-copy-replication-tagging_test.go

@@ -0,0 +1,143 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+// TestAPICopyObjectReplicaTaggingTimestampUnderKMS covers signed replica COPY
+// requests through encryption, metadata replacement and disk persistence. Both
+// the single-disk and 16-disk fixtures are single-pool backends.
+func TestAPICopyObjectReplicaTaggingTimestampUnderKMS(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS})
+}
+
+func testAPICopyObjectReplicaTaggingTimestampUnderKMS(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
+	// Ignore the host free-space percentage while retaining real disk I/O.
+	for _, pool := range obj.(*erasureServerPools).serverPools {
+		for _, set := range pool.sets {
+			original := set.getDisks
+			disks := append([]StorageAPI(nil), original()...)
+			for i := range disks {
+				disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			defer func() { set.getDisks = original }()
+		}
+	}
+	oldKMS, oldAuto := GlobalKMS, globalAutoEncryption
+	GlobalKMS = kms.NewStub("replica-tags-key")
+	globalAutoEncryption = false
+	defer func() { GlobalKMS, globalAutoEncryption = oldKMS, oldAuto }()
+	if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+		t.Fatal(err)
+	}
+	const tsKey = ReservedMetadataPrefixLower + TaggingTimestamp
+	stamp := time.Date(2026, 9, 15, 1, 0, 0, 123456789, time.UTC)
+	for _, mode := range []string{"none", "explicit-sse-s3", "explicit-kms", "auto-kms", "bucket-kms"} {
+		t.Run(instance+"/"+mode, func(t *testing.T) {
+			globalAutoEncryption = mode == "auto-kms"
+			if mode == "bucket-kms" {
+				sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>replica-tags-key</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
+				if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketSSEConfig, sseXML); err != nil {
+					t.Fatal(err)
+				}
+			}
+			const data = "encrypted replica copy remains readable"
+			oi, err := obj.PutObject(t.Context(), bucket, mode, mustGetPutObjReader(t, bytes.NewReader([]byte(data)), int64(len(data)), "", ""), ObjectOptions{
+				Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "key=old", tsKey: stamp.Format(time.RFC3339Nano)},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, event := range []struct {
+				name, tags, wantTags string
+				delta, wantDelta     time.Duration
+				missingTimestamp     bool
+			}{
+				{"newer", "key=new", "key=new", 2, 2, false},
+				{"stale", "key=stale", "key=new", 1, 2, false},
+				{"duplicate", "key=new", "key=new", 2, 2, false},
+				{"newer-again", "key=latest", "key=latest", 3, 3, false},
+				{"missing-timestamp", "key=unordered", "key=latest", 0, 3, true},
+			} {
+				headers := map[string]string{
+					xhttp.AmzCopySource:        "/" + bucket + "/" + mode + "?versionId=" + oi.VersionID,
+					xhttp.AmzMetadataDirective: "REPLACE", xhttp.AmzTagDirective: "REPLACE",
+					xhttp.AmzObjectTagging: event.tags, xhttp.MinIOSourceReplicationRequest: "true",
+					xhttp.AmzBucketReplicationStatus: "REPLICA", xhttp.MinIOSourceTaggingTimestamp: stamp.Add(event.delta).Format(time.RFC3339Nano),
+					xhttp.MinIOSourceMTime: oi.ModTime.Format(time.RFC3339Nano), xhttp.MinIOSourceETag: oi.ETag,
+				}
+				if event.missingTimestamp {
+					delete(headers, xhttp.MinIOSourceTaggingTimestamp)
+				}
+				if mode == "explicit-sse-s3" {
+					headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+				}
+				if mode == "explicit-kms" {
+					headers[xhttp.AmzServerSideEncryption] = "aws:kms"
+					headers[xhttp.AmzServerSideEncryptionKmsID] = "replica-tags-key"
+				}
+				req, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/"+mode+"?versionId="+oi.VersionID, 0, nil, creds.AccessKey, creds.SecretKey, headers)
+				if err != nil {
+					t.Fatal(err)
+				}
+				w := httptest.NewRecorder()
+				router.ServeHTTP(w, req)
+				if w.Code != http.StatusOK {
+					t.Fatalf("%s: COPY %d %s", event.name, w.Code, w.Body.String())
+				}
+				got, err := obj.GetObjectInfo(t.Context(), bucket, mode, ObjectOptions{VersionID: oi.VersionID})
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Logf("%s: tags=%q timestamp=%q kms=%v", event.name, got.UserTags, got.UserDefined[tsKey], crypto.S3KMS.IsEncrypted(got.UserDefined))
+				if got.UserTags != event.wantTags || got.UserDefined[tsKey] != stamp.Add(event.wantDelta).Format(time.RFC3339Nano) {
+					t.Errorf("%s: incorrect persisted tags/timestamp", event.name)
+				}
+				wantKMS := mode == "explicit-kms" || mode == "auto-kms" || mode == "bucket-kms"
+				if crypto.S3KMS.IsEncrypted(got.UserDefined) != wantKMS || crypto.S3.IsEncrypted(got.UserDefined) != (mode == "explicit-sse-s3") {
+					t.Errorf("%s: unexpected destination encryption", event.name)
+				}
+				if got.VersionID != oi.VersionID {
+					t.Errorf("%s: version=%q, want %q", event.name, got.VersionID, oi.VersionID)
+				}
+				req, err = newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/"+mode+"?versionId="+oi.VersionID, 0, nil, creds.AccessKey, creds.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				w = httptest.NewRecorder()
+				router.ServeHTTP(w, req)
+				if w.Code != http.StatusOK || w.Body.String() != data {
+					t.Fatalf("%s: GET %d %q", event.name, w.Code, w.Body.String())
+				}
+			}
+		})
+	}
+}

cmd/object-handlers-common.go

@@ -240,7 +240,10 @@ func checkPreconditionsPUT(ctx context.Context, w http.ResponseWriter, r *http.R
 	// updated. The predicate is the incoming request's restored SSE-C metadata,
 	// not what the destination happens to hold.
 	ssecReplica := isReplicaTrusted(r.Context()) && crypto.SSEC.IsEncrypted(opts.UserDefined)
-	if etagMatch && vidMatch && !ssecReplica {
+	// Matching content does not imply that its tag revision was delivered.
+	// Keep client preconditions above; relax only the internal duplicate check.
+	newerTags := isReplicaTrusted(r.Context()) && olderThan(objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], opts.ReplicationSourceTaggingTimestamp)
+	if etagMatch && vidMatch && !ssecReplica && !newerTags {
 		writeHeaders()
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL)
 		return true

cmd/object-handlers.go

@@ -576,8 +576,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		return
 	}
 
-	globalAccessTracker.note(bucket, object)
-
 	// Notify object accessed via a GET request.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectAccessedGet,
@@ -1799,6 +1797,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// source timestamp is newer than the stored one, and a stale update must
 	// leave the stored state in place instead of erasing it.
 	storedLock := storedObjectLockState(srcInfo.UserDefined)
+	storedTagTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
 
 	srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined, allowReplicationMetadata)
 	if err != nil {
@@ -1816,23 +1815,29 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	if objTags != "" {
-		lastTaggingTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]
-		if dstOpts.ReplicationRequest {
-			srcTimestamp := dstOpts.ReplicationSourceTaggingTimestamp
-			if !srcTimestamp.IsZero() {
-				ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastTaggingTimestamp)
-				// update tagging metadata only if replica  timestamp is newer than what's on disk
-				if err != nil || (err == nil && !ondiskTimestamp.After(srcTimestamp)) {
-					srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
-					srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
-				}
-			}
-		} else {
+	if dstOpts.ReplicationRequest {
+		srcTimestamp := dstOpts.ReplicationSourceTaggingTimestamp
+		if !srcTimestamp.IsZero() {
+			// An empty value with a timestamp is an ordered deletion. Recheck
+			// the captured state even if metadata REPLACE rebuilt the map.
+			srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
 			srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
-			srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+			reconcileStoredObjectTags(srcInfo.UserDefined, srcInfo.UserTags, storedTagTimestamp)
+		} else {
+			srcInfo.UserDefined[xhttp.AmzObjectTagging] = srcInfo.UserTags
+			if storedTagTimestamp != "" {
+				srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = storedTagTimestamp
+			} else {
+				delete(srcInfo.UserDefined, ReservedMetadataPrefixLower+TaggingTimestamp)
+			}
 		}
+	} else {
+		srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
+		srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
 	}
+	// SSE-C rotation snapshots reserved metadata before the tag decision. Its
+	// later merge must not put the old timestamp back over the accepted state.
+	delete(encMetadata, ReservedMetadataPrefixLower+TaggingTimestamp)
 
 	srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
 	srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
@@ -2315,6 +2320,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	if opts.ReplicationRequest && !opts.ReplicationSourceTaggingTimestamp.IsZero() {
+		metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = opts.ReplicationSourceTaggingTimestamp.UTC().Format(time.RFC3339Nano)
+	}
 
 	actualSize := size
 	var idxCb func() []byte
@@ -3762,11 +3770,11 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
+	stamp := UTCNow().Format(time.RFC3339Nano)
+	opts.UserDefined = map[string]string{ReservedMetadataPrefixLower + TaggingTimestamp: stamp}
 	if dsc.ReplicateAny() {
-		opts.UserDefined = make(map[string]string)
-		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
+		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = stamp
 		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
-		opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
 	}
 
 	// Put object tags
@@ -3865,9 +3873,10 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 	}
 
 	dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
+	stamp := UTCNow().Format(time.RFC3339Nano)
+	opts.UserDefined = map[string]string{ReservedMetadataPrefixLower + TaggingTimestamp: stamp}
 	if dsc.ReplicateAny() {
-		opts.UserDefined = make(map[string]string)
-		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
+		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = stamp
 		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 	}
 

cmd/object-multipart-handlers.go

@@ -312,6 +312,10 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	// Completion orders the upload's persisted tag state under the object lock.
+	if opts.ReplicationRequest && !opts.ReplicationSourceTaggingTimestamp.IsZero() {
+		metadata[ReservedMetadataPrefixLower+TaggingTimestamp] = opts.ReplicationSourceTaggingTimestamp.UTC().Format(time.RFC3339Nano)
+	}
 
 	if r.Header.Get(xhttp.IfMatch) != "" {
 		opts.HasIfMatch = true

cmd/peer-rest-server.go

@@ -204,7 +204,9 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(mss *grid.MSS) (np grid.NoP
 		return np, grid.NewRemoteErr(errors.New("service account name is missing"))
 	}
 
-	if err := globalIAMSys.DeleteServiceAccount(context.Background(), accessKey, false); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err := globalIAMSys.LoadServiceAccount(ctx, accessKey); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 
@@ -230,7 +232,8 @@ func (s *peerRESTServer) LoadServiceAccountHandler(mss *grid.MSS) (np grid.NoPay
 	return np, nerr
 }
 
-// DeleteUserHandler - deletes a user on the server.
+// DeleteUserHandler reloads the state committed by another node. A delayed
+// notification must not delete an identity recreated since that commit.
 func (s *peerRESTServer) DeleteUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
@@ -242,7 +245,9 @@ func (s *peerRESTServer) DeleteUserHandler(mss *grid.MSS) (np grid.NoPayload, ne
 		return np, grid.NewRemoteErr(errors.New("username is missing"))
 	}
 
-	if err := globalIAMSys.DeleteUser(context.Background(), accessKey, false); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err := globalIAMSys.LoadUserAfterDelete(ctx, accessKey); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 
@@ -271,7 +276,9 @@ func (s *peerRESTServer) LoadUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr
 		userType = stsUser
 	}
 
-	if err = globalIAMSys.LoadUser(context.Background(), objAPI, accessKey, userType); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err = globalIAMSys.LoadUser(ctx, objAPI, accessKey, userType); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 

cmd/replication-content-encoding_test.go

@@ -0,0 +1,276 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"encoding/xml"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio/internal/auth"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// Exercise authenticated handlers and actual disk metadata, including the
+// response headers consumers see after replication has completed.
+func TestAPIReplicaContentEncoding(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIReplicaContentEncoding})
+}
+
+func testAPIReplicaContentEncoding(obj ObjectLayer, instance, bucket string, router http.Handler, owner auth.Credentials, t *testing.T) {
+	ordinary := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject"`)
+	replicator := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
+	for _, mode := range []string{"ordinary", "untrusted-marker", "replica"} {
+		for _, tc := range []struct{ name, wire, want string }{
+			{"bare", "aws-chunked", ""}, {"mixed", "aws-chunked,gzip", "gzip"}, {"gzip", "gzip", "gzip"},
+		} {
+			for _, operation := range []string{"put", "copy-replace", "multipart"} {
+				t.Run(instance+"/"+mode+"/"+tc.name+"/"+operation, func(t *testing.T) {
+					object := mode + "/" + tc.name + "/" + operation
+					payload := replicaEncodingPayload(t, tc.want)
+					creds := ordinary
+					headers := map[string]string{xhttp.ContentEncoding: tc.wire, xhttp.ContentType: "application/octet-stream", "X-Amz-Meta-Source": "encoding-test"}
+					if mode != "ordinary" {
+						headers[xhttp.MinIOSourceReplicationRequest] = "true"
+					}
+					if mode == "replica" {
+						creds = replicator
+						headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
+					}
+					send := func(method, target string, data []byte, hdrs map[string]string) *httptest.ResponseRecorder {
+						t.Helper()
+						req, err := newTestSignedRequestV4(method, target, int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, hdrs)
+						if err != nil {
+							t.Fatal(err)
+						}
+						return replicaEncodingServe(t, router, req, http.StatusOK)
+					}
+					switch operation {
+					case "put":
+						if strings.Contains(tc.wire, "aws-chunked") {
+							req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), payload, creds, headers)
+							replicaEncodingServe(t, router, req, http.StatusOK)
+						} else {
+							send(http.MethodPut, getPutObjectURL("", bucket, object), payload, headers)
+						}
+					case "copy-replace":
+						source := object + "-source"
+						if _, err := obj.PutObject(t.Context(), bucket, source, mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{}); err != nil {
+							t.Fatal(err)
+						}
+						headers[xhttp.AmzCopySource] = url.QueryEscape("/" + bucket + "/" + source)
+						headers[xhttp.AmzMetadataDirective] = replaceDirective
+						send(http.MethodPut, getCopyObjectURL("", bucket, object), nil, headers)
+					case "multipart":
+						rec := send(http.MethodPost, getNewMultipartURL("", bucket, object), nil, headers)
+						var init InitiateMultipartUploadResponse
+						if err := xml.Unmarshal(rec.Body.Bytes(), &init); err != nil {
+							t.Fatal(err)
+						}
+						// Part/completion metadata must not replace the encoding saved at initiation.
+						partHeaders := map[string]string{xhttp.ContentEncoding: "br"}
+						if mode == "replica" {
+							partHeaders[xhttp.MinIOSourceReplicationRequest] = "true"
+							partHeaders[xhttp.AmzBucketReplicationStatus] = "REPLICA"
+						}
+						part := send(http.MethodPut, getPutObjectPartURL("", bucket, object, init.UploadID, "1"), payload, partHeaders)
+						partETags := part.Header()[xhttp.ETag]
+						if len(partETags) != 1 {
+							t.Fatalf("missing part ETag: %#v", part.Header())
+						}
+						complete, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(partETags[0])}}})
+						if err != nil {
+							t.Fatal(err)
+						}
+						send(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, init.UploadID), complete, partHeaders)
+					}
+					assertReplicaEncodingObject(t, obj, router, owner, bucket, object, tc.want, payload)
+					info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if got := info.UserDefined[xhttp.AmzBucketReplicationStatus]; (got == "REPLICA") != (mode == "replica") {
+						t.Errorf("replica status %q for mode %s", got, mode)
+					}
+					if info.ContentType != "application/octet-stream" {
+						t.Errorf("content-type=%q", info.ContentType)
+					}
+					if value, ok := caseInsensitiveMap(info.UserDefined).Lookup("x-amz-meta-source"); !ok || value != "encoding-test" {
+						t.Errorf("user metadata lost: %#v", info.UserDefined)
+					}
+				})
+			}
+		}
+	}
+	t.Run(instance+"/unauthorized-replica", func(t *testing.T) {
+		object := "denied-replica"
+		req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), []byte("denied"), ordinary, map[string]string{xhttp.ContentEncoding: "aws-chunked", xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA"})
+		rec := replicaEncodingServe(t, router, req, http.StatusForbidden)
+		var response APIErrorResponse
+		if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+			t.Fatal(err)
+		}
+		if response.Code != "AccessDenied" {
+			t.Fatalf("expected permission denial, got %s", response.Code)
+		}
+		if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil {
+			t.Error("denied replica created an object")
+		}
+	})
+}
+
+func replicaEncodingPayload(t *testing.T, encoding string) []byte {
+	t.Helper()
+	data := bytes.Repeat([]byte("replica encoding payload\n"), 128)
+	if encoding != "gzip" {
+		return data
+	}
+	var b bytes.Buffer
+	w := gzip.NewWriter(&b)
+	if _, err := w.Write(data); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return b.Bytes()
+}
+
+func replicaEncodingStream(t *testing.T, target string, data []byte, creds auth.Credentials, headers map[string]string) *http.Request {
+	t.Helper()
+	const chunkSize = 64
+	body := bytes.NewReader(data)
+	req, err := newTestStreamingRequest(http.MethodPut, target, int64(len(data)), chunkSize, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	now := UTCNow()
+	signature, err := signStreamingRequest(req, creds.AccessKey, creds.SecretKey, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req, err = assembleStreamingChunks(req, body, chunkSize, creds.SecretKey, signature, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return req
+}
+
+func replicaEncodingServe(t *testing.T, router http.Handler, req *http.Request, want int) *httptest.ResponseRecorder {
+	t.Helper()
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	if rec.Code != want {
+		t.Fatalf("%s %s: status=%d want=%d body=%s", req.Method, req.URL, rec.Code, want, rec.Body.String())
+	}
+	return rec
+}
+
+func assertReplicaEncodingObject(t *testing.T, obj ObjectLayer, router http.Handler, creds auth.Credentials, bucket, object, encoding string, data []byte) {
+	t.Helper()
+	info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.ContentEncoding != encoding {
+		t.Errorf("persisted content-encoding=%q want=%q", info.ContentEncoding, encoding)
+	}
+	if encoding == "" {
+		if _, present := info.UserDefined["content-encoding"]; present {
+			t.Error("transport-only content-encoding key persisted")
+		}
+	}
+	for _, method := range []string{http.MethodGet, http.MethodHead} {
+		req, err := newTestSignedRequestV4(method, getPutObjectURL("", bucket, object), 0, nil, creds.AccessKey, creds.SecretKey, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		rec := replicaEncodingServe(t, router, req, http.StatusOK)
+		if got := rec.Header().Get(xhttp.ContentEncoding); got != encoding {
+			t.Errorf("%s content-encoding=%q want=%q", method, got, encoding)
+		}
+		if encoding == "" {
+			if _, present := rec.Header()[xhttp.ContentEncoding]; present {
+				t.Errorf("%s sent an empty/transport encoding header", method)
+			}
+		}
+		if method == http.MethodGet && !bytes.Equal(rec.Body.Bytes(), data) {
+			t.Errorf("GET body differs: got %d bytes want %d", rec.Body.Len(), len(data))
+		}
+	}
+}
+
+func TestAPISnowballReplicaContentEncoding(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
+		for _, tc := range []struct {
+			name string
+			pax  map[string]string
+			want string
+		}{
+			{name: "no-pax"},
+			{name: "pax-without-encoding", pax: map[string]string{"minio.metadata.Content-Type": "application/octet-stream"}},
+			{name: "pax-bare", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked"}},
+			{name: "pax-mixed", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked,gzip"}, want: "gzip"},
+		} {
+			t.Run(instance+"/"+tc.name, func(t *testing.T) {
+				object := "snowball/" + tc.name
+				data := replicaEncodingPayload(t, tc.want)
+				var archive bytes.Buffer
+				tw := tar.NewWriter(&archive)
+				if err := tw.WriteHeader(&tar.Header{Name: object, Mode: 0o600, Size: int64(len(data)), PAXRecords: tc.pax}); err != nil {
+					t.Fatal(err)
+				}
+				if _, err := tw.Write(data); err != nil {
+					t.Fatal(err)
+				}
+				if err := tw.Close(); err != nil {
+					t.Fatal(err)
+				}
+				var ordinaryMetadata map[string]string
+				// An unauthorized entry in a REPLICA request is rejected. Compare the
+				// same archive across ordinary and authorized replica requests instead.
+				for _, replica := range []bool{false, true} {
+					headers := map[string]string{
+						xhttp.ContentEncoding: "aws-chunked", xhttp.AmzSnowballExtract: "true",
+						xhttp.ContentType: "application/x-tar", xhttp.CacheControl: "max-age=123",
+						"X-Amz-Meta-Archive": "outer-request",
+					}
+					if replica {
+						headers[xhttp.MinIOSourceReplicationRequest] = "true"
+						headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
+					}
+					req := replicaEncodingStream(t, getPutObjectURL("", bucket, "archive.tar"), archive.Bytes(), creds, headers)
+					replicaEncodingServe(t, router, req, http.StatusOK)
+					assertReplicaEncodingObject(t, obj, router, creds, bucket, object, tc.want, data)
+					info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					metadata := maps.Clone(info.UserDefined)
+					for _, key := range []string{xhttp.AmzBucketReplicationStatus, ReservedMetadataPrefixLower + ReplicaStatus, ReservedMetadataPrefixLower + ReplicaTimestamp, "etag"} {
+						delete(metadata, key)
+					}
+					if !replica {
+						ordinaryMetadata = metadata
+					} else if !reflect.DeepEqual(metadata, ordinaryMetadata) {
+						t.Errorf("replica inherited ordinary archive metadata: got %#v want %#v", metadata, ordinaryMetadata)
+					}
+				}
+			})
+		}
+	}})
+}

cmd/replication-delete-marker_test.go

@@ -106,6 +106,7 @@ func TestReplicateDeleteMarkerTargetSemantics(t *testing.T) {
 
 func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T, legacy bool) {
 	ctx := t.Context()
+	defer replicationTestCapacity(obj)()
 	const arn = "arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket"
 	const name = "marker"
 	version := mustGetUUID()
@@ -208,27 +209,12 @@ func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string
 		t.Errorf("purge scheduled as marker creation: version=%q marker=%q", deletion.VersionID, deletion.DeleteMarkerVersionID)
 	}
 	if legacy {
-		// Reproduce the old producer's state and let the existing scanner/heal
-		// path recover it. Upgrades must also finish purges already left pending.
+		// Old task shapes must complete directly; scanner/MRF recovery is
+		// covered separately by TestReplicationMRFMarkerRecovery.
 		deletion.VersionID, deletion.DeleteMarkerVersionID = "", version
-		replicateDelete(ctx, deletion, obj)
-		oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true})
-		if oi.VersionPurgeStatus != replication.VersionPurgePending {
-			t.Fatalf("legacy source purge = %s, want PENDING", oi.VersionPurgeStatus)
-		}
-		targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
-		if err != nil {
-			t.Fatal(err)
-		}
-		queueReplicationHeal(ctx, bucket, oi, replicationConfig{Config: &cfg, remotes: targets}, 0)
-		select {
-		case op := <-worker:
-			deletion = op.(DeletedObjectReplicationInfo)
-		case <-time.After(time.Second):
-			t.Fatal("legacy pending purge was not scheduled for healing")
-		}
 	}
-	result := replicateDelete(context.Background(), deletion, obj)
+
+	result := replicateDelete(context.Background(), deletion, markerPurgeUpdateLayer{ObjectLayer: obj, t: t})
 	if result.VersionPurgeStatus() != replication.VersionPurgeComplete {
 		t.Errorf("remote purge result = %s, want COMPLETE", result.VersionPurgeStatus())
 	}

cmd/replication-delete-mrf_test.go

@@ -0,0 +1,628 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	loghttp "github.com/minio/minio/internal/logger/target/http"
+	"github.com/minio/minio/internal/once"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+)
+
+type markerRecoveryCase struct {
+	name                                                                                                          string
+	legacy, creation, lostReply, partial, exhaust, lockFirst, invalidMRF, replicaSource, offline, unrecordedPurge bool
+	targets                                                                                                       int
+}
+
+func TestReplicationMRFMarkerRecovery(t *testing.T) {
+	for _, tc := range []markerRecoveryCase{
+		{name: "canonical", targets: 1},
+		{name: "lock-failure", lockFirst: true, targets: 1},
+		{name: "invalid-MRF-metadata", invalidMRF: true, targets: 1},
+		{name: "legacy", legacy: true, targets: 1},
+		{name: "unrecorded-purge", unrecordedPurge: true, targets: 2},
+		{name: "two-targets", legacy: true, targets: 2},
+		{name: "two-targets-offline", offline: true, targets: 2},
+		{name: "replica-source", replicaSource: true, targets: 1},
+		{name: "lost-reply-canonical", lostReply: true, targets: 1},
+		{name: "lost-reply-legacy", legacy: true, lostReply: true, targets: 1},
+		{name: "creation", creation: true, targets: 1},
+		{name: "partial-creation-block", partial: true, targets: 2},
+		{name: "retry-budget-and-scanner", exhaust: true, targets: 1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
+				testReplicationMRFMarkerRecovery(t, obj, backend, bucket, router, creds, tc)
+			}})
+		})
+	}
+}
+
+type markerRecoveryTarget struct {
+	arn, bucket       string
+	client            *minio.Client
+	reject, loseReply atomic.Bool
+	deletes           atomic.Int32
+}
+
+// Only adapt host capacity accounting, using the existing real-disk adapter.
+// Reads, object metadata, MRF files and writes all still use the fixture disks.
+func replicationTestCapacity(obj ObjectLayer) func() {
+	var restore []func()
+	for _, pool := range obj.(*erasureServerPools).serverPools {
+		for _, set := range pool.sets {
+			original := set.getDisks
+			disks := append([]StorageAPI(nil), original()...)
+			for i, disk := range disks {
+				if disk != nil {
+					disks[i] = tagTestCapacityDisk{StorageAPI: disk}
+				}
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			restore = append(restore, func() { set.getDisks = original })
+		}
+	}
+	return func() {
+		for _, fn := range restore {
+			fn()
+		}
+	}
+}
+
+func testReplicationMRFMarkerRecovery(t *testing.T, obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, tc markerRecoveryCase) {
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	defer replicationTestCapacity(obj)()
+	stats := NewReplicationStats(ctx, nil)
+	oldStats := globalReplicationStats.Swap(stats)
+	defer globalReplicationStats.Store(oldStats)
+	oldPool := globalReplicationPool
+	defer func() { globalReplicationPool = oldPool }()
+	const name = "marker"
+	version := mustGetUUID()
+	creationTime := UTCNow().Add(-time.Hour).Truncate(time.Second)
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
+		t.Fatal(err)
+	}
+	var targets []*markerRecoveryTarget
+	cfg := replication.Config{}
+	creationStates := make(map[string]replication.StatusType)
+	var creationInternal string
+	for i := 0; i < tc.targets; i++ {
+		target := &markerRecoveryTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()}
+		if err := obj.MakeBucket(ctx, target.bucket, MakeBucketOptions{}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := obj.PutObject(ctx, target.bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil {
+			t.Fatal(err)
+		}
+		if !tc.creation {
+			opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, ReplicationRequest: true, MTime: creationTime}
+			opts.SetReplicaStatus(replication.Replica)
+			if _, err := obj.DeleteObject(ctx, target.bucket, name, opts); err != nil {
+				t.Fatal(err)
+			}
+		}
+		remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
+			switch r.Method {
+			case http.MethodHead:
+				oi, err := obj.GetObjectInfo(r.Context(), target.bucket, name, opts)
+				if oi.DeleteMarker {
+					w.Header().Set(xhttp.AmzDeleteMarker, "true")
+					w.Header().Set(xhttp.AmzVersionID, oi.VersionID)
+				}
+				if err != nil {
+					writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err))
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			case http.MethodDelete:
+				target.deletes.Add(1)
+				if got := r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"; got != tc.creation {
+					t.Errorf("purge/creation wire flag=%v creation=%v", got, tc.creation)
+				}
+				if opts.VersionID == "" {
+					t.Error("missing remote versionId")
+				}
+				if target.reject.Load() {
+					w.WriteHeader(http.StatusForbidden)
+					fmt.Fprint(w, `<Error><Code>AccessDenied</Code></Error>`)
+					return
+				}
+				opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
+				opts.ReplicationRequest = true
+				opts.SetReplicaStatus(replication.Replica)
+				_, err := obj.DeleteObject(r.Context(), target.bucket, name, opts)
+				if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+					return
+				}
+				if target.loseReply.Swap(false) {
+					conn, _, err := w.(http.Hijacker).Hijack()
+					if err != nil {
+						t.Error(err)
+						return
+					}
+					conn.Close()
+					return
+				}
+				w.WriteHeader(http.StatusNoContent)
+			default:
+				t.Errorf("unexpected remote method %s", r.Method)
+			}
+		}))
+		defer remote.Close()
+		client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+		if err != nil {
+			t.Fatal(err)
+		}
+		target.client = client
+		globalBucketTargetSys.Lock()
+		globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()}
+		globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket})
+		globalBucketTargetSys.Unlock()
+		globalBucketTargetSys.hMutex.Lock()
+		globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true}
+		globalBucketTargetSys.hMutex.Unlock()
+		rule := configs[0].Rules[0]
+		rule.Priority = i + 1
+		rule.Destination = replication.Destination{ARN: target.arn, Bucket: target.bucket}
+		cfg.Rules = append(cfg.Rules, rule)
+		creationStates[target.arn] = replication.Completed
+		creationInternal += target.arn + "=COMPLETED;"
+		targets = append(targets, target)
+	}
+	if tc.targets == 1 {
+		cfg.RoleArn = targets[0].arn
+	}
+	if !tc.creation {
+		opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, MTime: creationTime, DeleteReplication: ReplicationState{Targets: creationStates, ReplicationStatusInternal: creationInternal, ReplicationTimeStamp: creationTime}}
+		if tc.replicaSource {
+			opts.DeleteReplication = ReplicationState{}
+			opts.SetReplicaStatus(replication.Replica)
+		}
+		if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
+			t.Fatal(err)
+		}
+	}
+	meta, err := globalBucketMetadataSys.Get(bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meta.replicationConfig = &cfg
+	globalBucketMetadataSys.Set(bucket, meta)
+	newPool := func() *ReplicationPool {
+		p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)}
+		globalReplicationPool = once.NewSingleton[ReplicationPool]()
+		globalReplicationPool.Set(p)
+		return p
+	}
+	p := newPool()
+	receive := func() DeletedObjectReplicationInfo {
+		select {
+		case op := <-p.workers[0]:
+			d, ok := op.(DeletedObjectReplicationInfo)
+			if !ok {
+				t.Fatalf("wrong queued operation %T", op)
+			}
+			return d
+		case <-time.After(3 * time.Second):
+			t.Fatal("no marker task from actual MRF/handler/scanner queue")
+			return DeletedObjectReplicationInfo{}
+		}
+	}
+	uri := "/" + bucket + "/" + name
+	if !tc.creation {
+		uri += "?versionId=" + version
+	}
+	req, err := newTestSignedRequestV4(http.MethodDelete, uri, 0, nil, creds.AccessKey, creds.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+	if w.Code != http.StatusNoContent {
+		t.Fatalf("source DELETE status %d: %s", w.Code, w.Body)
+	}
+	deletion := receive()
+	if tc.creation {
+		version = deletion.DeleteMarkerVersionID
+	} else if deletion.VersionID != version || deletion.DeleteMarkerVersionID != "" {
+		t.Fatalf("current producer emitted noncanonical purge: %+v", deletion)
+	}
+	if tc.unrecordedPurge {
+		// Exercise a task carrying purge state while the disk marker still has
+		// only creation metadata. Recreate only this isolated fixture marker.
+		if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true}); err != nil {
+			t.Fatal(err)
+		}
+		opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, MTime: creationTime, DeleteReplication: ReplicationState{Targets: creationStates, ReplicationStatusInternal: creationInternal, ReplicationTimeStamp: creationTime}}
+		if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if tc.legacy {
+		deletion.VersionID, deletion.DeleteMarkerVersionID = "", version
+	}
+	if tc.partial {
+		deletion.TargetArn = targets[len(targets)-1].arn
+	}
+	if tc.invalidMRF {
+		testReplicationMRFInvalidLookups(ctx, t, obj, p, newPool, deletion)
+		return
+	}
+	var auditStatuses <-chan string
+	if tc.name == "canonical" {
+		var stopAudit func()
+		auditStatuses, stopAudit = replicationTestAudit(ctx, t, bucket)
+		defer stopAudit()
+	}
+	assertAudit := func(want string) {
+		if auditStatuses == nil {
+			return
+		}
+		select {
+		case got := <-auditStatuses:
+			if got != want {
+				t.Fatalf("replication audit status=%q want %q", got, want)
+			}
+		case <-time.After(3 * time.Second):
+			t.Fatal("missing replication audit event")
+		}
+	}
+	deletion.OpType = replication.HealReplicationType // Exercise operation-status statistics too.
+	failing := targets[len(targets)-1]
+	failing.reject.Store(!tc.lostReply)
+	failing.loseReply.Store(tc.lostReply)
+	if tc.offline {
+		globalBucketTargetSys.hMutex.Lock()
+		globalBucketTargetSys.hc[failing.client.EndpointURL().Host] = epHealth{Online: false}
+		globalBucketTargetSys.hMutex.Unlock()
+	}
+	before, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version})
+	beforeCreation := before.ReplicationStatusInternal
+	beforeStamp := before.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp]
+	beforeReplica := before.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus]
+	beforeReplicaStamp := before.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp]
+	if !tc.creation && !tc.replicaSource && (len(replicationStatusesMap(beforeCreation)) != tc.targets || beforeStamp == "") {
+		t.Fatalf("missing seeded creation block: %+v", before)
+	}
+	if tc.replicaSource && (beforeReplica != "REPLICA" || beforeReplicaStamp == "") {
+		t.Fatalf("missing replica block: %+v", before)
+	}
+	updateObj := obj
+	if !tc.creation {
+		updateObj = markerPurgeUpdateLayer{ObjectLayer: obj, t: t}
+	}
+	rounds := 2
+	if tc.lostReply {
+		rounds = 1
+	}
+	if tc.exhaust {
+		rounds = mrfRetryLimit + 1
+	}
+	for round := 1; round <= rounds; round++ {
+		callObj := updateObj
+		if tc.lockFirst && round == 1 {
+			callObj = markerLockFailureLayer{ObjectLayer: updateObj}
+		}
+		result := replicateDelete(ctx, deletion, callObj)
+		switch {
+		case tc.lockFirst && round == 1:
+			if len(result.Targets) != 0 {
+				t.Fatal("lock failure attempted a target")
+			}
+		case tc.creation:
+			if result.ReplicationStatus() != replication.Failed {
+				t.Fatalf("creation result: %+v", result)
+			}
+		case result.VersionPurgeStatus() != replication.VersionPurgeFailed:
+			t.Fatalf("purge failure result: %+v", result)
+		}
+		assertAudit("FAILED")
+		if round == 1 && !tc.lockFirst && !tc.creation {
+			stats.RLock()
+			failed := stats.Cache[bucket].Stats[failing.arn].FailStats.SinceUptime
+			stats.RUnlock()
+			if failed.Count != 1 || failed.Bytes != 0 {
+				t.Fatalf("purge failure stats=%+v, want count 1 and zero bytes", failed)
+			}
+		}
+		oi, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version})
+		if !isErrMethodNotAllowed(err) || !oi.DeleteMarker {
+			t.Fatalf("source marker metadata/405 missing: %+v %v", oi, err)
+		}
+		if !tc.creation && (oi.ReplicationStatusInternal != beforeCreation || oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] != beforeStamp) {
+			t.Fatalf("purge rewrote creation block: before=%q/%q after=%q/%q", beforeCreation, beforeStamp, oi.ReplicationStatusInternal, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+		}
+		if !tc.creation && (oi.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] != beforeReplica || oi.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] != beforeReplicaStamp) {
+			t.Fatal("purge rewrote replica block")
+		}
+		if tc.targets == 2 && !tc.partial {
+			purges := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
+			if purges[targets[0].arn] != replication.VersionPurgeComplete || purges[failing.arn] != replication.VersionPurgeFailed {
+				t.Fatalf("incorrect persisted target states: %v", purges)
+			}
+			if targets[0].deletes.Load() != 1 {
+				t.Fatalf("successful target resent %d times", targets[0].deletes.Load())
+			}
+		}
+		if tc.partial {
+			select {
+			case <-p.mrfSaveCh:
+			default:
+				t.Fatal("partial failure missing MRF")
+			}
+			dsc := deletion.ReplicationState.ReplicateDecisionStr
+			deletion.ReplicationState = oi.ReplicationState()
+			deletion.ReplicationState.ReplicateDecisionStr = dsc
+			continue
+		}
+		if tc.exhaust && round > mrfRetryLimit {
+			if len(p.mrfSaveCh) != 0 || atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount) != 1 {
+				t.Fatalf("retry budget not applied: queued=%d drops=%d", len(p.mrfSaveCh), stats.mrfStats.TotalDroppedCount)
+			}
+			break
+		}
+		var entry MRFReplicateEntry
+		select {
+		case entry = <-p.mrfSaveCh:
+		default:
+			t.Fatal("failure did not enter MRF")
+		}
+		if entry.RetryCount != round || entry.versionID != version {
+			t.Fatalf("MRF entry lost identity/budget: %+v, round=%d", entry, round)
+		}
+		p.saveMRFEntries(ctx, map[string]MRFReplicateEntry{entry.versionID: entry})
+		record, err := p.loadMRF()
+		if err != nil || len(record.Entries) != 1 {
+			t.Fatalf("disk MRF missing: %+v %v", record, err)
+		}
+		if got := record.Entries[version]; got.RetryCount != round || got.Object != name || got.Bucket != bucket {
+			t.Fatalf("disk MRF mismatch: %+v", got)
+		}
+		p.saveMRFEntries(ctx, record.Entries) // loadMRF consumes the file; each replay uses a fresh disk record.
+		p = newPool()                         // no in-memory entries carried to the replacement pool.
+		if err := p.queueMRFHeal(); err != nil {
+			t.Fatal(err)
+		}
+		deletion = receive()
+		if deletion.RetryCount != round {
+			t.Fatalf("MRF retry count=%d want %d", deletion.RetryCount, round)
+		}
+		if !tc.creation && (deletion.VersionID != version || deletion.DeleteMarkerVersionID != "") {
+			t.Fatalf("MRF emitted wrong purge: %+v", deletion)
+		}
+	}
+	if tc.partial {
+		return
+	}
+	failing.reject.Store(false)
+	globalBucketTargetSys.hMutex.Lock()
+	globalBucketTargetSys.hc[failing.client.EndpointURL().Host] = epHealth{Online: true}
+	globalBucketTargetSys.hMutex.Unlock()
+	if tc.exhaust {
+		oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version})
+		QueueReplicationHeal(ctx, bucket, oi, 0) // Separately prove the existing scanner fallback after MRF exhaustion.
+		deletion = receive()
+		if deletion.RetryCount != 0 {
+			t.Fatal("scanner did not start a fresh retry budget")
+		}
+	}
+	result := replicateDelete(ctx, deletion, updateObj)
+	assertAudit("COMPLETED")
+	if tc.creation {
+		if result.ReplicationStatus() != replication.Completed {
+			t.Fatalf("creation recovery failed: %+v", result)
+		}
+	} else if result.VersionPurgeStatus() != replication.VersionPurgeComplete {
+		t.Fatalf("purge recovery failed: %+v", result)
+	}
+	for _, b := range append([]string{bucket}, func() []string {
+		var b []string
+		for _, target := range targets {
+			b = append(b, target.bucket)
+		}
+		return b
+	}()...) {
+		oi, err := obj.GetObjectInfo(ctx, b, name, ObjectOptions{VersionID: version})
+		if tc.creation {
+			if !oi.DeleteMarker || !isErrMethodNotAllowed(err) {
+				t.Fatalf("creation missing in %s: %+v %v", b, oi, err)
+			}
+		} else if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+			t.Fatalf("purge left marker in %s: %+v %v", b, oi, err)
+		}
+	}
+	if !tc.creation {
+		// Re-deliver an ambiguous purge directly. An absent marker must stay absent.
+		duplicate := deletion
+		duplicate.VersionID, duplicate.DeleteMarkerVersionID = "", version
+		duplicate.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{failing.arn: replication.VersionPurgePending}
+		duplicate.ReplicationState.VersionPurgeStatusInternal = ""
+		got := replicateDeleteToTarget(ctx, duplicate, &TargetClient{Client: failing.client, ARN: failing.arn, Bucket: failing.bucket})
+		if got.VersionPurgeStatus != replication.VersionPurgeComplete {
+			t.Fatalf("duplicate purge: %+v", got)
+		}
+		if oi, err := obj.GetObjectInfo(ctx, failing.bucket, name, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+			t.Fatalf("duplicate recreated marker: %+v %v", oi, err)
+		}
+	}
+	if len(p.mrfSaveCh) != 0 || len(p.workers[0]) != 0 {
+		t.Fatalf("completion left queued work: mrf=%d worker=%d", len(p.mrfSaveCh), len(p.workers[0]))
+	}
+	if !tc.creation && !tc.exhaust && !tc.lostReply {
+		stats.RLock()
+		got := stats.Cache[bucket].Stats[failing.arn].ReplicatedCount
+		size := stats.Cache[bucket].Stats[failing.arn].ReplicatedSize
+		stats.RUnlock()
+		if got != 1 || size != 0 {
+			t.Fatalf("purge completion statistic=%d want 1", got)
+		}
+	}
+	t.Logf("%s: %s recovered; %d target(s), persisted MRF, source/target metadata checked", backend, tc.name, tc.targets)
+}
+
+func TestReplicationDeleteQueueFullRetryBudget(t *testing.T) {
+	stats := NewReplicationStats(t.Context(), nil)
+	p := &ReplicationPool{ctx: t.Context(), objLayer: &replicationMRFTestObjectLayer{}, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 1), priority: "slow"}
+	d := DeletedObjectReplicationInfo{Bucket: "bucket", DeletedObject: DeletedObject{ObjectName: "marker", VersionID: mustGetUUID()}, RetryCount: 2}
+	p.queueReplicaDeleteTask(d)
+	if got := <-p.mrfSaveCh; got.RetryCount != 3 {
+		t.Fatalf("queue-full retry count=%d", got.RetryCount)
+	}
+	d.RetryCount = mrfRetryLimit
+	p.queueReplicaDeleteTask(d)
+	if len(p.mrfSaveCh) != 0 || atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount) != 1 {
+		t.Fatal("queue-full retry exceeded budget without a visible drop")
+	}
+}
+
+type markerLockFailureLayer struct{ ObjectLayer }
+
+func (markerLockFailureLayer) NewNSLock(string, ...string) RWLocker { return markerFailedLock{} }
+
+type markerFailedLock struct{ RWLocker }
+
+func (markerFailedLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) {
+	return LockContext{}, context.DeadlineExceeded
+}
+
+type markerLookupLayer struct {
+	ObjectLayer
+	mutate   func(ObjectInfo, error) (ObjectInfo, error)
+	lookedUp chan struct{}
+}
+
+func (l markerLookupLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+	defer close(l.lookedUp)
+	return l.mutate(oi, err)
+}
+
+func testReplicationMRFInvalidLookups(ctx context.Context, t *testing.T, obj ObjectLayer, p *ReplicationPool, newPool func() *ReplicationPool, d DeletedObjectReplicationInfo) {
+	for _, tc := range []struct {
+		name   string
+		mutate func(ObjectInfo, error) (ObjectInfo, error)
+	}{
+		{"empty-info", func(_ ObjectInfo, e error) (ObjectInfo, error) { return ObjectInfo{}, e }},
+		{"not-a-marker", func(o ObjectInfo, e error) (ObjectInfo, error) { o.DeleteMarker = false; return o, e }},
+		{"wrong-version", func(o ObjectInfo, e error) (ObjectInfo, error) { o.VersionID = mustGetUUID(); return o, e }},
+		{"wrong-bucket", func(o ObjectInfo, e error) (ObjectInfo, error) { o.Bucket = "different-bucket"; return o, e }},
+		{"wrong-object", func(o ObjectInfo, e error) (ObjectInfo, error) { o.Name = "different-object"; return o, e }},
+		{"zero-modtime", func(o ObjectInfo, e error) (ObjectInfo, error) { o.ModTime = time.Time{}; return o, e }},
+		{"missing", func(o ObjectInfo, _ error) (ObjectInfo, error) {
+			return o, ObjectNotFound{Bucket: o.Bucket, Object: o.Name}
+		}},
+		{"read-error", func(o ObjectInfo, _ error) (ObjectInfo, error) { return o, InsufficientReadQuorum{} }},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			entry := d.ToMRFEntry()
+			p.saveMRFEntries(ctx, map[string]MRFReplicateEntry{entry.versionID: entry})
+			fresh := newPool()
+			looked := make(chan struct{})
+			fresh.objLayer = markerLookupLayer{ObjectLayer: obj, mutate: tc.mutate, lookedUp: looked}
+			if err := fresh.queueMRFHeal(); err != nil {
+				t.Fatal(err)
+			}
+			select {
+			case <-looked:
+			case <-time.After(3 * time.Second):
+				t.Fatal("disk MRF entry was not read")
+			}
+			select {
+			case op := <-fresh.workers[0]:
+				t.Fatalf("invalid lookup scheduled %T", op)
+			case <-time.After(100 * time.Millisecond):
+			}
+		})
+	}
+}
+
+// Capture the actual internal audit event through the supported webhook sink.
+func replicationTestAudit(ctx context.Context, t *testing.T, bucket string) (<-chan string, func()) {
+	t.Helper()
+	if len(logger.AuditTargets()) != 0 {
+		t.Fatal("unexpected pre-existing test audit targets")
+	}
+	statuses := make(chan string, 16)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var entry struct {
+			API struct{ Name, Bucket, Status string }
+		}
+		if err := json.NewDecoder(r.Body).Decode(&entry); err != nil {
+			t.Error(err)
+		} else if entry.API.Name == ReplicateDeleteAPI && entry.API.Bucket == bucket {
+			statuses <- entry.API.Status
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	endpoint, err := xnet.ParseHTTPURL(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if errs := logger.UpdateAuditWebhooks(ctx, map[string]loghttp.Config{"r6": {Enabled: true, Name: "r6", Endpoint: endpoint, BatchSize: 1, QueueSize: 128, MaxRetry: 1, RetryIntvl: time.Millisecond, HTTPTimeout: time.Second}}); len(errs) > 0 {
+		t.Fatal(errs)
+	}
+	targets := logger.AuditTargets()
+	return statuses, func() {
+		logger.UpdateAuditWebhooks(ctx, nil)
+		for _, target := range targets {
+			target.Cancel()
+		}
+		server.Close()
+	}
+}
+
+type markerPurgeUpdateLayer struct {
+	ObjectLayer
+	t *testing.T
+}
+
+func (l markerPurgeUpdateLayer) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	l.t.Helper()
+	rs := opts.DeleteReplication
+	if rs.ReplicationStatusInternal != "" || rs.Targets != nil || rs.ReplicaStatus != "" || !rs.CompositeReplicationStatus().Empty() {
+		l.t.Fatalf("purge has a nonempty creation update: %+v", rs)
+	}
+	if rs.CompositeVersionPurgeStatus().Empty() {
+		l.t.Fatal("purge update lost its purge status")
+	}
+	return l.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+}

cmd/replication-delete-operation_test.go

@@ -0,0 +1,215 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// Purge uses the version-delete wire form regardless of the in-memory shape.
+// Creation status, purge status and successful resync markers are independent.
+func TestReplicateDeleteOperationExits(t *testing.T) {
+	for _, shape := range []string{"marker-creation", "legacy-marker-purge", "marker-purge", "object-purge"} {
+		purge := shape != "marker-creation"
+		for _, tc := range []struct {
+			name                 string
+			creation             replication.StatusType
+			priorPurge           VersionPurgeStatusType
+			headCode, deleteCode int
+			headError            string
+			offline, resync      bool
+		}{
+			{name: "pending-success", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 204},
+			{name: "completed-creation", creation: replication.Completed, priorPurge: replication.VersionPurgePending, headCode: 405, deleteCode: 204},
+			{name: "existing-marker", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 405, deleteCode: 204},
+			// Use the quorum S3 code without 503, which is classified as backend-down first.
+			{name: "head-read-quorum", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 400, headError: "SlowDownRead", deleteCode: 204},
+			{name: "replica-creation-status", creation: replication.Replica, priorPurge: replication.VersionPurgeFailed, headCode: 405, deleteCode: 204},
+			{name: "head-unavailable", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 503, deleteCode: 204},
+			{name: "head-forbidden", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 403, deleteCode: 204},
+			{name: "delete-forbidden", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 403},
+			{name: "delete-method-rejected", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 405},
+			{name: "delete-unavailable", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 503},
+			{name: "offline", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 204, offline: true},
+			{name: "retry-failed", creation: replication.Completed, priorPurge: replication.VersionPurgeFailed, headCode: 404, deleteCode: 204},
+			{name: "purge-complete", creation: replication.Pending, priorPurge: replication.VersionPurgeComplete, headCode: 404, deleteCode: 403},
+			{name: "resync-success", creation: replication.Completed, priorPurge: replication.VersionPurgePending, headCode: 405, deleteCode: 204, resync: true},
+			{name: "resync-already-purged", creation: replication.Pending, priorPurge: replication.VersionPurgeComplete, headCode: 404, deleteCode: 403, resync: true},
+			{name: "resync-failure", creation: replication.Completed, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 403, resync: true},
+		} {
+			t.Run(shape+"/"+tc.name, func(t *testing.T) {
+				version := mustGetUUID()
+				var heads, deletes atomic.Int32
+				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					if r.URL.Query().Get("versionId") != version {
+						t.Errorf("wrong request version: %s", r.URL)
+					}
+					switch r.Method {
+					case http.MethodHead:
+						heads.Add(1)
+						w.Header().Set(xhttp.AmzVersionID, version)
+						w.Header().Set(xhttp.LastModified, time.Now().UTC().Format(http.TimeFormat))
+						if tc.headCode == 405 {
+							w.Header().Set(xhttp.AmzDeleteMarker, "true")
+						}
+						if tc.headError != "" {
+							w.Header().Set("x-minio-error-code", tc.headError)
+						}
+						w.WriteHeader(tc.headCode)
+					case http.MethodDelete:
+						deletes.Add(1)
+						if got := r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"; got == purge {
+							t.Errorf("source delete-marker header=%v, purge=%v", got, purge)
+						}
+						w.WriteHeader(tc.deleteCode)
+						if tc.deleteCode != 204 {
+							fmt.Fprintf(w, `<Error><Code>%s</Code><Message>injected rejection</Message></Error>`, map[int]string{403: "AccessDenied", 405: "MethodNotAllowed", 503: "ServiceUnavailable"}[tc.deleteCode])
+						}
+					default:
+						t.Errorf("unexpected method %s", r.Method)
+					}
+				}))
+				defer server.Close()
+				client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+				if err != nil {
+					t.Fatal(err)
+				}
+				old := globalBucketTargetSys
+				globalBucketTargetSys = &BucketTargetSys{hc: map[string]epHealth{client.EndpointURL().Host: {Online: !tc.offline}}}
+				defer func() { globalBucketTargetSys = old }()
+				d := DeletedObjectReplicationInfo{Bucket: "source", DeletedObject: DeletedObject{ObjectName: "marker", DeleteMarker: shape != "object-purge"}}
+				if shape == "marker-creation" || shape == "legacy-marker-purge" {
+					d.DeleteMarkerVersionID = version
+				} else {
+					d.VersionID = version
+				}
+				d.ReplicationState.Targets = map[string]replication.StatusType{"arn1": tc.creation}
+				d.ReplicationState.ResetStatusesMap = map[string]string{"arn1": "previous-reset"}
+				if purge {
+					d.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{"arn1": tc.priorPurge}
+				}
+				if tc.resync {
+					d.OpType = replication.ExistingObjectReplicationType
+				}
+				got := replicateDeleteToTarget(t.Context(), d, &TargetClient{Client: client, ARN: "arn1", Bucket: "target", ResetID: "current-reset"})
+				var wantCreation replication.StatusType
+				var wantPurge VersionPurgeStatusType
+				var wantHeads, wantDeletes int32
+				success := false
+				if purge {
+					wantCreation = ""
+					wantPurge = replication.VersionPurgeComplete
+					switch {
+					case tc.priorPurge == replication.VersionPurgeComplete:
+						success = true
+					case tc.offline:
+						wantPurge = replication.VersionPurgeFailed
+					default:
+						wantDeletes = 1
+						success = tc.deleteCode == 204
+						if !success {
+							wantPurge = replication.VersionPurgeFailed
+						}
+					}
+				} else {
+					switch {
+					case tc.creation == replication.Completed && !tc.resync:
+						success = true
+					case tc.offline:
+					default:
+						wantHeads = 1
+						switch tc.headCode {
+						case 405:
+							success = true
+						case 403, 503:
+						default:
+							wantDeletes = 1
+							success = tc.deleteCode == 204
+						}
+					}
+					if success {
+						wantCreation = replication.Completed
+					} else {
+						wantCreation = replication.Failed
+					}
+				}
+				if got.PrevReplicationStatus != tc.creation {
+					t.Error("previous creation state changed")
+				}
+				if got.ReplicationStatus != wantCreation || got.VersionPurgeStatus != wantPurge {
+					t.Errorf("status=%+v, want creation=%s purge=%s", got, wantCreation, wantPurge)
+				}
+				if heads.Load() != wantHeads || deletes.Load() != wantDeletes {
+					t.Errorf("HEAD/DELETE=%d/%d, want %d/%d", heads.Load(), deletes.Load(), wantHeads, wantDeletes)
+				}
+				if (got.Err != nil) == success {
+					t.Errorf("success=%v, error=%v", success, got.Err)
+				}
+				if tc.resync && success {
+					if !strings.HasSuffix(got.ResyncTimestamp, ";current-reset") {
+						t.Errorf("successful resync missing reset: %q", got.ResyncTimestamp)
+					}
+				} else if got.ResyncTimestamp != "previous-reset" {
+					t.Errorf("unexpected reset: %q", got.ResyncTimestamp)
+				}
+			})
+		}
+	}
+}
+
+func TestReplicateDeletePurgeMissingTargetState(t *testing.T) {
+	// Classification belongs to the operation, even if this target has no
+	// previous purge entry (another target supplies the tracked purge state).
+	var deletes atomic.Int32
+	version := mustGetUUID()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete || r.URL.Query().Get("versionId") != version || r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true" {
+			t.Errorf("wrong purge request: %s %s %v", r.Method, r.URL, r.Header)
+		}
+		deletes.Add(1)
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+	client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	old := globalBucketTargetSys
+	globalBucketTargetSys = &BucketTargetSys{hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}}
+	defer func() { globalBucketTargetSys = old }()
+	d := DeletedObjectReplicationInfo{Bucket: "source", DeletedObject: DeletedObject{ObjectName: "marker", DeleteMarker: true, DeleteMarkerVersionID: version, ReplicationState: ReplicationState{Targets: map[string]replication.StatusType{"arn1": replication.Completed}, PurgeTargets: map[string]VersionPurgeStatusType{"arn2": replication.VersionPurgePending}}}}
+	got := replicateDeleteToTarget(t.Context(), d, &TargetClient{Client: client, ARN: "arn1", Bucket: "target", ResetID: "reset"})
+	if deletes.Load() != 1 || got.VersionPurgeStatus != replication.VersionPurgeComplete || got.ReplicationStatus != "" {
+		t.Fatalf("operation misclassified: %+v, deletes=%d", got, deletes.Load())
+	}
+	got.ResyncTimestamp = "resync;reset"
+	state := getReplicationState(replicatedInfos{Targets: []replicatedTargetInfo{got}}, ReplicationState{}, "")
+	if state.ResetStatusesMap[targetResetHeader("arn1")] != got.ResyncTimestamp {
+		t.Fatal("resync timestamp not preserved with nil reset map")
+	}
+}

cmd/replication-metadata_test.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"maps"
+	"net/http"
+	"net/textproto"
+	"reflect"
+	"strings"
+	"testing"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func TestExtractReplicationMetadataPreservesNormalizedMetadata(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		wire []string
+		want string
+	}{
+		{name: "absent"},
+		{name: "transport-only", wire: []string{"aws-chunked"}},
+		{name: "mixed", wire: []string{"aws-chunked,gzip"}, want: "gzip"},
+		{name: "gzip", wire: []string{"gzip"}, want: "gzip"},
+		{name: "transport-last", wire: []string{"gzip,aws-chunked"}, want: "gzip"},
+		{name: "multiple-values", wire: []string{"aws-chunked", "gzip"}, want: "gzip"},
+		// Preserve the existing exact-token grammar; whitespace is not normalized here.
+		{name: "space-before-gzip", wire: []string{"aws-chunked, gzip"}, want: " gzip"},
+		{name: "space-before-transport", wire: []string{"gzip, aws-chunked"}, want: "gzip, aws-chunked"},
+	} {
+		for _, lowercase := range []bool{false, true} {
+			name := tc.name + "/canonical"
+			if lowercase {
+				name = tc.name + "/lowercase"
+			}
+			t.Run(name, func(t *testing.T) {
+				header := http.Header{
+					"Content-Type":      []string{"application/octet-stream"},
+					"X-Amz-Meta-Source": []string{"raw"},
+					"X-Minio-Replication-Server-Side-Encryption-Sealed-Key":     []string{"sealed-key"},
+					"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
+					"X-Minio-Replication-Server-Side-Encryption-Iv":             []string{"iv"},
+					"X-Minio-Replication-Encrypted-Multipart":                   []string{""},
+					"X-Minio-Replication-Actual-Object-Size":                    []string{"1"},
+					ReplicationSsecChecksumHeader:                               []string{"checksum"},
+					xhttp.AmzMetaUnencryptedContentLength:                       []string{"injected-length"},
+					xhttp.AmzMetaUnencryptedContentMD5:                          []string{"injected-md5"},
+				}
+				if tc.wire != nil {
+					header[xhttp.ContentEncoding] = tc.wire
+				}
+				if lowercase {
+					h := make(http.Header, len(header))
+					for k, v := range header {
+						h[strings.ToLower(k)] = v
+					}
+					header = h
+				}
+				metadata, err := extractMetadata(t.Context(), textproto.MIMEHeader(header))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if metadata["content-encoding"] != tc.want {
+					t.Fatalf("ordinary encoding=%q want=%q", metadata["content-encoding"], tc.want)
+				}
+				for _, internal := range replicationToInternalHeaders {
+					if _, ok := metadata[internal]; ok {
+						t.Fatalf("ordinary request accepted internal field %s", internal)
+					}
+				}
+				// Callers own ordinary metadata and may transform it after extraction.
+				metadata["content-type"] = "application/wasm"
+				for k := range metadata {
+					if strings.EqualFold(k, "x-amz-meta-source") {
+						metadata[k] = "caller"
+					}
+				}
+				want := maps.Clone(metadata)
+				maps.Copy(want, map[string]string{
+					"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "sealed-key",
+					"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+					"X-Minio-Internal-Server-Side-Encryption-Iv":             "iv",
+					"X-Minio-Internal-Encrypted-Multipart":                   "",
+					"X-Minio-Internal-Actual-Object-Size":                    "1",
+					ReplicationSsecChecksumHeader:                            "checksum",
+				})
+				for range 2 {
+					if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
+						t.Fatal(err)
+					}
+					if !reflect.DeepEqual(metadata, want) {
+						t.Errorf("restoration changed normalized metadata: got %#v want %#v", metadata, want)
+					}
+				}
+				if tc.want == "" {
+					if _, present := metadata["content-encoding"]; present {
+						t.Error("transport-only content-encoding key restored")
+					}
+				}
+				for _, key := range []string{xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5} {
+					if _, present := caseInsensitiveMap(metadata).Lookup(key); present {
+						t.Errorf("redacted metadata restored: %s", key)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestExtractReplicationMetadataNilHeader(t *testing.T) {
+	metadata := map[string]string{"content-type": "application/wasm"}
+	want := maps.Clone(metadata)
+	if err := extractReplicationMetadataFromMime(t.Context(), nil, metadata); err != errInvalidArgument {
+		t.Fatalf("nil header: got %v want %v", err, errInvalidArgument)
+	}
+	if !reflect.DeepEqual(metadata, want) {
+		t.Fatalf("nil input changed metadata: %#v", metadata)
+	}
+}

cmd/replication-tagging-order_test.go

@@ -0,0 +1,599 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+const r5TagStamp = ReservedMetadataPrefixLower + TaggingTimestamp
+
+func r5Capacity(z *erasureServerPools) func() {
+	var restores []func()
+	for _, pool := range z.serverPools {
+		for _, set := range pool.sets {
+			old := set.getDisks
+			disks := append([]StorageAPI(nil), old()...)
+			for i := range disks {
+				disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			restores = append(restores, func() { set.getDisks = old })
+		}
+	}
+	return func() {
+		for _, restore := range restores {
+			restore()
+		}
+	}
+}
+
+func r5Request(t *testing.T, router http.Handler, cred auth.Credentials, method, path, body string, headers map[string]string) *httptest.ResponseRecorder {
+	t.Helper()
+	r, err := newTestSignedRequestV4(method, path, int64(len(body)), strings.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, r)
+	return w
+}
+
+func r5Stored(t *testing.T, obj interface {
+	GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error)
+}, bucket, name, vid, wantTags, wantStamp string,
+) ObjectInfo {
+	t.Helper()
+	oi, err := obj.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: vid})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if oi.UserTags != wantTags || oi.UserDefined[r5TagStamp] != wantStamp {
+		t.Fatalf("%s(%s): tags=%q stamp=%q, want %q %q", name, vid, oi.UserTags, oi.UserDefined[r5TagStamp], wantTags, wantStamp)
+	}
+	return oi
+}
+
+// The request bytes are signed and enter the real API and storage implementation.
+// Equal/stale retransmits may retain the existing 412 duplicate response.
+func r5Receive(t *testing.T, obj ObjectLayer, router http.Handler, cred auth.Credentials, bucket, operation string, source ObjectInfo, stamp string, afterInit func()) {
+	t.Helper()
+	opts, _, err := putReplicationOpts(t.Context(), "", source)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if operation == "multipart" {
+		opts.Internal.SourceMTime = time.Time{}
+	}
+	headers := make(map[string]string)
+	for k, vs := range opts.Header() {
+		if len(vs) > 0 {
+			headers[k] = vs[0]
+		}
+	}
+	if stamp == "" {
+		delete(headers, http.CanonicalHeaderKey(xhttp.MinIOSourceTaggingTimestamp))
+		delete(headers, xhttp.MinIOSourceTaggingTimestamp)
+	} else {
+		headers[http.CanonicalHeaderKey(xhttp.MinIOSourceTaggingTimestamp)] = stamp
+	}
+	path := "/" + bucket + "/" + source.Name + "?versionId=" + source.VersionID
+	var w *httptest.ResponseRecorder
+	switch operation {
+	case "copy", "copy-default":
+		maps.Copy(headers, getCopyObjMetadata(source, ""))
+		headers[xhttp.AmzCopySource] = "/" + bucket + "/" + source.Name + "?versionId=" + source.VersionID
+		if operation == "copy" {
+			headers[xhttp.AmzMetadataDirective] = "REPLACE"
+		}
+		w = r5Request(t, router, cred, http.MethodPut, path, "", headers)
+	case "put":
+		w = r5Request(t, router, cred, http.MethodPut, path, "data", headers)
+	case "multipart":
+		w = r5Request(t, router, cred, http.MethodPost, path+"&uploads", "", headers)
+		if w.Code == http.StatusPreconditionFailed {
+			return
+		}
+		if w.Code != http.StatusOK {
+			t.Fatalf("init: %d %s", w.Code, w.Body.String())
+		}
+		var init struct {
+			UploadID string `xml:"UploadId"`
+		}
+		if err := xml.Unmarshal(w.Body.Bytes(), &init); err != nil || init.UploadID == "" {
+			t.Fatalf("init XML: %v %s", err, w.Body.String())
+		}
+		mi, err := obj.GetMultipartInfo(t.Context(), bucket, source.Name, init.UploadID, ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if stamp != "" && mi.UserDefined[r5TagStamp] != stamp {
+			t.Fatalf("upload persisted stamp=%q, want %q", mi.UserDefined[r5TagStamp], stamp)
+		}
+		partPath := "/" + bucket + "/" + source.Name + "?uploadId=" + url.QueryEscape(init.UploadID)
+		ph := map[string]string{xhttp.MinIOSourceReplicationRequest: "true"}
+		part := r5Request(t, router, cred, http.MethodPut, partPath+"&partNumber=1", "data", ph)
+		if part.Code != http.StatusOK {
+			t.Fatalf("part: %d %s", part.Code, part.Body.String())
+		}
+		if afterInit != nil {
+			afterInit()
+		}
+		body := "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" + canonicalizeETag(part.Header()[xhttp.ETag][0]) + "</ETag></Part></CompleteMultipartUpload>"
+		ph[xhttp.MinIOSourceMTime] = source.ModTime.Format(time.RFC3339Nano)
+		ph[xhttp.MinIOSourceETag] = source.ETag
+		w = r5Request(t, router, cred, http.MethodPost, partPath, body, ph)
+	}
+	if w.Code != http.StatusOK && w.Code != http.StatusPreconditionFailed {
+		t.Fatalf("%s: %d %s", operation, w.Code, w.Body.String())
+	}
+}
+
+func TestAPITaggingReplicationOrdering(t *testing.T)    { r5APIOrdering(t, false) }
+func TestAPITaggingReplicationOrderingKMS(t *testing.T) { r5APIOrdering(t, true) }
+func r5APIOrdering(t *testing.T, encrypted bool) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		defer r5Capacity(obj.(*erasureServerPools))()
+		if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+			t.Fatal(err)
+		}
+		if encrypted {
+			prev := GlobalKMS
+			GlobalKMS = kms.NewStub("r5-tag-order")
+			defer func() { GlobalKMS = prev }()
+			sse := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>r5-tag-order</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
+			if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketSSEConfig, sse); err != nil {
+				t.Fatal(err)
+			}
+		}
+		base := time.Now().UTC().Add(-5 * time.Hour)
+		for _, op := range []string{"copy", "copy-default", "put", "multipart"} {
+			for _, version := range []string{"uuid", "null"} {
+				t.Run(instance+"/"+op+"/"+version, func(t *testing.T) {
+					name := op + "-" + version
+					vid := mustGetUUID()
+					if version == "null" {
+						vid = nullVersionID
+					}
+					original, err := obj.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true, VersionID: vid, UserDefined: map[string]string{xhttp.AmzObjectTagging: "key=original", r5TagStamp: base.Format(time.RFC3339Nano)}})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if op == "multipart" {
+						// The production sender uses multipart only for multipart
+						// sources. Preserve a real multipart ETag and part layout.
+						metadata := maps.Clone(original.UserDefined)
+						metadata[xhttp.AmzObjectTagging] = original.UserTags
+						mp, err := obj.NewMultipartUpload(t.Context(), bucket, name, ObjectOptions{Versioned: true, VersionID: vid, UserDefined: metadata})
+						if err != nil {
+							t.Fatal(err)
+						}
+						part, err := obj.PutObjectPart(t.Context(), bucket, name, mp.UploadID, 1, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{})
+						if err != nil {
+							t.Fatal(err)
+						}
+						original, err = obj.CompleteMultipartUpload(t.Context(), bucket, name, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
+						if err != nil {
+							t.Fatal(err)
+						}
+					}
+					// A later unrelated version must not contribute tags to an explicitly addressed UUID/null version.
+					latest, err := obj.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, strings.NewReader("other"), 5, "", ""), ObjectOptions{Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "key=latest", r5TagStamp: base.Add(10 * time.Hour).Format(time.RFC3339Nano)}})
+					if err != nil {
+						t.Fatal(err)
+					}
+					for _, event := range []struct {
+						name, tags string
+						hours      int
+						wantTags   string
+						wantHours  int
+					}{
+						{"delete", "", 3, "", 3}, {"stale", "key=stale", 2, "", 3}, {"equal-conflict", "key=conflict", 3, "", 3}, {"newer", "key=new", 4, "key=new", 4},
+					} {
+						t.Run(event.name, func(t *testing.T) {
+							source := original
+							source.VersionID = vid
+							source.UserDefined = maps.Clone(original.UserDefined)
+							source.UserTags = event.tags
+							stamp := base.Add(time.Duration(event.hours) * time.Hour).Format(time.RFC3339Nano)
+							source.UserDefined[r5TagStamp] = stamp
+							r5Receive(t, obj, router, cred, bucket, op, source, stamp, nil)
+							r5Stored(t, obj, bucket, name, vid, event.wantTags, base.Add(time.Duration(event.wantHours)*time.Hour).Format(time.RFC3339Nano))
+							r5Stored(t, obj, bucket, name, latest.VersionID, "key=latest", base.Add(10*time.Hour).Format(time.RFC3339Nano))
+						})
+					}
+					source := original
+					source.VersionID = vid
+					source.UserTags = "key=unversioned-event"
+					r5Receive(t, obj, router, cred, bucket, op, source, "", nil)
+					r5Stored(t, obj, bucket, name, vid, "key=new", base.Add(4*time.Hour).Format(time.RFC3339Nano))
+					get := r5Request(t, router, cred, http.MethodGet, "/"+bucket+"/"+name+"?versionId="+vid, "", nil)
+					if get.Code != http.StatusOK || get.Body.String() != "data" {
+						t.Fatalf("plaintext GET: %d %q", get.Code, get.Body.String())
+					}
+				})
+			}
+		}
+	}})
+}
+
+func TestAPITaggingMultipartCommitRechecksRevision(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		defer r5Capacity(obj.(*erasureServerPools))()
+		if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+			t.Fatal(err)
+		}
+		base := time.Now().UTC().Add(-time.Hour)
+		source := ObjectInfo{Name: "commit-recheck", VersionID: mustGetUUID(), ModTime: base, UserTags: "key=incoming", UserDefined: map[string]string{r5TagStamp: base.Format(time.RFC3339Nano)}}
+		later := base.Add(time.Minute).Format(time.RFC3339Nano)
+		r5Receive(t, obj, router, cred, bucket, "multipart", source, base.Format(time.RFC3339Nano), func() {
+			_, err := obj.PutObject(t.Context(), bucket, source.Name, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true, VersionID: source.VersionID, MTime: base, UserDefined: map[string]string{r5TagStamp: later}})
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+		r5Stored(t, obj, bucket, source.Name, source.VersionID, "", later)
+		t.Logf("%s: deletion committed between initiation and completion survived", instance)
+	}})
+}
+
+func TestAPILocalTaggingAlwaysAdvancesRevision(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		defer r5Capacity(obj.(*erasureServerPools))()
+		if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+			t.Fatal(err)
+		}
+		oi, err := obj.PutObject(t.Context(), bucket, "local-tags", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		path := "/" + bucket + "/local-tags?tagging&versionId=" + oi.VersionID
+		for n, method := range []string{http.MethodPut, http.MethodDelete, http.MethodDelete, http.MethodPut} {
+			body := ""
+			want := ""
+			status := http.StatusNoContent
+			if method == http.MethodPut {
+				status = http.StatusOK
+				body = "<Tagging><TagSet/></Tagging>"
+				if n == 0 {
+					want = "key=local"
+					body = "<Tagging><TagSet><Tag><Key>key</Key><Value>local</Value></Tag></TagSet></Tagging>"
+				}
+			}
+			before := time.Now().UTC()
+			w := r5Request(t, router, cred, method, path, body, nil)
+			if w.Code != status {
+				t.Fatalf("%s: %d %s", method, w.Code, w.Body.String())
+			}
+			now, err := obj.GetObjectInfo(t.Context(), bucket, "local-tags", ObjectOptions{VersionID: oi.VersionID})
+			if err != nil {
+				t.Fatal(err)
+			}
+			stamp, err := time.Parse(time.RFC3339Nano, now.UserDefined[r5TagStamp])
+			if err != nil || stamp.Before(before) || now.UserTags != want {
+				t.Fatalf("local %s: tags=%q timestamp=%q err=%v", method, now.UserTags, now.UserDefined[r5TagStamp], err)
+			}
+			if !now.ModTime.Equal(oi.ModTime) {
+				t.Fatal("tagging changed the object's data modification time")
+			}
+			t.Logf("%s mutation %d persisted tags=%q timestamp=%s", instance, n, now.UserTags, stamp)
+		}
+		// Ordinary COPY with empty REPLACE has the same local-deletion semantics.
+		before := time.Now().UTC()
+		headers := map[string]string{xhttp.AmzCopySource: "/" + bucket + "/local-tags?versionId=" + oi.VersionID, xhttp.AmzMetadataDirective: "REPLACE", xhttp.AmzTagDirective: "REPLACE"}
+		w := r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/copied-empty", "", headers)
+		if w.Code != http.StatusOK {
+			t.Fatalf("copy: %d %s", w.Code, w.Body.String())
+		}
+		copied, err := obj.GetObjectInfo(t.Context(), bucket, "copied-empty", ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		stamp, err := time.Parse(time.RFC3339Nano, copied.UserDefined[r5TagStamp])
+		if err != nil || stamp.Before(before) || copied.UserTags != "" {
+			t.Fatalf("local COPY: %v %+v", err, copied)
+		}
+	}})
+}
+
+func TestTaggingTimestampWire(t *testing.T) {
+	now := time.Now().UTC()
+	for _, tag := range []string{"", "key=value"} {
+		for _, stamp := range []string{"", now.Format(time.RFC3339Nano), "invalid"} {
+			t.Run(fmt.Sprintf("%s/%s", tag, stamp), func(t *testing.T) {
+				source := ObjectInfo{ModTime: now.Add(-time.Hour), UserTags: tag, UserDefined: map[string]string{}}
+				if stamp != "" {
+					source.UserDefined[r5TagStamp] = stamp
+				}
+				opts, _, err := putReplicationOpts(t.Context(), "", source)
+				if stamp == "invalid" {
+					if err == nil {
+						t.Fatal("invalid timestamp accepted")
+					}
+					return
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+				want := stamp
+				if want == "" && tag != "" {
+					want = source.ModTime.Format(time.RFC3339Nano)
+				}
+				if got := opts.Header().Get(xhttp.MinIOSourceTaggingTimestamp); got != want {
+					t.Fatalf("wire timestamp=%q want %q", got, want)
+				}
+			})
+		}
+	}
+}
+
+func TestAPIPoolsTaggingReplicaDeletion(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	defer r5Capacity(z)()
+	if err := newTestConfig(globalMinioDefaultRegion, z); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+		t.Fatal(err)
+	}
+	router := initTestAPIEndPoints(z, nil)
+	base := time.Now().UTC().Add(-5 * time.Hour)
+	for _, op := range []string{"copy", "copy-default", "put", "multipart"} {
+		for _, kind := range []string{"uuid", "null"} {
+			t.Run(op+"/"+kind, func(t *testing.T) {
+				vid := mustGetUUID()
+				if kind == "null" {
+					vid = nullVersionID
+				}
+				name := "pool-tags-" + op + "-" + kind
+				var source ObjectInfo
+				for pool := range 2 {
+					tag := "key=stale-pool"
+					ts := base
+					if pool == 1 {
+						tag = ""
+						ts = base.Add(3 * time.Hour)
+					}
+					source = putConsistencyObject(t, z, bucket, name, pool, "data", ObjectOptions{Versioned: true, VersionID: vid, MTime: base, UserDefined: map[string]string{xhttp.AmzObjectTagging: tag, r5TagStamp: ts.Format(time.RFC3339Nano)}})
+				}
+				// Clear all copies through the signed local handler, then replay a stale incoming state.
+				req := r5Request(t, router, globalActiveCred, http.MethodDelete, "/"+bucket+"/"+name+"?tagging&versionId="+vid, "", nil)
+				if req.Code != http.StatusNoContent {
+					t.Fatalf("DELETE: %d %s", req.Code, req.Body.String())
+				}
+				current, err := z.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: vid})
+				if err != nil {
+					t.Fatal(err)
+				}
+				deletedAt := current.UserDefined[r5TagStamp]
+				for pool := range 2 {
+					r5Stored(t, z.serverPools[pool], bucket, name, vid, "", deletedAt)
+				}
+				source.VersionID = vid
+				source.UserTags = "key=delayed"
+				source.UserDefined = map[string]string{r5TagStamp: base.Add(time.Hour).Format(time.RFC3339Nano)}
+				r5Receive(t, z, router, globalActiveCred, bucket, op, source, source.UserDefined[r5TagStamp], nil)
+				// The addressed version must remain readable through normal routing.
+				r5Stored(t, z, bucket, name, vid, "", deletedAt)
+				// Existing duplicate suppression may leave both identical tombstones; any retained copy must be correct.
+				for pool := range 2 {
+					got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: vid})
+					if isErrVersionNotFound(err) {
+						continue
+					}
+					if err != nil {
+						t.Fatal(err)
+					}
+					if got.UserTags != "" || got.UserDefined[r5TagStamp] != deletedAt {
+						t.Fatalf("pool %d: tags=%q stamp=%q want deletion %q", pool, got.UserTags, got.UserDefined[r5TagStamp], deletedAt)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestAPITaggingSSECRotationPreservesDeletionRevision(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		defer r5Capacity(obj.(*erasureServerPools))()
+		oldTLS := globalIsTLS
+		globalIsTLS = true
+		defer func() { globalIsTLS = oldTLS }()
+		if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+			t.Fatal(err)
+		}
+		keys := [][]byte{[]byte(strings.Repeat("a", 32)), []byte(strings.Repeat("b", 32)), []byte(strings.Repeat("c", 32)), []byte(strings.Repeat("d", 32))}
+		const name = "tag-rotation"
+		putCopyChecksumSource(t, router, cred, bucket, name, []byte("data"), ssecKeyHeaders(keys[0], false))
+		oi, err := obj.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		base := time.Now().UTC().Add(-time.Hour)
+		for i, event := range []struct {
+			tags      string
+			delta     int
+			wantTags  string
+			wantDelta int
+		}{
+			{"key=live", 1, "key=live", 1}, {"", 3, "", 3}, {"key=stale", 2, "", 3},
+		} {
+			h := ssecKeyHeaders(keys[i], true)
+			maps.Copy(h, ssecKeyHeaders(keys[i+1], false))
+			h[xhttp.AmzObjectTagging] = event.tags
+			h[xhttp.AmzTagDirective] = "REPLACE"
+			h[xhttp.MinIOSourceTaggingTimestamp] = base.Add(time.Duration(event.delta) * time.Minute).Format(time.RFC3339Nano)
+			sendReplicaLockCopy(t, router, cred, bucket, name, oi.VersionID, h)
+			r5Stored(t, obj, bucket, name, oi.VersionID, event.wantTags, base.Add(time.Duration(event.wantDelta)*time.Minute).Format(time.RFC3339Nano))
+		}
+		get := r5Request(t, router, cred, http.MethodGet, "/"+bucket+"/"+name+"?versionId="+oi.VersionID, "", ssecKeyHeaders(keys[3], false))
+		if get.Code != http.StatusOK || get.Body.String() != "data" {
+			t.Fatalf("%s GET after rotations: %d %q", instance, get.Code, get.Body.String())
+		}
+	}})
+}
+
+func TestTaggingRepeatedValueNeedsRevisionDelivery(t *testing.T) {
+	for _, tag := range []string{"", "key=same"} {
+		now := time.Now().UTC()
+		source := ObjectInfo{ModTime: now, UserTags: tag, UserDefined: map[string]string{r5TagStamp: now.Add(time.Hour).Format(time.RFC3339Nano)}}
+		target := minio.ObjectInfo{LastModified: now}
+		if tag != "" {
+			target.UserTags = map[string]string{"key": "same"}
+			target.UserTagCount = 1
+		}
+		if got := getReplicationAction(source, target, replication.MetadataReplicationType); got != replicateMetadata {
+			t.Errorf("equal tags %q suppress a newer revision: got %s; an intervening delayed deletion can win", tag, got)
+		}
+	}
+}
+
+func TestTaggingProductionCopyWireShape(t *testing.T) {
+	got := make(chan http.Header, 1)
+	peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got <- r.Header.Clone()
+		w.Header().Set(xhttp.ContentType, "application/xml")
+		w.Write([]byte(`<CopyObjectResult><LastModified>2026-09-15T01:00:00Z</LastModified><ETag>"abc"</ETag></CopyObjectResult>`))
+	}))
+	defer peer.Close()
+	c, err := minio.New(strings.TrimPrefix(peer.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Now().UTC()
+	oi := ObjectInfo{Name: "object", ModTime: now, ETag: "abc", VersionID: mustGetUUID()}
+	core := minio.Core{Client: c}
+	_, err = core.CopyObject(t.Context(), "bucket", "object", "bucket", "object", getCopyObjMetadata(oi, ""), minio.CopySrcOptions{VersionID: oi.VersionID}, minio.PutObjectOptions{Internal: minio.AdvancedPutOptions{SourceVersionID: oi.VersionID, ReplicationRequest: true, TaggingTimestamp: now}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := <-got
+	t.Logf("SDK wire: metadata-directive=%q tagging-directive=%q tagging=%q time=%q", h.Get(xhttp.AmzMetadataDirective), h.Get(xhttp.AmzTagDirective), h.Get(xhttp.AmzObjectTagging), h.Get(xhttp.MinIOSourceTaggingTimestamp))
+	if h.Get(xhttp.AmzMetadataDirective) != "" || h.Get(xhttp.AmzTagDirective) != "REPLACE" || h.Get(xhttp.MinIOSourceTaggingTimestamp) != now.Format(time.RFC3339Nano) {
+		t.Fatalf("unexpected SDK wire: %v", h)
+	}
+}
+
+func TestLocalTaggingCommitCannotRegressRevision(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	incoming := "2026-09-15T01:00:00Z"
+	newer := "2026-09-15T02:00:00Z"
+	newest := "2026-09-15T03:00:00Z"
+	vid := mustGetUUID()
+	name := "inverted-local-tags"
+	for pool := range 2 {
+		putConsistencyObject(t, z, bucket, name, pool, "data", ObjectOptions{Versioned: true, VersionID: vid, UserDefined: map[string]string{r5TagStamp: []string{newer, newest}[pool]}})
+	}
+	oi, err := z.PutObjectTags(t.Context(), bucket, name, "key=after-delete", ObjectOptions{VersionID: vid, UserDefined: map[string]string{r5TagStamp: incoming}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "2026-09-15T03:00:00.000000001Z"
+	for pool := range 2 {
+		r5Stored(t, z.serverPools[pool], bucket, name, vid, "key=after-delete", want)
+	}
+	if oi.UserDefined[r5TagStamp] != want {
+		t.Fatalf("response stamp=%q want %q", oi.UserDefined[r5TagStamp], want)
+	}
+	// The single-set guard also applies when the pools dispatcher is bypassed.
+	_, err = z.serverPools[0].PutObjectTags(t.Context(), bucket, name, "", ObjectOptions{VersionID: vid, UserDefined: map[string]string{r5TagStamp: incoming}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	r5Stored(t, z.serverPools[0], bucket, name, vid, "", "2026-09-15T03:00:00.000000002Z")
+}
+
+func TestTaggingReplicaContentDuplicateGuard(t *testing.T) {
+	stamp := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC)
+	for _, tc := range []struct {
+		name             string
+		delta            int
+		stored           string
+		trusted, replica bool
+		ifMatch, ifNone  string
+		wantSkip         bool
+	}{
+		{name: "newer", delta: 1, trusted: true, replica: true},
+		{name: "equal", trusted: true, replica: true, wantSkip: true},
+		{name: "older", delta: -1, trusted: true, replica: true, wantSkip: true},
+		{name: "invalid-stored", stored: "invalid", delta: 1, trusted: true, replica: true},
+		{name: "untrusted", delta: 1, wantSkip: true},
+		{name: "marker-only", delta: 1, trusted: true, wantSkip: true},
+		{name: "if-match-fails", delta: 1, trusted: true, replica: true, ifMatch: "other", wantSkip: true},
+		{name: "if-none-match-fails", delta: 1, trusted: true, replica: true, ifNone: "etag", wantSkip: true},
+		{name: "if-match-passes", delta: 1, trusted: true, replica: true, ifMatch: "etag"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := withReplicationTrust(t.Context(), tc.trusted, tc.replica)
+			r := httptest.NewRequest(http.MethodPut, "/bucket/object", nil).WithContext(ctx)
+			if tc.ifMatch != "" {
+				r.Header.Set(xhttp.IfMatch, tc.ifMatch)
+			}
+			if tc.ifNone != "" {
+				r.Header.Set(xhttp.IfNoneMatch, tc.ifNone)
+			}
+			stored := tc.stored
+			if stored == "" {
+				stored = stamp.Format(time.RFC3339Nano)
+			}
+			oi := ObjectInfo{ModTime: stamp, VersionID: mustGetUUID(), ETag: "etag", UserDefined: map[string]string{r5TagStamp: stored}}
+			opts := ObjectOptions{VersionID: oi.VersionID, PreserveETag: oi.ETag, ReplicationRequest: tc.trusted, ReplicationSourceTaggingTimestamp: stamp.Add(time.Duration(tc.delta) * time.Second)}
+			if skip := checkPreconditionsPUT(ctx, httptest.NewRecorder(), r, oi, opts); skip != tc.wantSkip {
+				t.Fatalf("skip=%v, want %v", skip, tc.wantSkip)
+			}
+		})
+	}
+}
+
+func TestAPITaggingUnqualifiedCopyOrdering(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		defer r5Capacity(obj.(*erasureServerPools))()
+		stamp := time.Now().UTC().Add(-time.Hour)
+		oi, err := obj.PutObject(t.Context(), bucket, "unqualified-tags", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: "key=stored", r5TagStamp: stamp.Format(time.RFC3339Nano)}})
+		if err != nil {
+			t.Fatal(err)
+		}
+		source := oi
+		source.UserTags = ""
+		source.UserDefined = maps.Clone(oi.UserDefined)
+		r5Receive(t, obj, router, cred, bucket, "copy", source, stamp.Format(time.RFC3339Nano), nil)
+		r5Stored(t, obj, bucket, oi.Name, "", "key=stored", stamp.Format(time.RFC3339Nano))
+		later := stamp.Add(time.Minute).Format(time.RFC3339Nano)
+		r5Receive(t, obj, router, cred, bucket, "copy-default", source, later, nil)
+		r5Stored(t, obj, bucket, oi.Name, "", "", later)
+		source.UserTags = "key=delayed"
+		r5Receive(t, obj, router, cred, bucket, "copy-default", source, stamp.Format(time.RFC3339Nano), nil)
+		r5Stored(t, obj, bucket, oi.Name, "", "", later)
+		t.Logf("%s: unqualified COPY keeps stored ties and ordered deletion", instance)
+	}})
+}

cmd/replication-tagging-sender_test.go

@@ -0,0 +1,154 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/once"
+)
+
+func r5ReplicationFixture(t *testing.T, obj ObjectLayer, bucket string, client *minio.Client) (chan ReplicationWorkerOperation, func()) {
+	t.Helper()
+	const arn = "arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket"
+	target := &TargetClient{Client: client, ARN: arn, Bucket: bucket}
+	globalBucketTargetSys.arnRemotesMap[arn] = arnTarget{Client: target, lastRefresh: UTCNow()}
+	globalBucketTargetSys.targetsMap[bucket] = []madmin.BucketTarget{{Arn: arn, TargetBucket: bucket}}
+	meta, err := globalBucketMetadataSys.Get(bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := configs[0]
+	cfg.RoleArn = arn
+	meta.replicationConfig = &cfg
+	globalBucketMetadataSys.Set(bucket, meta)
+	worker := make(chan ReplicationWorkerOperation, 10)
+	previous := globalReplicationPool
+	globalReplicationPool = once.NewSingleton[ReplicationPool]()
+	globalReplicationPool.Set(&ReplicationPool{ctx: t.Context(), objLayer: obj, workers: []chan ReplicationWorkerOperation{worker}, stats: globalReplicationStats.Load(), mrfSaveCh: make(chan MRFReplicateEntry, 10)})
+	return worker, func() { globalReplicationPool = previous }
+}
+
+func TestTaggingReplicationSenderRetryAndAcknowledgment(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		defer r5Capacity(obj.(*erasureServerPools))()
+		if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+			t.Fatal(err)
+		}
+		const name = "tagging-old-queue"
+		oi, err := obj.PutObject(t.Context(), bucket, name, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		requests := make(chan http.Header, 4)
+		var attempts atomic.Int32
+		peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set(xhttp.AmzVersionID, oi.VersionID)
+			w.Header().Set(xhttp.ETag, "\""+oi.ETag+"\"")
+			w.Header().Set(xhttp.LastModified, oi.ModTime.Format(http.TimeFormat))
+			w.Header().Set(xhttp.ContentType, oi.ContentType)
+			if r.Method == http.MethodHead {
+				w.Header().Set(xhttp.ContentLength, "4")
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+			requests <- r.Header.Clone()
+			w.Header().Set(xhttp.ContentType, "application/xml")
+			if attempts.Add(1) == 1 {
+				w.WriteHeader(http.StatusServiceUnavailable)
+				w.Write([]byte(`<Error><Code>SlowDown</Code><Message>retry fixture</Message></Error>`))
+				return
+			}
+			w.Write([]byte("<CopyObjectResult><LastModified>" + oi.ModTime.Format(time.RFC3339Nano) + "</LastModified><ETag>\"" + oi.ETag + "\"</ETag></CopyObjectResult>"))
+		}))
+		defer peer.Close()
+		client, err := minio.New(strings.TrimPrefix(peer.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+		if err != nil {
+			t.Fatal(err)
+		}
+		worker, cleanup := r5ReplicationFixture(t, obj, bucket, client)
+		defer cleanup()
+		body := `<Tagging><TagSet><Tag><Key>key</Key><Value>queued</Value></Tag></TagSet></Tagging>`
+		w := r5Request(t, router, cred, http.MethodPut, "/"+bucket+"/"+name+"?tagging&versionId="+oi.VersionID, body, nil)
+		if w.Code != http.StatusOK || len(worker) != 1 {
+			t.Fatalf("tagging PUT: %d %s queued=%d", w.Code, w.Body.String(), len(worker))
+		}
+		old := (<-worker).(ReplicateObjectInfo)
+		// Delete through the actual handler before processing the old task.
+		w = r5Request(t, router, cred, http.MethodDelete, "/"+bucket+"/"+name+"?tagging&versionId="+oi.VersionID, "", nil)
+		if w.Code != http.StatusNoContent || len(worker) != 1 {
+			t.Fatalf("tagging DELETE: %d %s queued=%d", w.Code, w.Body.String(), len(worker))
+		}
+		deleted, err := obj.GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: oi.VersionID})
+		if err != nil {
+			t.Fatal(err)
+		}
+		stamp := deleted.UserDefined[r5TagStamp]
+		for attempt := 0; attempt < 2; attempt++ {
+			result := replicateObject(t.Context(), old, obj)
+			want := replication.Failed
+			if attempt == 1 {
+				want = replication.Completed
+			}
+			if result.ReplicationStatus() != want {
+				t.Fatalf("attempt %d result=%+v want %s", attempt, result, want)
+			}
+			if len(result.Targets) != 1 || result.Targets[0].ReplicationAction != replicateMetadata || (result.Targets[0].Err != nil) != (attempt == 0) {
+				t.Fatalf("attempt %d reported wrong action/error: %+v", attempt, result)
+			}
+			r5Stored(t, obj, bucket, name, oi.VersionID, "", stamp)
+			select {
+			case h := <-requests:
+				if h.Get(xhttp.MinIOSourceTaggingTimestamp) != stamp || h.Get(xhttp.AmzObjectTagging) != "" || h.Get(xhttp.AmzTagDirective) != "REPLACE" || h.Get(xhttp.AmzMetadataDirective) != "" {
+					t.Fatalf("sender did not carry current deletion: %v", h)
+				}
+				t.Logf("%s attempt %d sent tags=%q timestamp=%s status=%s", instance, attempt, h.Get(xhttp.AmzObjectTagging), stamp, want)
+			default:
+				t.Fatal("no metadata COPY sent for same-empty target")
+			}
+		}
+		// With a real outgoing rule enabled, a signed incoming replica COPY
+		// must not queue another outgoing event and create a feedback loop.
+		before := len(worker)
+		r5Receive(t, obj, router, cred, bucket, "copy", deleted, stamp, nil)
+		if len(worker) != before {
+			t.Fatal("incoming replica COPY scheduled another outgoing event")
+		}
+		// The metadata sender must fail malformed stored revisions before COPY,
+		// just as the full retransmission option builder does.
+		_, err = obj.PutObjectTags(t.Context(), bucket, name, "", ObjectOptions{VersionID: oi.VersionID, UserDefined: map[string]string{r5TagStamp: "invalid"}})
+		if err != nil {
+			t.Fatal(err)
+		}
+		target := globalBucketTargetSys.GetRemoteTargetClient(bucket, globalBucketTargetSys.targetsMap[bucket][0].Arn)
+		invalid := old.replicateAll(t.Context(), obj, target)
+		if invalid.ReplicationStatus != replication.Failed || invalid.Err == nil || len(requests) != 0 {
+			t.Fatalf("invalid timestamp was not rejected before send: %+v", invalid)
+		}
+	}})
+}

cmd/server-main.go

@@ -501,7 +501,6 @@ func initAllSubsystems(ctx context.Context) {
 	globalTierConfigMgr = NewTierConfigMgr()
 
 	globalTransitionState = newTransitionState(GlobalContext)
-	globalAccessTierState = newAccessTierState(GlobalContext)
 	globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext)
 }
 
@@ -902,6 +901,8 @@ func serverMain(ctx *cli.Context) {
 		close(globalGridStart)
 		close(globalLockGridStart)
 
+		// The HTTP/1 listener preserves absolute header deadlines and renews the
+		// body read/write idle limits, so transfers may outlast IdleTimeout.
 		httpServer := xhttp.NewServer(getServerListenAddrs()).
 			UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 			UseTLSConfig(newTLSConfig(getCert)).
@@ -1070,10 +1071,6 @@ func serverMain(ctx *cli.Context) {
 		bootstrapTrace("globalTransitionState.Init", func() {
 			globalTransitionState.Init(newObject)
 		})
-		bootstrapTrace("globalAccessTierState.Init", func() {
-			globalAccessTierState.Init(newObject)
-			go globalAccessTracker.run(GlobalContext, newObject)
-		})
 
 		go func() {
 			// Initialize transition tier configuration manager

cmd/server_deadline_config_test.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/cli"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func TestServerReadHeaderTimeoutConfig(t *testing.T) {
+	for _, tc := range []struct {
+		name, env, idle string
+		args            []string
+		want            time.Duration
+		fmtgen          bool
+	}{
+		{name: "default", want: xhttp.DefaultReadHeaderTimeout},
+		{name: "flag", args: []string{"--read-header-timeout=100ms"}, want: 100 * time.Millisecond},
+		{name: "environment", env: "170ms", want: 170 * time.Millisecond},
+		{name: "flag-over-environment", env: "170ms", args: []string{"--read-header-timeout=80ms"}, want: 80 * time.Millisecond},
+		{name: "yaml-retains-flag", args: []string{"--config=testdata/config/1.yaml", "--read-header-timeout=100ms"}, want: 100 * time.Millisecond},
+		{name: "zero-fallback", args: []string{"--read-header-timeout=0s"}},
+		{name: "zero-idle-default-header", idle: "0s", want: xhttp.DefaultReadHeaderTimeout},
+		{name: "negative-idle-default-header", idle: "-1s", want: xhttp.DefaultReadHeaderTimeout},
+		{name: "fmt-gen-unregistered-duration", env: "100ms", fmtgen: true},
+		{name: "negative-disabled", args: []string{"--read-header-timeout=-1s"}, want: -time.Second},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			for _, key := range []string{"MINIO_ARGS", "MINIO_VOLUMES", "MINIO_ENDPOINTS", "MINIO_CONFIG", "MINIO_ERASURE_SET_DRIVE_COUNT"} {
+				t.Setenv(key, "")
+			}
+			t.Setenv("MINIO_READ_HEADER_TIMEOUT", tc.env)
+			if tc.env == "" {
+				if err := os.Unsetenv("MINIO_READ_HEADER_TIMEOUT"); err != nil {
+					t.Fatal(err)
+				}
+			}
+			idle := tc.idle
+			if idle == "" {
+				idle = "2s"
+			}
+			t.Setenv("MINIO_IDLE_TIMEOUT", idle)
+			idleWant, err := time.ParseDuration(idle)
+			if err != nil {
+				t.Fatal(err)
+			}
+			commandName, flags := "server", serverCmd.Flags
+			if tc.fmtgen {
+				commandName, flags = "fmt-gen", fmtGenFlags
+				idleWant = 0
+			}
+			var got serverCtxt
+			called := false
+			app := cli.NewApp()
+			app.Commands = []cli.Command{{Name: commandName, Flags: flags, Action: func(ctx *cli.Context) error {
+				called = true
+				if parsed := ctx.Duration("read-header-timeout"); parsed != tc.want {
+					t.Errorf("CLI read-header-timeout=%s, expected=%s", parsed, tc.want)
+				}
+				return buildServerCtxt(ctx, &got)
+			}}}
+			args := append([]string{"silo", commandName}, tc.args...)
+			args = append(args, t.TempDir())
+			if err := app.Run(args); err != nil {
+				t.Fatal(err)
+			}
+			if !called {
+				t.Fatal("server action did not run")
+			}
+			if got.ReadHeaderTimeout != tc.want {
+				t.Errorf("parsed ReadHeaderTimeout = %s, want %s", got.ReadHeaderTimeout, tc.want)
+			}
+			if got.IdleTimeout != idleWant {
+				t.Errorf("parsed IdleTimeout = %s, want %s", got.IdleTimeout, idleWant)
+			}
+		})
+	}
+}

cmd/site-replication-metadata-gate_test.go

@@ -0,0 +1,341 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+func TestBucketMetadataTombstoneExportAndInitialSync(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			recordBucketConfigPeer(t, cred)
+			old := globalSiteReplicationMetadataTombstones
+			defer func() { globalSiteReplicationMetadataTombstones = old }()
+			created := UTCNow().Add(-time.Hour)
+			deletedAt := created.Add(time.Minute)
+			for _, enabled := range []bool{false, true} {
+				globalSiteReplicationMetadataTombstones = enabled
+				meta := newBucketMetadata(bucket)
+				meta.Created = created
+				meta.defaultTimestamps()
+				for _, file := range []string{bucketPolicyConfig, bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile} {
+					_, at := replicatedBucketConfig(&meta, file)
+					*at = deletedAt
+				}
+				if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					t.Fatal(err)
+				}
+				// Force a disk reload instead of accepting the just-published cache.
+				globalBucketMetadataSys.Remove(bucket)
+				info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(t.Context(), obj, madmin.SRStatusOptions{Buckets: true})
+				if err != nil {
+					t.Fatal(err)
+				}
+				exported := info.Buckets[bucket]
+				if !exported.PolicyUpdatedAt.Equal(deletedAt) {
+					t.Fatal("Policy tombstone hidden by gate")
+				}
+				for _, at := range []time.Time{exported.TagConfigUpdatedAt, exported.SSEConfigUpdatedAt, exported.QuotaConfigUpdatedAt} {
+					if enabled && !at.Equal(deletedAt) || !enabled && !at.IsZero() {
+						t.Fatalf("gate=%v timestamp=%v", enabled, at)
+					}
+				}
+				for file, data := range bucketConfigTestData(bucket) {
+					event, send, err := initialBucketConfigReplicationEvent(meta, file)
+					wantSend := enabled && !bucketConfigUpdateOnly(file)
+					if err != nil || send != wantSend || send && !event.UpdatedAt.Equal(deletedAt) {
+						t.Fatalf("%s initial gate=%v: %+v %v %v", file, enabled, event, send, err)
+					}
+					baseline := newBucketMetadata(bucket)
+					baseline.Created = created
+					baseline.defaultTimestamps()
+					if _, send, err := initialBucketConfigReplicationEvent(baseline, file); err != nil || send {
+						t.Fatalf("empty baseline sent: %s", file)
+					}
+					value, _ := replicatedBucketConfig(&baseline, file)
+					*value = data
+					event, send, err = initialBucketConfigReplicationEvent(baseline, file)
+					if err != nil || !send || !event.UpdatedAt.Equal(created) {
+						t.Fatalf("historical baseline-live omitted: %s %v", file, err)
+					}
+				}
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataLegacyAndGeneration(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			old := globalSiteReplicationMetadataTombstones
+			defer func() { globalSiteReplicationMetadataTombstones = old }()
+			created := UTCNow().Add(-time.Hour)
+			for _, enabled := range []bool{false, true} {
+				globalSiteReplicationMetadataTombstones = enabled
+				for file, data := range bucketConfigTestData(bucket) {
+					meta := newBucketMetadata(bucket)
+					meta.Created = created
+					meta.defaultTimestamps()
+					if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+						t.Fatal(err)
+					}
+					counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+					setObjectLayer(counter)
+					event := newBucketConfigReplicationEvent(bucket, file, bucketConfigState{data: data, at: created.Add(-time.Second)})
+					rec := applySRBucketMetaViaAdmin(t, cred, event)
+					if rec.Code != http.StatusOK || counter.writes.Load() != 0 {
+						t.Fatalf("pre-creation apply: %s %d writes=%d", file, rec.Code, counter.writes.Load())
+					}
+					// A live baseline may initialize. The same-time nil cannot delete it.
+					event.UpdatedAt = created
+					rec = applySRBucketMetaViaAdmin(t, cred, event)
+					if rec.Code != http.StatusOK {
+						t.Fatalf("baseline apply: %s %s", file, rec.Body.String())
+					}
+					before := counter.writes.Load()
+					rec = applySRBucketMetaViaAdmin(t, cred, madmin.SRBucketMeta{Type: event.Type, Bucket: bucket, UpdatedAt: created})
+					if rec.Code != http.StatusOK || counter.writes.Load() != before {
+						t.Fatalf("nil baseline cleared configuration: %s", file)
+					}
+					event.UpdatedAt = time.Time{}
+					for range 2 {
+						rec = applySRBucketMetaViaAdmin(t, cred, event)
+						if rec.Code != http.StatusOK {
+							t.Fatalf("legacy-zero rejected: %s %s", file, rec.Body.String())
+						}
+					}
+					meta, err := loadBucketMetadata(t.Context(), obj, bucket)
+					if err != nil {
+						t.Fatal(err)
+					}
+					value, at := replicatedBucketConfig(&meta, file)
+					if len(*value) == 0 || !at.After(created) {
+						t.Fatalf("legacy-zero not reclocked: %s %v", file, *at)
+					}
+					setObjectLayer(obj)
+				}
+			}
+		})
+	}})
+}
+
+type bucketMetadataCreatedObjectLayer struct {
+	ObjectLayer
+	missing bool
+}
+
+func (o bucketMetadataCreatedObjectLayer) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
+	if opts.NoMetadata {
+		if o.missing {
+			return BucketInfo{}, BucketNotFound{Bucket: bucket}
+		}
+		// A physical bucket that reports no creation time either.
+		return BucketInfo{Name: bucket}, nil
+	}
+	return o.ObjectLayer.GetBucketInfo(ctx, bucket, opts)
+}
+
+func TestPeerBucketMetadataUnknownCreated(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			defer setObjectLayer(obj)
+			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
+			for _, mode := range []string{"unknown", "missing"} {
+				t.Run(mode, func(t *testing.T) {
+					setObjectLayer(obj)
+					if err := globalBucketMetadataSys.save(t.Context(), newBucketMetadata(bucket)); err != nil {
+						t.Fatal(err)
+					}
+					counter := &bucketConfigWriteCounter{ObjectLayer: bucketMetadataCreatedObjectLayer{ObjectLayer: obj, missing: mode == "missing"}}
+					setObjectLayer(counter)
+					stamp := UTCNow()
+					_, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketTaggingConfig, data, false, false, &stamp)
+					if err == nil || counter.writes.Load() != 0 {
+						t.Fatalf("unknown generation was invented: %v writes=%d", err, counter.writes.Load())
+					}
+				})
+			}
+		})
+	}})
+}
+
+// setPhysicalBucketCreated stamps the bucket directory on every local drive,
+// which is what StatVol reports as the physical creation time.
+func setPhysicalBucketCreated(t *testing.T, bucket string, at time.Time) {
+	t.Helper()
+	globalLocalDrivesMu.RLock()
+	drives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+	if len(drives) == 0 {
+		t.Fatal("no local drives registered")
+	}
+	for _, drive := range drives {
+		if err := os.Chtimes(pathJoin(drive.Endpoint().Path, bucket), at, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// Recovery has to run against the real ObjectLayer: a stub GetBucketInfo
+// returning the expected time would hide the cached zero creation time
+// overwriting it, which is what a bucket that never held a configuration has.
+func TestBucketMetadataPhysicalCreatedRecovery(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		// One known time on every drive, so the recovered value can be neither
+		// confused with UTCNow() nor dependent on which drive answers first.
+		physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
+		for _, missing := range []bool{false, true} {
+			for _, file := range replicatedBucketConfigs {
+				t.Run(backend+"/"+file+"/missing="+strconv.FormatBool(missing), func(t *testing.T) {
+					ctx := t.Context()
+					setPhysicalBucketCreated(t, bucket, physical)
+					if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
+						t.Fatal(err)
+					}
+					if missing {
+						if err := deleteConfig(ctx, obj, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile)); err != nil {
+							t.Fatal(err)
+						}
+					}
+					data := bucketConfigTestData(bucket)[file]
+					at, err := globalBucketMetadataSys.Update(ctx, bucket, file, data)
+					if err != nil {
+						t.Fatalf("bucket without a recorded creation time cannot update %s: %v", file, err)
+					}
+					got, err := readBucketMetadata(ctx, obj, bucket)
+					if err != nil || !got.Created.Equal(physical) || !at.After(physical) {
+						t.Fatalf("physical creation not persisted: created=%v physical=%v updated=%v err=%v", got.Created, physical, at, err)
+					}
+				})
+			}
+		}
+	}})
+}
+
+func TestBucketMetadataInitialSyncPhysicalCreated(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, cred.AccessKey, nil, newServiceAccountOpts{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { globalIAMSys.DeleteServiceAccount(context.Background(), serviceCred.AccessKey, false) })
+			physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
+			setPhysicalBucketCreated(t, bucket, physical)
+			meta := newBucketMetadata(bucket)
+			meta.TaggingConfigXML = bucketConfigTestData(bucket)[bucketTaggingConfig]
+			if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+				t.Fatal(err)
+			}
+			var mu sync.Mutex
+			var createdAt string
+			var events []madmin.SRBucketMeta
+			peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mu.Lock()
+				defer mu.Unlock()
+				if r.URL.Query().Get("operation") == string(madmin.MakeWithVersioningBktOp) {
+					createdAt = r.URL.Query().Get("createdAt")
+				}
+				if strings.HasSuffix(r.URL.Path, "/bucket-meta") {
+					var event madmin.SRBucketMeta
+					if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
+						t.Error(err)
+					}
+					events = append(events, event)
+				}
+				if r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions" {
+					_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "initial-peer", Instance: "initial-boot", Digest: "ack"}})
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer peer.Close()
+			// Exercise the complete outgoing sync sequence with real source
+			// storage. This peer acknowledges RPCs; it is not a second ObjectLayer.
+			c := &SiteReplicationSys{enabled: true, state: srState{
+				ServiceAccountAccessKey: serviceCred.AccessKey,
+				Peers:                   map[string]madmin.PeerInfo{"initial-peer": {DeploymentID: "initial-peer", Endpoint: peer.URL}},
+			}}
+			if err := c.syncToAllPeers(ctx, madmin.SRAddOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if createdAt != physical.Format(time.RFC3339Nano) {
+				t.Fatalf("peer creation time %q, want physical time %s", createdAt, physical)
+			}
+			for _, event := range events {
+				if event.Bucket == bucket && event.Type == madmin.SRBucketMetaTypeTags {
+					if event.Tags == nil || *event.Tags != base64.StdEncoding.EncodeToString(meta.TaggingConfigXML) || !event.UpdatedAt.Equal(physical) {
+						t.Fatalf("historical tags lost baseline: %+v", event)
+					}
+					return
+				}
+			}
+			t.Fatal("initial sync silently skipped historical tags")
+		})
+	}})
+}
+
+func TestPeerBucketMetadataPhysicalCreatedBoundary(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			physical := UTCNow().Truncate(time.Second)
+			setPhysicalBucketCreated(t, bucket, physical)
+			if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
+				t.Fatal(err)
+			}
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
+			at := physical.Add(-time.Hour)
+			_, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketTaggingConfig, data, false, false, &at)
+			if err != nil || counter.writes.Load() != 0 {
+				t.Fatalf("event before recovered creation must be skipped: err=%v writes=%d", err, counter.writes.Load())
+			}
+			// Physical mtime is only an approximation. A local correction can
+			// establish it; an earlier peer event cannot lower the bucket identity.
+			at, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := readBucketMetadata(ctx, obj, bucket)
+			if err != nil || !got.Created.Equal(physical) || !at.After(physical) || !bytes.Equal(got.TaggingConfigXML, data) {
+				t.Fatalf("local correction did not establish physical creation: %+v %v", got, err)
+			}
+		})
+	}})
+}

cmd/site-replication-metadata-heal_test.go

@@ -0,0 +1,323 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/target/testlogger"
+)
+
+func bucketConfigTestData(bucket string) map[string][]byte {
+	return map[string][]byte{
+		bucketPolicyConfig:     []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)),
+		bucketTaggingConfig:    []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`),
+		bucketSSEConfig:        []byte(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`),
+		bucketQuotaConfigFile:  []byte(`{"quota":1024,"quotatype":"hard"}`),
+		bucketVersioningConfig: enabledBucketVersioningConfig,
+		objectLockConfig:       enabledBucketObjectLockConfig,
+	}
+}
+
+type bucketConfigLogCapture struct {
+	testing.TB
+	mu    sync.Mutex
+	lines []string
+}
+
+func (c *bucketConfigLogCapture) Logf(format string, args ...any) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.lines = append(c.lines, fmt.Sprintf(format, args...))
+}
+
+func TestHealBucketConfigDiagnostics(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			recordBucketConfigPeer(t, cred)
+			capture := &bucketConfigLogCapture{TB: t}
+			defer testlogger.T.SetLogTB(capture)()
+			disabled := logger.DisableLog
+			logger.DisableLog = false
+			defer func() { logger.DisableLog = disabled }()
+			created := UTCNow().Add(-time.Hour)
+			local := globalDeploymentID()
+			bs := map[string]srBucketStatsSummary{
+				local:            bucketConfigTestInfo(bucket, bucketTaggingConfig, nil, created, created),
+				"missing-bucket": {},
+				"broken-rpc":     bucketConfigTestInfo(bucket, bucketTaggingConfig, nil, created, created),
+			}
+			info := srStatusInfo{Sites: map[string]madmin.PeerInfo{local: {}, "missing-bucket": {}, "unreachable": {}, "broken-rpc": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{bucket: bs}}
+			if err := globalSiteReplicationSys.healBucketConfig(t.Context(), bucket, bucketTaggingConfig, info); err != nil {
+				t.Fatal(err)
+			}
+			if len(capture.lines) != 0 {
+				t.Fatal("empty baselines produced diagnostics")
+			}
+			bs[local] = bucketConfigTestInfo(bucket, bucketTaggingConfig, bucketConfigTestData(bucket)[bucketTaggingConfig], created.Add(time.Minute), created)
+			for range 2 {
+				if err := globalSiteReplicationSys.healBucketConfig(t.Context(), bucket, bucketTaggingConfig, info); err != nil {
+					t.Fatal(err)
+				}
+			}
+			capture.mu.Lock()
+			defer capture.mu.Unlock()
+			var unreachable, peerError int
+			for _, line := range capture.lines {
+				switch {
+				case strings.Contains(line, "bucket metadata replication: unreachable"):
+					unreachable++
+				case strings.Contains(line, "bucket metadata replication: peer-error"):
+					peerError++
+				default:
+					t.Fatalf("unexpected diagnostic: %s", line)
+				}
+				if !strings.HasPrefix(line, "WARNING:") {
+					t.Fatalf("diagnostic is not a warning: %s", line)
+				}
+			}
+			if unreachable != 1 || peerError != 1 {
+				t.Fatalf("distinct reasons were lost or not deduplicated: unreachable=%d peer-error=%d", unreachable, peerError)
+			}
+		})
+	}})
+}
+
+func TestHealBucketConfigWithoutSourceDiagnostics(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			events := recordBucketConfigPeer(t, cred)
+			disabled := logger.DisableLog
+			logger.DisableLog = false
+			defer func() { logger.DisableLog = disabled }()
+			created := UTCNow().Add(-time.Hour)
+			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
+			for _, tc := range []struct {
+				name    string
+				data    []byte
+				at      time.Time
+				created time.Time
+				wantLog bool
+			}{
+				{"unknown-created", data, created.Add(time.Minute), time.Time{}, true},
+				{"malformed", []byte("<Tagging>"), created.Add(time.Minute), created, true},
+				{"before-created", data, created.Add(-time.Minute), created, true},
+				{"empty-baseline", nil, created, created, false},
+			} {
+				t.Run(tc.name, func(t *testing.T) {
+					capture := &bucketConfigLogCapture{TB: t}
+					defer testlogger.T.SetLogTB(capture)()
+					// Each case has its own diagnostic key; no source can be selected
+					// and neither local storage nor the recording peer may be written.
+					name := bucket + "-" + tc.name
+					local := globalDeploymentID()
+					info := srStatusInfo{
+						Sites: map[string]madmin.PeerInfo{local: {}, "metadata-peer": {}, "unreachable": {}},
+						BucketStats: map[string]map[string]srBucketStatsSummary{name: {
+							local:           bucketConfigTestInfo(name, bucketTaggingConfig, tc.data, tc.at, tc.created),
+							"metadata-peer": {},
+						}},
+					}
+					for range 2 {
+						if err := globalSiteReplicationSys.healBucketConfig(t.Context(), name, bucketTaggingConfig, info); err != nil {
+							t.Fatal(err)
+						}
+					}
+					capture.mu.Lock()
+					defer capture.mu.Unlock()
+					want := 0
+					if tc.wantLog {
+						want = 1
+					}
+					if len(capture.lines) != want {
+						t.Fatalf("got %d diagnostics, want %d: %v", len(capture.lines), want, capture.lines)
+					}
+					for _, line := range capture.lines {
+						if !strings.HasPrefix(line, "WARNING:") || !strings.Contains(line, "bucket metadata replication: indeterminate") {
+							t.Fatalf("unexpected diagnostic: %s", line)
+						}
+					}
+					if len(events()) != 0 {
+						t.Fatal("healing without a source sent a metadata RPC")
+					}
+				})
+			}
+		})
+	}})
+}
+
+// Construct independent wire fixtures, including timestamps hidden by legacy
+// exporters, without going through the production event or state helpers.
+func bucketConfigTestInfo(bucket, file string, data []byte, at, created time.Time) srBucketStatsSummary {
+	info := madmin.SRBucketInfo{Bucket: bucket, CreatedAt: created}
+	var payload *string
+	if len(data) != 0 {
+		encoded := base64.StdEncoding.EncodeToString(data)
+		payload = &encoded
+	}
+	switch file {
+	case bucketPolicyConfig:
+		info.Policy, info.PolicyUpdatedAt = data, at
+	case bucketTaggingConfig:
+		info.Tags, info.TagConfigUpdatedAt = payload, at
+	case bucketSSEConfig:
+		info.SSEConfig, info.SSEConfigUpdatedAt = payload, at
+	case bucketQuotaConfigFile:
+		info.QuotaConfig, info.QuotaConfigUpdatedAt = payload, at
+	case bucketVersioningConfig:
+		info.Versioning, info.VersioningConfigUpdatedAt = payload, at
+	case objectLockConfig:
+		info.ObjectLockConfig, info.ObjectLockConfigUpdatedAt = payload, at
+	}
+	return srBucketStatsSummary{meta: srBucketMetaInfo{SRBucketInfo: info}}
+}
+
+func TestLatestBucketConfigCandidates(t *testing.T) {
+	created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	for file, data := range bucketConfigTestData("bucket") {
+		t.Run(file, func(t *testing.T) {
+			info := srStatusInfo{Sites: map[string]madmin.PeerInfo{"a": {}, "b": {}, "c": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{"bucket": {}}}
+			bs := info.BucketStats["bucket"]
+			baseline := bucketConfigTestInfo("bucket", file, nil, created, created)
+			bs["a"], bs["b"], bs["c"] = baseline, baseline, baseline
+			if _, found := latestBucketConfig("bucket", file, info); found {
+				t.Fatal("empty baseline chosen as source")
+			}
+			// Unknown and unavailable IDs cannot contribute, even with future clocks.
+			bs[""], bs["unknown"] = bucketConfigTestInfo("bucket", file, data, created.Add(10*time.Hour), created), bucketConfigTestInfo("bucket", file, data, created.Add(20*time.Hour), created)
+			if _, found := latestBucketConfig("bucket", file, info); found {
+				t.Fatal("unknown source chosen")
+			}
+			liveBaseline := bucketConfigTestInfo("bucket", file, data, created, created)
+			modified := bucketConfigTestInfo("bucket", file, data, created.Add(time.Hour), created)
+			oldGeneration := bucketConfigTestInfo("bucket", file, data, created, created.Add(time.Second))
+			states := []srBucketStatsSummary{liveBaseline, modified, oldGeneration}
+			for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} {
+				bs["a"], bs["b"], bs["c"] = states[order[0]], states[order[1]], states[order[2]]
+				winner, found := latestBucketConfig("bucket", file, info)
+				if !found || !winner.at.Equal(created.Add(time.Hour)) || !bytes.Equal(winner.data, data) {
+					t.Fatalf("permutation %v chose %+v found=%v", order, winner, found)
+				}
+			}
+			bs["a"], bs["b"], bs["c"] = baseline, liveBaseline, baseline
+			if winner, found := latestBucketConfig("bucket", file, info); !found || !bytes.Equal(winner.data, data) || winner.real {
+				t.Fatal("historical baseline-live cannot initialize")
+			}
+			if !bucketConfigUpdateOnly(file) {
+				// A late-created empty baseline cannot beat a real earlier write.
+				bs["a"], bs["b"], bs["c"] = modified, bucketConfigTestInfo("bucket", file, nil, created.Add(2*time.Hour), created.Add(2*time.Hour)), baseline
+				if winner, found := latestBucketConfig("bucket", file, info); !found || len(winner.data) == 0 {
+					t.Fatal("default became a deletion")
+				}
+				bs["c"] = bucketConfigTestInfo("bucket", file, nil, created.Add(time.Hour), created)
+				if winner, found := latestBucketConfig("bucket", file, info); !found || len(winner.data) != 0 || !winner.real {
+					t.Fatal("real tombstone lost equal-time tie")
+				}
+			}
+		})
+	}
+}
+
+func TestHealBucketConfigSourceAndQuiescence(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			events := recordBucketConfigPeer(t, cred)
+			local := globalDeploymentID()
+			created := UTCNow().Add(-time.Hour)
+			oldAt, newAt := created.Add(time.Minute), created.Add(2*time.Minute)
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			for file, data := range bucketConfigTestData(bucket) {
+				t.Run(file, func(t *testing.T) {
+					meta := newBucketMetadata(bucket)
+					meta.Created = created
+					meta.defaultTimestamps()
+					value, at := replicatedBucketConfig(&meta, file)
+					*value, *at = data, newAt
+					if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+						t.Fatal(err)
+					}
+					bs := map[string]srBucketStatsSummary{
+						local:           bucketConfigTestInfo(bucket, file, data, newAt, created),
+						"metadata-peer": bucketConfigTestInfo(bucket, file, data, oldAt, created),
+						"":              {}, "unknown": bucketConfigTestInfo(bucket, file, nil, newAt.Add(time.Hour), created),
+						// Known but absent from c.state.Peers, forcing getAdminClient failure.
+						"broken": bucketConfigTestInfo(bucket, file, nil, created, created),
+					}
+					info := srStatusInfo{Sites: map[string]madmin.PeerInfo{local: {}, "metadata-peer": {}, "broken": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{bucket: bs}}
+					before := len(events())
+					writes := counter.writes.Load()
+					if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
+						t.Fatal(err)
+					}
+					got := events()[before:]
+					if len(got) != 1 || !got[0].UpdatedAt.Equal(newAt) {
+						t.Fatalf("same-payload timestamp heal did not reach healthy peer: %+v", got)
+					}
+					if file == bucketTaggingConfig && got[0].Tags == nil {
+						t.Fatal("tag event missing payload")
+					}
+					if counter.writes.Load() != writes {
+						t.Fatal("matching local state was rewritten")
+					}
+					bs["metadata-peer"], bs["broken"] = bs[local], bs[local]
+					if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
+						t.Fatal(err)
+					}
+					if len(events()) != before+1 || counter.writes.Load() != writes {
+						t.Fatal("stable second heal wrote or broadcast")
+					}
+					if bucketConfigUpdateOnly(file) {
+						return
+					}
+					bs["metadata-peer"] = bucketConfigTestInfo(bucket, file, nil, newAt.Add(time.Minute), created)
+					delete(bs, "broken")
+					if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
+						t.Fatal(err)
+					}
+					after, err := loadBucketMetadata(ctx, obj, bucket)
+					if err != nil {
+						t.Fatal(err)
+					}
+					value, at = replicatedBucketConfig(&after, file)
+					if len(*value) != 0 || !at.Equal(newAt.Add(time.Minute)) {
+						t.Fatalf("local heal lost deletion/source time: %s %v", *value, *at)
+					}
+					if file == bucketQuotaConfigFile {
+						q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
+						if err != nil || q.Quota != 0 {
+							t.Fatalf("quota cache not cleared: %+v %v", q, err)
+						}
+					}
+				})
+			}
+		})
+	}})
+}

cmd/site-replication-metadata.go

@@ -0,0 +1,192 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/logger"
+)
+
+// Read once at startup. Enable only after every participating node is fixed.
+var globalSiteReplicationMetadataTombstones bool
+
+func logBucketConfigReplication(ctx context.Context, bucket, file, reason string, at, created time.Time, detail string) {
+	// LogOnceIf compares error text as well as its key. Keep both stable; changing
+	// times and peer errors belong in ReqInfo, not in the error's message.
+	req := &logger.ReqInfo{API: "SiteReplicationMetadata", BucketName: bucket}
+	req.AppendTags("field", file)
+	req.AppendTags("sourceTime", at.UTC().Format(time.RFC3339Nano))
+	req.AppendTags("created", created.UTC().Format(time.RFC3339Nano))
+	req.AppendTags("detail", detail)
+	replLogOnceIf(logger.SetReqInfo(ctx, req), errors.New("bucket metadata replication: "+reason),
+		"bucket-metadata/"+bucket+"/"+file+"/"+reason, logger.WarningKind)
+}
+
+func initialBucketConfigReplicationEvent(meta BucketMetadata, file string) (madmin.SRBucketMeta, bool, error) {
+	data, at := replicatedBucketConfig(&meta, file)
+	state, err := newBucketConfigState(meta.Name, file, *data, *at, meta.Created, len(meta.ObjectLockConfigXML) != 0)
+	if err != nil {
+		return madmin.SRBucketMeta{}, false, err
+	}
+	if !state.candidate() || (len(state.data) == 0 && !globalSiteReplicationMetadataTombstones) {
+		return madmin.SRBucketMeta{}, false, nil
+	}
+	return newBucketConfigReplicationEvent(meta.Name, file, state), true, nil
+}
+
+func bucketConfigStateFromInfo(bucket, file string, meta madmin.SRBucketInfo) (bucketConfigState, error) {
+	var payload *string
+	var at time.Time
+	switch file {
+	case bucketPolicyConfig:
+		return newBucketConfigState(bucket, file, meta.Policy, meta.PolicyUpdatedAt, meta.CreatedAt, false)
+	case bucketTaggingConfig:
+		payload, at = meta.Tags, meta.TagConfigUpdatedAt
+	case bucketSSEConfig:
+		payload, at = meta.SSEConfig, meta.SSEConfigUpdatedAt
+	case bucketQuotaConfigFile:
+		payload, at = meta.QuotaConfig, meta.QuotaConfigUpdatedAt
+	case bucketVersioningConfig:
+		payload, at = meta.Versioning, meta.VersioningConfigUpdatedAt
+	case objectLockConfig:
+		payload, at = meta.ObjectLockConfig, meta.ObjectLockConfigUpdatedAt
+	}
+	var data []byte
+	if payload != nil {
+		var err error
+		data, err = base64.StdEncoding.DecodeString(*payload)
+		if err != nil {
+			return bucketConfigState{}, err
+		}
+	}
+	lockEnabled := meta.ObjectLockConfig != nil && len(*meta.ObjectLockConfig) != 0
+	return newBucketConfigState(bucket, file, data, at, meta.CreatedAt, lockEnabled)
+}
+
+func newBucketConfigReplicationEvent(bucket, file string, state bucketConfigState) madmin.SRBucketMeta {
+	event := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: state.at}
+	var payload *string
+	if len(state.data) != 0 {
+		encoded := base64.StdEncoding.EncodeToString(state.data)
+		payload = &encoded
+	}
+	switch file {
+	case bucketPolicyConfig:
+		event.Type, event.Policy = madmin.SRBucketMetaTypePolicy, state.data
+	case bucketTaggingConfig:
+		event.Type, event.Tags = madmin.SRBucketMetaTypeTags, payload
+	case bucketSSEConfig:
+		event.Type, event.SSEConfig = madmin.SRBucketMetaTypeSSEConfig, payload
+	case bucketQuotaConfigFile:
+		event.Type, event.Quota = madmin.SRBucketMetaTypeQuotaConfig, state.data
+	case bucketVersioningConfig:
+		event.Type, event.Versioning = madmin.SRBucketMetaTypeVersionConfig, payload
+	case objectLockConfig:
+		event.Type, event.ObjectLockConfig = madmin.SRBucketMetaTypeObjectLockConfig, payload
+	}
+	return event
+}
+
+func latestBucketConfig(bucket, file string, info srStatusInfo) (bucketConfigState, bool) {
+	var latest bucketConfigState
+	found := false
+	for id, status := range info.BucketStats[bucket] {
+		if _, known := info.Sites[id]; !known || id == "" {
+			continue
+		}
+		state, err := bucketConfigStateFromInfo(bucket, file, status.meta.SRBucketInfo)
+		if err != nil || !state.candidate() {
+			continue
+		}
+		if !found || compareBucketConfigStates(state, latest) > 0 {
+			latest, found = state, true
+		}
+	}
+	return latest, found
+}
+
+func (c *SiteReplicationSys) healBucketConfig(ctx context.Context, bucket, file string, info srStatusInfo) error {
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	latest, found := latestBucketConfig(bucket, file, info)
+	// Every reason keeps its own log key, so a site that did not report cannot
+	// deduplicate away an unusable peer state or a real heal RPC failure for
+	// the same bucket and field.
+	if found {
+		for id := range info.Sites {
+			if _, present := info.BucketStats[bucket][id]; !present {
+				logBucketConfigReplication(ctx, bucket, file, "unreachable", latest.at, time.Time{}, "peer "+id+" did not report")
+			}
+		}
+	}
+	for id, status := range info.BucketStats[bucket] {
+		if _, known := info.Sites[id]; !known || id == "" {
+			continue
+		}
+		target := status.meta.SRBucketInfo
+		current, currentErr := bucketConfigStateFromInfo(bucket, file, target)
+		// A peer without the bucket reports neither a field nor a creation
+		// time; bucket healing covers that normal transient. Report only a
+		// state that exists and still cannot be ordered.
+		if currentErr != nil || (!current.valid && (len(current.data) != 0 || !current.at.IsZero())) {
+			logBucketConfigReplication(ctx, bucket, file, "indeterminate", current.at, target.CreatedAt, "unusable peer "+id)
+		}
+		// Invalid existing state still needs a diagnosis when no source can
+		// be selected. Only propagation depends on having a valid source.
+		if !found || target.CreatedAt.IsZero() {
+			continue
+		}
+		if latest.at.Before(target.CreatedAt) {
+			logBucketConfigReplication(ctx, bucket, file, "before-created", latest.at, target.CreatedAt, "peer "+id)
+			continue
+		}
+		// Versioning can be normalized differently until Object Lock itself has
+		// converged. Compare what this target would actually persist.
+		incoming, err := newBucketConfigState(bucket, file, latest.data, latest.at, target.CreatedAt,
+			target.ObjectLockConfig != nil && len(*target.ObjectLockConfig) != 0)
+		if err != nil || !incoming.candidate() {
+			continue
+		}
+		if currentErr == nil && compareBucketConfigStates(incoming, current) <= 0 {
+			continue
+		}
+		if id == globalDeploymentID() {
+			_, err = globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, file, latest.data, false, false, &latest.at)
+		} else {
+			var client *madmin.AdminClient
+			client, err = c.getAdminClient(ctx, id)
+			if err == nil {
+				err = client.SRPeerReplicateBucketMeta(ctx, newBucketConfigReplicationEvent(bucket, file, incoming))
+			}
+		}
+		if err != nil {
+			// A missing credential or unreachable peer must not abandon the other
+			// targets simply because it happened to be visited first in this map.
+			logBucketConfigReplication(ctx, bucket, file, "peer-error", latest.at, target.CreatedAt, "peer "+id+": "+err.Error())
+		}
+	}
+	return nil
+}

cmd/site-replication-metadata_test.go

@@ -0,0 +1,779 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func TestPeerBucketMetadataSourceTimeAndDeletion(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPeerBucketMetadataSourceTimeAndDeletion})
+}
+
+func testPeerBucketMetadataSourceTimeAndDeletion(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+	ctx := t.Context()
+	created := UTCNow().Add(-time.Hour)
+	putAt := created.Add(10 * time.Minute)
+	delAt := putAt.Add(time.Minute)
+	enc := func(s string) *string { v := base64.StdEncoding.EncodeToString([]byte(s)); return &v }
+	base := newBucketMetadata(bucket)
+	base.SetCreatedAt(created)
+	base.defaultTimestamps()
+	quotaJSON, err := json.Marshal(madmin.BucketQuota{Quota: 1024, Type: madmin.HardQuota})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cases := []struct {
+		name, file string
+		put        madmin.SRBucketMeta
+		value      func(BucketMetadata) []byte
+		stamp      func(BucketMetadata) time.Time
+		deletable  bool
+	}{
+		{"policy", bucketPolicyConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypePolicy, Policy: []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))}, func(m BucketMetadata) []byte { return m.PolicyConfigJSON }, func(m BucketMetadata) time.Time { return m.PolicyConfigUpdatedAt }, true},
+		{"tags", bucketTaggingConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeTags, Tags: enc(`<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>`)}, func(m BucketMetadata) []byte { return m.TaggingConfigXML }, func(m BucketMetadata) time.Time { return m.TaggingConfigUpdatedAt }, true},
+		{"sse", bucketSSEConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeSSEConfig, SSEConfig: enc(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)}, func(m BucketMetadata) []byte { return m.EncryptionConfigXML }, func(m BucketMetadata) time.Time { return m.EncryptionConfigUpdatedAt }, true},
+		{"quota", bucketQuotaConfigFile, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeQuotaConfig, Quota: quotaJSON}, func(m BucketMetadata) []byte { return m.QuotaConfigJSON }, func(m BucketMetadata) time.Time { return m.QuotaConfigUpdatedAt }, true},
+		{"versioning", bucketVersioningConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeVersionConfig, Versioning: enc(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)}, func(m BucketMetadata) []byte { return m.VersioningConfigXML }, func(m BucketMetadata) time.Time { return m.VersioningConfigUpdatedAt }, false},
+		{"objectlock", objectLockConfig, newSRBucketObjectLockMeta(bucket, enc(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`), putAt), func(m BucketMetadata) []byte { return m.ObjectLockConfigXML }, func(m BucketMetadata) time.Time { return m.ObjectLockConfigUpdatedAt }, false},
+	}
+	for _, tc := range cases {
+		t.Run(backend+"/"+tc.name, func(t *testing.T) {
+			if err := globalBucketMetadataSys.save(ctx, base); err != nil {
+				t.Fatal(err)
+			}
+			item := tc.put
+			item.Bucket, item.UpdatedAt = bucket, putAt
+			apply := func(item madmin.SRBucketMeta) {
+				t.Helper()
+				rec := applySRBucketMetaViaAdmin(t, cred, item)
+				if rec.Code != http.StatusOK {
+					t.Fatalf("admin apply returned %d: %s", rec.Code, rec.Body.String())
+				}
+			}
+			read := func() BucketMetadata {
+				t.Helper()
+				m, err := loadBucketMetadata(ctx, obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				return m
+			}
+			apply(item)
+			put := read()
+			if len(tc.value(put)) == 0 {
+				t.Fatal("PUT did not establish live config")
+			}
+			if !tc.stamp(put).Equal(putAt) {
+				t.Errorf("SOURCE_TIME: persisted %s, want source %s", tc.stamp(put), putAt)
+			}
+			apply(madmin.SRBucketMeta{Type: item.Type, Bucket: bucket, UpdatedAt: delAt})
+			after := read()
+			if !tc.deletable {
+				if !bytes.Equal(tc.value(put), tc.value(after)) || !tc.stamp(put).Equal(tc.stamp(after)) {
+					t.Error("NIL_NOOP: update-only config changed")
+				}
+				t.Log("nil payload is correctly a no-op")
+				return
+			}
+			if len(tc.value(after)) != 0 {
+				t.Error("NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE")
+			}
+			if _, err := globalBucketMetadataSys.Delete(ctx, bucket, tc.file); err != nil {
+				t.Fatal(err)
+			}
+			tombstone := read()
+			if len(tc.value(tombstone)) != 0 {
+				t.Fatal("local DELETE failed to establish tombstone")
+			}
+			apply(item)
+			after = read()
+			if len(tc.value(after)) != 0 {
+				t.Error("STALE_RESURRECTION: older source PUT resurrected a locally deleted config")
+			} else {
+				t.Log("older source PUT did not resurrect config")
+			}
+		})
+	}
+}
+
+func TestPeerBucketMetadataBulkOrdering(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			newAt := meta.Created.Add(time.Hour)
+			newXML := `<Tagging><TagSet><Tag><Key>key</Key><Value>new</Value></Tag></TagSet></Tagging>`
+			meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = []byte(newXML), newAt
+			if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+				t.Fatal(err)
+			}
+			oldXML := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>`))
+			item := madmin.SRBucketMeta{Bucket: bucket, Tags: &oldXML, UpdatedAt: newAt.Add(-time.Minute)}
+			rec := applySRBucketMetaViaAdmin(t, cred, item)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("bulk apply returned %d: %s", rec.Code, rec.Body.String())
+			}
+			after, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(after.TaggingConfigXML, []byte(newXML)) || !after.TaggingConfigUpdatedAt.Equal(newAt) {
+				t.Errorf("BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value=%q time=%s", after.TaggingConfigXML, after.TaggingConfigUpdatedAt)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataOrderingUnderLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			oldAt, newAt := meta.Created.Add(time.Hour), meta.Created.Add(2*time.Hour)
+			oldXML := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>`))
+			newXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>newest</Value></Tag></TagSet></Tagging>`)
+			lockCtx, unlock, err := lockBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			locked := true
+			defer func() {
+				if locked {
+					unlock()
+				}
+			}()
+			ready := make(chan struct{}, 1)
+			hook := func(name string) {
+				if name == bucket {
+					select {
+					case ready <- struct{}{}:
+					default:
+					}
+				}
+			}
+			lockBucketMetadataAcquireHook.Store(&hook)
+			defer lockBucketMetadataAcquireHook.Store(nil)
+			done := make(chan error, 1)
+			go func() { done <- globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, bucket, &oldXML, oldAt) }()
+			select {
+			case <-ready:
+			case <-time.After(5 * time.Second):
+				t.Fatal("older event never reached metadata lock")
+			}
+			// A newer writer commits while holding the existing metadata lock.
+			// The old peer event has already checked the pre-commit cache.
+			meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = newXML, newAt
+			if err := globalBucketMetadataSys.saveMetadata(lockCtx, obj, &meta); err != nil {
+				t.Fatal(err)
+			}
+			unlock()
+			locked = false
+			select {
+			case err := <-done:
+				if err != nil {
+					t.Fatal(err)
+				}
+			case <-time.After(5 * time.Second):
+				t.Fatal("older event did not finish")
+			}
+			after, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(after.TaggingConfigXML) != string(newXML) {
+				t.Errorf("CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with %q", after.TaggingConfigXML)
+			}
+		})
+	}})
+}
+
+// Count actual metadata persistence, including writes that would be invisible
+// in a value-only assertion after a duplicate or rejected event.
+type bucketConfigWriteCounter struct {
+	ObjectLayer
+	writes atomic.Int64
+}
+
+func (o *bucketConfigWriteCounter) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
+	if bucket == minioMetaBucket && path.Base(object) == bucketMetadataFile {
+		o.writes.Add(1)
+	}
+	return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
+}
+
+func TestBucketConfigStateOrdering(t *testing.T) {
+	created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	live := []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>a</Value></Tag></TagSet></Tagging>`)
+	other := bytes.ReplaceAll(live, []byte("<Value>a"), []byte("<Value>b"))
+	state := func(data []byte, at, birth time.Time) bucketConfigState {
+		t.Helper()
+		s, err := newBucketConfigState("bucket", bucketTaggingConfig, data, at, birth, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return s
+	}
+	baseline := state(nil, created, created)
+	baselineLive := state(live, created, created)
+	put := state(live, created.Add(time.Second), created)
+	deleted := state(nil, put.at, created)
+	lateBaseline := state(other, created.Add(time.Hour), created.Add(time.Hour))
+	tests := []struct {
+		name          string
+		lower, higher bucketConfigState
+	}{
+		{"baseline initialization", baseline, baselineLive},
+		{"real before later baseline", lateBaseline, put},
+		{"delete wins tie", put, deleted},
+		{"live key tie", put, state(other, put.at, created)},
+		{"new PUT after deletion", deleted, state(live, put.at.Add(time.Second), created)},
+		{"baseline key ignores creation time", state(live, created.Add(time.Hour), created.Add(time.Hour)), state(other, created, created)},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if compareBucketConfigStates(tc.lower, tc.higher) >= 0 || compareBucketConfigStates(tc.higher, tc.lower) <= 0 {
+				t.Fatal("ordering is not antisymmetric or chose the wrong winner")
+			}
+		})
+	}
+	for _, s := range []bucketConfigState{baseline, state(live, created.Add(-time.Second), created), state(live, created, time.Time{})} {
+		if s.candidate() {
+			t.Fatal("default, pre-creation, or unknown-generation state became a source")
+		}
+	}
+	for _, file := range []string{bucketVersioningConfig, objectLockConfig} {
+		s, err := newBucketConfigState("bucket", file, nil, put.at, created, false)
+		if err != nil || s.candidate() {
+			t.Fatalf("empty update-only candidate: %+v %v", s, err)
+		}
+	}
+}
+
+func TestBucketPolicyReplicationKey(t *testing.T) {
+	// Permute independent set arrays and parse separately, as two sites would.
+	a := []byte(`{"Version":"2012-10-17","Statement":[{"Sid":"one","Effect":"Allow","Principal":{"AWS":["b","a"]},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::bucket/b*","arn:aws:s3:::bucket/a*"],"Condition":{"StringLike":{"s3:prefix":["b*","a*"]}}},{"Sid":"two","Effect":"Deny","Principal":"*","NotAction":["s3:GetObject","s3:PutObject"],"NotResource":["arn:aws:s3:::bucket/d*","arn:aws:s3:::bucket/c*"]}]}`)
+	// Use a condition valid for object actions.
+	a = bytes.ReplaceAll(a, []byte("s3:prefix"), []byte("aws:UserAgent"))
+	b := []byte(`{"Statement":[{"NotResource":["arn:aws:s3:::bucket/c*","arn:aws:s3:::bucket/d*"],"NotAction":["s3:PutObject","s3:GetObject"],"Principal":"*","Effect":"Deny","Sid":"two"},{"Condition":{"StringLike":{"aws:UserAgent":["a*","b*"]}},"Resource":["arn:aws:s3:::bucket/a*","arn:aws:s3:::bucket/b*"],"Action":["s3:PutObject","s3:GetObject"],"Principal":{"AWS":["a","b"]},"Effect":"Allow","Sid":"one"}],"Version":"2012-10-17"}`)
+	_, key, err := bucketConfigPayload("bucket", bucketPolicyConfig, a, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range 30 {
+		_, next, err := bucketConfigPayload("bucket", bucketPolicyConfig, b, false)
+		if err != nil || !bytes.Equal(key, next) {
+			t.Fatalf("independent parse %d: %s != %s (%v)", i, key, next, err)
+		}
+	}
+	_, sid, err := bucketConfigPayload("bucket", bucketPolicyConfig, bytes.ReplaceAll(a, []byte(`"one"`), []byte(`"other"`)), false)
+	if err != nil || bytes.Equal(key, sid) {
+		t.Fatalf("Sid difference lost: %v", err)
+	}
+	n, err := canonicalBucketPolicyJSON([]byte(`{"n":[9007199254740993,9007199254740992]}`))
+	if err != nil || string(n) != `{"n":[9007199254740992,9007199254740993]}` {
+		t.Fatalf("integer precision lost: %s %v", n, err)
+	}
+}
+
+func TestBucketPolicyReplicationStatusLegacyOrder(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			// Statement order here is the reverse of the canonical encoder's,
+			// which is what an upgraded peer stores: the permutation must not
+			// be reported as a permanent mismatch.
+			legacy := []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"allow","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"},{"Sid":"deny","Effect":"Deny","Principal":"*","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket, bucket))
+			source, err := policy.ParseBucketPolicyConfig(bytes.NewReader(legacy), bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			meta := newBucketMetadata(bucket)
+			meta.Created = UTCNow().Add(-time.Hour)
+			meta.PolicyConfigJSON = legacy
+			meta.PolicyConfigUpdatedAt = meta.Created.Add(time.Minute)
+			event, send, err := initialBucketConfigReplicationEvent(meta, bucketPolicyConfig)
+			if err != nil || !send {
+				t.Fatalf("legacy initial event: %v send=%v", err, send)
+			}
+			target := newBucketMetadata(bucket)
+			target.Created = meta.Created
+			if err := globalBucketMetadataSys.save(ctx, target); err != nil {
+				t.Fatal(err)
+			}
+			if rec := applySRBucketMetaViaAdmin(t, cred, event); rec.Code != http.StatusOK {
+				t.Fatalf("peer apply: %d %s", rec.Code, rec.Body.String())
+			}
+			received, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !isBktPolicyReplicated(2, []*policy.BucketPolicy{source, received}) {
+				t.Fatal("equivalent legacy and received policy reported as permanently mismatched")
+			}
+			changed, err := policy.ParseBucketPolicyConfig(bytes.NewReader(legacy), bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			changed.Statements[0].SID = "different"
+			if isBktPolicyReplicated(2, []*policy.BucketPolicy{source, changed}) {
+				t.Fatal("distinct policy state reported as replicated")
+			}
+			if isBktPolicyReplicated(2, []*policy.BucketPolicy{source, nil}) || !isBktPolicyReplicated(2, []*policy.BucketPolicy{nil, nil}) {
+				t.Fatal("per-site presence accounting changed")
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataWireAtomicity(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			stamp := meta.Created.Add(time.Hour)
+			policyData := []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))
+			meta.PolicyConfigJSON, meta.PolicyConfigUpdatedAt = policyData, stamp
+			meta.QuotaConfigJSON, meta.QuotaConfigUpdatedAt = []byte(`{"quota":1024,"quotatype":"hard"}`), stamp
+			meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>a</Value></Tag></TagSet></Tagging>`), stamp
+			if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+				t.Fatal(err)
+			}
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			read := func() BucketMetadata {
+				t.Helper()
+				m, err := loadBucketMetadata(ctx, obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				return m
+			}
+			apply := func(fragment string, at time.Time, ok bool) {
+				t.Helper()
+				raw := fmt.Sprintf(`{"bucket":%q,"updatedAt":%q,%s}`, bucket, at.Format(time.RFC3339Nano), fragment)
+				var item madmin.SRBucketMeta
+				if err := json.Unmarshal([]byte(raw), &item); err != nil {
+					t.Fatal(err)
+				}
+				rec := applySRBucketMetaViaAdmin(t, cred, item)
+				if (rec.Code == http.StatusOK) != ok {
+					t.Fatalf("%s: %d %s", raw, rec.Code, rec.Body.String())
+				}
+			}
+			// Omitted/null *string and empty update-only payloads are all no-ops.
+			apply(`"policy":null,"quota":null,"tags":null,"versioningConfig":"","objectLockConfig":""`, stamp.Add(time.Minute), true)
+			current := read()
+			if len(current.PolicyConfigJSON) != 0 || len(current.QuotaConfigJSON) == 0 {
+				t.Fatal("explicit null semantics lost")
+			}
+			if q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err != nil || q.Quota != 0 {
+				t.Fatalf("zero quota cache: %+v %v", q, err)
+			}
+			before := counter.writes.Load()
+			apply(`"tags":null,"versioningConfig":"","objectLockConfig":""`, stamp.Add(2*time.Minute), true)
+			if counter.writes.Load() != before {
+				t.Fatal("omitted fields or empty update-only wrote metadata")
+			}
+			// An invalid second field cannot persist the valid deletion in this bulk.
+			apply(`"tags":"","quota":{"quota":1,"quotatype":"invalid"}`, stamp.Add(2*time.Minute), false)
+			if counter.writes.Load() != before || len(read().TaggingConfigXML) == 0 {
+				t.Fatal("invalid bulk partially committed")
+			}
+			apply(`"tags":""`, stamp.Add(2*time.Minute), true)
+			before = counter.writes.Load()
+			apply(`"tags":""`, stamp.Add(2*time.Minute), true)
+			apply(`"tags":""`, stamp.Add(time.Minute), true)
+			if counter.writes.Load() != before {
+				t.Fatal("duplicate or old event wrote metadata")
+			}
+			localAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, []byte(`{}`))
+			if err != nil || !localAt.After(current.QuotaConfigUpdatedAt) {
+				t.Fatalf("local future monotonic time: %v %v", localAt, err)
+			}
+			deletedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketQuotaConfigFile)
+			if err != nil || !deletedAt.After(localAt) {
+				t.Fatalf("adjacent local time: %v %v", deletedAt, err)
+			}
+			if q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err != nil || q.Quota != 0 {
+				t.Fatalf("deleted quota cache: %+v %v", q, err)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketAdoptionRebasesOnlyDefaults(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		for _, shift := range []time.Duration{-time.Hour, 0, time.Hour, 3 * time.Hour} {
+			t.Run(fmt.Sprintf("%s/%s", backend, shift), func(t *testing.T) {
+				created := UTCNow().Add(-3 * time.Hour)
+				meta := newBucketMetadata(bucket)
+				meta.Created = created
+				meta.defaultTimestamps()
+				meta.QuotaConfigUpdatedAt = time.Time{}
+				meta.TaggingConfigUpdatedAt = created.Add(2 * time.Hour) // real deletion
+				meta.EncryptionConfigXML = []byte(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
+				meta.EncryptionConfigUpdatedAt = created.Add(2 * time.Hour)
+				if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					t.Fatal(err)
+				}
+				if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucket, MakeBucketOptions{CreatedAt: created.Add(shift)}); err != nil {
+					t.Fatal(err)
+				}
+				got, err := readBucketMetadata(t.Context(), obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !got.PolicyConfigUpdatedAt.Equal(got.Created) || !got.QuotaConfigUpdatedAt.Equal(got.Created) {
+					t.Fatal("default turned into tombstone or invalid source")
+				}
+				if !got.TaggingConfigUpdatedAt.Equal(meta.TaggingConfigUpdatedAt) || !got.EncryptionConfigUpdatedAt.Equal(meta.EncryptionConfigUpdatedAt) || !bytes.Equal(got.EncryptionConfigXML, meta.EncryptionConfigXML) {
+					t.Fatal("actual state changed during adoption")
+				}
+				if shift > 2*time.Hour {
+					// Preserve history, but do not promote state from an earlier
+					// bucket generation to a new valid source by retimestamping it.
+					for _, file := range []string{bucketTaggingConfig, bucketSSEConfig} {
+						data, at := replicatedBucketConfig(&got, file)
+						state, err := newBucketConfigState(bucket, file, *data, *at, got.Created, false)
+						if err != nil || state.candidate() {
+							t.Fatalf("pre-generation history became a source: %s %v", file, err)
+						}
+						// As a target, that invalid history must yield to a valid
+						// state from the adopted generation, including a live tag
+						// replacing the preserved earlier-generation deletion.
+						incoming := bucketConfigTestData(bucket)[file]
+						incomingAt := got.Created.Add(time.Minute)
+						changed, err := applyBucketConfig(&got, file, incoming, incomingAt)
+						if err != nil || !changed || !at.Equal(incomingAt) || !bytes.Equal(*data, incoming) {
+							t.Fatalf("adopted generation did not replace invalid target: %s %v", file, err)
+						}
+					}
+				}
+			})
+		}
+	}})
+}
+
+func recordBucketConfigPeer(t *testing.T, cred auth.Credentials) func() []madmin.SRBucketMeta {
+	t.Helper()
+	var mu sync.Mutex
+	var events []madmin.SRBucketMeta
+	remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var event madmin.SRBucketMeta
+		if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
+			t.Error(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		mu.Lock()
+		events = append(events, event)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(remote.Close)
+	serviceCred, err := auth.CreateCredentials("metadata-source-time-svc", "metadata-source-time-service-secret")
+	if err != nil {
+		t.Fatal(err)
+	}
+	serviceCred.ParentUser = cred.AccessKey
+	globalSiteReplicatorCred.Set(serviceCred.SecretKey)
+	t.Cleanup(func() { globalSiteReplicatorCred.Set("") })
+	if _, err = globalIAMSys.store.AddServiceAccount(t.Context(), serviceCred); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { globalIAMSys.DeleteServiceAccount(context.Background(), serviceCred.AccessKey, false) })
+	globalSiteReplicationSys.Lock()
+	oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
+	globalSiteReplicationSys.enabled = true
+	globalSiteReplicationSys.state = srState{Name: "metadata-source-test", ServiceAccountAccessKey: serviceCred.AccessKey, Peers: map[string]madmin.PeerInfo{
+		globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
+		"metadata-peer":      {Name: "remote", DeploymentID: "metadata-peer", Endpoint: remote.URL},
+	}}
+	globalSiteReplicationSys.Unlock()
+	t.Cleanup(func() {
+		globalSiteReplicationSys.Lock()
+		globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
+		globalSiteReplicationSys.Unlock()
+	})
+	return func() []madmin.SRBucketMeta {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]madmin.SRBucketMeta(nil), events...)
+	}
+}
+
+// The final metadata lock is taken after reading the ZIP. Commit another
+// writer immediately before that acquisition, using the existing lock wrapper.
+type importInterleavingObjectLayer struct {
+	ObjectLayer
+	once  atomic.Bool
+	write func()
+}
+
+func (o *importInterleavingObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket != minioMetaBucket || len(objects) != 1 || path.Base(objects[0]) != "metadata.lock" {
+		return lock
+	}
+	return metadataObservedRWLocker{RWLocker: lock, onLock: func(context.Context) {
+		if o.once.CompareAndSwap(false, true) {
+			o.write()
+		}
+	}}
+}
+
+func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"PutBucketPolicy", "GetBucketPolicy", "PutBucketVersioning"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			events := recordBucketConfigPeer(t, cred)
+			putPolicy := func(data []byte) {
+				t.Helper()
+				req, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucket), int64(len(data)), bytes.NewReader(data), cred.AccessKey, cred.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				rec := httptest.NewRecorder()
+				router.ServeHTTP(rec, req)
+				if rec.Code != http.StatusNoContent {
+					t.Fatalf("empty policy PUT: %d %s", rec.Code, rec.Body.String())
+				}
+			}
+			putPolicy([]byte(`{"Version":"2012-10-17","Statement":[]}`))
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := events()
+			if len(got) != 1 || got[0].Type != madmin.SRBucketMetaTypePolicy || len(got[0].Policy) != 0 || len(meta.PolicyConfigJSON) != 0 || !meta.PolicyConfigUpdatedAt.Equal(got[0].UpdatedAt) {
+				t.Fatalf("empty policy event/disk mismatch: %+v", got)
+			}
+			req, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", bucket), 0, nil, cred.AccessKey, cred.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := httptest.NewRecorder()
+			router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusNotFound {
+				t.Fatalf("empty policy GET: %d %s", rec.Code, rec.Body.String())
+			}
+			// Supported negative sets must survive the same serializer on PUT,
+			// peer apply, GET and export instead of failing on empty Action.
+			putPolicy([]byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","NotAction":["s3:DeleteObject"],"NotResource":["arn:aws:s3:::%s/private/*"]}]}`, bucket)))
+			req, err = newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", bucket), 0, nil, cred.AccessKey, cred.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec = httptest.NewRecorder()
+			router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK || !bytes.Contains(rec.Body.Bytes(), []byte(`"NotAction"`)) || !bytes.Contains(rec.Body.Bytes(), []byte(`"NotResource"`)) {
+				t.Fatalf("negative policy set GET: %d %s", rec.Code, rec.Body.String())
+			}
+
+			corsAdminRequest(t, cred, http.MethodPut, "/set-bucket-quota?bucket="+bucket, []byte(`{}`))
+			meta, err = loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got = events()
+			last := got[len(got)-1]
+			if last.Type != madmin.SRBucketMetaTypeQuotaConfig || len(last.Quota) == 0 || !last.UpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+				t.Fatalf("zero quota event: %+v", last)
+			}
+
+			injectedAt := UTCNow().Add(2 * time.Hour)
+			interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() {
+				data := []byte(`{"quota":2048,"quotatype":"hard"}`)
+				_, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketQuotaConfigFile, data, false, false, &injectedAt)
+				if err != nil {
+					t.Error(err)
+				}
+			}}
+			setObjectLayer(interleaving)
+			defer setObjectLayer(obj)
+			before := len(events())
+			rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
+				bucket + "/" + bucketPolicyConfig:     []byte(`{"Version":"2012-10-17","Statement":[]}`),
+				bucket + "/quota.json":                []byte(`{}`),
+				bucket + "/" + bucketTaggingConfig:    []byte(`<Tagging><TagSet><Tag><Key>imported</Key><Value>yes</Value></Tag></TagSet></Tagging>`),
+				bucket + "/" + objectLockConfig:       enabledBucketObjectLockConfig,
+				bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
+			}))
+			report := corsImportReport(t, rec).Buckets[bucket]
+			if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" {
+				t.Fatalf("import: %+v", report)
+			}
+			meta, err = loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !meta.QuotaConfigUpdatedAt.After(injectedAt) || !meta.TaggingConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) || !meta.PolicyConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+				t.Fatalf("import time %v must exceed intervening %v; policy=%v tags=%v", meta.QuotaConfigUpdatedAt, injectedAt, meta.PolicyConfigUpdatedAt, meta.TaggingConfigUpdatedAt)
+			}
+			if !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || !meta.VersioningConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) || !meta.ObjectLockConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+				t.Fatal("import normalization or common source time lost")
+			}
+			got = events()[before:]
+			if len(got) != 2 {
+				t.Fatalf("import events: %+v", got)
+			}
+			for _, event := range got {
+				if !event.UpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+					t.Fatal("import hook used ZIP start time")
+				}
+				if event.Type == madmin.SRBucketMetaTypePolicy {
+					if len(event.Policy) != 0 {
+						t.Fatal("empty policy import is not deletion")
+					}
+				} else if event.Tags == nil || len(event.Quota) == 0 {
+					t.Fatalf("bulk omitted imported state: %+v", event)
+				}
+				if event.Type == "" {
+					if event.Versioning == nil || event.ObjectLockConfig == nil {
+						t.Fatal("normalized import fields omitted")
+					}
+					payload, err := base64.StdEncoding.DecodeString(*event.Versioning)
+					if err != nil || !bytes.Equal(payload, enabledBucketVersioningConfig) {
+						t.Fatal("import sent pre-normalization versioning")
+					}
+				}
+			}
+			beforeMeta, beforeEvents := meta, len(events())
+			rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
+				bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>only</Key><Value>tags</Value></Tag></TagSet></Tagging>`),
+			}))
+			if status := corsImportReport(t, rec).Buckets[bucket]; status.Err != "" || status.Tagging.Err != "" {
+				t.Fatalf("tags-only import: %+v", status)
+			}
+			meta, err = loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(meta.PolicyConfigJSON, beforeMeta.PolicyConfigJSON) || !meta.PolicyConfigUpdatedAt.Equal(beforeMeta.PolicyConfigUpdatedAt) || !bytes.Equal(meta.QuotaConfigJSON, beforeMeta.QuotaConfigJSON) || !meta.QuotaConfigUpdatedAt.Equal(beforeMeta.QuotaConfigUpdatedAt) {
+				t.Fatal("tags-only import changed Policy or Quota")
+			}
+			got = events()[beforeEvents:]
+			if len(got) != 1 || got[0].Tags == nil || got[0].Policy != nil || got[0].Quota != nil || got[0].Versioning != nil || got[0].ObjectLockConfig != nil || !got[0].UpdatedAt.Equal(meta.TaggingConfigUpdatedAt) {
+				t.Fatalf("tags-only import hook leaked unspecified fields: %+v", got)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataNormalizesBeforeComparison(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			meta, err := loadBucketMetadata(t.Context(), obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			versioning := base64.StdEncoding.EncodeToString([]byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>skip/</Prefix></ExcludedPrefixes><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`))
+			lock := base64.StdEncoding.EncodeToString(enabledBucketObjectLockConfig)
+			item := madmin.SRBucketMeta{Bucket: bucket, ObjectLockConfig: &lock, Versioning: &versioning, UpdatedAt: meta.Created.Add(time.Hour)}
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			for range 2 {
+				rec := applySRBucketMetaViaAdmin(t, cred, item)
+				if rec.Code != http.StatusOK {
+					t.Fatalf("bulk: %d %s", rec.Code, rec.Body.String())
+				}
+			}
+			if counter.writes.Load() != 1 {
+				t.Fatalf("normalized duplicate persisted %d times", counter.writes.Load())
+			}
+			meta, err = loadBucketMetadata(t.Context(), obj, bucket)
+			if err != nil || !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || !meta.VersioningConfigUpdatedAt.Equal(item.UpdatedAt) {
+				t.Fatalf("normalization: %s %v", meta.VersioningConfigXML, err)
+			}
+			// Ordinary local updates use the identical effective document in their
+			// commit result, even if Object Lock changed since the handler's precheck.
+			data, _ := base64.StdEncoding.DecodeString(versioning)
+			result, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketVersioningConfig, data, false, false, nil)
+			if err != nil || !bytes.Equal(result.meta.VersioningConfigXML, enabledBucketVersioningConfig) {
+				t.Fatalf("local commit snapshot: %s %v", result.meta.VersioningConfigXML, err)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataEqualTimeArrivalOrders(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			created := UTCNow().Add(-time.Hour)
+			a := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>a</Value></Tag></TagSet></Tagging>`)
+			b := bytes.ReplaceAll(a, []byte("<Value>a"), []byte("<Value>b"))
+			for _, pair := range [][2][]byte{{a, b}, {b, a}, {a, nil}, {nil, a}} {
+				meta := newBucketMetadata(bucket)
+				meta.Created = created
+				meta.defaultTimestamps()
+				if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					t.Fatal(err)
+				}
+				for _, data := range pair {
+					var payload *string
+					if data != nil {
+						encoded := base64.StdEncoding.EncodeToString(data)
+						payload = &encoded
+					}
+					rec := applySRBucketMetaViaAdmin(t, cred, madmin.SRBucketMeta{Bucket: bucket, Type: madmin.SRBucketMetaTypeTags, Tags: payload, UpdatedAt: created.Add(time.Minute)})
+					if rec.Code != http.StatusOK {
+						t.Fatalf("equal time apply: %d %s", rec.Code, rec.Body.String())
+					}
+				}
+				got, err := readBucketMetadata(t.Context(), obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				want := b
+				if pair[0] == nil || pair[1] == nil {
+					want = nil
+				}
+				if !bytes.Equal(got.TaggingConfigXML, want) || !got.TaggingConfigUpdatedAt.Equal(created.Add(time.Minute)) {
+					t.Fatalf("arrival order changed winner: %q at %v", got.TaggingConfigXML, got.TaggingConfigUpdatedAt)
+				}
+			}
+		})
+	}})
+}

cmd/site-replication_test.go

@@ -133,28 +133,3 @@ func TestSRBucketMetaCorsRoundTrip(t *testing.T) {
 		t.Fatalf("expected nil Cors for deletion, got %q", *gotDel.Cors)
 	}
 }
-
-// TestIsBucketMetadataEqualCors covers the pointer-comparison helper used by
-// the CORS heal path to decide whether a peer already holds the latest config.
-func TestIsBucketMetadataEqualCors(t *testing.T) {
-	a := base64.StdEncoding.EncodeToString([]byte("config-a"))
-	b := base64.StdEncoding.EncodeToString([]byte("config-b"))
-
-	cases := []struct {
-		name string
-		one  *string
-		two  *string
-		want bool
-	}{
-		{"both nil", nil, nil, true},
-		{"one nil", &a, nil, false},
-		{"other nil", nil, &b, false},
-		{"equal", &a, &a, true},
-		{"different", &a, &b, false},
-	}
-	for _, tc := range cases {
-		if got := isBucketMetadataEqual(tc.one, tc.two); got != tc.want {
-			t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
-		}
-	}
-}

cmd/sts-handlers.go

@@ -624,6 +624,10 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 
 	claims[expClaim] = UTCNow().Add(duration).Unix()
 	claims[parentClaim] = user.AccessKey
+	if err := setIAMParentRevocationClaim(ctx, globalIAMSys.store, user.AccessKey, claims); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
+		return
+	}
 
 	tokenRevokeType := r.Form.Get(stsRevokeTokenType)
 	if tokenRevokeType != "" {

cmd/testdata/data-usage-v9/README.md

@@ -0,0 +1,27 @@
+# Data-usage v9 retirement fixture
+
+Generated using unmodified `scanDataFolder` and `dataUsageCache.serializeTo` from
+Server commit `89637554d60c27cfc51d2281d0a4fe15e415f06d` (Go 1.27.1, darwin/arm64).
+The scanner visits three real local files; its size callback supplies synthetic
+version/delete-marker/remote-tier summaries, following `TestDataUsageCacheSerialize`.
+It is a scanner/cache compatibility fixture, not a distributed object-store test.
+
+The old scanner counts 74,962 bytes, 3 objects, 5 versions and 2 delete markers.
+Its nonzero retired `hts` totals 9,426 bytes. Remote tier `COLD` contains 65,536
+bytes, one version and one object. The cache includes nested children, both
+histograms, a fixed timestamp and scanner cycle 42. The JSON is the old scanner's
+complete expected cache (including `HotTierSize`, which the new reader ignores).
+
+Binary SHA-256: `4c9c7e94cea7758fe9b498b19f639188764b2787582783e6cc950f2c44d52a8b`.
+
+To regenerate, create a detached worktree at that exact source commit, copy
+`generate.go.txt` to `cmd/retirement-fixture_test.go`, and run:
+
+```sh
+SILO_RETIRE_FIXTURE_DIR=/absolute/output/directory go test ./cmd -run '^TestGenerateAccessRetirementV9Fixture$' -count=1 -v
+```
+
+The historical production writer adds the version byte, compresses with zstd
+and encodes msgp, including the nonzero `hts` field. Do not regenerate using the
+retired implementation or by changing the header of a v8 payload. Map order may
+change serialized bytes across regeneration; compare the decoded full cache.