chore: align IAM file notices with contribution policy

Signed-off-by: Feng Ruohang <rh@vonng.com>

.github/workflows/docker-release.yml

@@ -140,7 +140,7 @@ jobs:
             mkdir -p "${context}/dockerscripts"
             tar -xzf "${archive}" -C "${context}" silo
             cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
-            cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
+            cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
               "${context}/dockerscripts/"
           done
 

.github/workflows/test-release.yml

@@ -10,7 +10,7 @@ on:
       - "Dockerfile.distroless"
       - "cmd/healthcheck-main.go"
       - "cmd/main.go"
-      - "dockerscripts/download-static-curl.sh"
+      - "dockerscripts/build-static-curl.sh"
       - "dockerscripts/docker-entrypoint.sh"
       - "dockerscripts/docker-entrypoint_test.sh"
       - "silo.service"
@@ -41,6 +41,28 @@ permissions:
   contents: read
 
 jobs:
+  curl:
+    name: Static curl (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-latest
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v7
+      - name: Build and exercise curl in an empty runtime
+        run: |
+          docker build --build-arg TARGETARCH=${{ matrix.arch }} \
+            --target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
+          docker run --rm silo-curl-test --version | tee curl-version.txt
+          grep -F 'curl 8.22.0 ' curl-version.txt
+          grep -F 'HTTP2' curl-version.txt
+          docker run --rm silo-curl-test --fail --silent --show-error \
+            --connect-timeout 15 --max-time 60 https://curl.se/robots.txt
+
   validate:
     runs-on: ubuntu-latest
     steps:
@@ -490,7 +512,7 @@ jobs:
           bash -n buildscripts/package/lifecycle_test.sh
           buildscripts/package/lifecycle_test.sh
           bash -n dockerscripts/docker-entrypoint_test.sh
-          bash -n dockerscripts/download-static-curl.sh
+          bash -n dockerscripts/build-static-curl.sh
           dockerscripts/docker-entrypoint_test.sh
           go run ./buildscripts/rebrand-guard
           buildscripts/verify-rebrand.sh

.github/workflows/vulncheck.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Install govulncheck
         run: |
-          go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
+          go install golang.org/x/vuln/cmd/govulncheck@v1.8.0
           echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
 
       - name: Run govulncheck

CHANGELOG.md

@@ -0,0 +1,104 @@
+# Changelog
+
+## Unreleased
+
+The entries below describe source changes on main since the latest published Server.
+**The latest published Server remains 20260903.** These changes are not in its
+binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/)
+and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
+
+### Authorization and security
+
+- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
+  copy of another object accessible to the signer (SN-2026-011). The latest
+  public Server is affected; the fix is on main. See [the advisory ledger](docs/security/advisories.md).
+- Align signed request fields with policy conditions and enforce header-only
+  presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
+- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
+  from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
+  both denies if the previous combined restriction must survive upgrades or
+  rollback. Saved policies are not rewritten. Deploy with the matching Console
+  and pkg; see [the migration guide](docs/iam/password-permissions.md).
+
+### Object storage and replication
+
+- Evaluate conditional multipart completion against the logical current object
+  across all pools while holding the existing object lock. A stale `If-Match`
+  can no longer replace newer data in another pool, and the current ETag is no
+  longer rejected because the upload resides next to an older copy. Conditions
+  are evaluated once; a current delete marker counts as an absent object.
+  **Availability change:** if metadata cannot be read from any pool, conditional
+  completion fails even when another pool can still serve GET/HEAD. This also
+  applies when the unreadable pool may not hold the object: absence cannot be
+  verified. Retry after the pool recovers. Unconditional completion and the
+  single-pool path retain their existing behavior.
+
+- Reconcile ordinary single-object version DELETE across all pools, including
+  null versions, delete markers and unqualified directory-marker DELETE. This
+  applies the deletion to every resolved pool copy under existing quorum
+  rules. Pending outbound delete replication retains versions until the
+  existing replication worker completes their purge; a successful response
+  does not imply immediate physical removal from every drive. Unreadable
+  pools now consistently return 503 instead of depending on pool traversal
+  order; insufficient read quorum returns `SlowDownRead`. This extends the
+  existing failure surface. Retry after recovery.
+  Cleanup failures also return an error. Batch deletion already fans out across
+  pools; replication and scanner cleanup keep their existing contracts. See
+  [scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope).
+
+- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including
+  its tracker, mover, scanner hooks, configuration, XML actions and metrics.
+  Accept and ignore retired configuration/XML and preserve ordinary statistics
+  when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md).
+  The [decision record](docs/investigations/access-tiering-revert.md) preserves
+  the feature's introduction, subsequent fixes, rollback scope and review history.
+- Preserve the independent multi-pool write, metadata, healing and conditional
+  deletion fixes from PR #178, including shared remote-tier reference protection.
+- Enforce `If-Match` on DELETE, preserve retention and independently ordered
+  Object Lock/tag updates, and correctly retransmit encrypted replicas.
+- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
+  compression, honor key-rotation checksums, and complete attributes pagination.
+- Repair federated CopyObject checksums, destination timestamps, reserved
+  metadata, encrypted-object forwarding, legal hold and KMS context.
+- Make resync counters, target selection, cancellation and worker lifetimes
+  reflect actual work; complete delete-marker purges and report bounded MRF drops.
+- Converge bucket metadata with deterministic source state, deletion tombstones,
+  creation time recovery and diagnostics. The mixed-version export gate requires
+  coordinated upgrades before tombstones are exported. See [the #77 record](docs/investigations/issue-77-current.md).
+- Include per-bucket CORS in metadata export/import, close metadata publication
+  and logger races, and report effective bucket quotas in metrics.
+
+### Console, dependencies and delivery
+
+- Restore embedded Console login over loopback TLS, trusted-proxy handling and
+  all four WebSocket connection limits. Preserve Go TLS defaults across transports.
+- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console
+  `v0.0.0-20260913015128-417559bb2c97` and MC
+  `v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements.
+- Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/*
+  modules and security fixes including bounded AMQP frame handling. Keep Go
+  1.27.1 and go-systemd v22.6.0's NetBSD compatibility replacement.
+- Refresh container base digests and build static curl 8.22.0 from verified
+  source for both Linux architectures. Pin the actual mcli 20260913 archives and
+  hashes. Helm's client image follows that release; its Server image still names
+  the latest published Server 20260903.
+
+The dependency update passed the final candidate's Go, vulnerability and Test
+Release workflows; native curl builds passed on both architectures. A local
+ARM64 image passed startup, health, S3 transfer and embedded Console checks.
+These checks do not publish a Server tag or production image and do not replace
+cluster upgrade/rollback acceptance for the next release. Dated investigations
+retain the exact source and runtime boundaries they tested.
+
+## RELEASE.2026-09-03T13-18-01Z
+
+Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
+[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
+[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
+
+This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
+mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
+Installing the newer standalone mcli or Console does not replace components
+inside this existing Server binary or image.
+
+Earlier releases: [release archive](https://github.com/pgsty/silo/releases).

CONTRIBUTORS.md

@@ -62,7 +62,7 @@ the Git history and [NOTICE](NOTICE); this record covers activity in the PGSTY r
 | :-- | :-- | :-- |
 | [@Vonng](https://github.com/Vonng) | Maintains SILO, Console, mcli, shared packages, releases, and documentation | 85 merged PRs across five repositories; [complete maintainer record](#maintainer-record) |
 | [@h5vx](https://github.com/h5vx) | Implemented per-bucket CORS configuration and enforcement | [pgsty/silo#71](https://github.com/pgsty/silo/pull/71) — Merged ([e4e3007da](https://github.com/pgsty/silo/commit/e4e3007da6d7d1198a6a050e34f84566d40a9654)) |
-| [@mrjavadseydi](https://github.com/mrjavadseydi) | Fixed effective bucket quota metrics; proposed access-frequency ILM | [pgsty/silo#132](https://github.com/pgsty/silo/pull/132) — Merged ([ad873c735](https://github.com/pgsty/silo/commit/ad873c73571b121293791f13f8dc46ddf5264d60))<br>[pgsty/silo#60](https://github.com/pgsty/silo/pull/60) — Open |
+| [@mrjavadseydi](https://github.com/mrjavadseydi) | Fixed effective bucket quota metrics; proposed access-frequency ILM | [pgsty/silo#132](https://github.com/pgsty/silo/pull/132) — Merged ([ad873c735](https://github.com/pgsty/silo/commit/ad873c73571b121293791f13f8dc46ddf5264d60))<br>[pgsty/silo#60](https://github.com/pgsty/silo/pull/60) — Merged; access-frequency feature subsequently removed |
 | [@Dansyuqri](https://github.com/Dansyuqri) | Added ChecksumType to multipart completion responses | [pgsty/silo#57](https://github.com/pgsty/silo/pull/57) — Merged ([a96116b12](https://github.com/pgsty/silo/commit/a96116b128bbf2aa42f85eafbf75eb6636cd36ee)) |
 | [@ycjlin](https://github.com/ycjlin) | Fixed missing-bucket ListObjects semantics | [pgsty/silo#37](https://github.com/pgsty/silo/pull/37) — Merged ([49c8aeac4](https://github.com/pgsty/silo/commit/49c8aeac403916f52f8588bbe8ee42753d86eeef)) |
 | [@pinginfo](https://github.com/pinginfo) | Repaired bucket notification streaming | [pgsty/silo#34](https://github.com/pgsty/silo/pull/34) — Merged ([b7f52ca43](https://github.com/pgsty/silo/commit/b7f52ca4336bc45a48b81b39c8983a5e6882a6fa)) |

Dockerfile.goreleaser

@@ -1,4 +1,15 @@
-FROM golang:1.27.1-alpine AS build
+FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS curl-build
+ARG TARGETARCH
+COPY dockerscripts/build-static-curl.sh /build/build-static-curl
+RUN /bin/sh /build/build-static-curl
+
+# Exercise the exact shipped curl without a dynamic loader or shared libraries.
+FROM scratch AS curl-runtime
+COPY --from=curl-build /go/bin/curl /curl
+COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ENTRYPOINT ["/curl"]
+
+FROM golang:1.27.1-alpine@sha256:cf6fca6641884b8433441b2b0652976f975e1d0fdd26d177eaaf8596087f3125 AS build
 
 ARG TARGETARCH
 
@@ -6,9 +17,9 @@ ENV GOPATH=/go
 ENV CGO_ENABLED=0
 
 ARG MC_REPO=pgsty/mc
-ARG MC_VERSION=RELEASE.2026-09-03T07-13-05Z
-ARG MC_AMD64_SHA256=cd7fcd449bb6b52e2eb727431ba6975b1e5d90df011a75869020ea9ac9e2b2a8
-ARG MC_ARM64_SHA256=7962afc37c3e60e5758b19e819cb62d2f340ee655fad7b067e2ac9bc5716c2e8
+ARG MC_VERSION=RELEASE.2026-09-13T00-00-00Z
+ARG MC_AMD64_SHA256=9d2a92de9c7b887d9b944fe9ddce68d23f1b6df3415092e737594e56593f5e2b
+ARG MC_ARM64_SHA256=3d82e9ea6c601c4cb44fe5dd5f2ad1b7d7d64369378110f9ada9c524688a452a
 
 RUN apk add -U --no-cache \
     ca-certificates \
@@ -56,18 +67,14 @@ RUN apk add -U --no-cache \
     chmod +x /go/bin/mcli && \
     ln -sf mcli /go/bin/mc
 
-COPY dockerscripts/download-static-curl.sh /build/download-static-curl
-RUN chmod +x /build/download-static-curl && \
-    /build/download-static-curl
-
-FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
+FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs
 RUN dnf -y install ca-certificates && \
     update-ca-trust && \
     cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
-FROM registry.access.redhat.com/ubi9/ubi-micro:latest
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6
 
 LABEL org.opencontainers.image.title="Silo" \
       org.opencontainers.image.description="S3-Interface Libre Object Storage" \
@@ -88,7 +95,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY silo /usr/bin/silo
 COPY --from=build /go/bin/mcli /usr/bin/mcli
-COPY --from=build /go/bin/curl* /usr/bin/
+COPY --from=curl-build /go/bin/curl /usr/bin/curl
+COPY --from=curl-build /go/share/curl /licenses/curl
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY LICENSE /licenses/LICENSE
 COPY NOTICE /licenses/NOTICE

Makefile

@@ -216,7 +216,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
 			--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
 		mkdir -p "$$context/dockerscripts"; \
 		cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
-		cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
+		cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
 			"$$context/dockerscripts/"; \
 		docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
 			-f "$$context/Dockerfile.goreleaser" "$$context"

README.md

@@ -35,6 +35,14 @@
 > [!NOTE]
 > Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
 
+## Current release and main branch
+
+The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z).
+As of 2026-09-13, the main branch has newer security, storage, Console and
+shared-package changes that have not shipped in a Server release. See
+[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
+for the exact release/source boundary, including SN-2026-011 and password-policy migration.
+
 ## Overview
 
 PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.

README_ZH.md

@@ -35,6 +35,13 @@
 > [!NOTE]
 > 2026-08-06，本仓库由 `pgsty/minio` 更名为 `pgsty/silo`，默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支，以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
 
+## 当前发行版与主分支
+
+最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。
+截至 2026-09-13，主分支已合入更新的安全、存储、Console 与共享包改动，但尚未发布新 Server。
+准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/)，
+其中包括 SN-2026-011 修复状态与密码权限迁移要求。
+
 ## 概述
 
 上游停止社区发行后，Silo 为开源 MinIO 服务端维护一条持续可用的版本线：构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。

buildscripts/install-mcli.sh

@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
 	exit 0
 fi
 
-release=${MCLI_RELEASE:-RELEASE.2026-09-03T07-13-05Z}
+release=${MCLI_RELEASE:-RELEASE.2026-09-13T00-00-00Z}
 version_hyphen=${release#RELEASE.}
 package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
 if [ "${package_version}" = "${version_hyphen}" ]; then

buildscripts/rebrand-guard/compat-baseline.json

@@ -289,16 +289,6 @@
     "MINIO_IDENTITY_TLS_SKIP_VERIFY",
     "MINIO_IDENTITY_TLS_STS_EXPIRY",
     "MINIO_IDLE_TIMEOUT",
-    "MINIO_ILM_ACCESS_BINS",
-    "MINIO_ILM_ACCESS_BIN_WIDTH",
-    "MINIO_ILM_ACCESS_FLUSH",
-    "MINIO_ILM_ACCESS_MAX_SIZE",
-    "MINIO_ILM_ACCESS_MAX_TRACKED",
-    "MINIO_ILM_ACCESS_MIN_RESIDENCY",
-    "MINIO_ILM_ACCESS_POOLS",
-    "MINIO_ILM_ACCESS_PROMOTE_WATERMARK",
-    "MINIO_ILM_ACCESS_TIERING",
-    "MINIO_ILM_ACCESS_WORKERS",
     "MINIO_ILM_EXPIRATION_WORKERS",
     "MINIO_ILM_TRANSITION_WORKERS",
     "MINIO_INTERFACE",
@@ -512,6 +502,7 @@
     "MINIO_SITE_COMMENT",
     "MINIO_SITE_NAME",
     "MINIO_SITE_REGION",
+    "MINIO_SITE_REPLICATION_METADATA_TOMBSTONES",
     "MINIO_STORAGE_CLASS_COMMENT",
     "MINIO_STORAGE_CLASS_INLINE_BLOCK",
     "MINIO_STORAGE_CLASS_OPTIMIZE",
@@ -744,7 +735,6 @@
     "/idp/ldap/policy/{operation}",
     "/idp/openid/list-access-keys-bulk",
     "/ilm",
-    "/ilm/access",
     "/import-bucket-metadata",
     "/import-iam",
     "/import-iam-v2",
@@ -839,6 +829,7 @@
     "/site-replication/peer/bucket-ops",
     "/site-replication/peer/edit",
     "/site-replication/peer/iam-item",
+    "/site-replication/peer/iam-revisions",
     "/site-replication/peer/idp-settings",
     "/site-replication/peer/join",
     "/site-replication/peer/remove",
@@ -887,6 +878,7 @@
     "/v2/metrics/cluster",
     "/v2/metrics/node",
     "/v2/metrics/resource",
+    "/v3/site-replication/peer/iam-revisions",
     "/var/vcap/bosh",
     "/verifybinary",
     "/version",
@@ -914,6 +906,7 @@
     ".minio.sys/config/config.json",
     ".minio.sys/config/hello.txt",
     ".minio.sys/config/iam/${username}/identity.json",
+    ".minio.sys/config/ilm/access",
     ".minio.sys/format.json",
     ".minio.sys/multipart",
     ".minio.sys/multipart/bucket/object/uploads.json",

buildscripts/verify-rebrand.sh

@@ -36,7 +36,7 @@ for file in \
 	buildscripts/verify-helm-migration.sh \
 	Dockerfile.goreleaser \
 	Dockerfile.distroless \
-	dockerscripts/download-static-curl.sh \
+	dockerscripts/build-static-curl.sh \
 	dockerscripts/docker-entrypoint.sh \
 	helm/silo/Chart.yaml \
 	helm/silo/values.yaml \
@@ -101,7 +101,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
 require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
 require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
 require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
-require_text dockerscripts/download-static-curl.sh "sha256sum -c"
+require_text dockerscripts/build-static-curl.sh "sha256sum -c"
 require_text helm/silo/Chart.yaml "name: silo"
 require_text helm/silo/values.yaml "repository: pgsty/silo"
 require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"

cmd/admin-bucket-handlers.go

@@ -76,7 +76,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	quotaConfig, err := parseBucketQuota(bucket, data)
+	_, err = parseBucketQuota(bucket, data)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -94,9 +94,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		Quota:     data,
 		UpdatedAt: updatedAt,
 	}
-	if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
-		bucketMeta.Quota = nil
-	}
 
 	// Call site replication hook.
 	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
@@ -438,7 +435,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				configData, err := json.Marshal(config)
+				configData, err := canonicalBucketPolicy(config)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
@@ -922,7 +919,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			configData, err := json.Marshal(bucketPolicy)
+			configData, err := canonicalBucketPolicy(bucketPolicy)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
 				continue
@@ -1081,18 +1078,39 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		var merged BucketMetadata
+		var commitAt time.Time
 		err := func() error {
 			lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
 			if err != nil {
 				return err
 			}
 			defer unlock()
-			merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, true)
+			merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, false)
 			if err != nil {
 				return err
 			}
+			if err := ensureBucketMetadataCreated(lockCtx, objectAPI, &merged); err != nil {
+				return err
+			}
+			commitAt = UTCNow()
+			for _, file := range replicatedBucketConfigs {
+				if _, ok := fields[file]; ok {
+					commitAt = localBucketConfigUpdatedAt(merged, file, commitAt)
+				}
+			}
 			applyImportedBucketMetadata(&merged, *meta, fields)
-			return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, merged)
+			for _, file := range replicatedBucketConfigs {
+				if _, ok := fields[file]; !ok {
+					continue
+				}
+				data, at := replicatedBucketConfig(&merged, file)
+				payload, _, err := bucketConfigPayload(bucket, file, *data, len(merged.ObjectLockConfigXML) != 0)
+				if err != nil {
+					return err
+				}
+				*data, *at = payload, commitAt
+			}
+			return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, &merged)
 		}()
 		if err != nil {
 			rpt.SetStatus(bucket, "", err)
@@ -1100,7 +1118,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		*meta = merged
 		globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
-		hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: updatedAt}
+		hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: commitAt}
 		var hookNeeded bool
 		if _, ok := fields[bucketQuotaConfigFile]; ok {
 			hook.Quota = meta.QuotaConfigJSON
@@ -1108,7 +1126,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		if _, ok := fields[bucketPolicyConfig]; ok {
 			hook.Policy = meta.PolicyConfigJSON
-			hookNeeded = true
+			hookNeeded = hookNeeded || len(hook.Policy) != 0
 		}
 		if _, ok := fields[bucketVersioningConfig]; ok {
 			hook.Versioning = enc(meta.VersioningConfigXML)
@@ -1129,6 +1147,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		if hookNeeded {
 			err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
 		}
+		if _, ok := fields[bucketPolicyConfig]; ok && len(meta.PolicyConfigJSON) == 0 {
+			// An omitted bulk Policy cannot express deletion.
+			err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type: madmin.SRBucketMetaTypePolicy, Bucket: bucket, UpdatedAt: commitAt,
+			}))
+		}
 		if _, ok := fields[bucketCorsConfig]; ok {
 			// CORS carries its own timestamp, so it replicates through the
 			// dedicated event rather than the shared bucket metadata hook. It

cmd/admin-router.go

@@ -388,6 +388,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
+		adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))

cmd/bucket-cors-site-replication_test.go

@@ -467,25 +467,6 @@ func testAdversarialHealCorsPropagatesNewerEqualValueTimestamp(obj ObjectLayer,
 	}
 }
 
-func TestAdversarialBucketMetadataComparisonIsBase64CaseSensitive(t *testing.T) {
-	upper := "QQ=="
-	lower := "qQ=="
-	upperBytes, err := base64.StdEncoding.Strict().DecodeString(upper)
-	if err != nil {
-		t.Fatal(err)
-	}
-	lowerBytes, err := base64.StdEncoding.Strict().DecodeString(lower)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if string(upperBytes) == string(lowerBytes) {
-		t.Fatal("test inputs unexpectedly decode to the same bytes")
-	}
-	if isBucketMetadataEqual(&upper, &lower) {
-		t.Fatal("different decoded payloads were treated as equal")
-	}
-}
-
 func TestSiteReplicationStatusDetectsCorsTimestampMismatch(t *testing.T) {
 	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
 		t:          t,

cmd/bucket-lifecycle-audit.go

@@ -39,7 +39,6 @@ const (
 	lcEventSrc_s3PutObject
 	lcEventSrc_s3CopyObject
 	lcEventSrc_s3CompleteMultipartUpload
-	lcEventSrc_AccessTier
 )
 
 //revive:enable:var-naming

cmd/bucket-metadata-replication.go

@@ -0,0 +1,330 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"sort"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/tags"
+	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// Only these fields share the site-replication source-time ordering contract.
+// Bulk apply/import process Object Lock before Versioning, whose effective
+// document depends on it. Periodic heal retains its existing type order.
+var replicatedBucketConfigs = [...]string{
+	objectLockConfig, bucketVersioningConfig, bucketPolicyConfig,
+	bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile,
+}
+
+func replicatedBucketConfig(meta *BucketMetadata, file string) (*[]byte, *time.Time) {
+	switch file {
+	case bucketPolicyConfig:
+		return &meta.PolicyConfigJSON, &meta.PolicyConfigUpdatedAt
+	case bucketTaggingConfig:
+		return &meta.TaggingConfigXML, &meta.TaggingConfigUpdatedAt
+	case bucketSSEConfig:
+		return &meta.EncryptionConfigXML, &meta.EncryptionConfigUpdatedAt
+	case bucketQuotaConfigFile:
+		return &meta.QuotaConfigJSON, &meta.QuotaConfigUpdatedAt
+	case bucketVersioningConfig:
+		return &meta.VersioningConfigXML, &meta.VersioningConfigUpdatedAt
+	case objectLockConfig:
+		return &meta.ObjectLockConfigXML, &meta.ObjectLockConfigUpdatedAt
+	}
+	return nil, nil
+}
+
+// Callers that only need to know whether a file is under the contract must not
+// probe replicatedBucketConfig with a throwaway BucketMetadata.
+func isReplicatedBucketConfig(file string) bool {
+	for _, replicated := range replicatedBucketConfigs {
+		if replicated == file {
+			return true
+		}
+	}
+	return false
+}
+
+func bucketConfigUpdateOnly(file string) bool {
+	return file == bucketVersioningConfig || file == objectLockConfig
+}
+
+// Reuse the persistence rule before comparison, so an accepted Versioning
+// event and the document Save actually writes have the same comparison key.
+func effectiveBucketVersioning(data []byte, lockEnabled bool) []byte {
+	if lockEnabled {
+		config, err := versioning.ParseConfig(bytes.NewReader(data))
+		if err != nil || !config.Enabled() || config.PrefixesExcluded() {
+			return enabledBucketVersioningConfig
+		}
+	}
+	return data
+}
+
+// A parsed policy still contains map-backed sets with nondeterministic Marshal
+// order. Sort every set array recursively, including statements and conditions.
+// RawMessage keeps integer values intact; decoding through float64 would not.
+func canonicalBucketPolicyJSON(data json.RawMessage) (json.RawMessage, error) {
+	data = bytes.TrimSpace(data)
+	if len(data) == 0 {
+		return nil, nil
+	}
+	switch data[0] {
+	case '{':
+		var obj map[string]json.RawMessage
+		if err := json.Unmarshal(data, &obj); err != nil {
+			return nil, err
+		}
+		for key, value := range obj {
+			var err error
+			obj[key], err = canonicalBucketPolicyJSON(value)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return json.Marshal(obj)
+	case '[':
+		var arr []json.RawMessage
+		if err := json.Unmarshal(data, &arr); err != nil {
+			return nil, err
+		}
+		for i := range arr {
+			var err error
+			arr[i], err = canonicalBucketPolicyJSON(arr[i])
+			if err != nil {
+				return nil, err
+			}
+		}
+		sort.Slice(arr, func(i, j int) bool { return bytes.Compare(arr[i], arr[j]) < 0 })
+		return json.Marshal(arr)
+	default:
+		var compact bytes.Buffer
+		if err := json.Compact(&compact, data); err != nil {
+			return nil, err
+		}
+		return compact.Bytes(), nil
+	}
+}
+
+// Encode the validated policy fields explicitly: BPStatement's required
+// Action/Resource tags otherwise try to marshal empty sets for the supported
+// NotAction/NotResource alternatives. This stays within the existing schema.
+func canonicalBucketPolicy(cfg *policy.BucketPolicy) ([]byte, error) {
+	if cfg.IsEmpty() {
+		return nil, nil
+	}
+	doc := map[string]any{"Version": cfg.Version}
+	if cfg.ID != "" {
+		doc["ID"] = cfg.ID
+	}
+	statements := make([]map[string]any, 0, len(cfg.Statements))
+	for _, st := range cfg.Statements {
+		statement := map[string]any{"Effect": st.Effect, "Principal": st.Principal}
+		if st.SID != "" {
+			statement["Sid"] = st.SID
+		}
+		if len(st.Actions) != 0 {
+			statement["Action"] = st.Actions
+		}
+		if len(st.NotActions) != 0 {
+			statement["NotAction"] = st.NotActions
+		}
+		if len(st.Resources) != 0 {
+			statement["Resource"] = st.Resources
+		}
+		if len(st.NotResources) != 0 {
+			statement["NotResource"] = st.NotResources
+		}
+		if len(st.Conditions) != 0 {
+			statement["Condition"] = st.Conditions
+		}
+		statements = append(statements, statement)
+	}
+	doc["Statement"] = statements
+	data, err := json.Marshal(doc)
+	if err != nil {
+		return nil, err
+	}
+	return canonicalBucketPolicyJSON(data)
+}
+
+// Validate with the same parsers as Save. Policy's established empty-policy
+// semantics are deletion; a parsed zero quota is still a live document.
+func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]byte, []byte, error) {
+	if len(data) == 0 {
+		return nil, nil, nil
+	}
+	var err error
+	key := data
+	switch file {
+	case bucketPolicyConfig:
+		var cfg *policy.BucketPolicy
+		cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
+		if err == nil {
+			if cfg.IsEmpty() {
+				return nil, nil, nil
+			}
+			key, err = canonicalBucketPolicy(cfg)
+		}
+	case bucketQuotaConfigFile:
+		cfg, parseErr := parseBucketQuota(bucket, data)
+		err = parseErr
+		if err == nil {
+			key, err = json.Marshal(cfg)
+		}
+	case bucketTaggingConfig:
+		_, err = tags.ParseBucketXML(bytes.NewReader(data))
+	case bucketSSEConfig:
+		_, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(data))
+	case objectLockConfig:
+		_, err = objectlock.ParseObjectLockConfig(bytes.NewReader(data))
+	case bucketVersioningConfig:
+		data = effectiveBucketVersioning(data, lockEnabled)
+		key = data
+		_, err = versioning.ParseConfig(bytes.NewReader(data))
+	}
+	return data, key, err
+}
+
+type bucketConfigState struct {
+	data, key   []byte
+	at          time.Time
+	real, valid bool
+}
+
+func newBucketConfigState(bucket, file string, data []byte, at, created time.Time, lockEnabled bool) (bucketConfigState, error) {
+	data, key, err := bucketConfigPayload(bucket, file, data, lockEnabled)
+	if err != nil {
+		return bucketConfigState{}, err
+	}
+	if at.IsZero() {
+		at = created
+	}
+	valid := !created.IsZero() && !at.Before(created)
+	modified := valid && at.After(created)
+	if bucketConfigUpdateOnly(file) && len(data) == 0 {
+		modified = false
+	}
+	return bucketConfigState{data: data, key: key, at: at, real: modified, valid: valid}, nil
+}
+
+func (s bucketConfigState) candidate() bool {
+	return s.valid && (s.real || len(s.data) != 0)
+}
+
+func compareBucketConfigStates(a, b bucketConfigState) int {
+	if a.valid != b.valid {
+		if a.valid {
+			return 1
+		}
+		return -1
+	}
+	if a.real != b.real {
+		if a.real {
+			return 1
+		}
+		return -1
+	}
+	if a.real {
+		if n := a.at.Compare(b.at); n != 0 {
+			return n
+		}
+		// At equal source time a real deletion wins, preventing resurrection.
+		if (len(a.data) == 0) != (len(b.data) == 0) {
+			if len(a.data) == 0 {
+				return 1
+			}
+			return -1
+		}
+	}
+	return bytes.Compare(a.key, b.key)
+}
+
+func localBucketConfigUpdatedAt(meta BucketMetadata, file string, now time.Time) time.Time {
+	_, at := replicatedBucketConfig(&meta, file)
+	for _, lower := range []time.Time{meta.Created, *at} {
+		if !now.After(lower) {
+			now = lower.Add(time.Nanosecond)
+		}
+	}
+	return now.UTC()
+}
+
+func ensureBucketMetadataCreated(ctx context.Context, obj ObjectLayer, meta *BucketMetadata) error {
+	if !meta.Created.IsZero() {
+		return nil
+	}
+	info, err := obj.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true})
+	if err != nil {
+		return err
+	}
+	if info.Created.IsZero() {
+		return errors.New("bucket metadata creation time is unknown")
+	}
+	meta.Created = info.Created.UTC()
+	return nil
+}
+
+// applyBucketConfig runs under metadata.lock, on freshly loaded metadata. It
+// changes only the selected field; the caller persists once after all checks.
+func applyBucketConfig(meta *BucketMetadata, file string, data []byte, at time.Time) (bool, error) {
+	if bucketConfigUpdateOnly(file) && len(data) == 0 {
+		return false, nil
+	}
+	current, currentAt := replicatedBucketConfig(meta, file)
+	lockEnabled := len(meta.ObjectLockConfigXML) != 0
+	incoming, err := newBucketConfigState(meta.Name, file, data, at, meta.Created, lockEnabled)
+	if err != nil {
+		return false, err
+	}
+	if !incoming.candidate() {
+		return false, nil
+	}
+	local, err := newBucketConfigState(meta.Name, file, *current, *currentAt, meta.Created, lockEnabled)
+	if err != nil {
+		return false, err
+	}
+	if compareBucketConfigStates(incoming, local) <= 0 {
+		return false, nil
+	}
+	*current, *currentAt = bytes.Clone(incoming.data), incoming.at.UTC()
+	return true, nil
+}
+
+func rebaseBucketConfigDefaults(meta *BucketMetadata, oldCreated time.Time) {
+	if meta.Created.Equal(oldCreated) {
+		return
+	}
+	// These six fields alone use Created to distinguish a baseline from a
+	// tombstone. Preserve actual source times when adopting an existing bucket.
+	for _, file := range replicatedBucketConfigs {
+		_, at := replicatedBucketConfig(meta, file)
+		if at.IsZero() || at.Equal(oldCreated) {
+			*at = meta.Created
+		}
+	}
+}

cmd/bucket-metadata-sys.go

@@ -126,19 +126,42 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 	}
 }
 
-func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool) (updatedAt time.Time, err error) {
+// bucketMetadataUpdate returns the committed snapshot to the caller. meta and
+// updatedAt hold the saved state only when changed is true. Local writes always
+// change state, because localBucketConfigUpdatedAt is strictly greater than the
+// current field time, so their handlers can broadcast meta without rechecking.
+type bucketMetadataUpdate struct {
+	meta      BucketMetadata
+	updatedAt time.Time
+	changed   bool
+}
+
+func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket, configFile string, configData []byte, parse, lifecycleDelete bool) (time.Time, error) {
+	result, err := sys.updateAndParseMetadata(ctx, bucket, configFile, configData, parse, lifecycleDelete, nil)
+	return result.updatedAt, err
+}
+
+func (sys *BucketMetadataSys) updateAndParseMetadata(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool, sourceTime *time.Time) (result bucketMetadataUpdate, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return updatedAt, errServerNotInitialized
+		return result, errServerNotInitialized
 	}
 
 	if isMinioMetaBucketName(bucket) {
-		return updatedAt, errInvalidArgument
+		return result, errInvalidArgument
+	}
+	// Load deletions without parsed caches (notably quota), and compare the
+	// six replicated fields against the raw document under the same lock.
+	if isReplicatedBucketConfig(configFile) {
+		parse = false
+		if bucketConfigUpdateOnly(configFile) && len(configData) == 0 {
+			return result, nil
+		}
 	}
 	notifyCtx := ctx
 	ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
-		return updatedAt, err
+		return result, err
 	}
 
 	err = func() error {
@@ -158,55 +181,73 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 				return err
 			}
 		}
-		updatedAt = UTCNow()
-		switch configFile {
-		case bucketPolicyConfig:
-			meta.PolicyConfigJSON = configData
-			meta.PolicyConfigUpdatedAt = updatedAt
-		case bucketNotificationConfig:
-			meta.NotificationConfigXML = configData
-			meta.NotificationConfigUpdatedAt = updatedAt
-		case bucketLifecycleConfig:
-			meta.LifecycleConfigXML = configData
-			meta.LifecycleConfigUpdatedAt = updatedAt
-		case bucketSSEConfig:
-			meta.EncryptionConfigXML = configData
-			meta.EncryptionConfigUpdatedAt = updatedAt
-		case bucketTaggingConfig:
-			meta.TaggingConfigXML = configData
-			meta.TaggingConfigUpdatedAt = updatedAt
-		case bucketQuotaConfigFile:
-			meta.QuotaConfigJSON = configData
-			meta.QuotaConfigUpdatedAt = updatedAt
-		case objectLockConfig:
-			meta.ObjectLockConfigXML = configData
-			meta.ObjectLockConfigUpdatedAt = updatedAt
-		case bucketVersioningConfig:
-			meta.VersioningConfigXML = configData
-			meta.VersioningConfigUpdatedAt = updatedAt
-		case bucketReplicationConfig:
-			meta.ReplicationConfigXML = configData
-			meta.ReplicationConfigUpdatedAt = updatedAt
-		case bucketTargetsFile:
-			meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
-				bucket:            meta.Name,
-				bucketTargetsFile: bucketTargetsFile,
-			})
-			if err != nil {
-				return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+		updatedAt := UTCNow()
+		if isReplicatedBucketConfig(configFile) {
+			if err := ensureBucketMetadataCreated(ctx, objAPI, &meta); err != nil {
+				var at time.Time
+				if sourceTime != nil {
+					at = *sourceTime
+				}
+				logBucketConfigReplication(ctx, bucket, configFile, "indeterminate", at, meta.Created, err.Error())
+				return err
+			}
+			if sourceTime == nil || sourceTime.IsZero() {
+				updatedAt = localBucketConfigUpdatedAt(meta, configFile, updatedAt)
+				if sourceTime != nil {
+					logBucketConfigReplication(ctx, bucket, configFile, "legacy-zero", *sourceTime, meta.Created, "assigned local source time")
+				}
+			} else {
+				updatedAt = sourceTime.UTC()
+			}
+			if updatedAt.Before(meta.Created) {
+				logBucketConfigReplication(ctx, bucket, configFile, "before-created", updatedAt, meta.Created, "peer event")
+				return nil
+			}
+			changed, err := applyBucketConfig(&meta, configFile, configData, updatedAt)
+			if err != nil {
+				return err
+			}
+			if !changed {
+				return nil
+			}
+		} else {
+			switch configFile {
+			case bucketNotificationConfig:
+				meta.NotificationConfigXML = configData
+				meta.NotificationConfigUpdatedAt = updatedAt
+			case bucketLifecycleConfig:
+				meta.LifecycleConfigXML = configData
+				meta.LifecycleConfigUpdatedAt = updatedAt
+			case bucketReplicationConfig:
+				meta.ReplicationConfigXML = configData
+				meta.ReplicationConfigUpdatedAt = updatedAt
+			case bucketTargetsFile:
+				meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
+					bucket:            meta.Name,
+					bucketTargetsFile: bucketTargetsFile,
+				})
+				if err != nil {
+					return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+				}
+				meta.BucketTargetsConfigUpdatedAt = updatedAt
+				meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
+			default:
+				return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 			}
-			meta.BucketTargetsConfigUpdatedAt = updatedAt
-			meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
-		default:
-			return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 		}
-		return sys.saveMetadata(ctx, objAPI, meta)
+		if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
+			return err
+		}
+		result = bucketMetadataUpdate{meta: meta, updatedAt: updatedAt, changed: true}
+		return nil
 	}()
 	if err != nil {
-		return updatedAt, err
+		return result, err
 	}
-	globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
-	return updatedAt, nil
+	if result.changed {
+		globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket)
+	}
+	return result, nil
 }
 
 func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
@@ -219,7 +260,7 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
 		return errInvalidArgument
 	}
 
-	if err := sys.saveMetadata(ctx, objAPI, meta); err != nil {
+	if err := sys.saveMetadata(ctx, objAPI, &meta); err != nil {
 		return err
 	}
 
@@ -229,7 +270,7 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
 
 // saveMetadata persists and publishes metadata locally. Callers performing a
 // read-modify-write must hold metadata.lock and release it before peer fan-out.
-func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta BucketMetadata) error {
+func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta *BucketMetadata) error {
 	// A writer may have queued for metadata.lock before DeleteBucket completed.
 	// Recheck the physical bucket under that lock, before recreating metadata.
 	if _, err := objAPI.GetBucketInfo(ctx, meta.Name, BucketOptions{NoMetadata: true}); err != nil {
@@ -238,7 +279,7 @@ func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLay
 	if err := meta.Save(ctx, objAPI); err != nil {
 		return err
 	}
-	sys.Set(meta.Name, meta)
+	sys.Set(meta.Name, *meta)
 	return nil
 }
 
@@ -351,7 +392,7 @@ func (sys *BucketMetadataSys) UpdateExpiryLCConfig(ctx context.Context, bucket s
 		}
 		meta.LifecycleConfigXML = configData
 		meta.LifecycleConfigUpdatedAt = UTCNow()
-		return sys.saveMetadata(ctx, objAPI, meta)
+		return sys.saveMetadata(ctx, objAPI, &meta)
 	}()
 	if err != nil {
 		return err

cmd/bucket-metadata.go

@@ -388,15 +388,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	} else {
 		b.objectLockConfig = nil
 	}
-	if b.objectLockConfig != nil {
-		// Object Lock requires every object to be versioned. Whatever the lock
-		// document contains, a suspended or prefix-excluded versioning document
-		// is replaced by plain Enabled versioning; Save persists the result.
-		config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
-		if versioningErr != nil || !config.Enabled() || config.PrefixesExcluded() {
-			b.VersioningConfigXML = enabledBucketVersioningConfig
-		}
-	}
+	b.VersioningConfigXML = effectiveBucketVersioning(b.VersioningConfigXML, b.objectLockConfig != nil)
 
 	if len(b.VersioningConfigXML) != 0 {
 		b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))

cmd/bucket-metadata_test.go

@@ -39,3 +39,28 @@ func TestBucketMetadataCorsRoundTrip(t *testing.T) {
 		t.Fatalf("CorsConfigUpdatedAt not preserved")
 	}
 }
+
+// A persisted retired extension must not prevent the whole bucket's metadata
+// from loading, including unrelated versioning and ordinary lifecycle rules.
+func TestBucketMetadataRetiredAccessTiering(t *testing.T) {
+	meta := newBucketMetadata("retired-access")
+	meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><AccessTierQuota>500GiB</AccessTierQuota><Rule><ID>access</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AccessTransition><Window>10m</Window><PromoteAfterAccesses>10</PromoteAfterAccesses></AccessTransition></Rule><Rule><ID>ordinary</ID><Status>Enabled</Status><Filter><Prefix>expired/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
+	meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
+	data, err := meta.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := newBucketMetadata(meta.Name)
+	if _, err := got.UnmarshalMsg(data); err != nil {
+		t.Fatal(err)
+	}
+	if err := got.parseAllConfigs(t.Context(), nil); err != nil {
+		t.Fatalf("bucket metadata failed to load: %v", err)
+	}
+	if got.lifecycleConfig == nil || got.lifecycleConfig.HasActiveRules("logs/") || !got.lifecycleConfig.HasActiveRules("expired/") {
+		t.Fatal("unexpected lifecycle behavior")
+	}
+	if got.versioningConfig == nil || !got.versioningConfig.Enabled() {
+		t.Fatal("unrelated versioning lost")
+	}
+}

cmd/bucket-policy-handlers.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"encoding/json"
 	"io"
 	"net/http"
 
@@ -100,13 +99,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	configData, err := json.Marshal(bucketPolicy)
+	configData, err := canonicalBucketPolicy(bucketPolicy)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+	result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketPolicyConfig, configData, false, false, nil)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -116,8 +115,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
-		Policy:    bucketPolicyBytes,
-		UpdatedAt: updatedAt,
+		Policy:    result.meta.PolicyConfigJSON,
+		UpdatedAt: result.updatedAt,
 	}))
 
 	// Success.
@@ -200,7 +199,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	configData, err := json.Marshal(config)
+	configData, err := canonicalBucketPolicy(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-versioning-handler.go

@@ -97,7 +97,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+	result, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketVersioningConfig, configData, false, false, nil)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -107,12 +107,12 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 	//
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
-	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	cfgStr := base64.StdEncoding.EncodeToString(result.meta.VersioningConfigXML)
 	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,
-		UpdatedAt:  updatedAt,
+		UpdatedAt:  result.updatedAt,
 	}))
 
 	writeSuccessResponseHeadersOnly(w)

cmd/common-main.go

@@ -900,6 +900,7 @@ func serverHandleEnvVars() {
 	}
 
 	globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
+	globalSiteReplicationMetadataTombstones = env.Get("MINIO_SITE_REPLICATION_METADATA_TOMBSTONES", config.EnableOff) == config.EnableOn
 }
 
 func loadRootCredentials() auth.Credentials {

cmd/config-current.go

@@ -227,7 +227,7 @@ func initHelp() {
 		},
 		config.HelpKV{
 			Key:         config.ILMSubSys,
-			Description: "manage ILM settings for expiration, transition, and access-tier workers",
+			Description: "manage ILM settings for expiration and transition workers",
 			Optional:    true,
 		},
 	}
@@ -704,9 +704,6 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			if globalExpiryState != nil {
 				globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
 			}
-			if globalAccessTierState != nil {
-				globalAccessTierState.UpdateWorkers(ilmCfg.AccessWorkers)
-			}
 			globalILMConfig.update(ilmCfg)
 		}
 	}

cmd/data-scanner.go

@@ -899,7 +899,6 @@ type scannerItem struct {
 	objectName  string // Only the object name without prefixes.
 	replication replicationConfig
 	lifeCycle   *lifecycle.Lifecycle
-	poolIdx     int
 	Typ         fs.FileMode
 	heal        struct {
 		enabled bool
@@ -910,7 +909,6 @@ type scannerItem struct {
 
 type sizeSummary struct {
 	totalSize       int64
-	hotTierSize     int64
 	versions        uint64
 	deleteMarkers   uint64
 	replicatedSize  int64
@@ -1161,14 +1159,6 @@ eventLoop:
 		globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents)
 	}
 	i.alertExcessiveVersions(remainingVersions, cumulativeSize)
-	if globalILMConfig.accessTieringEnabled() {
-		for idx, oi := range objInfos {
-			if oi.IsLatest && events[idx].Action == lifecycle.NoneAction {
-				applyAccessTransition(ctx, i, oi)
-				break
-			}
-		}
-	}
 }
 
 func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
@@ -1484,8 +1474,6 @@ const (
 	ILMFreeVersionDelete = "ilm:free-version-delete"
 	// ILMTransition - audit trail for ILM transitioning.
 	ILMTransition = " ilm:transition"
-	// ILMAccessTier - audit trail for moving objects between server pools.
-	ILMAccessTier = "ilm:access-tier"
 )
 
 func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
@@ -1497,8 +1485,6 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
 		apiName = "ILMFreeVersionDelete"
 	case ILMTransition:
 		apiName = "ILMTransition"
-	case ILMAccessTier:
-		apiName = "ILMAccessTier"
 	}
 	auditLogInternal(ctx, AuditLogOptions{
 		Event:     event,

cmd/data-usage-cache-v9_test.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"reflect"
+	"testing"
+)
+
+// Read a real pre-removal scanner cache with nonzero hot-tier bytes. A v8
+// payload with a relabeled header would not exercise the ignored hts field.
+func TestDataUsageCacheReadV9(t *testing.T) {
+	raw, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.bin")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(raw) == 0 || raw[0] != 9 {
+		t.Fatal("fixture is not v9")
+	}
+	expected, err := os.ReadFile("testdata/data-usage-v9/data-usage-v9.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var want, got dataUsageCache
+	if err := json.Unmarshal(expected, &want); err != nil {
+		t.Fatal(err)
+	}
+	if err := got.deserialize(bytes.NewReader(raw)); err != nil {
+		t.Fatal(err)
+	}
+	if !got.Info.LastUpdate.Equal(want.Info.LastUpdate) {
+		t.Fatal("cache timestamp changed")
+	}
+	// msgp restores local time while JSON preserves the UTC representation.
+	want.Info.LastUpdate = got.Info.LastUpdate
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("v9 ordinary cache fields changed:\ngot: %#v\nwant: %#v", got, want)
+	}
+	flat := got.flatten(*got.root())
+	if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 {
+		t.Fatalf("statistics changed: %+v", flat)
+	}
+	if flat.AllTierStats == nil || flat.AllTierStats.Tiers["COLD"] != (tierStats{TotalSize: 65536, NumVersions: 1, NumObjects: 1}) {
+		t.Fatalf("remote tier statistics changed: %+v", flat.AllTierStats)
+	}
+	buckets := []BucketInfo{{Name: "v9-bucket"}}
+	if gotInfo, wantInfo := got.dui(dataUsageRoot, buckets), want.dui(dataUsageRoot, buckets); !reflect.DeepEqual(gotInfo, wantInfo) {
+		t.Fatalf("bucket usage aggregation changed: %+v != %+v", gotInfo, wantInfo)
+	}
+	var buf bytes.Buffer
+	if err := got.serializeTo(&buf); err != nil {
+		t.Fatal(err)
+	}
+	if buf.Bytes()[0] != 8 {
+		t.Fatalf("wrote cache version %d, want 8", buf.Bytes()[0])
+	}
+	var roundtrip dataUsageCache
+	if err := roundtrip.deserialize(&buf); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, roundtrip) {
+		t.Fatal("v8 round trip lost ordinary fields")
+	}
+	if err := roundtrip.deserialize(bytes.NewReader(raw[:len(raw)/2])); err == nil {
+		t.Fatal("truncated v9 cache accepted")
+	}
+	raw[0] = 10
+	if err := roundtrip.deserialize(bytes.NewReader(raw)); err == nil {
+		t.Fatal("unknown cache version accepted")
+	}
+}

cmd/data-usage-cache.go

@@ -61,7 +61,6 @@ type dataUsageEntry struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
 	Size          int64             `msg:"sz"`
-	HotTierSize   int64             `msg:"hts"`
 	Objects       uint64            `msg:"os"`
 	Versions      uint64            `msg:"vs"` // Versions that are not delete markers.
 	DeleteMarkers uint64            `msg:"dms"`
@@ -135,8 +134,8 @@ func (ts tierStats) add(u tierStats) tierStats {
 	}
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -200,30 +199,14 @@ type dataUsageEntryV7 struct {
 	Compacted     bool              `msg:"c"`
 }
 
-// dataUsageEntryV8 is the on-disk shape before access-tier accounting was
-// introduced. Keep it so caches written by the previous release decode
-// without being discarded.
-type dataUsageEntryV8 struct {
-	Children dataUsageHashMap `msg:"ch"`
-	// These fields do no include any children.
-	Size          int64             `msg:"sz"`
-	Objects       uint64            `msg:"os"`
-	Versions      uint64            `msg:"vs"`
-	DeleteMarkers uint64            `msg:"dms"`
-	ObjSizes      sizeHistogram     `msg:"szs"`
-	ObjVersions   versionsHistogram `msg:"vh"`
-	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
-	Compacted     bool              `msg:"c"`
-}
-
 // dataUsageCache contains a cache of data usage entries latest version.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Cache map[string]dataUsageEntry
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -261,12 +244,6 @@ type dataUsageCacheV7 struct {
 	Cache map[string]dataUsageEntryV7
 }
 
-// dataUsageCacheV8 contains a cache of data usage entries version 8.
-type dataUsageCacheV8 struct {
-	Info  dataUsageCacheInfo
-	Cache map[string]dataUsageEntryV8
-}
-
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -295,7 +272,6 @@ type dataUsageCacheInfo struct {
 
 func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Size += summary.totalSize
-	e.HotTierSize += summary.hotTierSize
 	e.Versions += summary.versions
 	e.DeleteMarkers += summary.deleteMarkers
 	e.ObjSizes.add(summary.totalSize)
@@ -315,7 +291,6 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Versions += other.Versions
 	e.DeleteMarkers += other.DeleteMarkers
 	e.Size += other.Size
-	e.HotTierSize += other.HotTierSize
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
@@ -456,7 +431,6 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	flat := d.flatten(*e)
 	dui := DataUsageInfo{
 		LastUpdate:              d.Info.LastUpdate,
-		ScannerCycle:            d.Info.NextCycle,
 		ObjectsTotalCount:       flat.Objects,
 		VersionsTotalCount:      flat.Versions,
 		DeleteMarkersTotalCount: flat.DeleteMarkers,
@@ -807,7 +781,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		flat := d.flatten(*e)
 		bui := BucketUsageInfo{
 			Size:                    uint64(flat.Size),
-			HotTierSize:             uint64(max(flat.HotTierSize, 0)),
 			VersionsCount:           flat.Versions,
 			ObjectsCount:            flat.Objects,
 			DeleteMarkersCount:      flat.DeleteMarkers,
@@ -1007,8 +980,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 9
-	dataUsageCacheVerV8      = 8
+	dataUsageCacheVerCurrent = 8
+	dataUsageCacheVerV9      = 9 // Retired access-tier cache; only adds the ignored "hts" entry key.
 	dataUsageCacheVerV7      = 7
 	dataUsageCacheVerV6      = 6
 	dataUsageCacheVerV5      = 5
@@ -1210,34 +1183,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		}
 
 		return nil
-	case dataUsageCacheVerV8:
-		// Zstd compressed.
-		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
-		if err != nil {
-			return err
-		}
-		defer dec.Close()
-		dold := &dataUsageCacheV8{}
-		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
-			return err
-		}
-		d.Info = dold.Info
-		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
-		for k, v := range dold.Cache {
-			d.Cache[k] = dataUsageEntry{
-				Children:      v.Children,
-				Size:          v.Size,
-				Objects:       v.Objects,
-				Versions:      v.Versions,
-				DeleteMarkers: v.DeleteMarkers,
-				ObjSizes:      v.ObjSizes,
-				ObjVersions:   v.ObjVersions,
-				AllTierStats:  v.AllTierStats,
-				Compacted:     v.Compacted,
-			}
-		}
-		return nil
-	case dataUsageCacheVerCurrent:
+	case dataUsageCacheVerCurrent, dataUsageCacheVerV9:
 		// Zstd compressed.
 		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
 		if err != nil {

cmd/data-usage-cache_gen.go

@@ -1591,145 +1591,6 @@ func (z *dataUsageCacheV7) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *dataUsageCacheV8) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Info":
-			err = z.Info.DecodeMsg(dc)
-			if err != nil {
-				err = msgp.WrapError(err, "Info")
-				return
-			}
-		case "Cache":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Cache")
-				return
-			}
-			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV8, zb0002)
-			} else if len(z.Cache) > 0 {
-				clear(z.Cache)
-			}
-			for zb0002 > 0 {
-				zb0002--
-				var za0001 string
-				za0001, err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Cache")
-					return
-				}
-				var za0002 dataUsageEntryV8
-				err = za0002.DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
-					return
-				}
-				z.Cache[za0001] = za0002
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageCacheV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Info":
-			bts, err = z.Info.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Info")
-				return
-			}
-		case "Cache":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Cache")
-				return
-			}
-			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV8, zb0002)
-			} else if len(z.Cache) > 0 {
-				clear(z.Cache)
-			}
-			for zb0002 > 0 {
-				var za0002 dataUsageEntryV8
-				zb0002--
-				var za0001 string
-				za0001, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache")
-					return
-				}
-				bts, err = za0002.UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
-					return
-				}
-				z.Cache[za0001] = za0002
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageCacheV8) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
-	if z.Cache != nil {
-		for za0001, za0002 := range z.Cache {
-			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
-		}
-	}
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -1762,12 +1623,6 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Size")
 				return
 			}
-		case "hts":
-			z.HotTierSize, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "HotTierSize")
-				return
-			}
 		case "os":
 			z.Objects, err = dc.ReadUint64()
 			if err != nil {
@@ -1946,12 +1801,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// check for omitted fields
-	zb0001Len := uint32(10)
-	var zb0001Mask uint16 /* 10 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x100
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -1981,16 +1836,6 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Size")
 			return
 		}
-		// write "hts"
-		err = en.Append(0xa3, 0x68, 0x74, 0x73)
-		if err != nil {
-			return
-		}
-		err = en.WriteInt64(z.HotTierSize)
-		if err != nil {
-			err = msgp.WrapError(err, "HotTierSize")
-			return
-		}
 		// write "os"
 		err = en.Append(0xa2, 0x6f, 0x73)
 		if err != nil {
@@ -2055,7 +1900,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 				return
 			}
 		}
-		if (zb0001Mask & 0x100) == 0 { // if not omitted
+		if (zb0001Mask & 0x80) == 0 { // if not omitted
 			// write "ats"
 			err = en.Append(0xa3, 0x61, 0x74, 0x73)
 			if err != nil {
@@ -2136,12 +1981,12 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// check for omitted fields
-	zb0001Len := uint32(10)
-	var zb0001Mask uint16 /* 10 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x100
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -2158,9 +2003,6 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 		// string "sz"
 		o = append(o, 0xa2, 0x73, 0x7a)
 		o = msgp.AppendInt64(o, z.Size)
-		// string "hts"
-		o = append(o, 0xa3, 0x68, 0x74, 0x73)
-		o = msgp.AppendInt64(o, z.HotTierSize)
 		// string "os"
 		o = append(o, 0xa2, 0x6f, 0x73)
 		o = msgp.AppendUint64(o, z.Objects)
@@ -2182,7 +2024,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 		for za0002 := range z.ObjVersions {
 			o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 		}
-		if (zb0001Mask & 0x100) == 0 { // if not omitted
+		if (zb0001Mask & 0x80) == 0 { // if not omitted
 			// string "ats"
 			o = append(o, 0xa3, 0x61, 0x74, 0x73)
 			if z.AllTierStats == nil {
@@ -2246,12 +2088,6 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Size")
 				return
 			}
-		case "hts":
-			z.HotTierSize, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "HotTierSize")
-				return
-			}
 		case "os":
 			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
 			if err != nil {
@@ -2429,7 +2265,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
 	if z.AllTierStats == nil {
 		s += msgp.NilSize
 	} else {
@@ -3422,438 +3258,6 @@ func (z *dataUsageEntryV7) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *dataUsageEntryV8) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var zb0001Mask uint8 /* 1 bits */
-	_ = zb0001Mask
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "ch":
-			err = z.Children.DecodeMsg(dc)
-			if err != nil {
-				err = msgp.WrapError(err, "Children")
-				return
-			}
-		case "sz":
-			z.Size, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "Size")
-				return
-			}
-		case "os":
-			z.Objects, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "Objects")
-				return
-			}
-		case "vs":
-			z.Versions, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "Versions")
-				return
-			}
-		case "dms":
-			z.DeleteMarkers, err = dc.ReadUint64()
-			if err != nil {
-				err = msgp.WrapError(err, "DeleteMarkers")
-				return
-			}
-		case "szs":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "ObjSizes")
-				return
-			}
-			if zb0002 != uint32(dataUsageBucketLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
-				return
-			}
-			for za0001 := range z.ObjSizes {
-				z.ObjSizes[za0001], err = dc.ReadUint64()
-				if err != nil {
-					err = msgp.WrapError(err, "ObjSizes", za0001)
-					return
-				}
-			}
-		case "vh":
-			var zb0003 uint32
-			zb0003, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "ObjVersions")
-				return
-			}
-			if zb0003 != uint32(dataUsageVersionLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
-				return
-			}
-			for za0002 := range z.ObjVersions {
-				z.ObjVersions[za0002], err = dc.ReadUint64()
-				if err != nil {
-					err = msgp.WrapError(err, "ObjVersions", za0002)
-					return
-				}
-			}
-		case "ats":
-			if dc.IsNil() {
-				err = dc.ReadNil()
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				z.AllTierStats = nil
-			} else {
-				if z.AllTierStats == nil {
-					z.AllTierStats = new(allTierStats)
-				}
-				var zb0004 uint32
-				zb0004, err = dc.ReadMapHeader()
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "AllTierStats")
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "ts":
-						var zb0005 uint32
-						zb0005, err = dc.ReadMapHeader()
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats", "Tiers")
-							return
-						}
-						if z.AllTierStats.Tiers == nil {
-							z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
-						} else if len(z.AllTierStats.Tiers) > 0 {
-							clear(z.AllTierStats.Tiers)
-						}
-						for zb0005 > 0 {
-							zb0005--
-							var za0003 string
-							za0003, err = dc.ReadString()
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers")
-								return
-							}
-							var za0004 tierStats
-							var zb0006 uint32
-							zb0006, err = dc.ReadMapHeader()
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-								return
-							}
-							for zb0006 > 0 {
-								zb0006--
-								field, err = dc.ReadMapKeyPtr()
-								if err != nil {
-									err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-									return
-								}
-								switch msgp.UnsafeString(field) {
-								case "ts":
-									za0004.TotalSize, err = dc.ReadUint64()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
-										return
-									}
-								case "nv":
-									za0004.NumVersions, err = dc.ReadInt()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
-										return
-									}
-								case "no":
-									za0004.NumObjects, err = dc.ReadInt()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
-										return
-									}
-								default:
-									err = dc.Skip()
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-										return
-									}
-								}
-							}
-							z.AllTierStats.Tiers[za0003] = za0004
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats")
-							return
-						}
-					}
-				}
-			}
-			zb0001Mask |= 0x1
-		case "c":
-			z.Compacted, err = dc.ReadBool()
-			if err != nil {
-				err = msgp.WrapError(err, "Compacted")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	// Clear omitted fields.
-	if (zb0001Mask & 0x1) == 0 {
-		z.AllTierStats = nil
-	}
-
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageEntryV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var zb0001Mask uint8 /* 1 bits */
-	_ = zb0001Mask
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "ch":
-			bts, err = z.Children.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Children")
-				return
-			}
-		case "sz":
-			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Size")
-				return
-			}
-		case "os":
-			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Objects")
-				return
-			}
-		case "vs":
-			z.Versions, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Versions")
-				return
-			}
-		case "dms":
-			z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "DeleteMarkers")
-				return
-			}
-		case "szs":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ObjSizes")
-				return
-			}
-			if zb0002 != uint32(dataUsageBucketLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
-				return
-			}
-			for za0001 := range z.ObjSizes {
-				z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "ObjSizes", za0001)
-					return
-				}
-			}
-		case "vh":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ObjVersions")
-				return
-			}
-			if zb0003 != uint32(dataUsageVersionLen) {
-				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
-				return
-			}
-			for za0002 := range z.ObjVersions {
-				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "ObjVersions", za0002)
-					return
-				}
-			}
-		case "ats":
-			if msgp.IsNil(bts) {
-				bts, err = msgp.ReadNilBytes(bts)
-				if err != nil {
-					return
-				}
-				z.AllTierStats = nil
-			} else {
-				if z.AllTierStats == nil {
-					z.AllTierStats = new(allTierStats)
-				}
-				var zb0004 uint32
-				zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "AllTierStats")
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "AllTierStats")
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "ts":
-						var zb0005 uint32
-						zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats", "Tiers")
-							return
-						}
-						if z.AllTierStats.Tiers == nil {
-							z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
-						} else if len(z.AllTierStats.Tiers) > 0 {
-							clear(z.AllTierStats.Tiers)
-						}
-						for zb0005 > 0 {
-							var za0004 tierStats
-							zb0005--
-							var za0003 string
-							za0003, bts, err = msgp.ReadStringBytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers")
-								return
-							}
-							var zb0006 uint32
-							zb0006, bts, err = msgp.ReadMapHeaderBytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-								return
-							}
-							for zb0006 > 0 {
-								zb0006--
-								field, bts, err = msgp.ReadMapKeyZC(bts)
-								if err != nil {
-									err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-									return
-								}
-								switch msgp.UnsafeString(field) {
-								case "ts":
-									za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
-										return
-									}
-								case "nv":
-									za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
-										return
-									}
-								case "no":
-									za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
-										return
-									}
-								default:
-									bts, err = msgp.Skip(bts)
-									if err != nil {
-										err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
-										return
-									}
-								}
-							}
-							z.AllTierStats.Tiers[za0003] = za0004
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "AllTierStats")
-							return
-						}
-					}
-				}
-			}
-			zb0001Mask |= 0x1
-		case "c":
-			z.Compacted, bts, err = msgp.ReadBoolBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Compacted")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	// Clear omitted fields.
-	if (zb0001Mask & 0x1) == 0 {
-		z.AllTierStats = nil
-	}
-
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageEntryV8) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
-	if z.AllTierStats == nil {
-		s += msgp.NilSize
-	} else {
-		s += 1 + 3 + msgp.MapHeaderSize
-		if z.AllTierStats.Tiers != nil {
-			for za0003, za0004 := range z.AllTierStats.Tiers {
-				_ = za0004
-				s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
-			}
-		}
-	}
-	s += 2 + msgp.BoolSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{

cmd/data-usage-utils.go

@@ -46,8 +46,7 @@ type BucketTargetUsageInfo struct {
 // - total objects in a bucket
 // - object size histogram per bucket
 type BucketUsageInfo struct {
-	Size        uint64 `json:"size"`
-	HotTierSize uint64 `json:"hotTierSize,omitempty"`
+	Size uint64 `json:"size"`
 	// Following five fields suffixed with V1 are here for backward compatibility
 	// Total Size for objects that have not yet been replicated
 	ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
@@ -79,10 +78,6 @@ type DataUsageInfo struct {
 	// LastUpdate is the timestamp of when the data usage info was last updated.
 	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
-	// ScannerCycle changes only after a complete scanner pass. Background
-	// consumers use it to distinguish a partial cache update from a baseline
-	// that has visited every bucket and server pool.
-	ScannerCycle uint32 `json:"scannerCycle,omitempty"`
 
 	// Objects total count across all buckets
 	ObjectsTotalCount uint64 `json:"objectsCount"`

cmd/erasure-multipart.go

@@ -1179,7 +1179,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		switch {
 		case gerr == nil:
 			reconcileStoredObjectLock(fi.Metadata, storedObjectLockState(curr.UserDefined))
-			reconcileStoredObjectTags(fi.Metadata, curr.UserDefined)
+			reconcileStoredObjectTags(fi.Metadata, curr.UserTags, curr.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
 		case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
 			// No existing version to order against: keep the upload's own accepted
 			// lock, including a pre-upgrade upload that persisted values without

cmd/erasure-object.go

@@ -135,7 +135,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 
 	if dstOpts.ReplicaLockReconcile {
 		reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(fi.Metadata))
-		reconcileStoredObjectTags(srcInfo.UserDefined, fi.Metadata)
+		reconcileStoredObjectTags(srcInfo.UserDefined, fi.Metadata[xhttp.AmzObjectTagging], fi.Metadata[ReservedMetadataPrefixLower+TaggingTimestamp])
 	}
 
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
@@ -1311,7 +1311,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		// existing version contributes independently ordered lock and tags.
 		if opts.ReplicaLockReconcile && err == nil {
 			reconcileStoredObjectLock(opts.UserDefined, storedObjectLockState(obj.UserDefined))
-			reconcileStoredObjectTags(opts.UserDefined, obj.UserDefined)
+			reconcileStoredObjectTags(opts.UserDefined, obj.UserTags, obj.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
 		}
 	}
 

cmd/erasure-server-pool-consistency.go

@@ -126,10 +126,9 @@ func mergedPoolObjectInfo(copies []PoolObjInfo) ObjectInfo {
 		stamp, _ := time.Parse(time.RFC3339Nano, ts)
 		if olderThan(oi.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp], stamp) {
 			oi.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = ts
-			oi.UserDefined[xhttp.AmzObjectTagging] = copy.ObjInfo.UserDefined[xhttp.AmzObjectTagging]
+			oi.UserTags = copy.ObjInfo.UserTags
 		}
 	}
-	oi.UserTags = oi.UserDefined[xhttp.AmzObjectTagging]
 	return oi
 }
 
@@ -190,6 +189,12 @@ func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, obj
 			changes[key] = ""
 		}
 	}
+	// Metadata callbacks may explicitly replace tags in the raw write map.
+	// Otherwise retain the merged value from the ObjectInfo read model.
+	tags, ok := updated.UserDefined[xhttp.AmzObjectTagging]
+	if !ok {
+		tags = updated.UserTags
+	}
 	state := storedObjectLockState(updated.UserDefined)
 	opts.VersionID = updated.VersionID
 	if opts.VersionID == "" {
@@ -199,11 +204,13 @@ func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, obj
 	opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
 		maps.Copy(oi.UserDefined, changes)
 		replaceObjectLockMetadata(oi.UserDefined, state)
-		for _, key := range []string{xhttp.AmzObjectTagging, ReservedMetadataPrefixLower + TaggingTimestamp} {
-			value, exists := updated.UserDefined[key]
-			if exists || oi.UserDefined[key] != "" {
-				oi.UserDefined[key] = value
-			}
+		// Reassemble the tag value and its ordering timestamp for storage.
+		if tags != "" || oi.UserTags != "" {
+			oi.UserDefined[xhttp.AmzObjectTagging] = tags
+		}
+		key := ReservedMetadataPrefixLower + TaggingTimestamp
+		if value, exists := updated.UserDefined[key]; exists || oi.UserDefined[key] != "" {
+			oi.UserDefined[key] = value
 		}
 		return ReplicateDecision{}, nil
 	}
@@ -220,16 +227,18 @@ func (z *erasureServerPools) updatePoolMetadata(ctx context.Context, bucket, obj
 	return primary, nil
 }
 
-func reconcileStoredObjectTags(metadata, stored map[string]string) {
+// Pass the stored tag value explicitly: ObjectInfo.UserDefined excludes it,
+// whereas FileInfo.Metadata retains the raw storage key.
+func reconcileStoredObjectTags(metadata map[string]string, storedTags, storedTimestamp string) {
 	key := ReservedMetadataPrefixLower + TaggingTimestamp
-	stamp, err := time.Parse(time.RFC3339Nano, stored[key])
+	stamp, err := time.Parse(time.RFC3339Nano, storedTimestamp)
 	if err != nil {
 		return
 	}
 	incoming, err := time.Parse(time.RFC3339Nano, metadata[key])
 	if err != nil || !stamp.Before(incoming) {
-		metadata[key] = stored[key]
-		metadata[xhttp.AmzObjectTagging] = stored[xhttp.AmzObjectTagging]
+		metadata[key] = storedTimestamp
+		metadata[xhttp.AmzObjectTagging] = storedTags
 	}
 }
 
@@ -287,33 +296,51 @@ func (z *erasureServerPools) retireReplicaCopies(ctx context.Context, bucket, ob
 	return nil
 }
 
-// deleteObjectConditional evaluates the condition once against the logical
+// deleteObjectReconciled evaluates any condition once against the logical
 // version, then removes all its copies under the same lock as pooled writers.
-func (z *erasureServerPools) deleteObjectConditional(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	primary := copies[0]
-	if opts.CheckPrecondFn(primary.ObjInfo) {
+	if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(primary.ObjInfo) {
 		return ObjectInfo{}, PreConditionFailed{}
 	}
 	opts.CheckPrecondFn = nil
 	opts.NoLock = true
 	if opts.EvalRetentionBypassFn != nil || opts.EvalMetadataFn != nil {
-		versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
-		if err != nil {
-			return ObjectInfo{}, err
+		logical := primary.ObjInfo
+		var gerr error
+		switch {
+		case logical.DeleteMarker:
+			// Markers can be deleted by version ID. Match the set layer's
+			// callback inputs instead of rejecting them as metadata updates.
+			gerr = toObjectErr(errMethodNotAllowed, bucket, object)
+			if opts.VersionID == "" || opts.DeleteMarker {
+				gerr = toObjectErr(errFileNotFound, bucket, object)
+			}
+		case opts.VersionID != "":
+			// An addressed version already resolved every copy above.
+			logical = mergedPoolObjectInfo(copies)
+		default:
+			versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
+			if err != nil {
+				return ObjectInfo{}, err
+			}
+			logical = mergedPoolObjectInfo(versions)
 		}
-		logical := mergedPoolObjectInfo(versions)
+		// Keep the retention gate first. These callbacks independently evaluate
+		// the logical version and run once before any deletion; the handler only
+		// sweeps metadata's transition state after a successful delete.
 		if opts.EvalRetentionBypassFn != nil {
-			if err := opts.EvalRetentionBypassFn(logical, nil); err != nil {
+			if err := opts.EvalRetentionBypassFn(logical, gerr); err != nil {
 				return ObjectInfo{}, err
 			}
 			opts.EvalRetentionBypassFn = nil
 		}
 		if opts.EvalMetadataFn != nil {
-			decision, err := opts.EvalMetadataFn(&logical, nil)
+			decision, err := opts.EvalMetadataFn(&logical, gerr)
 			if err != nil {
 				return ObjectInfo{}, err
 			}

cmd/erasure-server-pool-consistency_test.go

@@ -24,12 +24,16 @@ import (
 	"fmt"
 	"io"
 	"maps"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	madmin "github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
 
@@ -62,6 +66,490 @@ func putConsistencyObject(t *testing.T, z *erasureServerPools, bucket, object st
 	return oi
 }
 
+func consistencyRequest(t *testing.T, router http.Handler, method, bucket, object, version string) *httptest.ResponseRecorder {
+	t.Helper()
+	url := getGetObjectURL("", bucket, object)
+	if version != "" {
+		url += "?versionId=" + version
+	}
+	req, err := newTestSignedRequestV4(method, url, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	return rec
+}
+
+// Exercise the handler's real replication/retention callbacks, including the
+// MethodNotAllowed metadata returned for an explicitly addressed delete marker.
+func TestPoolsDeleteVersionAPI(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, kind := range []string{"uuid", "null", "marker"} {
+		for primary := range 2 {
+			t.Run(fmt.Sprintf("%s/primary=%d", kind, primary), func(t *testing.T) {
+				object := fmt.Sprintf("%s-%d", kind, primary)
+				opts := ObjectOptions{Versioned: kind != "null", MTime: UTCNow().Add(-time.Hour)}
+				var addressed ObjectInfo
+				if kind == "marker" {
+					opts.VersionID, opts.DeleteMarker = mustGetUUID(), true
+					addressed, err = z.serverPools[primary].DeleteObject(ctx, bucket, object, opts)
+					if err != nil {
+						t.Fatal(err)
+					}
+				} else {
+					addressed = putConsistencyObject(t, z, bucket, object, primary, "payload", opts)
+				}
+				version := addressed.VersionID
+				if version == "" {
+					version = nullVersionID
+				}
+				opts.VersionID = version
+				opts.MTime = addressed.ModTime.Add(-time.Minute)
+				if kind == "marker" {
+					opts.DeleteMarker = true
+					if _, err := z.serverPools[1-primary].DeleteObject(ctx, bucket, object, opts); err != nil {
+						t.Fatal(err)
+					}
+				} else {
+					putConsistencyObject(t, z, bucket, object, 1-primary, "payload", opts)
+				}
+				latest := putConsistencyObject(t, z, bucket, object, 1-primary, "keep-latest", ObjectOptions{Versioned: true})
+				rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, version)
+				if rec.Code != http.StatusNoContent {
+					t.Fatalf("DELETE: %d %s", rec.Code, rec.Body.String())
+				}
+				if kind == "marker" && (strings.Join(rec.Header()[xhttp.AmzDeleteMarker], "") != "true" || strings.Join(rec.Header()[xhttp.AmzVersionID], "") != version) {
+					t.Errorf("lost deleted marker response headers: %v", rec.Header())
+				}
+				for i, pool := range z.serverPools {
+					if _, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) {
+						t.Errorf("pool %d retained addressed version: %v", i, err)
+					}
+				}
+				for _, method := range []string{http.MethodGet, http.MethodHead} {
+					if rec := consistencyRequest(t, router, method, bucket, object, version); rec.Code != http.StatusNotFound {
+						t.Errorf("%s after DELETE: %d %s", method, rec.Code, rec.Body.String())
+					}
+				}
+				if got, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err != nil || got.VersionID != latest.VersionID {
+					t.Errorf("DELETE changed another version: %+v, %v", got, err)
+				}
+				if rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, version); rec.Code != http.StatusNoContent {
+					t.Errorf("idempotent retry: %d %s", rec.Code, rec.Body.String())
+				}
+			})
+		}
+	}
+	if rec := consistencyRequest(t, router, http.MethodDelete, bucket, "absent-key", mustGetUUID()); rec.Code != http.StatusNoContent {
+		t.Errorf("absent key: %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+type consistencyReadFaultDisk struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d consistencyReadFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+	if volume == d.bucket && path == d.object {
+		return FileInfo{}, errDiskNotFound
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d consistencyReadFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) {
+	if volume == d.bucket && path == d.object {
+		return RawFileInfo{}, errDiskNotFound
+	}
+	return d.StorageAPI.ReadXL(ctx, volume, path, readData)
+}
+
+func TestPoolsDeleteVersionUnreadablePool(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, holding := range []bool{false, true} {
+		for failed := range 2 {
+			t.Run(fmt.Sprintf("holding=%t/pool=%d", holding, failed), func(t *testing.T) {
+				object := fmt.Sprintf("unreadable-%t-%d", holding, failed)
+				oi := putConsistencyObject(t, z, bucket, object, 1-failed, "payload", ObjectOptions{Versioned: true})
+				if holding {
+					putConsistencyObject(t, z, bucket, object, failed, "payload", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime})
+				}
+				set := z.serverPools[failed].getHashedSet(object)
+				getDisks := set.getDisks
+				faulty := append([]StorageAPI(nil), getDisks()...)
+				for i := range faulty {
+					faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+				}
+				set.getDisks = func() []StorageAPI { return faulty }
+				defer func() { set.getDisks = getDisks }()
+				if rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, oi.VersionID); rec.Code != http.StatusServiceUnavailable {
+					t.Errorf("unreadable pool DELETE: %d %s", rec.Code, rec.Body.String())
+				}
+				if _, err := z.serverPools[1-failed].GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: oi.VersionID}); err != nil {
+					t.Errorf("DELETE lost the readable copy: %v", err)
+				}
+				set.getDisks = getDisks
+				if rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, oi.VersionID); rec.Code != http.StatusNoContent {
+					t.Errorf("recovered pool retry: %d %s", rec.Code, rec.Body.String())
+				}
+				for i, pool := range z.serverPools {
+					if _, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: oi.VersionID}); !isErrVersionNotFound(err) {
+						t.Errorf("pool %d retained version after retry: %v", i, err)
+					}
+				}
+			})
+		}
+	}
+}
+
+type consistencyDeleteCountDisk struct {
+	StorageAPI
+	bucket, object         string
+	metadataReads, deletes *atomic.Int32
+}
+
+func (d consistencyDeleteCountDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+	if volume == d.bucket && path == d.object {
+		d.metadataReads.Add(1)
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d consistencyDeleteCountDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
+	if volume == d.bucket && path == d.object {
+		d.deletes.Add(1)
+	}
+	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, force, opts)
+}
+
+func TestPoolsDeleteVersionSingleCopy(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for primary := range 2 {
+		t.Run(fmt.Sprintf("pool=%d", primary), func(t *testing.T) {
+			object := fmt.Sprintf("single-copy-%d", primary)
+			oi := putConsistencyObject(t, z, bucket, object, primary, "payload", ObjectOptions{Versioned: true})
+			var metadataReads, deletes [2]atomic.Int32
+			for i, pool := range z.serverPools {
+				set := pool.getHashedSet(object)
+				getDisks := set.getDisks
+				disks := append([]StorageAPI(nil), getDisks()...)
+				for j := range disks {
+					disks[j] = consistencyDeleteCountDisk{StorageAPI: disks[j], bucket: bucket, object: object, metadataReads: &metadataReads[i], deletes: &deletes[i]}
+				}
+				set.getDisks = func() []StorageAPI { return disks }
+				defer func() { set.getDisks = getDisks }()
+			}
+			metadata, retention := 0, 0
+			_, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{
+				Versioned: true, VersionID: oi.VersionID,
+				EvalMetadataFn: func(current *ObjectInfo, err error) (ReplicateDecision, error) {
+					metadata++
+					if err != nil || current.VersionID != oi.VersionID {
+						t.Errorf("metadata callback: %+v, %v", current, err)
+					}
+					// Count preflight reads before the physical delete reads its own metadata.
+					for i := range metadataReads {
+						if got := metadataReads[i].Load(); got != 16 {
+							t.Errorf("pool %d metadata reads before callbacks = %d; want once per disk (16)", i, got)
+						}
+					}
+					return ReplicateDecision{}, nil
+				},
+				EvalRetentionBypassFn: func(current ObjectInfo, err error) error {
+					retention++
+					return err
+				},
+			})
+			if err != nil || metadata != 1 || retention != 1 {
+				t.Fatalf("DELETE: %v, metadata=%d retention=%d", err, metadata, retention)
+			}
+			if deletes[primary].Load() != 16 || deletes[1-primary].Load() != 0 {
+				t.Errorf("physical deletes per pool = %d, %d; want once per holding disk only", deletes[0].Load(), deletes[1].Load())
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionReplicationPurge(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for _, marker := range []bool{false, true} {
+		t.Run(fmt.Sprintf("marker=%t", marker), func(t *testing.T) {
+			object := fmt.Sprintf("local-replication-purge-%t", marker)
+			version := mustGetUUID()
+			for _, pool := range z.serverPools {
+				opts := ObjectOptions{Versioned: true, VersionID: version, DeleteMarker: marker}
+				var err error
+				if marker {
+					_, err = pool.DeleteObject(t.Context(), bucket, object, opts)
+				} else {
+					_, err = pool.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("payload"), 7, "", ""), opts)
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			dsc := ReplicateDecision{}
+			dsc.Set(newReplicateTargetDecision("arn1", true, false))
+			opts := ObjectOptions{Versioned: true, VersionID: version}
+			for attempt := range 2 {
+				calls := 0
+				opts.EvalMetadataFn = func(current *ObjectInfo, gerr error) (ReplicateDecision, error) {
+					calls++
+					wantMethodNotAllowed := marker || attempt > 0
+					if (wantMethodNotAllowed && !isErrMethodNotAllowed(gerr)) || (!wantMethodNotAllowed && gerr != nil) {
+						t.Errorf("pending purge callback lost set-layer read error: %v", gerr)
+					}
+					return dsc, nil
+				}
+				got, err := z.DeleteObject(t.Context(), bucket, object, opts)
+				if err != nil || calls != 1 || got.VersionPurgeStatus != replication.VersionPurgePending || got.replicationDecision != dsc.String() {
+					t.Fatalf("replication scheduling result: %+v, %v, calls=%d", got, err, calls)
+				}
+				for i, pool := range z.serverPools {
+					got, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version})
+					if (err != nil && (!got.DeleteMarker || !isErrMethodNotAllowed(err))) || got.VersionPurgeStatus != replication.VersionPurgePending {
+						t.Errorf("pool %d lost pending purge: %+v, %v", i, got, err)
+					}
+				}
+			}
+			// The local worker completes the purge without ReplicationRequest.
+			// Both pending copies must be removed, including a pending marker purge.
+			opts.EvalMetadataFn = nil
+			opts.DeleteReplication = ReplicationState{
+				VersionPurgeStatusInternal: "arn1=COMPLETED;",
+				PurgeTargets:               map[string]VersionPurgeStatusType{"arn1": replication.VersionPurgeComplete},
+			}
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) {
+					t.Errorf("pool %d retained completed replication purge: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionCleanupFailure(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for primary := range 2 {
+		t.Run(fmt.Sprintf("primary=%d", primary), func(t *testing.T) {
+			object := fmt.Sprintf("cleanup-failure-%d", primary)
+			oi := putConsistencyObject(t, z, bucket, object, primary, "payload", ObjectOptions{Versioned: true})
+			putConsistencyObject(t, z, bucket, object, 1-primary, "payload", ObjectOptions{
+				Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime.Add(-time.Minute),
+			})
+			set := z.serverPools[1-primary].getHashedSet(object)
+			getDisks := set.getDisks
+			faulty := append([]StorageAPI(nil), getDisks()...)
+			for i := range faulty {
+				faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
+			}
+			set.getDisks = func() []StorageAPI { return faulty }
+			defer func() { set.getDisks = getDisks }()
+			opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID}
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err == nil {
+				t.Error("acknowledged DELETE despite failed secondary cleanup")
+			}
+			if got, err := z.serverPools[primary].GetObjectInfo(t.Context(), bucket, object, opts); err != nil || got.ETag != oi.ETag {
+				t.Errorf("lost authoritative copy after secondary failure: %+v, %v", got, err)
+			}
+			set.getDisks = getDisks
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(t.Context(), bucket, object, opts); !isErrVersionNotFound(err) {
+					t.Errorf("pool %d retained version after retry: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionCallbacks(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	for _, marker := range []bool{false, true} {
+		t.Run(fmt.Sprintf("marker=%t", marker), func(t *testing.T) {
+			object := fmt.Sprintf("callbacks-%t", marker)
+			old, recent := "2026-09-09T09:00:00Z", "2026-09-09T10:00:00Z"
+			oi := putConsistencyObject(t, z, bucket, object, 0, "payload", ObjectOptions{
+				Versioned: true, UserDefined: poolLockMetadata("GOVERNANCE", "OFF", recent, old),
+			})
+			putConsistencyObject(t, z, bucket, object, 1, "payload", ObjectOptions{
+				Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime,
+				UserDefined: poolLockMetadata("", "ON", old, recent),
+			})
+			if marker {
+				markerOpts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow()}
+				for _, pool := range z.serverPools {
+					var err error
+					oi, err = pool.DeleteObject(t.Context(), bucket, object, markerOpts)
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			check := func(current ObjectInfo, err error) {
+				t.Helper()
+				if current.VersionID != oi.VersionID || current.DeleteMarker != marker || (marker && !isErrMethodNotAllowed(err)) || (!marker && err != nil) {
+					t.Errorf("wrong callback version or read error: %+v, %v", current, err)
+				}
+				if !marker {
+					state := storedObjectLockState(current.UserDefined)
+					if state.mode != "GOVERNANCE" || state.legalHold != "ON" {
+						t.Errorf("callback did not reconcile independently ordered lock state: %+v", state)
+					}
+				}
+			}
+			for _, reject := range []string{"retention", "metadata", "none"} {
+				metadata, retention := 0, 0
+				denied := errors.New("callback denied deletion")
+				opts := ObjectOptions{
+					Versioned: true, VersionID: oi.VersionID,
+					EvalRetentionBypassFn: func(current ObjectInfo, err error) error {
+						retention++
+						check(current, err)
+						if reject == "retention" {
+							return denied
+						}
+						return nil
+					},
+					EvalMetadataFn: func(current *ObjectInfo, err error) (ReplicateDecision, error) {
+						metadata++
+						check(*current, err)
+						if reject == "metadata" {
+							return ReplicateDecision{}, denied
+						}
+						return ReplicateDecision{}, nil
+					},
+				}
+				_, err := z.DeleteObject(t.Context(), bucket, object, opts)
+				if reject == "none" {
+					if err != nil || metadata != 1 || retention != 1 {
+						t.Fatalf("DELETE: %v, metadata=%d retention=%d", err, metadata, retention)
+					}
+				} else if !errors.Is(err, denied) {
+					t.Fatalf("%s rejection was lost: %v", reject, err)
+				}
+				for i, pool := range z.serverPools {
+					current, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
+					if reject == "none" {
+						if !isErrVersionNotFound(err) {
+							t.Errorf("pool %d retained deleted version: %v", i, err)
+						}
+					} else if current.VersionID != oi.VersionID || (err != nil && (!marker || !isErrMethodNotAllowed(err))) {
+						t.Errorf("pool %d changed despite %s rejection: %+v, %v", i, reject, current, err)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteVersionSpecialCalls(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	t.Run("incoming-marker-replication", func(t *testing.T) {
+		const object = "incoming-marker"
+		opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, ReplicationRequest: true}
+		opts.SetReplicaStatus(replication.Replica)
+		if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+			t.Fatalf("replication could not create an absent marker: %v", err)
+		}
+		if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: opts.VersionID}); !isErrMethodNotAllowed(err) || !got.DeleteMarker {
+			t.Fatalf("replicated marker missing: %+v, %v", got, err)
+		}
+	})
+	for _, rebalance := range []bool{false, true} {
+		t.Run(fmt.Sprintf("movement/rebalance=%t", rebalance), func(t *testing.T) {
+			object := fmt.Sprintf("movement-%t", rebalance)
+			putConsistencyObject(t, z, bucket, object, 1, "destination", ObjectOptions{Versioned: true})
+			opts := ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow()}
+			if _, err := z.serverPools[0].DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			if rebalance {
+				z.rebalMu.Lock()
+				z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}, {}}}
+				z.rebalMu.Unlock()
+				defer func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() }()
+			} else {
+				z.poolMetaMutex.Lock()
+				z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+				z.poolMetaMutex.Unlock()
+				defer func() { z.poolMetaMutex.Lock(); z.poolMeta.Pools[0].Decommission = nil; z.poolMetaMutex.Unlock() }()
+			}
+			// These are the marker-copy options used by rebalance/decommission;
+			// Source cleanup belongs to the mover.
+			opts.DataMovement, opts.SrcPoolIdx = true, 0
+			opts.SkipRebalancing, opts.SkipDecommissioned = rebalance, !rebalance
+			if _, err := z.DeleteObject(t.Context(), bucket, object, opts); err != nil {
+				t.Fatal(err)
+			}
+			for i, pool := range z.serverPools {
+				if got, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: opts.VersionID}); !isErrMethodNotAllowed(err) || !got.DeleteMarker {
+					t.Errorf("movement lost marker in pool %d: %+v, %v", i, got, err)
+				}
+			}
+		})
+	}
+	for _, expiration := range []bool{false, true} {
+		t.Run(fmt.Sprintf("scanner/expiration=%t", expiration), func(t *testing.T) {
+			object := fmt.Sprintf("scanner-%t", expiration)
+			oi := putConsistencyObject(t, z, bucket, object, 0, "payload", ObjectOptions{Versioned: true})
+			set := z.serverPools[1].getHashedSet(object)
+			getDisks := set.getDisks
+			faulty := append([]StorageAPI(nil), getDisks()...)
+			for i := range faulty {
+				faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+			}
+			set.getDisks = func() []StorageAPI { return faulty }
+			defer func() { set.getDisks = getDisks }()
+			opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID, InclFreeVersions: !expiration, Expiration: ExpirationOptions{Expire: expiration}}
+			_, err := z.DeleteObject(t.Context(), bucket, object, opts)
+			if expiration {
+				// No lifecycle rule authorizes expiration. Preserve its existing
+				// version-not-found result, even with an unrelated unreadable pool.
+				if !isErrVersionNotFound(err) {
+					t.Errorf("expiration changed its scanner contract: %v", err)
+				}
+			} else if err != nil {
+				t.Errorf("free-version cleanup was forced through all-pool resolution: %v", err)
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteUnversionedFanout(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const object = "unversioned-fanout"
+	for i := range z.serverPools {
+		putConsistencyObject(t, z, bucket, object, i, "payload", ObjectOptions{})
+	}
+	if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	for i, pool := range z.serverPools {
+		if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
+			t.Errorf("unversioned fanout left pool %d readable: %v", i, err)
+		}
+	}
+}
+
 func TestPoolsConditionalDeleteVersionSelection(t *testing.T) {
 	z, bucket := consistencyPools(t)
 	const object = "split-versions"
@@ -108,7 +596,7 @@ func TestPoolsConditionalDeleteReportsOtherPoolFailure(t *testing.T) {
 	getDisks := set.getDisks
 	faulty := append([]StorageAPI(nil), getDisks()...)
 	for i := range faulty {
-		faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+		faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
 	}
 	set.getDisks = func() []StorageAPI { return faulty }
 	defer func() { set.getDisks = getDisks }()
@@ -135,9 +623,8 @@ func testPoolsConditionalDeleteWriter(t *testing.T, multipart bool) {
 	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
 	defer cancel()
 	reader := mustGetPutObjReader(t, bytes.NewBufferString("after"), 5, "", "")
-	destination := 1
 	write := func() error {
-		_, err := z.PutObject(ctx, bucket, object, reader, ObjectOptions{DataMovement: true, SrcPoolIdx: 0, DstPoolIdx: &destination})
+		_, err := z.PutObject(ctx, bucket, object, reader, ObjectOptions{})
 		return err
 	}
 	if multipart {
@@ -548,7 +1035,7 @@ func TestPoolsReplicaCleanupFailureCanRetry(t *testing.T) {
 	getDisks := set.getDisks
 	faulty := append([]StorageAPI(nil), getDisks()...)
 	for i := range faulty {
-		faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
+		faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
 	}
 	set.getDisks = func() []StorageAPI { return faulty }
 	defer func() { set.getDisks = getDisks }()
@@ -582,6 +1069,8 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		failPrimaryDelete bool
 		differentRemote   bool
 		restored          bool
+		unconditional     bool
+		skipFreeVersion   bool
 	}{
 		{name: "metadata-copy"},
 		{name: "restored-metadata-copy", restored: true},
@@ -589,6 +1078,9 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		{name: "failed-primary-delete", deleting: true, failPrimaryDelete: true},
 		{name: "failed-primary-delete-distinct-reference", deleting: true, failPrimaryDelete: true, differentRemote: true},
 		{name: "successful-delete", deleting: true},
+		{name: "ordinary-failed-primary-delete", deleting: true, unconditional: true, failPrimaryDelete: true},
+		{name: "ordinary-successful-delete", deleting: true, unconditional: true},
+		{name: "ordinary-skip-free-version", deleting: true, unconditional: true, skipFreeVersion: true},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			z, bucket := consistencyPools(t)
@@ -622,16 +1114,20 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 				getDisks := set.getDisks
 				faulty := append([]StorageAPI(nil), getDisks()...)
 				for i := range faulty {
-					faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
+					faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
 				}
 				set.getDisks = func() []StorageAPI { return faulty }
 				defer func() { set.getDisks = getDisks }()
 			}
 			if test.deleting {
-				_, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{
-					Versioned: true, VersionID: oi.VersionID,
+				opts := ObjectOptions{
+					Versioned: true, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
 					CheckPrecondFn: func(info ObjectInfo) bool { return info.ETag != current.ETag },
-				})
+				}
+				if test.unconditional {
+					opts.CheckPrecondFn = nil
+				}
+				_, err := z.DeleteObject(t.Context(), bucket, object, opts)
 				if (err != nil) != test.failPrimaryDelete {
 					t.Fatalf("unexpected authoritative delete result: %v", err)
 				}
@@ -656,6 +1152,7 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 				t.Fatalf("lost retained authoritative copy: %v", err)
 			}
 			for pool, wantFree := range []bool{primaryDeleted, test.differentRemote} {
+				wantFree = wantFree && !test.skipFreeVersion
 				for _, disk := range z.serverPools[pool].getHashedSet(object).getDisks() {
 					data, err := disk.ReadAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile))
 					if errors.Is(err, errFileNotFound) && !wantFree {
@@ -680,3 +1177,132 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
 		})
 	}
 }
+
+// Fault injection shared by general multi-pool regressions.
+type consistencyDeleteFaultDisk struct {
+	StorageAPI
+	bucket, object, version string
+}
+
+func (d consistencyDeleteFaultDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
+	if volume == d.bucket && path == d.object && fi.VersionID == d.version {
+		return errDiskFull
+	}
+	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
+}
+
+// Exercise the surviving production rebalance copy, then interrupt the workflow
+// before its later source cleanup. Repeated versions are reachable without the
+// retired access-tier mover, and an API delete must remove both copies.
+func TestPoolsDeleteVersionAfterInterruptedRebalance(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for source := range 2 {
+		t.Run(fmt.Sprintf("source=%d", source), func(t *testing.T) {
+			object := fmt.Sprintf("interrupted-rebalance-%d", source)
+			original := putConsistencyObject(t, z, bucket, object, source, "original", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+			stats := []*rebalanceStats{{}, {}}
+			stats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
+			z.rebalMu.Lock()
+			z.rebalMeta = &rebalanceMeta{PoolStats: stats}
+			z.rebalMu.Unlock()
+			defer func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() }()
+			gr, err := z.serverPools[source].GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{VersionID: original.VersionID, NoLock: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := z.rebalanceObject(ctx, source, bucket, gr); err != nil {
+				t.Fatalf("rebalance copy: %v", err)
+			}
+			// Simulate interruption after rebalanceObject, before the enclosing loop
+			// removes the source version stack. Stop rebalance before the API request.
+			z.rebalMu.Lock()
+			z.rebalMeta = nil
+			z.rebalMu.Unlock()
+			for i, pool := range z.serverPools {
+				got, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: original.VersionID})
+				if err != nil || got.VersionID != original.VersionID || !got.ModTime.Equal(original.ModTime) {
+					t.Fatalf("copy missing/changed in pool %d: %+v, %v", i, got, err)
+				}
+			}
+			later, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewBufferString("later"), 5, "", ""), ObjectOptions{Versioned: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, original.VersionID)
+			if rec.Code != http.StatusNoContent {
+				t.Fatalf("DELETE: %d %s", rec.Code, rec.Body.String())
+			}
+			for i, pool := range z.serverPools {
+				if _, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: original.VersionID}); !isErrVersionNotFound(err) {
+					t.Errorf("pool %d retained old version: %v", i, err)
+				}
+			}
+			if got, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err != nil || got.VersionID != later.VersionID {
+				t.Fatalf("other version changed: %+v, %v", got, err)
+			}
+			for _, method := range []string{http.MethodGet, http.MethodHead} {
+				if rec := consistencyRequest(t, router, method, bucket, object, original.VersionID); rec.Code != http.StatusNotFound {
+					t.Errorf("%s returned %d", method, rec.Code)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsDeleteDirectoryMarker(t *testing.T) {
+	z, _ := consistencyPools(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for holding := range 2 {
+		for _, unreadable := range []bool{false, true} {
+			t.Run(fmt.Sprintf("holding=%d/unreadable=%t", holding, unreadable), func(t *testing.T) {
+				object := fmt.Sprintf("directory-%d-%t/", holding, unreadable)
+				encoded := encodeDirObject(object)
+				putConsistencyObject(t, z, bucket, encoded, holding, "", ObjectOptions{})
+				set := z.serverPools[1-holding].getHashedSet(encoded)
+				getDisks := set.getDisks
+				defer func() { set.getDisks = getDisks }()
+				if unreadable {
+					disks := append([]StorageAPI(nil), getDisks()...)
+					for i := range disks {
+						disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: encoded}
+					}
+					set.getDisks = func() []StorageAPI { return disks }
+				}
+				// No explicit versionId: delOpts permanently deletes the null version.
+				rec := consistencyRequest(t, router, http.MethodDelete, bucket, object, "")
+				if unreadable {
+					if rec.Code != http.StatusServiceUnavailable {
+						t.Fatalf("unreadable pool: %d %s", rec.Code, rec.Body.String())
+					}
+					if _, err := z.serverPools[holding].GetObjectInfo(ctx, bucket, encoded, ObjectOptions{VersionID: nullVersionID}); err != nil {
+						t.Fatalf("readable directory marker lost: %v", err)
+					}
+					set.getDisks = getDisks
+					rec = consistencyRequest(t, router, http.MethodDelete, bucket, object, "")
+				}
+				if rec.Code != http.StatusNoContent {
+					t.Fatalf("DELETE: %d %s", rec.Code, rec.Body.String())
+				}
+				for i, pool := range z.serverPools {
+					if _, err := pool.GetObjectInfo(ctx, bucket, encoded, ObjectOptions{VersionID: nullVersionID}); !isErrVersionNotFound(err) {
+						t.Errorf("pool %d retained null version: %v", i, err)
+					}
+				}
+				if rec := consistencyRequest(t, router, http.MethodHead, bucket, object, ""); rec.Code != http.StatusNotFound {
+					t.Errorf("directory still visible: %d", rec.Code)
+				}
+			})
+		}
+	}
+}

cmd/erasure-server-pool-multipart-cond_test.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+	"time"
+)
+
+// A multipart completion's If-Match must be evaluated against the logical
+// latest object across pools, not against the copy local to a pool.
+//
+// Multi-pool write placement is not sticky (getPoolIdx picks by available
+// space even for existing objects), so an upload and a newer overwrite of
+// the same name routinely end up in different pools. Uploads are pinned to
+// their pools directly: routing through z.NewMultipartUpload would make the
+// placement depend on the space-weighted random choice.
+func TestPoolsMultipartConditionalUsesLogicalLatest(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	ctx := t.Context()
+
+	ifMatch := func(etag string) (opts ObjectOptions) {
+		return ObjectOptions{
+			HasIfMatch: true,
+			CheckPrecondFn: func(oi ObjectInfo) bool {
+				return oi.ETag != etag
+			},
+		}
+	}
+
+	uploadPart := func(t *testing.T, bucket, object, uploadID string) []CompletePart {
+		t.Helper()
+		pi, err := z.PutObjectPart(ctx, bucket, object, uploadID, 1,
+			mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return []CompletePart{{PartNumber: 1, ETag: pi.ETag}}
+	}
+
+	// Scenario A: the uploaded If-Match carries the stale ETag of the pool-0
+	// copy while the logical latest object lives in pool 1. The completion
+	// must fail with 412 instead of shadowing the newer logical state.
+	objectA := "cond-mp-stale-etag"
+	base := time.Now()
+	oldA := putConsistencyObject(t, z, bucket, objectA, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpA, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectA, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerA := putConsistencyObject(t, z, bucket, objectA, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+
+	latest, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerA.ETag {
+		t.Fatalf("logical latest should be the pool-1 copy: got %s want %s", latest.ETag, newerA.ETag)
+	}
+
+	if _, err = z.CompleteMultipartUpload(ctx, bucket, objectA, mpA.UploadID,
+		uploadPart(t, bucket, objectA, mpA.UploadID), ifMatch(oldA.ETag)); err == nil {
+		t.Fatal("If-Match with the stale pool-0 ETag must not complete over the newer pool-1 object")
+	} else if _, ok := err.(PreConditionFailed); !ok {
+		t.Fatalf("expected PreconditionFailed, got %v", err)
+	}
+
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerA.ETag {
+		t.Fatalf("the newer pool-1 object must remain the logical latest, got %s", latest.ETag)
+	}
+
+	// Scenario B: the uploaded If-Match carries the logical latest ETag (the
+	// pool-1 copy) while the upload sits next to the stale pool-0 copy. The
+	// precondition is satisfied, so completion must succeed and its result
+	// must become the logical latest.
+	objectB := "cond-mp-latest-etag"
+	putConsistencyObject(t, z, bucket, objectB, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpB, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectB, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerB := putConsistencyObject(t, z, bucket, objectB, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	oiB, err := z.CompleteMultipartUpload(ctx, bucket, objectB, mpB.UploadID,
+		uploadPart(t, bucket, objectB, mpB.UploadID), ifMatch(newerB.ETag))
+	if err != nil {
+		t.Fatalf("If-Match with the logical latest ETag must complete, got %v", err)
+	}
+	if oiB.ETag == "" {
+		t.Fatal("completion returned an empty ETag")
+	}
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectB, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != oiB.ETag {
+		t.Fatalf("the completed object must be the logical latest: got %s want %s", latest.ETag, oiB.ETag)
+	}
+
+	// Scenario C: the upload lives in pool 1 with the newer copy while pool 0
+	// holds the stale one. A set-local evaluation order would let pool 0's
+	// stale copy fail the request before pool 1 is reached; the logical
+	// latest ETag must complete.
+	objectC := "cond-mp-upload-other-pool"
+	putConsistencyObject(t, z, bucket, objectC, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpC, err := z.serverPools[1].NewMultipartUpload(ctx, bucket, objectC, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerC := putConsistencyObject(t, z, bucket, objectC, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	if _, err = z.CompleteMultipartUpload(ctx, bucket, objectC, mpC.UploadID,
+		uploadPart(t, bucket, objectC, mpC.UploadID), ifMatch(newerC.ETag)); err != nil {
+		t.Fatalf("If-Match with the logical latest ETag must complete regardless of upload pool, got %v", err)
+	}
+
+	// Scenario D: pool 1 holds the newer copy but cannot be read. An
+	// unreadable pool may contain the newest state, so the unverifiable
+	// condition must fail the request rather than pass it against pool 0's
+	// stale ETag.
+	objectD := "cond-mp-unreadable-pool"
+	oldD := putConsistencyObject(t, z, bucket, objectD, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpD, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectD, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerD := putConsistencyObject(t, z, bucket, objectD, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectD, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerD.ETag {
+		t.Fatalf("logical latest before faulting pool 1 should be its copy: got %s want %s", latest.ETag, newerD.ETag)
+	}
+	set := z.serverPools[1].getHashedSet(objectD)
+	getDisks := set.getDisks
+	faulty := append([]StorageAPI(nil), getDisks()...)
+	for i := range faulty {
+		faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: objectD}
+	}
+	set.getDisks = func() []StorageAPI { return faulty }
+	defer func() { set.getDisks = getDisks }()
+
+	_, err = z.CompleteMultipartUpload(ctx, bucket, objectD, mpD.UploadID,
+		uploadPart(t, bucket, objectD, mpD.UploadID), ifMatch(oldD.ETag))
+	if !isErrReadQuorum(err) {
+		t.Fatalf("expected an insufficient read quorum error, got %v", err)
+	}
+}

cmd/erasure-server-pool-multipart-http_test.go

@@ -0,0 +1,341 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func multipartConditionRequest(t *testing.T, router http.Handler, method, target, body string, headers map[string]string) *httptest.ResponseRecorder {
+	t.Helper()
+	req, err := newTestSignedRequestV4(method, target, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	return rec
+}
+
+// The server writes the wire spelling ETag directly into Header; unlike a
+// network response, httptest's map has not canonicalized it to Etag.
+func multipartConditionResponseETag(rec *httptest.ResponseRecorder) string {
+	for key, values := range rec.Header() {
+		if strings.EqualFold(key, xhttp.ETag) && len(values) > 0 {
+			return strings.Trim(values[0], "\"")
+		}
+	}
+	return ""
+}
+
+func multipartConditionUpload(t *testing.T, z *erasureServerPools, bucket, object string, owner int, opts ObjectOptions) (string, []CompletePart) {
+	t.Helper()
+	mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, object, opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	part, err := z.serverPools[owner].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}
+}
+
+func multipartConditionCompleteBody(parts []CompletePart) string {
+	data, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
+	if err != nil {
+		panic(err)
+	}
+	return string(data)
+}
+
+func multipartConditionError(t *testing.T, rec *httptest.ResponseRecorder, code string) {
+	t.Helper()
+	decoder := xml.NewDecoder(strings.NewReader(rec.Body.String()))
+	var response APIErrorResponse
+	if err := decoder.Decode(&response); err != nil || response.Code != code {
+		t.Fatalf("expected %s error, got %q: %v", code, rec.Body.String(), err)
+	}
+	if err := decoder.Decode(&response); err != io.EOF {
+		t.Fatalf("expected exactly one error response, got %q: %v", rec.Body.String(), err)
+	}
+}
+
+// State is deliberately placed per pool; the final request uses signed HTTP
+// and the real handler, precondition callback, erasure metadata and rename.
+func TestPoolsMultipartConditionalHTTPMatrix(t *testing.T) {
+	z, _ := consistencyPools(t)
+	bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for owner := range 2 {
+		for _, localOld := range []bool{false, true} {
+			for _, condition := range []string{"match-old", "match-current", "none-match"} {
+				t.Run(fmt.Sprintf("owner=%d/old=%t/%s", owner, localOld, condition), func(t *testing.T) {
+					object := fmt.Sprintf("http-%d-%t-%s", owner, localOld, condition)
+					oldETag := "old-does-not-exist"
+					if localOld {
+						oldETag = putConsistencyObject(t, z, bucket, object, owner, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)}).ETag
+					}
+					id, parts := multipartConditionUpload(t, z, bucket, object, owner, ObjectOptions{})
+					current := putConsistencyObject(t, z, bucket, object, 1-owner, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
+					head := multipartConditionRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object), "", nil)
+					if head.Code != 200 || multipartConditionResponseETag(head) != current.ETag {
+						t.Fatalf("bad HEAD: %d %v", head.Code, head.Header())
+					}
+					h := map[string]string{xhttp.IfMatch: "\"" + oldETag + "\""}
+					want := 412
+					if condition == "match-current" {
+						h[xhttp.IfMatch] = "\"" + current.ETag + "\""
+						want = 200
+					}
+					if condition == "none-match" {
+						h = map[string]string{xhttp.IfNoneMatch: "*"}
+					}
+					rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, id), multipartConditionCompleteBody(parts), h)
+					t.Logf("HEAD current=%s, requested=%v, complete HTTP=%d", current.ETag, h, rec.Code)
+					if rec.Code != want {
+						t.Errorf("want HTTP %d, got %d: %s", want, rec.Code, rec.Body.String())
+					}
+					if want == 412 {
+						multipartConditionError(t, rec, "PreconditionFailed")
+						got := multipartConditionRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object), "", nil)
+						if got.Code != 200 || got.Body.String() != "current" {
+							t.Errorf("rejected request must preserve current data: %d %q", got.Code, got.Body.String())
+						}
+						if _, err := z.serverPools[owner].ListObjectParts(t.Context(), bucket, object, id, 0, 10, ObjectOptions{}); err != nil {
+							t.Errorf("rejected request consumed upload: %v", err)
+						}
+					}
+				})
+			}
+		}
+	}
+}
+
+func TestPoolsMultipartConditionalHTTPAbsentObject(t *testing.T) {
+	for _, deleted := range []bool{false, true} {
+		for _, match := range []bool{false, true} {
+			t.Run(fmt.Sprintf("delete-marker=%t/if-match=%t", deleted, match), func(t *testing.T) {
+				z, _ := consistencyPools(t)
+				bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: deleted})
+				if err != nil {
+					t.Fatal(err)
+				}
+				object := "http-absent"
+				if deleted {
+					putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+					_, err = z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)})
+					if err != nil {
+						t.Fatal(err)
+					}
+				}
+				id, parts := multipartConditionUpload(t, z, bucket, object, 0, ObjectOptions{Versioned: deleted})
+				headers := map[string]string{xhttp.IfNoneMatch: "*"}
+				want := http.StatusOK
+				if match {
+					headers = map[string]string{xhttp.IfMatch: "\"missing\""}
+					want = http.StatusNotFound
+				}
+				rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, id), multipartConditionCompleteBody(parts), headers)
+				if rec.Code != want {
+					t.Fatalf("expected HTTP %d, got %d: %s", want, rec.Code, rec.Body.String())
+				}
+				if match {
+					multipartConditionError(t, rec, "NoSuchKey")
+					if _, err := z.serverPools[0].ListObjectParts(t.Context(), bucket, object, id, 0, 10, ObjectOptions{}); err != nil {
+						t.Errorf("rejected request consumed upload: %v", err)
+					}
+				}
+			})
+		}
+	}
+}
+
+// All writes below use ordinary signed S3 requests. The result must be correct
+// for every placement; the matrix above deterministically covers split pools.
+func TestPoolsMultipartConditionalHTTPNormalRouting(t *testing.T) {
+	z, _ := consistencyPools(t)
+	bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	object := "normal-routing"
+	url := getPutObjectURL("", bucket, object)
+	old := multipartConditionRequest(t, router, http.MethodPut, url, "old-data", nil)
+	if old.Code != http.StatusOK {
+		t.Fatalf("initial PUT %d: %s", old.Code, old.Body.String())
+	}
+	init := multipartConditionRequest(t, router, http.MethodPost, url+"?uploads", "", nil)
+	if init.Code != http.StatusOK {
+		t.Fatalf("init %d: %s", init.Code, init.Body.String())
+	}
+	var mp InitiateMultipartUploadResponse
+	if err := xml.Unmarshal(init.Body.Bytes(), &mp); err != nil {
+		t.Fatal(err)
+	}
+	part := multipartConditionRequest(t, router, http.MethodPut, getPutObjectPartURL("", bucket, object, mp.UploadID, "1"), "replacement", nil)
+	if part.Code != http.StatusOK {
+		t.Fatalf("part %d: %s", part.Code, part.Body.String())
+	}
+	parts := []CompletePart{{PartNumber: 1, ETag: multipartConditionResponseETag(part)}}
+	newer := multipartConditionRequest(t, router, http.MethodPut, url, "newer-data", nil)
+	if newer.Code != http.StatusOK {
+		t.Fatalf("new PUT %d: %s", newer.Code, newer.Body.String())
+	}
+	head := multipartConditionRequest(t, router, http.MethodHead, url, "", nil)
+	if head.Code != http.StatusOK || multipartConditionResponseETag(head) != multipartConditionResponseETag(newer) {
+		t.Fatalf("HEAD did not pick new object: %d %v", head.Code, head.Header())
+	}
+	rec := multipartConditionRequest(t, router, http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, mp.UploadID), multipartConditionCompleteBody(parts), map[string]string{xhttp.IfMatch: "\"" + multipartConditionResponseETag(old) + "\""})
+	if rec.Code != http.StatusPreconditionFailed {
+		t.Errorf("stale If-Match should be HTTP 412, got %d: %s", rec.Code, rec.Body.String())
+	}
+	get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil)
+	if get.Code != http.StatusOK || get.Body.String() != "newer-data" {
+		t.Errorf("conditional completion changed newer data: %d %q", get.Code, get.Body.String())
+	}
+}
+
+func TestPoolsMultipartConditionalUnreadablePool(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	object := "quorum-with-readable-copy"
+	old := putConsistencyObject(t, z, bucket, object, 0, "readable", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
+	putConsistencyObject(t, z, bucket, object, 1, "hidden-newer", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
+	id, parts := multipartConditionUpload(t, z, bucket, object, 0, ObjectOptions{})
+	set := z.serverPools[1].getHashedSet(object)
+	original := set.getDisks
+	disks := append([]StorageAPI(nil), original()...)
+	for i := range disks {
+		disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
+	}
+	set.getDisks = func() []StorageAPI { return disks }
+	defer func() { set.getDisks = original }()
+	read, readErr := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+	t.Logf("ordinary GET lookup: etag=%s err=%v", read.ETag, readErr)
+	called := 0
+	_, err := z.CompleteMultipartUpload(t.Context(), bucket, object, id, parts, ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { called++; return oi.ETag != old.ETag }})
+	if !isErrReadQuorum(err) {
+		t.Errorf("conditional write must fail on unreadable pool, got %v", err)
+	}
+	if called != 0 {
+		t.Errorf("callback evaluated without complete state: %d calls", called)
+	}
+}
+
+func TestPoolsMultipartConditionalLatestVersionAndCallbackOnce(t *testing.T) {
+	for _, explicit := range []bool{false, true} {
+		t.Run(fmt.Sprintf("explicit=%t", explicit), func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			object := "tie-version"
+			opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour), Versioned: explicit}
+			if explicit {
+				opts.VersionID = mustGetUUID()
+			}
+			current := putConsistencyObject(t, z, bucket, object, 0, "first", opts)
+			putConsistencyObject(t, z, bucket, object, 1, "second", opts)
+			if explicit {
+				current = putConsistencyObject(t, z, bucket, object, 1, "latest-other-version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)})
+			}
+			id, parts := multipartConditionUpload(t, z, bucket, object, 1, opts)
+			called := 0
+			opts.CheckPrecondFn = func(oi ObjectInfo) bool { called++; return oi.ETag != current.ETag }
+			opts.MTime = time.Time{}
+			opts.HasIfMatch = true
+			_, err := z.CompleteMultipartUpload(t.Context(), bucket, object, id, parts, opts)
+			if err != nil {
+				t.Errorf("logical current object should match: %v", err)
+			}
+			if called != 1 {
+				t.Errorf("condition evaluated %d times; want exactly once", called)
+			}
+		})
+	}
+}
+
+func TestPoolsMultipartConditionalConcurrentCompletes(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	object := "concurrent-completes"
+	old := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
+	putConsistencyObject(t, z, bucket, object, 1, "old", ObjectOptions{MTime: old.ModTime})
+	ids := make([]string, 2)
+	parts := make([][]CompletePart, 2)
+	for i := range 2 {
+		ids[i], parts[i] = multipartConditionUpload(t, z, bucket, object, i, ObjectOptions{})
+	}
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var gate, releaseOnce sync.Once
+	defer releaseOnce.Do(func() { close(release) })
+	errs := make([]error, 2)
+	var wg sync.WaitGroup
+	// Let pool 1's completion hold the object lock before pool 0's starts.
+	// Once pool 1 commits, a set-local read in pool 0 would still see the
+	// old ETag and incorrectly accept the second completion.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, errs[1] = z.CompleteMultipartUpload(t.Context(), bucket, object, ids[1], parts[1], ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool {
+			gate.Do(func() { close(entered); <-release })
+			return oi.ETag != old.ETag
+		}})
+	}()
+	select {
+	case <-entered:
+	case <-time.After(10 * time.Second):
+		t.Fatal("first completion did not enter its condition callback")
+	}
+	started := make(chan struct{})
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		close(started)
+		_, errs[0] = z.CompleteMultipartUpload(t.Context(), bucket, object, ids[0], parts[0], ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != old.ETag }})
+	}()
+	<-started
+	releaseOnce.Do(func() { close(release) })
+	wg.Wait()
+	success, failed := 0, 0
+	for _, err := range errs {
+		var p PreConditionFailed
+		switch {
+		case err == nil:
+			success++
+		case errors.As(err, &p):
+			failed++
+		default:
+			t.Errorf("unexpected completion error %v", err)
+		}
+	}
+	if success != 1 || failed != 1 {
+		t.Errorf("CAS writers: success=%d conditional failures=%d errors=%v", success, failed, errs)
+	}
+}

cmd/erasure-server-pool-multipart-preconditions_test.go

@@ -0,0 +1,135 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestPoolsMultipartConditionMatrix(t *testing.T) {
+	for owner := range 2 {
+		for _, withOld := range []bool{false, true} {
+			for _, condition := range []string{"match-current", "match-old", "none-match-any"} {
+				t.Run(fmt.Sprintf("upload-pool=%d/old-copy=%t/%s", owner, withOld, condition), func(t *testing.T) {
+					z, bucket := consistencyPools(t)
+					object := "conditional-multipart"
+					oldETag := "arbitrary-old"
+					if withOld {
+						old := putConsistencyObject(t, z, bucket, object, owner, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
+						oldETag = old.ETag
+					}
+					mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					part, err := z.serverPools[owner].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					current := putConsistencyObject(t, z, bucket, object, 1-owner, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)})
+					visible, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil || visible.ETag != current.ETag {
+						t.Fatalf("invalid current state: %v", err)
+					}
+					opts := ObjectOptions{HasIfMatch: condition != "none-match-any", CheckPrecondFn: func(oi ObjectInfo) bool {
+						switch condition {
+						case "match-current":
+							return oi.ETag != current.ETag
+						case "match-old":
+							return oi.ETag != oldETag
+						default:
+							return true
+						}
+					}}
+					_, err = z.CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, opts)
+					if condition == "match-current" {
+						if err != nil {
+							t.Errorf("correct logical If-Match rejected: %v", err)
+						}
+					} else {
+						var expected PreConditionFailed
+						if !errors.As(err, &expected) {
+							t.Errorf("logical condition must fail, got %v", err)
+						}
+					}
+				})
+			}
+		}
+	}
+}
+
+func TestPoolsMultipartConditionBoundaries(t *testing.T) {
+	for _, state := range []string{"missing", "latest-delete-marker", "unreadable-other-pool"} {
+		for _, match := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/if-match=%t", state, match), func(t *testing.T) {
+				z, bucket := consistencyPools(t)
+				object := "conditional-boundary"
+				versioned := state == "latest-delete-marker"
+				if versioned {
+					putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+					if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)}); err != nil {
+						t.Fatal(err)
+					}
+				}
+				mp, err := z.serverPools[0].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{Versioned: versioned})
+				if err != nil {
+					t.Fatal(err)
+				}
+				part, err := z.serverPools[0].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewBufferString("new"), 3, "", ""), ObjectOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+				if state == "unreadable-other-pool" {
+					set := z.serverPools[1].getHashedSet(object)
+					getDisks := set.getDisks
+					faulty := append([]StorageAPI(nil), getDisks()...)
+					for i := range faulty {
+						faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
+					}
+					set.getDisks = func() []StorageAPI { return faulty }
+					defer func() { set.getDisks = getDisks }()
+				}
+				opts := ObjectOptions{Versioned: versioned, HasIfMatch: match, CheckPrecondFn: func(ObjectInfo) bool { return true }}
+				_, err = z.CompleteMultipartUpload(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, opts)
+				switch {
+				case state == "unreadable-other-pool":
+					if !isErrReadQuorum(err) {
+						t.Errorf("unreadable pool must not mean absence: %v", err)
+					}
+				case match:
+					if !isErrObjectNotFound(err) {
+						t.Errorf("If-Match against logical absence should report absence: %v", err)
+					}
+				default:
+					if err != nil {
+						t.Errorf("If-None-Match against logical absence must succeed: %v", err)
+					}
+				}
+				if err != nil {
+					if _, lerr := z.serverPools[0].ListObjectParts(t.Context(), bucket, object, mp.UploadID, 0, 10, ObjectOptions{}); lerr != nil {
+						t.Errorf("failed condition consumed upload: %v", lerr)
+					}
+				}
+			})
+		}
+	}
+}

cmd/erasure-server-pool-tags_test.go

@@ -0,0 +1,459 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"strings"
+	"testing"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// The fixture places copies directly in real erasure pools. It models a
+// duplicated version; it does not claim to exercise a rebalance workflow.
+func TestPoolsMetadataUpdatePreservesTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, test := range []struct {
+		name         string
+		tags, stamps [2]string
+		winner       int
+		single       bool
+	}{
+		{name: "newer-secondary", tags: [2]string{"key=old", "key=new"}, stamps: [2]string{old, recent}, winner: 1},
+		{name: "newer-primary", tags: [2]string{"key=new", "key=old"}, stamps: [2]string{recent, old}},
+		{name: "empty-secondary", tags: [2]string{"key=old", ""}, stamps: [2]string{old, recent}, winner: 1},
+		{name: "empty-primary", tags: [2]string{"", "key=old"}, stamps: [2]string{recent, old}},
+		{name: "single-copy-primary", tags: [2]string{"key=only", ""}, stamps: [2]string{recent, ""}, single: true},
+		{name: "single-copy-secondary", tags: [2]string{"", "key=only"}, stamps: [2]string{"", recent}, winner: 1, single: true},
+		{name: "legacy-single-copy", tags: [2]string{"key=legacy", ""}, single: true},
+		{name: "legacy-duplicates", tags: [2]string{"key=legacy", "key=legacy"}},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			object := test.name
+			var original ObjectInfo
+			for pool := range 2 {
+				if test.single && pool != test.winner {
+					continue
+				}
+				metadata := map[string]string{
+					xhttp.AmzObjectTagging: test.tags[pool],
+					"copy-local":           fmt.Sprint(pool),
+				}
+				if test.stamps[pool] != "" {
+					metadata[timestamp] = test.stamps[pool]
+				}
+				original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
+				})
+				got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
+				if err != nil || got.UserTags != test.tags[pool] || got.UserDefined[timestamp] != test.stamps[pool] {
+					t.Fatalf("pool %d fixture: tags=%q timestamp=%q err=%v", pool, got.UserTags, got.UserDefined[timestamp], err)
+				}
+				if _, exists := got.UserDefined[xhttp.AmzObjectTagging]; exists {
+					t.Fatal("fixture must use the cleaned ObjectInfo representation")
+				}
+			}
+			wantTags, wantStamp := test.tags[test.winner], test.stamps[test.winner]
+			called := 0
+			got, err := z.PutObjectMetadata(t.Context(), bucket, object, ObjectOptions{
+				VersionID: original.VersionID, MTime: original.ModTime,
+				EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
+					called++
+					if current.UserTags != wantTags || current.UserDefined[timestamp] != wantStamp {
+						t.Errorf("callback tags=%q timestamp=%q; want %q %q", current.UserTags, current.UserDefined[timestamp], wantTags, wantStamp)
+					}
+					current.UserDefined["unrelated-update"] = "preserved"
+					return ReplicateDecision{}, nil
+				},
+			})
+			if err != nil || called != 1 {
+				t.Fatalf("metadata update: %v, callbacks=%d", err, called)
+			}
+			if got.UserTags != wantTags || got.UserDefined[timestamp] != wantStamp {
+				t.Errorf("response tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], wantTags, wantStamp)
+			}
+			for pool := range 2 {
+				got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
+				if test.single && pool != test.winner {
+					if !isErrVersionNotFound(err) {
+						t.Errorf("metadata update created another copy: %v", err)
+					}
+					continue
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Logf("pool %d persisted tags=%q timestamp=%q", pool, got.UserTags, got.UserDefined[timestamp])
+				if got.UserTags != wantTags || got.UserDefined[timestamp] != wantStamp {
+					t.Errorf("pool %d persisted tags=%q timestamp=%q; want %q %q", pool, got.UserTags, got.UserDefined[timestamp], wantTags, wantStamp)
+				}
+				if got.UserDefined["unrelated-update"] != "preserved" || got.UserDefined["copy-local"] != fmt.Sprint(pool) {
+					t.Errorf("pool %d lost unrelated metadata: %v", pool, got.UserDefined)
+				}
+			}
+		})
+	}
+}
+
+func TestReplicaWritesPreserveTagOrdering(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	// Pool allocation checks the host's used-space percentage. Present only
+	// its free space as fixture capacity; all reads and writes still use the
+	// real disks. This keeps unrelated host disk usage out of the tag test.
+	for _, pool := range z.serverPools {
+		for _, set := range pool.sets {
+			getDisks := set.getDisks
+			disks := append([]StorageAPI(nil), getDisks()...)
+			for i := range disks {
+				disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			t.Cleanup(func() { set.getDisks = getDisks })
+		}
+	}
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, path := range []struct {
+		name, operation string
+		direct          bool
+		winner          int
+	}{
+		{name: "set-put", operation: "put", direct: true},
+		{name: "set-multipart", operation: "multipart", direct: true},
+		{name: "set-copy", operation: "copy", direct: true},
+		{name: "pools-put", operation: "put", winner: 1},
+		{name: "pools-multipart", operation: "multipart", winner: 1},
+		{name: "pools-copy-primary", operation: "copy"},
+		{name: "pools-copy-secondary", operation: "copy", winner: 1},
+	} {
+		for _, test := range []struct {
+			name, storedTags, incomingTags, storedStamp, incomingStamp, wantTags string
+		}{
+			{"stored-newer", "key=stored", "key=incoming", recent, old, "key=stored"},
+			{"stored-deleted", "", "key=incoming", recent, old, ""},
+			{"incoming-newer", "key=stored", "key=incoming", old, recent, "key=incoming"},
+			{"incoming-deleted", "key=stored", "", old, recent, ""},
+		} {
+			t.Run(path.name+"/"+test.name, func(t *testing.T) {
+				object := path.name + "-" + test.name
+				var original ObjectInfo
+				for pool := range 2 {
+					if path.direct && pool != 0 {
+						continue
+					}
+					metadata := map[string]string{
+						xhttp.AmzObjectTagging: "key=older-copy",
+						timestamp:              "2026-09-09T08:00:00Z",
+					}
+					if pool == path.winner {
+						metadata[xhttp.AmzObjectTagging] = test.storedTags
+						metadata[timestamp] = test.storedStamp
+					}
+					original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
+						Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
+					})
+				}
+				opts := ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, ReplicaLockReconcile: true,
+					UserDefined: map[string]string{
+						xhttp.AmzObjectTagging: test.incomingTags,
+						timestamp:              test.incomingStamp,
+					},
+				}
+				var got ObjectInfo
+				var err error
+				switch path.operation {
+				case "put":
+					put := z.PutObject
+					if path.direct {
+						put = z.serverPools[0].PutObject
+					}
+					got, err = put(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), opts)
+				case "multipart":
+					// Persist the incoming tags with the upload, before completion
+					// reconciles the destination version through its real resolver.
+					mp, err := z.serverPools[0].NewMultipartUpload(t.Context(), bucket, object, opts)
+					if err != nil {
+						t.Fatal(err)
+					}
+					part, err := z.serverPools[0].PutObjectPart(t.Context(), bucket, object, mp.UploadID, 1,
+						mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					complete := z.CompleteMultipartUpload
+					if path.direct {
+						complete = z.serverPools[0].CompleteMultipartUpload
+					}
+					got, err = complete(t.Context(), bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{
+						Versioned: true, MTime: original.ModTime, ReplicaLockReconcile: true,
+					})
+					if err != nil {
+						t.Fatal(err)
+					}
+				case "copy":
+					src := original
+					src.metadataOnly = true
+					src.UserDefined = maps.Clone(opts.UserDefined)
+					copyObject := z.CopyObject
+					if path.direct {
+						copyObject = z.serverPools[0].CopyObject
+					}
+					got, err = copyObject(t.Context(), bucket, object, bucket, object, src, ObjectOptions{VersionID: original.VersionID}, opts)
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+				if got.UserTags != test.wantTags || got.UserDefined[timestamp] != recent {
+					t.Errorf("response tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], test.wantTags, recent)
+				}
+				copies := 0
+				for pool := range 2 {
+					got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: original.VersionID})
+					if isErrVersionNotFound(err) {
+						continue
+					}
+					if err != nil {
+						t.Fatal(err)
+					}
+					copies++
+					if got.UserTags != test.wantTags || got.UserDefined[timestamp] != recent {
+						t.Errorf("pool %d persisted tags=%q timestamp=%q; want %q %q", pool, got.UserTags, got.UserDefined[timestamp], test.wantTags, recent)
+					}
+				}
+				if copies != 1 {
+					t.Errorf("replacement left %d copies; want 1", copies)
+				}
+			})
+		}
+	}
+}
+
+type tagTestCapacityDisk struct{ StorageAPI }
+
+func (d tagTestCapacityDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (DiskInfo, error) {
+	info, err := d.StorageAPI.DiskInfo(ctx, opts)
+	info.Total, info.Used = info.Free, 0
+	return info, err
+}
+
+func TestMergedPoolObjectInfoTagOrdering(t *testing.T) {
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, test := range []struct {
+		name, firstStamp, secondStamp, secondTags string
+		winner                                    int
+	}{
+		{"newer", old, recent, "key=second", 1},
+		{"newer-removal", old, recent, "", 1},
+		{"equal", recent, recent, "key=second", 0},
+		{"unordered", "", "", "key=second", 0},
+		{"missing-first", "", recent, "key=second", 1},
+		{"missing-second", recent, "", "key=second", 0},
+		{"invalid-first", "invalid", recent, "key=second", 1},
+		{"invalid-second", recent, "invalid", "key=second", 0},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			tagValues := []string{"key=first", test.secondTags}
+			stamps := []string{test.firstStamp, test.secondStamp}
+			copies := make([]PoolObjInfo, 2)
+			before := make([]map[string]string, 2)
+			for i := range copies {
+				fi := FileInfo{Metadata: map[string]string{xhttp.AmzObjectTagging: tagValues[i]}}
+				if stamps[i] != "" {
+					fi.Metadata[timestamp] = stamps[i]
+				}
+				copies[i] = PoolObjInfo{Index: i, ObjInfo: fi.ToObjectInfo("bucket", "object", true)}
+				before[i] = maps.Clone(copies[i].ObjInfo.UserDefined)
+			}
+			got := mergedPoolObjectInfo(copies)
+			if got.UserTags != tagValues[test.winner] || got.UserDefined[timestamp] != stamps[test.winner] {
+				t.Errorf("merged tags=%q timestamp=%q; want %q %q", got.UserTags, got.UserDefined[timestamp], tagValues[test.winner], stamps[test.winner])
+			}
+			if _, exists := got.UserDefined[xhttp.AmzObjectTagging]; exists {
+				t.Error("merged ObjectInfo leaked the raw tagging key into UserDefined")
+			}
+			for i := range copies {
+				if !maps.Equal(copies[i].ObjInfo.UserDefined, before[i]) || copies[i].ObjInfo.UserTags != tagValues[i] {
+					t.Errorf("merge mutated input copy %d", i)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsMetadataCallbackReplacesTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	for _, test := range []struct{ name, tags string }{
+		{"replace", "key=callback"},
+		{"remove", ""},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var original ObjectInfo
+			for pool := range 2 {
+				original = putConsistencyObject(t, z, bucket, test.name, pool, "data", ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime,
+					UserDefined: map[string]string{
+						xhttp.AmzObjectTagging: []string{"key=old", "key=new"}[pool],
+						timestamp:              []string{"2026-09-09T09:00:00Z", "2026-09-09T10:00:00Z"}[pool],
+					},
+				})
+			}
+			const updatedStamp = "2026-09-09T11:00:00Z"
+			got, err := z.PutObjectMetadata(t.Context(), bucket, test.name, ObjectOptions{
+				VersionID: original.VersionID, MTime: original.ModTime,
+				EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
+					if current.UserTags != "key=new" {
+						t.Errorf("callback read tags=%q; want key=new", current.UserTags)
+					}
+					if _, exists := current.UserDefined[xhttp.AmzObjectTagging]; exists {
+						t.Error("callback received the raw tagging key")
+					}
+					current.UserDefined[xhttp.AmzObjectTagging] = test.tags
+					current.UserDefined[timestamp] = updatedStamp
+					return ReplicateDecision{}, nil
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.UserTags != test.tags || got.UserDefined[timestamp] != updatedStamp {
+				t.Errorf("callback update response tags=%q timestamp=%q", got.UserTags, got.UserDefined[timestamp])
+			}
+			for pool := range 2 {
+				got, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, test.name, ObjectOptions{VersionID: original.VersionID})
+				if err != nil || got.UserTags != test.tags || got.UserDefined[timestamp] != updatedStamp {
+					t.Errorf("pool %d did not persist callback tags: tags=%q timestamp=%q err=%v", pool, got.UserTags, got.UserDefined[timestamp], err)
+				}
+			}
+		})
+	}
+}
+
+func TestReconcileStoredObjectTagOrdering(t *testing.T) {
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, test := range []struct {
+		name, storedStamp, incomingStamp, storedTags string
+		wantStored                                   bool
+	}{
+		{"stored-newer", recent, old, "key=stored", true},
+		{"incoming-newer", old, recent, "key=stored", false},
+		{"equal", recent, recent, "key=stored", true},
+		{"equal-removal", recent, recent, "", true},
+		{"missing-stored", "", recent, "key=stored", false},
+		{"missing-incoming", recent, "", "key=stored", true},
+		{"invalid-stored", "invalid", recent, "key=stored", false},
+		{"invalid-incoming", recent, "invalid", "key=stored", true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			metadata := map[string]string{
+				xhttp.AmzObjectTagging: "key=incoming",
+				timestamp:              test.incomingStamp,
+				"unrelated":            "preserved",
+			}
+			reconcileStoredObjectTags(metadata, test.storedTags, test.storedStamp)
+			wantTags, wantStamp := "key=incoming", test.incomingStamp
+			if test.wantStored {
+				wantTags, wantStamp = test.storedTags, test.storedStamp
+			}
+			if metadata[xhttp.AmzObjectTagging] != wantTags || metadata[timestamp] != wantStamp || metadata["unrelated"] != "preserved" {
+				t.Errorf("reconciled metadata=%v; want tags=%q timestamp=%q and unrelated field preserved", metadata, wantTags, wantStamp)
+			}
+		})
+	}
+}
+
+func TestPoolsMetadataUpdatePreservesAbsentTags(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	const (
+		old       = "2026-09-09T09:00:00Z"
+		recent    = "2026-09-09T10:00:00Z"
+		timestamp = ReservedMetadataPrefixLower + TaggingTimestamp
+	)
+	for _, removal := range []bool{false, true} {
+		t.Run(fmt.Sprintf("timestamp-only-removal=%t", removal), func(t *testing.T) {
+			object := fmt.Sprintf("absent-tags-%t", removal)
+			var original ObjectInfo
+			checkStored := func(pool int, wantKey bool, wantTags, wantStamp string) {
+				t.Helper()
+				infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(object).getDisks(), "", bucket, object, original.VersionID, false, false)
+				for disk, info := range infos {
+					if errs[disk] != nil {
+						t.Fatal(errs[disk])
+					}
+					tags, exists := info.Metadata[xhttp.AmzObjectTagging]
+					if exists != wantKey || tags != wantTags || info.Metadata[timestamp] != wantStamp {
+						t.Errorf("pool %d disk %d raw tagging key=%t value=%q stamp=%q; want %t %q %q", pool, disk, exists, tags, info.Metadata[timestamp], wantKey, wantTags, wantStamp)
+					}
+				}
+			}
+			for pool := range 2 {
+				metadata := map[string]string{}
+				if removal {
+					metadata[timestamp] = recent
+					if pool == 1 {
+						metadata[xhttp.AmzObjectTagging] = "key=old"
+						metadata[timestamp] = old
+					}
+				}
+				original = putConsistencyObject(t, z, bucket, object, pool, "data", ObjectOptions{
+					Versioned: true, VersionID: original.VersionID, MTime: original.ModTime, UserDefined: metadata,
+				})
+				checkStored(pool, removal && pool == 1, metadata[xhttp.AmzObjectTagging], metadata[timestamp])
+			}
+			_, err := z.PutObjectMetadata(t.Context(), bucket, object, ObjectOptions{
+				VersionID: original.VersionID, MTime: original.ModTime,
+				EvalMetadataFn: func(current *ObjectInfo, _ error) (ReplicateDecision, error) {
+					current.UserDefined["unrelated-update"] = "preserved"
+					return ReplicateDecision{}, nil
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			wantStamp := ""
+			if removal {
+				wantStamp = recent
+			}
+			for pool := range 2 {
+				// A previously non-empty key needs an explicit empty value to
+				// propagate deletion; an absent key should remain absent.
+				checkStored(pool, removal && pool == 1, "", wantStamp)
+			}
+		})
+	}
+}

cmd/erasure-server-pool.go

@@ -615,17 +615,7 @@ func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucke
 	})
 }
 
-func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) {
-	if dstPoolIdx != nil {
-		if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
-			return -1, errInvalidArgument
-		}
-		if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
-			return -1, toObjectErr(errDiskFull)
-		}
-		return *dstPoolIdx, nil
-	}
-
+func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
 	idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object)
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
@@ -644,23 +634,13 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 // getPoolIdx returns the found previous object and its corresponding pool idx,
 // if none are found falls back to most available space pool, this function is
 // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
-func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) {
-	return z.getWritePoolIdx(ctx, bucket, object, size, dstPoolIdx, false)
+func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
+	return z.getWritePoolIdx(ctx, bucket, object, size, false)
 }
 
 // getWritePoolIdx keeps the write-allocation policy when the caller already
 // holds the pools-layer object lock and must not reacquire a set read lock.
-func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int, noLock bool) (idx int, err error) {
-	if dstPoolIdx != nil {
-		if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
-			return -1, errInvalidArgument
-		}
-		if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
-			return -1, toObjectErr(errDiskFull)
-		}
-		return *dstPoolIdx, nil
-	}
-
+func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock bool) (idx int, err error) {
 	pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{
 		NoLock:             noLock,
 		SkipDecommissioned: true,
@@ -683,13 +663,6 @@ func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object
 	return idx, nil
 }
 
-func dataMovementDstPool(opts ObjectOptions) *int {
-	if !opts.DataMovement {
-		return nil
-	}
-	return opts.DstPoolIdx
-}
-
 func (z *erasureServerPools) Shutdown(ctx context.Context) error {
 	g := errgroup.WithNErrs(len(z.serverPools))
 
@@ -1158,16 +1131,10 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 
 	object = encodeDirObject(object)
 	if z.SinglePool() {
-		idx, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
+		_, err := z.getPoolIdx(ctx, bucket, object, data.Size())
 		if err != nil {
 			return ObjectInfo{}, err
 		}
-		if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx {
-			return ObjectInfo{}, DataMovementOverwriteErr{
-				Bucket: bucket, Object: object, VersionID: opts.VersionID,
-				Err: errDataMovementSrcDstPoolSame,
-			}
-		}
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
 
@@ -1182,7 +1149,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 	}
 	opts.NoLock = true
 
-	idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts), true)
+	idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -1238,30 +1205,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
 	}
 
-	// Access-tier moves must recreate delete markers on the explicitly
-	// selected destination. The regular data-movement path discovers a pool
-	// from existing object state, which is ambiguous while both source and
-	// destination temporarily contain the version stack.
-	if dstPoolIdx := dataMovementDstPool(opts); dstPoolIdx != nil {
-		if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
-			return ObjectInfo{}, errInvalidArgument
-		}
-		if *dstPoolIdx == opts.SrcPoolIdx {
-			return ObjectInfo{}, DataMovementOverwriteErr{
-				Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID,
-				Err: errDataMovementSrcDstPoolSame,
-			}
-		}
-		if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
-			return ObjectInfo{}, toObjectErr(errDiskFull)
-		}
-		objInfo, err = z.serverPools[*dstPoolIdx].DeleteObject(ctx, bucket, object, opts)
-		objInfo.Name = decodeDirObject(object)
-		return objInfo, err
-	}
-
-	if !z.SinglePool() && opts.CheckPrecondFn != nil {
-		return z.deleteObjectConditional(ctx, bucket, object, opts)
+	// Reconcile ordinary addressed-version deletes independently of pool movement.
+	reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
+		!opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
+	if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
+		return z.deleteObjectReconciled(ctx, bucket, object, opts)
 	}
 
 	gopts := opts
@@ -1499,7 +1447,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		}
 		stored := mergedPoolObjectInfo(copies)
 		reconcileStoredObjectLock(srcInfo.UserDefined, storedObjectLockState(stored.UserDefined))
-		reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserDefined)
+		reconcileStoredObjectTags(srcInfo.UserDefined, stored.UserTags, stored.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp])
 		idx := copies[0].Index
 		oi, err := z.serverPools[idx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		if err == nil {
@@ -1508,16 +1456,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		return oi, err
 	}
 
-	poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size, dataMovementDstPool(dstOpts))
+	poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
-	if dataMovementDstPool(dstOpts) != nil && poolIdx == dstOpts.SrcPoolIdx {
-		return ObjectInfo{}, DataMovementOverwriteErr{
-			Bucket: dstBucket, Object: dstObject, VersionID: dstOpts.VersionID,
-			Err: errDataMovementSrcDstPoolSame,
-		}
-	}
 
 	if !z.SinglePool() && dstOpts.ReplicaLockReconcile {
 		dstOpts.replicaObjectInfo = z.replicaObjectInfo
@@ -1977,41 +1919,29 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}()
 
 	if z.SinglePool() {
-		idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts))
-		if err != nil {
-			return nil, err
-		}
-		if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx {
-			return nil, DataMovementOverwriteErr{
-				Bucket: bucket, Object: object, VersionID: opts.VersionID,
-				Err: errDataMovementSrcDstPoolSame,
-			}
-		}
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
 
-	if dataMovementDstPool(opts) == nil {
-		for idx, pool := range z.serverPools {
-			if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
-				continue
-			}
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
+			continue
+		}
 
-			result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
-			if err != nil {
-				return nil, err
-			}
-			// If there is a multipart upload with the same bucket/object name,
-			// create the new multipart in the same pool, this will avoid
-			// creating two multiparts uploads in two different pools.
-			if len(result.Uploads) != 0 {
-				return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
-			}
+		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
+		if err != nil {
+			return nil, err
+		}
+		// If there is a multipart upload with the same bucket/object name,
+		// create the new multipart in the same pool, this will avoid
+		// creating two multiparts uploads in two different pools
+		if len(result.Uploads) != 0 {
+			return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
 		}
 	}
 
 	// any parallel writes on the object will block for this poolIdx
 	// to return since this holds a read lock on the namespace.
-	idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts))
+	idx, err := z.getPoolIdx(ctx, bucket, object, -1)
 	if err != nil {
 		return nil, err
 	}
@@ -2045,7 +1975,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 	}
 
 	if z.SinglePool() {
-		_, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
+		_, err := z.getPoolIdx(ctx, bucket, object, data.Size())
 		if err != nil {
 			return PartInfo{}, err
 		}
@@ -2224,6 +2154,47 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 			defer lk.Unlock(lkctx)
 		}
 		opts.NoLock = true
+
+		// A conditional completion must be evaluated against the logical
+		// latest object across pools, under the object write lock held for
+		// this operation. The pool hosting the upload may only hold a stale
+		// duplicate, so its set-local check would both accept an outdated
+		// ETag and reject the current one. An unreadable pool is not
+		// absence: it may hold the newest copy, so a read that cannot be
+		// verified fails the request instead of passing the condition.
+		// Once satisfied, the callback is cleared so the set layer does not
+		// re-evaluate it against its local copy.
+		if opts.CheckPrecondFn != nil {
+			copies, lerr := z.objectPoolInfos(ctx, bucket, encodeDirObject(object), ObjectOptions{
+				// Conditions always compare the logical current object,
+				// independently of the completion's destination version.
+				VersionID:        "",
+				Versioned:        opts.Versioned,
+				VersionSuspended: opts.VersionSuspended,
+				NoAuditLog:       true,
+			})
+			var latest ObjectInfo
+			if lerr == nil {
+				latest = copies[0].ObjInfo
+				if latest.DeleteMarker {
+					// A delete-marker latest reads as an absent key, matching
+					// the set layer's getObjectInfo.
+					lerr = toObjectErr(errFileNotFound, bucket, object)
+				}
+			}
+			if lerr == nil && opts.CheckPrecondFn(latest) {
+				return ObjectInfo{}, PreConditionFailed{}
+			}
+			if lerr != nil && !isErrVersionNotFound(lerr) && !isErrObjectNotFound(lerr) {
+				return ObjectInfo{}, lerr
+			}
+			// if object doesn't exist return error for If-Match conditional requests
+			// If-None-Match should be allowed to proceed for non-existent objects
+			if lerr != nil && opts.HasIfMatch && (isErrObjectNotFound(lerr) || isErrVersionNotFound(lerr)) {
+				return ObjectInfo{}, lerr
+			}
+			opts.CheckPrecondFn = nil
+		}
 	}
 
 	// Hold write locks to verify uploaded parts, also disallows any
@@ -2276,6 +2247,11 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, o
 	if err != nil {
 		return bucketInfo, toObjectErr(err, bucket)
 	}
+	// Physical existence/creation probes must not be overwritten by cached
+	// metadata, which can legitimately lack Created on an unmigrated bucket.
+	if opts.NoMetadata {
+		return bucketInfo, nil
+	}
 
 	meta, err := globalBucketMetadataSys.Get(bucket)
 	if err == nil {
@@ -3214,7 +3190,7 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
 		defer ns.Unlock(lkctx)
 		opts.NoLock = true
 	}
-	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size, dataMovementDstPool(opts))
+	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
 	if err != nil {
 		return err
 	}

cmd/iam-credential-retention_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+func TestIAMCredentialRetention(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			secret, err := getTokenSigningKey()
+			mustIAM(t, err)
+			parent := "external-idp-parent"
+			credential := func(exp time.Time) auth.Credentials {
+				cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": exp.Unix(), parentClaim: parent}, secret)
+				mustIAM(t, err)
+				cred.ParentUser = parent
+				return cred
+			}
+			// Disablement of an external identity must include cached STS,
+			// which are kept separately from regular and service accounts.
+			cred := credential(UTCNow().Add(time.Hour))
+			_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
+			mustIAM(t, err)
+			mustIAM(t, sys.store.DeleteUsers(ctx, []string{parent}))
+			r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
+			mustIAM(t, err)
+			if !r.Deleted || !r.ExpiresAt.Equal(cred.Expiration.Add(globalMaxSkewTime)) || r.Credentials.SessionToken != "" || r.Credentials.SecretKey != "" {
+				t.Fatal("early STS revocation lost its retention boundary or retained a secret")
+			}
+			if _, ok := sys.store.GetUser(cred.AccessKey); ok {
+				t.Fatal("external disablement left the STS cache live")
+			}
+			_, err = sys.SetTempUser(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), cred.AccessKey, cred, "")
+			if !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("same revoked token was reissued by replay: %v", err)
+			}
+			var mp MappedPolicy
+			err = sys.store.loadIAMConfig(ctx, &mp, getMappedPolicyPath(cred.AccessKey, stsUser, false))
+			if !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("random STS key produced a permanent mapping: %v", err)
+			}
+
+			// Seed genuinely expired immutable tokens, as an ordinary startup
+			// loader sees them. Natural expiry leaves no permanent tombstone.
+			expired := credential(UTCNow().Add(-time.Hour))
+			path := getUserIdentityPath(expired.AccessKey, stsUser)
+			mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: expired, UpdatedAt: UTCNow().Add(-2 * time.Hour)}, path))
+			_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
+			var u UserIdentity
+			if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("natural expiration retained a random key: %v", err)
+			}
+			// A retained early-revocation record is collectable only after the
+			// immutable token's expiration plus the skew allowance.
+			tomb := UserIdentity{Version: 1, Deleted: true, UpdatedAt: UTCNow().Add(-2 * time.Hour), ExpiresAt: expired.Expiration.Add(globalMaxSkewTime)}
+			mustIAM(t, sys.store.saveIAMConfig(ctx, &tomb, path))
+			_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
+			if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("expired STS revocation not collected: %v", err)
+			}
+			if _, ok := sys.store.revisionIndex().snapshot()[path]; ok {
+				t.Fatal("expired STS retained an index entry")
+			}
+		})
+	}
+}
+
+func TestIAMPolicyDeletionRemainsExplicit(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			mustIAM(t, sys.DeletePolicy(ctx, "misspelled-policy", true))
+			r, err := loadIAMRevision(ctx, sys.store, getPolicyDocPath("misspelled-policy"))
+			mustIAM(t, err)
+			if r.Deleted {
+				t.Fatal("local nonexistent policy created a tombstone")
+			}
+			p, err := sys.store.GetPolicy("readwrite")
+			mustIAM(t, err)
+			if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil {
+				t.Fatal("local pristine builtin policy became deletable")
+			}
+			_, err = sys.SetPolicy(ctx, "readwrite", p)
+			mustIAM(t, err)
+			mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true))
+			mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+			if _, err := sys.store.GetPolicy("readwrite"); !errors.Is(err, errNoSuchPolicy) {
+				t.Fatalf("reload restored an explicitly deleted override: %v", err)
+			}
+			_, err = sys.SetPolicy(ctx, "readwrite", p)
+			mustIAM(t, err)
+			if _, err := sys.store.GetPolicy("readwrite"); err != nil {
+				t.Fatal("explicit policy recreation failed", err)
+			}
+			mustIAM(t, globalSiteReplicationSys.PeerAddPolicyHandler(ctx, "remote-unknown-policy", nil, UTCNow()))
+			r, err = loadIAMRevision(ctx, sys.store, getPolicyDocPath("remote-unknown-policy"))
+			mustIAM(t, err)
+			if !r.Deleted {
+				t.Fatal("replicated unknown deletion lost its version")
+			}
+		})
+	}
+}

cmd/iam-etcd-store.go

@@ -26,7 +26,6 @@ import (
 	"sync"
 	"time"
 
-	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
@@ -62,6 +61,7 @@ type IAMEtcdStore struct {
 	sync.RWMutex
 
 	*iamCache
+	index iamRevisionIndex
 
 	usersSysType UsersSysType
 
@@ -69,13 +69,17 @@ type IAMEtcdStore struct {
 }
 
 func newIAMEtcdStore(client *etcd.Client, usersSysType UsersSysType) *IAMEtcdStore {
-	return &IAMEtcdStore{
+	store := &IAMEtcdStore{
 		iamCache:     newIamCache(),
 		client:       client,
 		usersSysType: usersSysType,
 	}
+	store.revisions = &store.index
+	return store
 }
 
+func (ies *IAMEtcdStore) revisionIndex() *iamRevisionIndex { return &ies.index }
+
 func (ies *IAMEtcdStore) rlock() *iamCache {
 	ies.RLock()
 	return ies.iamCache
@@ -103,6 +107,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
 	if err != nil {
 		return err
 	}
+	plain := data
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, itemPath),
@@ -111,24 +116,28 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
 			return err
 		}
 	}
-	return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
+	if err := saveKeyEtcd(ctx, ies.client, itemPath, data, opts...); err != nil {
+		return err
+	}
+	ies.index.observe(itemPath, plain)
+	return nil
 }
 
-func getIAMConfig(item any, data []byte, itemPath string) error {
-	data, err := decryptData(data, itemPath)
+func (ies *IAMEtcdStore) decodeIAMConfig(item any, data []byte, path string) error {
+	data, err := decryptData(data, path)
 	if err != nil {
 		return err
 	}
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	ies.index.observe(path, data)
 	return json.Unmarshal(data, item)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item any, path string) error {
-	data, err := readKeyEtcd(ctx, ies.client, path)
+	data, err := ies.loadIAMConfigBytes(ctx, path)
 	if err != nil {
 		return err
 	}
-	return getIAMConfig(item, data, path)
+	return json.Unmarshal(data, item)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) {
@@ -136,11 +145,19 @@ func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([
 	if err != nil {
 		return nil, err
 	}
-	return decryptData(data, path)
+	data, err = decryptData(data, path)
+	if err == nil {
+		ies.index.observe(path, data)
+	}
+	return data, err
 }
 
 func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error {
-	return deleteKeyEtcd(ctx, ies.client, path)
+	if err := deleteKeyEtcd(ctx, ies.client, path); err != nil {
+		return err
+	}
+	ies.index.forget(path)
+	return nil
 }
 
 func (ies *IAMEtcdStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, _ int) error {
@@ -162,6 +179,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
 		return err
 	}
 
+	if p.Deleted {
+		return errNoSuchPolicy
+	}
 	m[policy] = p
 	return nil
 }
@@ -181,7 +201,11 @@ func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValu
 		return err
 	}
 
+	ies.index.observe(string(kvs.Key), data)
 	policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key)))
+	if p.Deleted {
+		return errNoSuchPolicy
+	}
 	m[policy] = p
 	return nil
 }
@@ -207,7 +231,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]Policy
 
 func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
 	var u UserIdentity
-	err := getIAMConfig(&u, userkv.Value, string(userkv.Key))
+	err := ies.decodeIAMConfig(&u, userkv.Value, string(userkv.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -219,10 +243,14 @@ func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue,
 }
 
 func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error {
+	if u.Deleted {
+		if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
+			bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
+		}
+		return errNoSuchUser
+	}
 	if u.Credentials.IsExpired() {
-		// Delete expired identity.
-		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
+		bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
 		return nil
 	}
 	if u.Credentials.AccessKey == "" {
@@ -231,16 +259,17 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
-			if u.Credentials.IsTemp() {
-				// We should delete such that the client can re-request
-				// for the expiring credentials.
-				deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-				deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
-			}
+			// A temporarily unavailable signing key is not proof of expiration.
 			return nil
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
+	if err := checkIAMParentRevision(ctx, ies, u.Credentials); err != nil {
+		if errors.Is(err, errIAMStaleUpdate) {
+			return errNoSuchUser
+		}
+		return err
+	}
 	if u.Credentials.Description == "" {
 		u.Credentials.Description = u.Credentials.Comment
 	}
@@ -258,6 +287,9 @@ func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userTyp
 		}
 		return "", err
 	}
+	if u.Deleted {
+		return "", errNoSuchUser
+	}
 	return u.Credentials.SecretKey, nil
 }
 
@@ -274,6 +306,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 }
 
 func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
+	ctx = withIAMExpirationCleanup(ctx)
 	var basePrefix string
 	switch userType {
 	case svcUser:
@@ -312,6 +345,9 @@ func (ies *IAMEtcdStore) loadGroup(ctx context.Context, group string, m map[stri
 		}
 		return err
 	}
+	if gi.Deleted {
+		return errNoSuchGroup
+	}
 	m[group] = gi
 	return nil
 }
@@ -349,13 +385,16 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
 		}
 		return err
 	}
+	if !ies.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+		return errNoSuchPolicy
+	}
 	m.Store(name, p)
 	return nil
 }
 
-func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
+func (ies *IAMEtcdStore) getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
 	var p MappedPolicy
-	err := getIAMConfig(&p, kv.Value, string(kv.Key))
+	err := ies.decodeIAMConfig(&p, kv.Value, string(kv.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -363,6 +402,9 @@ func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy],
 		return err
 	}
 	name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json")
+	if !ies.index.mappingAllowed(string(kv.Key), p) {
+		return errNoSuchPolicy
+	}
 	m.Store(name, p)
 	return nil
 }
@@ -392,61 +434,13 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Parse all policies mapping to create the proper data model
 	for _, kv := range r.Kvs {
-		if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if err = ies.getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
 			return err
 		}
 	}
 	return nil
 }
 
-func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
-	return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
-}
-
-func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
-	return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
-}
-
-func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
-	return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
-}
-
-func (ies *IAMEtcdStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
-	return ies.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
-}
-
-func (ies *IAMEtcdStore) deletePolicyDoc(ctx context.Context, name string) error {
-	err := ies.deleteIAMConfig(ctx, getPolicyDocPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
-	err := ies.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
-	err := ies.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
-	if err == errConfigNotFound {
-		err = errNoSuchUser
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteGroupInfo(ctx context.Context, name string) error {
-	err := ies.deleteIAMConfig(ctx, getGroupInfoPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchGroup
-	}
-	return err
-}
-
 func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWatchEvent {
 	ch := make(chan iamWatchEvent)
 

cmd/iam-group-revision.go

@@ -0,0 +1,164 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"maps"
+	"slices"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+type (
+	iamGroupGrantsKey   struct{}
+	iamGroupMutationKey struct{}
+	iamGroupMutation    struct {
+		Members    []string
+		Remove     bool
+		StatusOnly bool
+	}
+)
+
+// Merge the intended mutation with the record read under the distributed
+// revision lock, not the older cache used to prepare the request.
+func mergeIAMGroupMutation(ctx context.Context, previous GroupInfo, next *GroupInfo) {
+	op, ok := ctx.Value(iamGroupMutationKey{}).(iamGroupMutation)
+	if !ok || previous.Deleted || previous.Version == 0 {
+		return
+	}
+	members := set.CreateStringSet(previous.Members...)
+	grants := maps.Clone(previous.MemberGrants)
+	if grants == nil {
+		grants = make(map[string]time.Time)
+	}
+	switch {
+	case op.StatusOnly:
+		// Only the status changes.
+	case op.Remove:
+		for _, member := range op.Members {
+			members.Remove(member)
+			delete(grants, member)
+		}
+		next.Status = previous.Status
+	default:
+		requested := set.CreateStringSet(next.Members...)
+		for _, member := range op.Members {
+			if !requested.Contains(member) {
+				continue
+			}
+			at := next.MemberGrants[member]
+			if at.Before(grants[member]) {
+				continue
+			}
+			members.Add(member)
+			grants[member] = at
+		}
+		next.Status = previous.Status
+	}
+	next.Members, next.MemberGrants = members.ToSlice(), grants
+	slices.Sort(next.Members)
+}
+
+// A non-nil map is supplied by the versioned peer envelope, including for
+// snapshots. Missing times are unknown, never the snapshot's newer timestamp.
+func withIAMGroupGrants(ctx context.Context, grants map[string]time.Time) context.Context {
+	return context.WithValue(ctx, iamGroupGrantsKey{}, grants)
+}
+
+func (c *iamCache) effectiveGroupMembers(gi GroupInfo) []string {
+	var members []string
+	for _, member := range gi.Members {
+		if c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
+			members = append(members, member)
+		}
+	}
+	return members
+}
+
+func (c *iamCache) effectiveUserGroups(user string) []string {
+	var groups []string
+	for group := range c.iamUserGroupMemberships[user] {
+		gi, ok := c.iamGroupsMap[group]
+		r := c.revisions.get(getGroupInfoPath(group))
+		if r.RevokedBefore.After(gi.RevokedBefore) {
+			gi.RevokedBefore = r.RevokedBefore
+		}
+		if ok && !r.Deleted && c.groupMemberAllowed(user, gi.MemberGrants[user], gi.RevokedBefore) {
+			groups = append(groups, group)
+		}
+	}
+	return groups
+}
+
+func (c *iamCache) addGroupMembers(ctx context.Context, gi GroupInfo, members []string) (GroupInfo, error) {
+	grants, versioned := ctx.Value(iamGroupGrantsKey{}).(map[string]time.Time)
+	if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(gi.RevokedBefore) {
+		gi.RevokedBefore = boundary
+	}
+	origin, replicated := iamReplicationTime(ctx)
+	gi.Members = slices.Clone(gi.Members)
+	gi.MemberGrants = maps.Clone(gi.MemberGrants)
+	if gi.MemberGrants == nil {
+		gi.MemberGrants = make(map[string]time.Time)
+	}
+	current := set.CreateStringSet(gi.Members...)
+	gi.UpdatedAt = UTCNow()
+	if replicated {
+		gi.UpdatedAt = origin
+	}
+	for _, member := range members {
+		at := gi.UpdatedAt
+		r := c.userRevocation(member)
+		if replicated {
+			switch {
+			case versioned:
+				at = grants[member]
+				if at.After(origin) {
+					return gi, errInvalidArgument
+				}
+			case !r.RevokedBefore.IsZero() || r.Deleted || !gi.RevokedBefore.IsZero():
+				// Legacy snapshots cannot prove a post-revocation grant.
+				continue
+			case current.Contains(member):
+				continue
+			}
+			if !c.groupMemberAllowed(member, at, gi.RevokedBefore) {
+				continue
+			}
+		} else {
+			if current.Contains(member) && c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
+				continue // Editing the group is not reissuing every grant.
+			}
+			if !at.After(gi.RevokedBefore) {
+				at = gi.RevokedBefore.Add(time.Nanosecond)
+			}
+			if !at.After(r.RevokedBefore) {
+				at = r.RevokedBefore.Add(time.Nanosecond)
+			}
+			if !at.After(gi.MemberGrants[member]) {
+				at = gi.MemberGrants[member].Add(time.Nanosecond)
+			}
+			if at.After(gi.UpdatedAt) {
+				gi.UpdatedAt = at
+			}
+		}
+		u, ok := c.iamUsersMap[member]
+		if !ok {
+			return gi, errNoSuchUser
+		}
+		if u.Credentials.IsTemp() || u.Credentials.IsServiceAccount() {
+			return gi, errIAMActionNotAllowed
+		}
+		if previous := gi.MemberGrants[member]; previous.After(at) {
+			continue
+		}
+		current.Add(member)
+		gi.MemberGrants[member] = at
+	}
+	gi.Members = current.ToSlice()
+	slices.Sort(gi.Members)
+	return gi, nil
+}

cmd/iam-healing-leadership_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestIAMHealingResumesAfterLeadershipLoss(t *testing.T) {
+	previous := globalLeaderLock
+	locks := make(chan LockContext)
+	globalLeaderLock = &sharedLock{lockContext: locks}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	c := &SiteReplicationSys{}
+	go func() { c.startHealRoutine(ctx, nil); close(done) }()
+	t.Cleanup(func() {
+		cancel()
+		select {
+		case <-done:
+		case locks <- LockContext{ctx: ctx}:
+		}
+		<-done
+		globalLeaderLock = previous
+	})
+	first, loseFirst := context.WithCancel(ctx)
+	defer loseFirst()
+	select {
+	case locks <- LockContext{ctx: first}:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not acquire its first leader context")
+	}
+	loseFirst() // A transient quorum loss cancels the distributed lease.
+	select {
+	case <-done:
+		t.Fatal("healer permanently exited after temporary leadership loss")
+	case locks <- LockContext{ctx: ctx}:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not wait for reacquired leadership")
+	}
+	// Shutdown must also interrupt the wait for leadership after lease loss.
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not stop with its owning context")
+	}
+}
+
+func TestIAMHealingLeadershipWaitCancels(t *testing.T) {
+	previous := globalLeaderLock
+	locks := make(chan LockContext)
+	globalLeaderLock = &sharedLock{lockContext: locks}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	c := &SiteReplicationSys{}
+	go func() { c.startHealRoutine(ctx, nil); close(done) }()
+	cancel()
+	t.Cleanup(func() {
+		select {
+		case <-done:
+		case locks <- LockContext{ctx: ctx}:
+		}
+		<-done
+		globalLeaderLock = previous
+	})
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("healer ignored shutdown while waiting for leadership")
+	}
+}

cmd/iam-healing-performance_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+// Measures steady-state index traversal, sorting and the capability request.
+// The network peer acknowledges real batches but performs no disk I/O; this
+// benchmark deliberately does not claim durable catch-up throughput.
+func BenchmarkIAMRevisionConvergedHealing(b *testing.B) {
+	for _, n := range []int{1000, 10000} {
+		b.Run(fmt.Sprint(n), func(b *testing.B) {
+			ctx, sys, _ := prepareIAMRevisionFixture(b)
+			_, err := sys.CreateUser(ctx, "benchmark-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
+			mustIAM(b, err)
+			for i := range n {
+				at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
+				data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
+				mustIAM(b, err)
+				sys.store.revisionIndex().observe(getUserIdentityPath(fmt.Sprintf("deleted-%06d", i), regUser), data)
+			}
+			var puts atomic.Int64
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.Path == "/minio/health/live" {
+					w.WriteHeader(http.StatusOK)
+					return
+				}
+				if r.Method == http.MethodPut {
+					puts.Add(1)
+				}
+				_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: "benchmark-peer", Digest: "constant"}})
+			}))
+			defer server.Close()
+			c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "benchmark-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
+			mustIAM(b, c.healIAMDeletions(ctx))
+			before := puts.Load()
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				mustIAM(b, c.healIAMDeletions(ctx))
+			}
+			b.StopTimer()
+			b.ReportMetric(float64(puts.Load()-before)/float64(b.N), "PUT/op")
+			if puts.Load() != before {
+				b.Fatal("steady-state healing replayed acknowledged records")
+			}
+		})
+	}
+}

cmd/iam-object-store.go

@@ -45,6 +45,7 @@ type IAMObjectStore struct {
 	sync.RWMutex
 
 	*iamCache
+	index iamRevisionIndex
 
 	usersSysType UsersSysType
 
@@ -52,13 +53,17 @@ type IAMObjectStore struct {
 }
 
 func newIAMObjectStore(objAPI ObjectLayer, usersSysType UsersSysType) *IAMObjectStore {
-	return &IAMObjectStore{
+	store := &IAMObjectStore{
 		iamCache:     newIamCache(),
 		objAPI:       objAPI,
 		usersSysType: usersSysType,
 	}
+	store.revisions = &store.index
+	return store
 }
 
+func (iamOS *IAMObjectStore) revisionIndex() *iamRevisionIndex { return &iamOS.index }
+
 func (iamOS *IAMObjectStore) rlock() *iamCache {
 	iamOS.RLock()
 	return iamOS.iamCache
@@ -87,6 +92,7 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
 	if err != nil {
 		return err
 	}
+	plain := data
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, objPath),
@@ -95,7 +101,11 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
 			return err
 		}
 	}
-	return saveConfig(ctx, iamOS.objAPI, objPath, data)
+	if err := saveConfig(ctx, iamOS.objAPI, objPath, data); err != nil {
+		return err
+	}
+	iamOS.index.observe(objPath, plain)
+	return nil
 }
 
 func decryptData(data []byte, objPath string) ([]byte, error) {
@@ -133,6 +143,7 @@ func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context,
 	if err != nil {
 		return nil, meta, err
 	}
+	iamOS.index.observe(objPath, data)
 	return data, meta, nil
 }
 
@@ -146,7 +157,11 @@ func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item any, objPat
 }
 
 func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) error {
-	return deleteConfig(ctx, iamOS.objAPI, path)
+	if err := deleteConfig(ctx, iamOS.objAPI, path); err != nil {
+		return err
+	}
+	iamOS.index.forget(path)
+	return nil
 }
 
 func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error {
@@ -171,6 +186,10 @@ func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy
 			return err
 		}
 
+		if p.Deleted {
+			return errNoSuchPolicy
+		}
+
 		if p.Version == 0 {
 			// This means that policy was in the old version (without any
 			// timestamp info). We fetch the mod time of the file and save
@@ -200,6 +219,10 @@ func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (Pol
 		return p, err
 	}
 
+	if p.Deleted {
+		return PolicyDoc{}, errNoSuchPolicy
+	}
+
 	if p.Version == 0 {
 		// This means that policy was in the old version (without any
 		// timestamp info). We fetch the mod time of the file and save
@@ -245,6 +268,9 @@ func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, use
 		}
 		return "", err
 	}
+	if u.Deleted {
+		return "", errNoSuchUser
+	}
 	return u.Credentials.SecretKey, nil
 }
 
@@ -258,10 +284,15 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
 		return u, err
 	}
 
+	if u.Deleted {
+		if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
+			bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
+		}
+		return UserIdentity{}, errNoSuchUser
+	}
+
 	if u.Credentials.IsExpired() {
-		// Delete expired identity - ignoring errors here.
-		iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
-		iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
+		bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
 		return u, errNoSuchUser
 	}
 
@@ -272,16 +303,18 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
-			if u.Credentials.IsTemp() {
-				// We should delete such that the client can re-request
-				// for the expiring credentials.
-				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
-				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
-			}
-			return u, errNoSuchUser
+			// During startup the site signing key may not be available yet.
+			// Reject this load without deleting a credential that has not expired.
+			return UserIdentity{}, errNoSuchUser
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
+	if err := checkIAMParentRevision(ctx, iamOS, u.Credentials); err != nil {
+		if errors.Is(err, errIAMStaleUpdate) {
+			return UserIdentity{}, errNoSuchUser
+		}
+		return UserIdentity{}, err
+	}
 
 	if u.Credentials.Description == "" {
 		u.Credentials.Description = u.Credentials.Comment
@@ -320,6 +353,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 }
 
 func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
+	ctx = withIAMExpirationCleanup(ctx)
 	var basePrefix string
 	switch userType {
 	case svcUser:
@@ -354,6 +388,9 @@ func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[
 		}
 		return err
 	}
+	if g.Deleted {
+		return errNoSuchGroup
+	}
 	m[group] = g
 	return nil
 }
@@ -391,6 +428,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
 			goto retry
 		}
 
+		if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+			return errNoSuchPolicy
+		}
 		m.Store(name, p)
 		return nil
 	}
@@ -405,6 +445,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name
 		}
 		return p, err
 	}
+	if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+		return MappedPolicy{}, errNoSuchPolicy
+	}
 	return p, nil
 }
 
@@ -824,54 +867,6 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	return nil
 }
 
-func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
-	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
-}
-
-func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
-	return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
-}
-
-func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
-	return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
-}
-
-func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
-	return iamOS.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
-}
-
-func (iamOS *IAMObjectStore) deletePolicyDoc(ctx context.Context, name string) error {
-	err := iamOS.deleteIAMConfig(ctx, getPolicyDocPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
-	err := iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
-	err := iamOS.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
-	if err == errConfigNotFound {
-		err = errNoSuchUser
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteGroupInfo(ctx context.Context, name string) error {
-	err := iamOS.deleteIAMConfig(ctx, getGroupInfoPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchGroup
-	}
-	return err
-}
-
 // Lists objects in the minioMetaBucket at the given path prefix. All returned
 // items have the pathPrefix removed from their names.
 func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr[string] {

cmd/iam-peer-reload_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/grid"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// Two independent IAM caches share the same real object backend, as sibling
+// nodes do. Deliver the actual peer handler only after the source committed.
+func TestIAMPeerDeleteNotificationReloadsCommittedState(t *testing.T) {
+	for _, name := range []string{"deleted", "recreated", "recreated_without_grant"} {
+		recreate := name != "deleted"
+		t.Run(name, func(t *testing.T) {
+			resetTestGlobals()
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			obj, disk, err := prepareFS(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.RemoveAll(disk)
+			defer obj.Shutdown(ctx)
+			defer resetTestGlobals()
+			globalObjLayerMutex.Lock()
+			globalObjectAPI = obj
+			globalObjLayerMutex.Unlock()
+			must := func(err error) {
+				t.Helper()
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			source := globalIAMSys
+			const user = "peer-reload-user"
+			req := madmin.AddOrUpdateUserReq{SecretKey: "original-test-password", Status: madmin.AccountEnabled}
+			_, err = source.CreateUser(ctx, user, req)
+			must(err)
+			_, err = source.PolicyDBSet(ctx, user, "readwrite", regUser, false)
+			must(err)
+			_, err = source.AddUsersToGroup(ctx, "peer-reload-group", []string{user})
+			must(err)
+			_, err = source.PolicyDBSet(ctx, "peer-reload-group", "readwrite", regUser, true)
+			must(err)
+			svc, _, err := source.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{accessKey: "peer-reload-service", secretKey: "service-test-password"})
+			must(err)
+			signingKey, err := getTokenSigningKey()
+			must(err)
+			sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: user}, signingKey)
+			must(err)
+			sts.ParentUser = user
+			_, err = source.SetTempUser(ctx, sts.AccessKey, sts, "")
+			must(err)
+
+			siblingStore := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			must(siblingStore.LoadIAMCache(ctx, true))
+			must(siblingStore.UserNotificationHandler(ctx, sts.AccessKey, stsUser))
+			for _, key := range []string{user, svc.AccessKey, sts.AccessKey} {
+				if _, ok := siblingStore.GetUser(key); !ok {
+					t.Fatalf("fixture did not load %s", key)
+				}
+			}
+			must(source.DeleteUser(ctx, user, false))
+			if recreate {
+				req.SecretKey = "recreated-test-password"
+				_, err = source.CreateUser(ctx, user, req)
+				must(err)
+				if name == "recreated" {
+					_, err = source.PolicyDBSet(ctx, user, "readonly", regUser, false)
+					must(err)
+				}
+			}
+
+			sibling := &IAMSys{store: siblingStore, usersSysType: MinIOUsersSysType}
+			globalIAMSys = sibling
+			defer func() { globalIAMSys = source }()
+			server := &peerRESTServer{}
+			for range 2 {
+				_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+				if remoteErr != nil {
+					t.Fatal(remoteErr)
+				}
+			}
+			if recreate {
+				u, ok := siblingStore.GetUser(user)
+				if !ok || u.Credentials.SecretKey != req.SecretKey {
+					t.Fatal("delayed deletion notification removed the recreated user")
+				}
+				loaded := make(map[string]UserIdentity)
+				must(source.store.loadUser(ctx, user, regUser, loaded))
+				if loaded[user].Credentials.SecretKey != req.SecretKey {
+					t.Fatal("notification changed the persisted recreated identity")
+				}
+				if allowed := sibling.IsAllowed(policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}); allowed != (name == "recreated") {
+					t.Fatal("notification did not load the recreated user's current grant")
+				}
+			}
+			if sibling.IsAllowed(policy.Args{AccountName: user, Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) {
+				t.Fatal("notification retained an old direct or group grant")
+			}
+			for _, key := range []string{svc.AccessKey, sts.AccessKey} {
+				if _, ok := siblingStore.GetUser(key); ok {
+					t.Fatalf("notification retained a revoked child: %s", key)
+				}
+			}
+			if !recreate {
+				for _, key := range []string{user} {
+					if _, ok := siblingStore.GetUser(key); ok {
+						t.Fatalf("notification retained a revoked cached identity: %s", key)
+					}
+				}
+				if sibling.IsAllowed(policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}) {
+					t.Fatal("notification retained the user's old grant")
+				}
+				cache := siblingStore.rlock()
+				member := cache.iamUserGroupMemberships[user].Contains("peer-reload-group")
+				siblingStore.runlock()
+				if member {
+					t.Fatal("notification retained the deleted user's group membership")
+				}
+			}
+		})
+	}
+}

cmd/iam-performance_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// Uses APIs shared with the pre-revision tree so the same benchmark can be
+// overlaid on that tree for a comparable local baseline.
+func prepareIAMPerformanceFixture(b *testing.B) (context.Context, *IAMSys) {
+	b.Helper()
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		b.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err != nil {
+		b.Fatal(err)
+	}
+	initAllSubsystems(ctx)
+	globalIAMSys.initStore(obj, nil)
+	if err := globalIAMSys.Load(ctx, true); err != nil {
+		b.Fatal(err)
+	}
+	b.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
+	return ctx, globalIAMSys
+}
+
+func BenchmarkIAMCachedCredential(b *testing.B) {
+	for _, kind := range []string{"user", "service", "sts"} {
+		b.Run(kind, func(b *testing.B) {
+			ctx, sys := prepareIAMPerformanceFixture(b)
+			const parent = "benchmark-parent"
+			_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
+			if err != nil {
+				b.Fatal(err)
+			}
+			key := parent
+			if kind == "service" {
+				c, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "benchmark-service", secretKey: "benchmark-service-password"})
+				if err != nil {
+					b.Fatal(err)
+				}
+				key = c.AccessKey
+			}
+			if kind == "sts" {
+				secret, err := getTokenSigningKey()
+				if err != nil {
+					b.Fatal(err)
+				}
+				c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+				if err != nil {
+					b.Fatal(err)
+				}
+				c.ParentUser = parent
+				if _, err := sys.SetTempUser(ctx, c.AccessKey, c, ""); err != nil {
+					b.Fatal(err)
+				}
+				key = c.AccessKey
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				if _, ok := sys.store.GetUser(key); !ok {
+					b.Fatal("credential missing")
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkIAMSetTempUser(b *testing.B) {
+	ctx, sys := prepareIAMPerformanceFixture(b)
+	const parent = "benchmark-sts-parent"
+	_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
+	if err != nil {
+		b.Fatal(err)
+	}
+	secret, err := getTokenSigningKey()
+	if err != nil {
+		b.Fatal(err)
+	}
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cred.ParentUser = parent
+	b.ReportAllocs()
+	b.ResetTimer()
+	for b.Loop() {
+		if _, err := sys.SetTempUser(ctx, cred.AccessKey, cred, "readwrite"); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// Run with -benchtime=1x. Preparation is outside the timer; each measured load
+// sees a fresh set of expired reusable service-account records.
+func BenchmarkIAMColdLoadExpiredServices(b *testing.B) {
+	for _, count := range []int{100, 1000} {
+		b.Run(fmt.Sprint(count), func(b *testing.B) {
+			ctx, sys := prepareIAMPerformanceFixture(b)
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				b.StopTimer()
+				for j := 0; j < count; j++ {
+					key := fmt.Sprintf("expired-benchmark-%d-%d", i, j)
+					u := UserIdentity{Version: 1, UpdatedAt: UTCNow().Add(-2 * time.Hour), Credentials: auth.Credentials{AccessKey: key, SecretKey: "expired-benchmark-password", ParentUser: "absent-idp-parent", Expiration: UTCNow().Add(-time.Hour), Status: auth.AccountOn}}
+					if err := sys.store.saveIAMConfig(ctx, &u, getUserIdentityPath(key, svcUser)); err != nil {
+						b.Fatal(err)
+					}
+				}
+				b.StartTimer()
+				if err := sys.store.LoadIAMCache(ctx, true); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

cmd/iam-replay_test.go

@@ -0,0 +1,168 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func TestReviewIAMRevokedUserReplay(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "review-revoked-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
+	created, err := globalIAMSys.CreateUser(ctx, user, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	policyAt, err := globalIAMSys.PolicyDBSet(ctx, user, "readwrite", regUser, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "review-bucket", ObjectName: "review-object"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("seed must allow object read")
+	}
+	if err := globalIAMSys.DeleteUser(ctx, user, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("deletion did not remove initial permission")
+	}
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Errorf("revoked user restored by an older replicated create, GetUserInfo error = %v", err)
+	}
+	if err := globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: user, UserType: int(regUser), Policy: "readwrite"}, policyAt); err != nil {
+		t.Fatal(err)
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Error("older replicated identity and policy events restored revoked S3 read permission")
+	}
+}
+
+func TestReviewIAMSourceTimestampOrder(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "review-ordered-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour)
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Fatalf("newer source deletion skipped after delayed creation, GetUserInfo error = %v", err)
+	}
+}
+
+// A user's old group grant must not return after deletion and deliberate recreation.
+func TestR3CandidateOldGroupReplayAfterRecreation(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user, group := "r3-group-member", "r3-granting-group"
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
+	peer := &globalSiteReplicationSys
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
+	add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
+	must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
+	must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("fixture must grant through group")
+	}
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("recreation must start without deleted group membership")
+	}
+	must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("old group event restored the deleted user's read grant after recreation and durable reload")
+	}
+}
+
+// The user delete is also a revocation of its earlier group memberships.
+func TestR3CandidateLateDeleteRetainsOldGroupGrant(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user, group := "r3-late-member", "r3-late-group"
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
+	peer := &globalSiteReplicationSys
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
+	must(peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin.Add(time.Minute)))
+	must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("fixture must grant through group")
+	}
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if _, ok := globalIAMSys.GetUser(ctx, user); !ok {
+		t.Fatal("newer identity must survive")
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("late user deletion retained the older group grant on the recreated identity")
+	}
+}

cmd/iam-replication-protocol.go

@@ -0,0 +1,321 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+const (
+	iamRevisionProtocol  = 1
+	iamRevisionPeerPath  = "/v3/site-replication/peer/iam-revisions"
+	iamUserBoundaryType  = "silo-user-revocation"
+	iamGroupBoundaryType = "silo-group-revocation"
+	maxIAMRevisionBatch  = 128
+)
+
+var iamRevisionInstance = mustGetUUID()
+
+type iamUserBoundary struct {
+	User   string    `json:"user"`
+	Before time.Time `json:"before"`
+}
+
+type iamGroupBoundary struct {
+	Group  string    `json:"group"`
+	Before time.Time `json:"before"`
+}
+
+// The server owns this additive protocol, without changing the client SDK or
+// overloading a policy/document field. Old servers reject the dedicated route
+// before applying any change that would lose revocation or member metadata.
+type iamReplicationItem struct {
+	madmin.SRIAMItem
+	GroupGrants     map[string]time.Time `json:"groupGrants,omitempty"`
+	GroupSnapshot   bool                 `json:"groupSnapshot,omitempty"`
+	UserRevocation  *iamUserBoundary     `json:"userRevocation,omitempty"`
+	GroupRevocation *iamGroupBoundary    `json:"groupRevocation,omitempty"`
+	RevokedBefore   time.Time            `json:"revokedBefore,omitempty"`
+}
+
+type iamRevisionBatch struct {
+	Version int                  `json:"version"`
+	Items   []iamReplicationItem `json:"items"`
+}
+
+type iamRevisionStatus struct {
+	Version  int    `json:"version"`
+	Node     string `json:"node"`
+	Instance string `json:"instance"`
+	Digest   string `json:"digest"`
+}
+
+type iamRevisionResponse struct {
+	iamRevisionStatus
+	Errors []string `json:"errors,omitempty"`
+}
+
+type iamRevisionBatchError struct{ failures []string }
+
+func (e *iamRevisionBatchError) Error() string {
+	return "IAM revision batch: " + strings.Join(e.failures, "; ")
+}
+
+type iamRevisionProgress struct {
+	Instances    map[string]string
+	Acknowledged map[string]string
+}
+
+type iamRevisionMetrics struct {
+	healFailures       atomic.Uint64
+	healLastSuccess    atomic.Int64
+	healDurationMillis atomic.Int64
+}
+
+func iamRevisionDigest(items map[string]iamRevision) string {
+	paths := make([]string, 0, len(items))
+	for path := range items {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	h := sha256.New()
+	for _, path := range paths {
+		r := items[path]
+		fmt.Fprintf(h, "%q %s %t %s\n", path, r.timestamp().UTC().Format(time.RFC3339Nano), r.Deleted, r.RevokedBefore.UTC().Format(time.RFC3339Nano))
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func (store *IAMStoreSys) iamRevisionStatus() iamRevisionStatus {
+	node := globalLocalNodeName
+	if node == "" {
+		node = "local"
+	}
+	return iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: iamRevisionInstance, Digest: store.revisionIndex().digest()}
+}
+
+func executeIAMRevisionRequest(ctx context.Context, client *madmin.AdminClient, method string, batch *iamRevisionBatch) (status iamRevisionStatus, err error) {
+	var content []byte
+	if batch != nil {
+		content, err = json.Marshal(batch)
+		if err != nil {
+			return status, err
+		}
+	}
+	resp, err := client.ExecuteMethod(ctx, method, madmin.RequestData{RelPath: iamRevisionPeerPath, QueryValues: url.Values{"api-version": {madmin.SiteReplAPIVersion}}, Content: content})
+	if resp != nil {
+		defer xhttp.DrainBody(resp.Body)
+	}
+	if err != nil {
+		return status, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		var remote madmin.ErrorResponse
+		if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&remote) == nil && remote.Code != "" {
+			return status, remote
+		}
+		return status, fmt.Errorf("IAM revision protocol requires upgraded peers: %s", resp.Status)
+	}
+	var response iamRevisionResponse
+	if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&response); err != nil {
+		return status, err
+	}
+	status = response.iamRevisionStatus
+	if status.Version != iamRevisionProtocol || status.Node == "" || status.Instance == "" || status.Digest == "" {
+		return status, errors.New("peer did not acknowledge the IAM revision protocol")
+	}
+	if len(response.Errors) != 0 {
+		return status, &iamRevisionBatchError{failures: response.Errors}
+	}
+	return status, nil
+}
+
+type (
+	iamRecordBoundaryKey struct{}
+	iamGroupSnapshotKey  struct{}
+)
+
+func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) {
+	out := iamReplicationItem{SRIAMItem: item}
+	if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil {
+		var key string
+		if item.SvcAccChange.Create != nil {
+			key = item.SvcAccChange.Create.AccessKey
+		} else if item.SvcAccChange.Update != nil {
+			key = item.SvcAccChange.Update.AccessKey
+		}
+		if key != "" {
+			r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(key, svcUser))
+			if err != nil {
+				return out, err
+			}
+			if r.Deleted || r.timestamp().After(item.UpdatedAt) {
+				return out, errIAMStaleUpdate
+			}
+			out.RevokedBefore = r.RevokedBefore
+		}
+	}
+	if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && !item.GroupInfo.UpdateReq.IsRemove {
+		out.GroupSnapshot = true
+		var gi GroupInfo
+		if err := globalIAMSys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath(item.GroupInfo.UpdateReq.Group)); err != nil {
+			return out, err
+		}
+		// The matching persisted snapshot carries member grant times. If a
+		// later write won before sending, propagate that whole newer state.
+		if gi.Deleted {
+			return out, errIAMStaleUpdate
+		}
+		out.UpdatedAt = gi.UpdatedAt
+		out.RevokedBefore = gi.RevokedBefore
+		out.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: item.GroupInfo.UpdateReq.Group, Status: madmin.GroupStatus(gi.Status)}}
+		cache := globalIAMSys.store.rlock()
+		out.GroupInfo.UpdateReq.Members = cache.effectiveGroupMembers(gi)
+		globalIAMSys.store.runlock()
+		out.GroupGrants = make(map[string]time.Time, len(out.GroupInfo.UpdateReq.Members))
+		for _, member := range out.GroupInfo.UpdateReq.Members {
+			out.GroupGrants[member] = gi.MemberGrants[member]
+		}
+	}
+	if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, getGroupInfoPath(item.GroupInfo.UpdateReq.Group))
+		if err != nil {
+			return out, err
+		}
+		if !r.Deleted && !r.RevokedBefore.IsZero() {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+			out.UpdatedAt = r.RevokedBefore
+		}
+	}
+	if item.Type == madmin.SRIAMItemIAMUser && item.IAMUser != nil {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(item.IAMUser.AccessKey, regUser))
+		if err != nil {
+			return out, err
+		}
+		if item.IAMUser.IsDeleteReq && !r.Deleted && !r.RevokedBefore.IsZero() {
+			out.Type = iamUserBoundaryType
+			out.IAMUser = nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+			out.UpdatedAt = r.RevokedBefore
+		} else if !item.IAMUser.IsDeleteReq {
+			if r.Deleted || r.timestamp().After(item.UpdatedAt) {
+				return out, errIAMStaleUpdate
+			}
+			out.RevokedBefore = r.RevokedBefore
+		}
+	}
+	return out, nil
+}
+
+func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error {
+	if item.GroupInfo != nil {
+		if item.GroupSnapshot {
+			ctx = context.WithValue(ctx, iamGroupSnapshotKey{}, true)
+		}
+		// A nil map also explicitly denotes unknown legacy grants. Do not
+		// turn an unrelated group edit into a new grant after a revocation.
+		ctx = withIAMGroupGrants(ctx, item.GroupGrants)
+	}
+	if !item.RevokedBefore.IsZero() {
+		if item.RevokedBefore.After(item.UpdatedAt) {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore)
+	}
+	switch item.Type {
+	case iamUserBoundaryType:
+		if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		return iamReplicationError(globalIAMSys.DeleteUser(withIAMReplicationTime(ctx, item.UserRevocation.Before), item.UserRevocation.User, true))
+	case iamGroupBoundaryType:
+		if item.GroupRevocation == nil || item.GroupRevocation.Group == "" || item.GroupRevocation.Before.IsZero() {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		_, err := globalIAMSys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, item.GroupRevocation.Before), item.GroupRevocation.Group, nil)
+		return iamReplicationError(err)
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
+		}
+		p, err := policy.ParseConfig(bytes.NewReader(item.Policy))
+		if err != nil {
+			return err
+		}
+		if p.IsEmpty() {
+			p = nil
+		}
+		return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, p, item.UpdatedAt)
+	case madmin.SRIAMItemSvcAcc:
+		return globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
+	case madmin.SRIAMItemPolicyMapping:
+		return globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
+	case madmin.SRIAMItemSTSAcc:
+		return globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
+	case madmin.SRIAMItemIAMUser:
+		return globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
+	case madmin.SRIAMItemGroupInfo:
+		return globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
+	default:
+		return errSRInvalidRequest(errInvalidArgument)
+	}
+}
+
+func (a adminAPIHandlers) SRPeerIAMRevisions(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	if obj, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction); obj == nil {
+		return
+	}
+	var failures []string
+	if r.Method == http.MethodPut {
+		var batch iamRevisionBatch
+		if err := parseJSONBody(ctx, r.Body, &batch, ""); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if batch.Version != iamRevisionProtocol || len(batch.Items) == 0 || len(batch.Items) > maxIAMRevisionBatch {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
+			return
+		}
+		for i, item := range batch.Items {
+			if err := applyIAMReplicationItem(ctx, item); err != nil {
+				failures = append(failures, fmt.Sprintf("item %d (%s): %v", i, item.Type, err))
+			}
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: globalIAMSys.store.iamRevisionStatus(), Errors: failures})
+}
+
+// A site endpoint can balance requests across nodes sharing durable IAM state.
+// Switching between known node incarnations preserves ACKs; a new incarnation
+// conservatively invalidates them so restoring an old backend cannot inherit
+// acknowledgements from before the restore.
+func (p *iamRevisionProgress) observePeer(status iamRevisionStatus) {
+	if p.Instances == nil {
+		p.Instances = make(map[string]string)
+	}
+	if p.Instances[status.Node] != status.Instance || p.Acknowledged == nil {
+		p.Instances[status.Node] = status.Instance
+		p.Acknowledged = make(map[string]string)
+	}
+}

cmd/iam-replication-protocol_test.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+func TestIAMRevisionProtocolDoesNotFallBackToLegacy(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
+			t.Errorf("unsafe fallback path: %s", r.URL.Path)
+		}
+		w.WriteHeader(http.StatusNotFound)
+		_, _ = w.Write([]byte(`{"Code":"NotImplemented","Message":"old server"}`))
+	}))
+	defer server.Close()
+	client, err := madmin.New(strings.TrimPrefix(server.URL, "http://"), "test-access", "valid-test-secret", false)
+	mustIAM(t, err)
+	_, err = executeIAMRevisionRequest(context.Background(), client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{{SRIAMItem: madmin.SRIAMItem{Type: iamUserBoundaryType}, UserRevocation: &iamUserBoundary{User: "recreated", Before: UTCNow()}}}})
+	if err == nil || requests.Load() != 1 {
+		t.Fatalf("old peer must reject without fallback, err=%v requests=%d", err, requests.Load())
+	}
+}
+
+type iamNoHealingScanStore struct{ IAMStorageAPI }
+
+func (s *iamNoHealingScanStore) listIAMConfigPaths(context.Context) ([]string, error) {
+	panic("healing must use the loaded revision index")
+}
+
+func TestIAMRevisionHealingAcknowledgements(t *testing.T) {
+	for _, balanced := range []bool{false, true} {
+		t.Run(fmt.Sprintf("load_balanced_%t", balanced), func(t *testing.T) { testIAMRevisionHealingAcknowledgements(t, balanced) })
+	}
+}
+
+func testIAMRevisionHealingAcknowledgements(t *testing.T, balanced bool) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "ack-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	for i := range maxIAMRevisionBatch*2 + 1 {
+		at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
+		mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath(fmt.Sprintf("ack-%04d", i), regUser)))
+	}
+	sys.store.IAMStorageAPI = &iamNoHealingScanStore{IAMStorageAPI: sys.store.IAMStorageAPI}
+	var mu sync.Mutex
+	var puts, gets int
+	var applied int
+	instance := "boot-1"
+	failSecondBatch := true
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		mu.Lock()
+		defer mu.Unlock()
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
+			t.Errorf("unexpected request: %s", r.URL.Path)
+			w.WriteHeader(404)
+			return
+		}
+		var failures []string
+		if r.Method == http.MethodGet {
+			gets++
+		} else {
+			puts++
+			var batch iamRevisionBatch
+			if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+				t.Error(err)
+				w.WriteHeader(400)
+				return
+			}
+			if len(batch.Items) > maxIAMRevisionBatch {
+				t.Error("batch exceeds limit")
+			}
+			if failSecondBatch && puts == 2 {
+				failures = []string{"injected item error"}
+			} else {
+				applied += len(batch.Items)
+			}
+		}
+		node := "node-1"
+		if balanced {
+			node = fmt.Sprintf("node-%d", (gets+puts)%2+1)
+		}
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: node + instance, Digest: fmt.Sprintf("%d", applied)}, Errors: failures})
+	}))
+	defer server.Close()
+	c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "ack-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
+	if err := c.healIAMDeletions(ctx); err == nil {
+		t.Fatal("item failure was hidden")
+	}
+	mu.Lock()
+	if puts != 3 || applied != maxIAMRevisionBatch+1 {
+		t.Errorf("failed middle batch blocked later revocations: puts=%d applied=%d", puts, applied)
+	}
+	failSecondBatch = false
+	applied++ // Unrelated remote mutation changes its digest.
+	mu.Unlock()
+	at := UTCNow()
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath("ack-new-local", regUser)))
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	if puts != 5 {
+		t.Errorf("did not resume at unacknowledged batch: puts=%d", puts)
+	}
+	mu.Unlock()
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	if puts != 5 || gets != 3 {
+		t.Errorf("converged records were replayed: puts=%d gets=%d", puts, gets)
+	}
+	instance = "boot-2"
+	mu.Unlock()
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	defer mu.Unlock()
+	if puts != 8 {
+		t.Fatalf("peer restart reused an old acknowledgement: puts=%d", puts)
+	}
+}
+
+func TestIAMRevisionIndexRebuildsFromStorage(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	const user = "index-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-parent-password", Status: madmin.AccountEnabled}
+	_, err := sys.CreateUser(ctx, user, req)
+	mustIAM(t, err)
+	mustIAM(t, sys.DeleteUser(ctx, user, false))
+	before := sys.store.revisionIndex().snapshot()
+	store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+	mustIAM(t, store.LoadIAMCache(ctx, true))
+	if iamRevisionDigest(before) != iamRevisionDigest(store.revisionIndex().snapshot()) {
+		t.Fatal("ordinary IAM loading did not restore the deletion index")
+	}
+	_, err = store.AddUser(ctx, user, req)
+	mustIAM(t, err)
+	r := store.revisionIndex().get(getUserIdentityPath(user, regUser))
+	if r.Deleted || r.RevokedBefore.IsZero() {
+		t.Fatal("recreation discarded the retained boundary")
+	}
+	if r.Credentials.SecretKey != "" || r.Credentials.SessionToken != "" {
+		t.Fatal("index retained credentials")
+	}
+}

cmd/iam-revision-boundary_test.go

@@ -0,0 +1,393 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+func prepareIAMRevisionFixture(t testing.TB, backend ...string) (context.Context, *IAMSys, ObjectLayer) {
+	t.Helper()
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	disks, err := getRandomDisks(1)
+	mustIAM(t, err)
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	mustIAM(t, err)
+	initAllSubsystems(ctx)
+	// Deliberately omit the periodic refresh goroutine. Fault injection can
+	// replace this fixture's storage interface without racing initialization.
+	var client *etcd.Client
+	if len(backend) != 0 && backend[0] == "etcd" {
+		endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+		if endpoint == "" {
+			cancel()
+			obj.Shutdown(context.Background())
+			os.RemoveAll(disks[0])
+			t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+		}
+		client, err = etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
+		mustIAM(t, err)
+		prefix := fmt.Sprintf("/silo-boundary-test/%d/", time.Now().UnixNano())
+		client.KV = namespace.NewKV(client.KV, prefix)
+		client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
+		t.Cleanup(func() { client.Delete(context.Background(), "", etcd.WithPrefix()); client.Close() })
+	}
+	globalIAMSys.initStore(obj, client)
+	mustIAM(t, globalIAMSys.Load(ctx, true))
+	t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
+	return ctx, globalIAMSys, obj
+}
+
+func mustIAM(t testing.TB, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+var errIAMInjectedWrite = errors.New("injected IAM persistence failure")
+
+type iamFailingCleanupStore struct {
+	IAMStorageAPI
+	parentPath   string
+	beforeCommit bool
+}
+
+func (s *iamFailingCleanupStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
+	if s.beforeCommit || path != s.parentPath {
+		return errIAMInjectedWrite
+	}
+	return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
+}
+
+func TestIAMRevocationCommitBoundary(t *testing.T) {
+	for _, before := range []bool{true, false} {
+		name := "after_identity_commit"
+		if before {
+			name = "before_identity_commit"
+		}
+		t.Run(name, func(t *testing.T) {
+			ctx, sys, obj := prepareIAMRevisionFixture(t)
+			const user = "commit-boundary-user"
+			origin := UTCNow().Add(-time.Hour)
+			req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+			_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
+			mustIAM(t, err)
+			_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(time.Minute)), user, "readwrite", regUser, false)
+			mustIAM(t, err)
+			_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(time.Minute)), "commit-group", []string{user})
+			mustIAM(t, err)
+			_, err = sys.PolicyDBSet(ctx, "commit-group", "readwrite", regUser, true)
+			mustIAM(t, err)
+			child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), user, nil, newServiceAccountOpts{accessKey: "commit-child", secretKey: "valid-child-password"})
+			mustIAM(t, err)
+			args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+			if !sys.IsAllowed(args) {
+				t.Fatal("fixture has no grant")
+			}
+			siblingStore := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			mustIAM(t, siblingStore.LoadIAMCache(ctx, true))
+			sibling := &IAMSys{store: siblingStore, usersSysType: MinIOUsersSysType}
+			tg, err := grid.SetupTestGrid(2)
+			mustIAM(t, err)
+			defer tg.Cleanup()
+			var notifications atomic.Int32
+			mustIAM(t, deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+				notifications.Add(1)
+				if err := sibling.LoadUserAfterDelete(ctx, r.Get(peerRESTUser)); err != nil {
+					return grid.NoPayload{}, grid.NewRemoteErr(err)
+				}
+				return grid.NoPayload{}, nil
+			}))
+			host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+			mustIAM(t, err)
+			globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{host: host, gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}}}
+			original := sys.store.IAMStorageAPI
+			sys.store.IAMStorageAPI = &iamFailingCleanupStore{IAMStorageAPI: original, parentPath: getUserIdentityPath(user, regUser), beforeCommit: before}
+			boundary := origin.Add(2 * time.Minute)
+			err = sys.DeleteUser(withIAMReplicationTime(ctx, boundary), user, true)
+			if !errors.Is(err, errIAMInjectedWrite) {
+				t.Fatalf("expected write failure, got %v", err)
+			}
+			sys.store.IAMStorageAPI = original
+			r, err := loadIAMRevision(ctx, original, getUserIdentityPath(user, regUser))
+			mustIAM(t, err)
+			if before {
+				if r.Deleted || !sys.IsAllowed(args) || !sibling.IsAllowed(args) || notifications.Load() != 0 {
+					t.Fatal("failure before commit changed the identity or grant")
+				}
+				return
+			}
+			if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
+				t.Fatal("cleanup failure lost durable revocation")
+			}
+			if sys.IsAllowed(args) || sibling.IsAllowed(args) || notifications.Load() != 1 {
+				t.Fatal("cleanup failure retained old permission")
+			}
+			// Subsequent fixture writes need no additional RPC handlers.
+			globalNotificationSys = &NotificationSys{}
+			// Recreate after the partial cleanup. The old mapping, group member
+			// and child still exist in storage; none may authorize this identity.
+			_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), user, req)
+			mustIAM(t, err)
+			reloaded := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			mustIAM(t, reloaded.LoadIAMCache(ctx, true))
+			fresh := &IAMSys{store: reloaded, usersSysType: MinIOUsersSysType}
+			if fresh.IsAllowed(args) {
+				t.Fatal("cold reload restored partially cleaned-up grants")
+			}
+			if _, ok := reloaded.GetUser(child.AccessKey); ok {
+				t.Fatal("cold reload restored the old child")
+			}
+			gd, err := reloaded.GetGroupDescription("commit-group")
+			mustIAM(t, err)
+			if len(gd.Members) != 0 {
+				t.Fatalf("listing exposed a revoked group relation: %v", gd.Members)
+			}
+			_, err = sys.AddUsersToGroup(ctx, "commit-group", []string{user})
+			mustIAM(t, err)
+			if !sys.IsAllowed(args) {
+				t.Fatal("explicit new group grant was not accepted")
+			}
+		})
+	}
+}
+
+func TestIAMGroupGrantVersionsSurviveSnapshotsAndRecreation(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	for _, user := range []string{"grant-alice", "grant-bob"} {
+		_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
+		mustIAM(t, err)
+	}
+	grant := origin.Add(time.Minute)
+	_, err := sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), "grant-group", []string{"grant-alice"})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(ctx, "grant-group", "readwrite", regUser, true)
+	mustIAM(t, err)
+	boundary := origin.Add(2 * time.Minute)
+	mustIAM(t, sys.DeleteUser(withIAMReplicationTime(ctx, boundary), "grant-alice", false))
+	_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), "grant-alice", req)
+	mustIAM(t, err)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), "grant-group", []string{"grant-bob"})
+	mustIAM(t, err)
+	_, err = sys.SetGroupStatus(withIAMReplicationTime(ctx, origin.Add(5*time.Minute)), "grant-group", true)
+	mustIAM(t, err)
+	var gi GroupInfo
+	mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
+	if !gi.MemberGrants["grant-alice"].Equal(grant) {
+		t.Fatal("unrelated group edits refreshed an old grant")
+	}
+	args := policy.Args{AccountName: "grant-alice", Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	for _, stale := range []time.Time{grant, boundary, {}} {
+		item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(6 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: "grant-group", Members: []string{"grant-alice", "grant-bob"}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{"grant-alice": stale, "grant-bob": origin.Add(4 * time.Minute)}}
+		mustIAM(t, applyIAMReplicationItem(ctx, item))
+		mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+		if sys.IsAllowed(args) {
+			t.Fatalf("snapshot restored revoked grant %s", stale)
+		}
+		gd, err := sys.GetGroupDescription("grant-group")
+		mustIAM(t, err)
+		if len(gd.Members) != 1 || gd.Members[0] != "grant-bob" {
+			t.Fatalf("inconsistent effective members: %v", gd.Members)
+		}
+	}
+	// Only an explicit post-revocation grant restores access.
+	freshAt, err := sys.AddUsersToGroup(ctx, "grant-group", []string{"grant-alice"})
+	mustIAM(t, err)
+	if !sys.IsAllowed(args) {
+		t.Fatal("explicit regrant rejected")
+	}
+	mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+	mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
+	if !gi.MemberGrants["grant-alice"].Equal(freshAt) {
+		t.Fatal("new grant version was not persisted")
+	}
+	if !gi.MemberGrants["grant-bob"].Equal(origin.Add(4 * time.Minute)) {
+		t.Fatal("regranting Alice changed Bob's grant")
+	}
+}
+
+func TestIAMGroupRevocationCommitAndRecreation(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) { testIAMGroupRevocationCommitAndRecreation(t, backend) })
+	}
+}
+
+func testIAMGroupRevocationCommitAndRecreation(t *testing.T, backend string) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
+	origin := UTCNow().Add(-time.Hour)
+	user, group := "group-boundary-user", "group-boundary"
+	_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	grant, boundary := origin.Add(time.Minute), origin.Add(2*time.Minute)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
+	mustIAM(t, err)
+	// A newer mapping must not veto the authoritative group deletion.
+	_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, boundary), group, nil)
+	mustIAM(t, err)
+	r, err := loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
+	mustIAM(t, err)
+	if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
+		t.Fatal("newer mapping swallowed group deletion")
+	}
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), group, nil)
+	mustIAM(t, err)
+	for _, at := range []time.Time{grant, boundary, {}} {
+		item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(5 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{user: at}}
+		mustIAM(t, applyIAMReplicationItem(ctx, item))
+		gd, err := sys.GetGroupDescription(group)
+		mustIAM(t, err)
+		if len(gd.Members) != 0 {
+			t.Fatalf("group recreation restored grant %s", at)
+		}
+	}
+	_, err = sys.AddUsersToGroup(ctx, group, []string{user})
+	mustIAM(t, err)
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	if !sys.IsAllowed(args) {
+		t.Fatal("explicit group regrant was rejected")
+	}
+	// The newer live snapshot may arrive before an older group deletion.
+	lateBoundary := origin.Add(6 * time.Minute)
+	_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, lateBoundary), group, nil)
+	mustIAM(t, err)
+	r, err = loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
+	mustIAM(t, err)
+	if r.Deleted || !r.RevokedBefore.Equal(lateBoundary) {
+		t.Fatal("late deletion lost the live group's revocation boundary")
+	}
+	// The old mapping is now revoked; a new explicit mapping restores access.
+	if sys.IsAllowed(args) {
+		t.Fatal("late group boundary retained an old mapping")
+	}
+	_, err = sys.PolicyDBSet(ctx, group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+	if es, ok := sys.store.IAMStorageAPI.(*IAMEtcdStore); ok {
+		store.IAMStorageAPI = newIAMEtcdStore(es.client, MinIOUsersSysType)
+	}
+	mustIAM(t, store.LoadIAMCache(ctx, true))
+	fresh := &IAMSys{store: store, usersSysType: MinIOUsersSysType}
+	if !fresh.IsAllowed(args) {
+		t.Fatal("reload lost explicit grants after a retained group boundary")
+	}
+	item, err := globalSiteReplicationSys.replicationItem(ctx, madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group}}, UpdatedAt: r.timestamp()})
+	mustIAM(t, err)
+	if !item.RevokedBefore.Equal(lateBoundary) || !item.GroupGrants[user].After(lateBoundary) {
+		t.Fatal("group snapshot lost revision metadata")
+	}
+}
+
+// A committed revision is observable before all cached dependents have been
+// cleaned up. Every authorization read must apply that boundary in this window.
+func TestIAMCachedMappingHonorsCommittedRevision(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	parent := "cached-external-parent"
+	_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), parent, "readwrite", stsUser, false)
+	mustIAM(t, err)
+	policies, err := sys.PolicyDBGet(parent)
+	mustIAM(t, err)
+	if len(policies) == 0 {
+		t.Fatal("fixture has no STS-parent mapping")
+	}
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Deleted: true, UpdatedAt: origin.Add(time.Minute)}, getMappedPolicyPath(parent, stsUser, false)))
+	policies, err = sys.PolicyDBGet(parent)
+	mustIAM(t, err)
+	if len(policies) != 0 {
+		t.Fatal("cached STS mapping ignored its own namespace tombstone")
+	}
+
+	user, group := "cached-group-user", "cached-group"
+	_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	grant := origin.Add(5 * time.Minute)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	if !sys.IsAllowed(args) {
+		t.Fatal("fixture has no group grant")
+	}
+	// A late deletion preserves the newer member grant but revokes the older
+	// policy mapping. Simulate the interval before mapping cleanup completes.
+	gi := GroupInfo{Version: 1, Status: statusEnabled, Members: []string{user}, MemberGrants: map[string]time.Time{user: grant}, UpdatedAt: grant, RevokedBefore: origin.Add(2 * time.Minute)}
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &gi, getGroupInfoPath(group)))
+	if sys.IsAllowed(args) {
+		t.Fatal("cached group mapping ignored the committed group boundary")
+	}
+	gd, err := sys.GetGroupDescription(group)
+	mustIAM(t, err)
+	if gd.Policy != "" {
+		t.Fatal("group listing exposed a revoked mapping")
+	}
+}
+
+type (
+	iamExpiryLockFailure struct {
+		ObjectLayer
+		path string
+	}
+	iamFailedExpiryLock struct{ RWLocker }
+)
+
+func (o *iamExpiryLockFailure) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path+".revision-lock" {
+		return &iamFailedExpiryLock{RWLocker: lock}
+	}
+	return lock
+}
+
+func (l *iamFailedExpiryLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) {
+	return LockContext{}, errIAMInjectedWrite
+}
+
+func TestIAMExpiredCredentialCleanupDoesNotBlockLoading(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "healthy-user", madmin.AddOrUpdateUserReq{SecretKey: "healthy-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(ctx, "healthy-user", "readwrite", regUser, false)
+	mustIAM(t, err)
+	c, _, err := sys.NewServiceAccount(ctx, "healthy-user", nil, newServiceAccountOpts{accessKey: "expired-service", secretKey: "expired-service-password"})
+	mustIAM(t, err)
+	c.Expiration = UTCNow().Add(-time.Hour)
+	path := getUserIdentityPath(c.AccessKey, svcUser)
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: c, UpdatedAt: UTCNow()}, path))
+	// A cold loader sees the existing version but cannot acquire the cleanup
+	// write lock. Healthy users must still load; the expired one stays denied.
+	fresh := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(&iamExpiryLockFailure{ObjectLayer: obj, path: path}, MinIOUsersSysType)}
+	mustIAM(t, fresh.LoadIAMCache(ctx, true))
+	if _, ok := fresh.GetUser("healthy-user"); !ok {
+		t.Fatal("cleanup failure prevented healthy IAM state from loading")
+	}
+	if _, ok := fresh.GetUser(c.AccessKey); ok {
+		t.Fatal("cleanup failure admitted an expired service account")
+	}
+	r, err := loadIAMRevision(ctx, fresh, path)
+	mustIAM(t, err)
+	if r.Deleted || !r.Credentials.IsExpired() {
+		t.Fatal("failed cleanup lost the existing expired revision")
+	}
+}

cmd/iam-revision-index.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+// This index is rebuilt by the existing IAM loaders and updated by successful
+// storage operations. It avoids a second full IAM walk during every heal pass.
+// It is an optimization of the durable records, never a reason to delete them.
+// The index contains no secrets or grants.
+type iamParentRevision struct {
+	deleted bool
+	before  time.Time
+}
+
+type iamRevisionIndex struct {
+	mu         sync.RWMutex
+	items      map[string]iamRevision
+	parents    map[string]iamParentRevision
+	floors     map[string]time.Time
+	generation uint64
+}
+
+func (idx *iamRevisionIndex) observe(path string, data []byte) {
+	if !strings.HasPrefix(path, iamConfigPrefix+"/") {
+		return
+	}
+	var r iamRevision
+	if json.Unmarshal(data, &r) != nil {
+		return // The caller reports malformed data using its normal decoder.
+	}
+	r.Credentials = auth.Credentials{ParentUser: r.Credentials.ParentUser, Expiration: r.Credentials.Expiration}
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+	if strings.HasPrefix(path, iamConfigUsersPrefix) {
+		// Keep a compact name-keyed view for the authentication hot path;
+		// constructing a config path on every S3 request allocates needlessly.
+		defer func() {
+			name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+			if current, ok := idx.items[path]; ok {
+				if idx.parents == nil {
+					idx.parents = make(map[string]iamParentRevision)
+				}
+				idx.parents[name] = iamParentRevision{deleted: current.Deleted, before: current.RevokedBefore}
+			} else {
+				delete(idx.parents, name)
+			}
+		}()
+	}
+	if floor, ok := idx.floors[path]; ok && r.timestamp().Before(floor) {
+		return
+	}
+	if previous, ok := idx.items[path]; ok {
+		// A concurrent read that began before a write must not roll it back.
+		if previous.timestamp().After(r.timestamp()) || (previous.Deleted && !r.Deleted && !r.timestamp().After(previous.timestamp())) {
+			return
+		}
+		if previous.RevokedBefore.After(r.RevokedBefore) {
+			r.RevokedBefore = previous.RevokedBefore
+		}
+		if previous.timestamp().Equal(r.timestamp()) && previous.Deleted == r.Deleted && previous.RevokedBefore.Equal(r.RevokedBefore) {
+			return
+		}
+	}
+	if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
+		if _, tracked := idx.items[path]; tracked {
+			delete(idx.items, path)
+			idx.generation++
+		}
+		delete(idx.floors, path)
+		return
+	}
+	if !r.Deleted && r.RevokedBefore.IsZero() {
+		_, tracked := idx.items[path]
+		_, hasFloor := idx.floors[path]
+		if tracked || hasFloor {
+			if idx.floors == nil {
+				idx.floors = make(map[string]time.Time)
+			}
+			idx.floors[path] = r.timestamp()
+		}
+		if tracked {
+			delete(idx.items, path)
+			idx.generation++
+		}
+		return
+	}
+	if idx.items == nil {
+		idx.items = make(map[string]iamRevision)
+	}
+	idx.items[path] = r
+	delete(idx.floors, path)
+	idx.generation++
+}
+
+func (idx *iamRevisionIndex) get(path string) iamRevision {
+	if idx == nil {
+		return iamRevision{}
+	}
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return idx.items[path]
+}
+
+func (idx *iamRevisionIndex) snapshot() map[string]iamRevision {
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+	for path, r := range idx.items {
+		if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
+			delete(idx.items, path)
+			delete(idx.floors, path)
+			idx.generation++
+		}
+	}
+	return maps.Clone(idx.items)
+}
+
+func (idx *iamRevisionIndex) count() int {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return len(idx.items)
+}
+
+// A process-local generation plus the protocol's instance ID is sufficient
+// for acknowledgements. Avoid hashing the entire index on every IAM write.
+func (idx *iamRevisionIndex) digest() string {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return fmt.Sprintf("%x:%x", idx.generation, len(idx.items))
+}
+
+func (idx *iamRevisionIndex) forget(path string) {
+	idx.mu.Lock()
+	if _, ok := idx.items[path]; ok {
+		delete(idx.items, path)
+		idx.generation++
+	}
+	delete(idx.floors, path)
+	if strings.HasPrefix(path, iamConfigUsersPrefix) {
+		delete(idx.parents, strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile))
+	}
+	idx.mu.Unlock()
+}
+
+func (c *iamCache) userRevocation(user string) iamRevision {
+	r := c.revisions.parentRevision(user)
+	if u, ok := c.iamUsersMap[user]; ok && u.RevokedBefore.After(r.RevokedBefore) {
+		r.RevokedBefore = u.RevokedBefore
+	}
+	return r
+}
+
+func (c *iamCache) groupMemberAllowed(member string, grantedAt, groupBoundary time.Time) bool {
+	r := c.userRevocation(member)
+	return !r.Deleted && (r.RevokedBefore.IsZero() || grantedAt.After(r.RevokedBefore)) && (groupBoundary.IsZero() || grantedAt.After(groupBoundary))
+}
+
+func iamMappingParentPath(path string) string {
+	kind, name, ok := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+	if !ok {
+		return ""
+	}
+	name = strings.TrimSuffix(name, ".json")
+	switch kind {
+	case "users", "sts-users":
+		return getUserIdentityPath(name, regUser)
+	case "service-accounts":
+		return getUserIdentityPath(name, svcUser)
+	case "groups":
+		return getGroupInfoPath(name)
+	}
+	return ""
+}
+
+func (idx *iamRevisionIndex) mappingAllowed(path string, mp MappedPolicy) bool {
+	if mp.Deleted || idx.get(path).Deleted {
+		return false
+	}
+	r := idx.get(iamMappingParentPath(path))
+	return !r.Deleted && (r.RevokedBefore.IsZero() || mp.UpdatedAt.After(r.RevokedBefore))
+}
+
+// Apply the persisted commit boundary even before dependent cache cleanup has
+// completed. The map namespace is part of the authorization record's identity.
+func (c *iamCache) cachedMappedPolicy(name string, userType IAMUserType, isGroup bool) (MappedPolicy, bool) {
+	var mp MappedPolicy
+	var ok bool
+	switch {
+	case isGroup:
+		mp, ok = c.iamGroupPolicyMap.Load(name)
+	case userType == stsUser:
+		mp, ok = c.iamSTSPolicyMap.Load(name)
+	default:
+		mp, ok = c.iamUserPolicyMap.Load(name)
+	}
+	if !ok || !c.revisions.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), mp) {
+		return MappedPolicy{}, false
+	}
+	return mp, true
+}
+
+func (idx *iamRevisionIndex) parentRevision(user string) iamRevision {
+	if idx == nil {
+		return iamRevision{}
+	}
+	idx.mu.RLock()
+	p := idx.parents[user]
+	idx.mu.RUnlock()
+	return iamRevision{Deleted: p.deleted, RevokedBefore: p.before}
+}

cmd/iam-revision-lock_test.go

@@ -0,0 +1,305 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+type iamRevisionLockObserver struct {
+	ObjectLayer
+	path    string
+	waiting chan struct{}
+	once    sync.Once
+}
+
+func (o *iamRevisionLockObserver) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path {
+		return &iamRevisionObservedLock{RWLocker: lock, observe: func() { o.once.Do(func() { close(o.waiting) }) }}
+	}
+	return lock
+}
+
+type iamRevisionObservedLock struct {
+	RWLocker
+	observe func()
+}
+
+func (l *iamRevisionObservedLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	l.observe()
+	return l.RWLocker.GetLock(ctx, timeout)
+}
+
+type iamRevisionWatchObserver struct {
+	etcd.Watcher
+	waiting chan struct{}
+	once    sync.Once
+}
+
+func (w *iamRevisionWatchObserver) Watch(ctx context.Context, key string, opts ...etcd.OpOption) etcd.WatchChan {
+	w.once.Do(func() { close(w.waiting) })
+	return w.Watcher.Watch(ctx, key, opts...)
+}
+
+// Simulate an unavailable cleanup RPC. Mutex.Lock calls Delete after its wait
+// is canceled; that RPC must inherit a deadline too, not Client.Ctx() forever.
+type iamRevisionCleanupBlocker struct {
+	etcd.KV
+	release chan struct{}
+}
+
+func (b *iamRevisionCleanupBlocker) Delete(ctx context.Context, key string, opts ...etcd.OpOption) (*etcd.DeleteResponse, error) {
+	if strings.Contains(key, "/iam-revision-locks/") {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-b.release:
+		}
+	}
+	return b.KV.Delete(ctx, key, opts...)
+}
+
+type iamRevisionReadBlocker struct {
+	IAMStorageAPI
+	path    string
+	after   int
+	waiting chan struct{}
+}
+
+func (b *iamRevisionReadBlocker) loadIAMConfig(ctx context.Context, item any, path string) error {
+	if path == b.path {
+		b.after--
+		if b.after == 0 {
+			close(b.waiting)
+			<-ctx.Done()
+			return ctx.Err()
+		}
+	}
+	return b.IAMStorageAPI.loadIAMConfig(ctx, item, path)
+}
+
+func TestIAMRevisionReadDoesNotBlockAuthentication(t *testing.T) {
+	for _, stage := range []struct {
+		name   string
+		offset time.Duration
+	}{{"deletion", time.Minute}, {"retained_revocation", -time.Minute}} {
+		t.Run(stage.name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			disks, err := getRandomDisks(1)
+			if err != nil {
+				t.Fatal(err)
+			}
+			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() {
+				obj.Shutdown(context.Background())
+				os.RemoveAll(disks[0])
+			})
+			store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			const user = "read-blocked-parent"
+			created, err := store.AddUser(ctx, user, madmin.AddOrUpdateUserReq{SecretKey: "original-password", Status: madmin.AccountEnabled})
+			if err != nil {
+				t.Fatal(err)
+			}
+			blocked := &iamRevisionReadBlocker{IAMStorageAPI: store.IAMStorageAPI, path: getUserIdentityPath(user, regUser), after: 1, waiting: make(chan struct{})}
+			store.IAMStorageAPI = blocked
+			done := make(chan error, 1)
+			go func() {
+				done <- store.DeleteUser(withIAMReplicationTime(ctx, created.Add(stage.offset)), user, regUser)
+			}()
+			defer func() { cancel(); <-done }()
+			select {
+			case <-blocked.waiting:
+			case <-time.After(5 * time.Second):
+				t.Fatal("revision read was not attempted")
+			}
+			read := make(chan bool, 1)
+			go func() {
+				u, ok := store.GetUser(user)
+				read <- ok && u.Credentials.SecretKey == "original-password"
+			}()
+			select {
+			case ok := <-read:
+				if !ok {
+					t.Fatal("pending revision read changed the cached identity")
+				}
+			case <-time.After(time.Second):
+				t.Fatal("revision read blocked cached authentication")
+			}
+		})
+	}
+}
+
+func TestIAMRevisionLockContention(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+			if backend == "etcd" && endpoint == "" {
+				t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+			}
+			for _, outcome := range []string{"release", "cancel", "default_timeout"} {
+				t.Run(outcome, func(t *testing.T) {
+					resetTestGlobals()
+					t.Cleanup(resetTestGlobals)
+					ctx, cancel := context.WithCancel(context.Background())
+					defer cancel()
+					oldTimeout := defaultContextTimeout
+					defaultContextTimeout = 2 * time.Second
+					t.Cleanup(func() { defaultContextTimeout = oldTimeout })
+					must := func(err error) {
+						t.Helper()
+						if err != nil {
+							t.Fatal(err)
+						}
+					}
+					const user = "contended-user"
+					path := getUserIdentityPath(user, regUser)
+					waiting := make(chan struct{})
+					var store *IAMStoreSys
+					var hold func() func()
+					unblockCleanup := func() {}
+					if backend == "object" {
+						disks, err := getRandomDisks(1)
+						must(err)
+						obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+						must(err)
+						t.Cleanup(func() {
+							obj.Shutdown(context.Background())
+							os.RemoveAll(disks[0])
+						})
+						observed := &iamRevisionLockObserver{ObjectLayer: obj, path: path + ".revision-lock", waiting: waiting}
+						store = &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+						hold = func() func() {
+							lock := obj.NewNSLock(minioMetaBucket, observed.path)
+							lc, err := lock.GetLock(ctx, newDynamicTimeout(time.Second, time.Second))
+							must(err)
+							store.IAMStorageAPI.(*IAMObjectStore).objAPI = observed
+							return func() { lock.Unlock(lc) }
+						}
+					} else {
+						client, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: time.Second})
+						must(err)
+						t.Cleanup(func() { client.Close() })
+						prefix := fmt.Sprintf("/silo-lock-test/%d/", time.Now().UnixNano())
+						client.KV = namespace.NewKV(client.KV, prefix)
+						client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
+						store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(client, MinIOUsersSysType)}
+						hold = func() func() {
+							session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+							must(err)
+							lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+							must(lock.Lock(ctx))
+							client.Watcher = &iamRevisionWatchObserver{Watcher: client.Watcher, waiting: waiting}
+							blocker := &iamRevisionCleanupBlocker{KV: client.KV, release: make(chan struct{})}
+							client.KV = blocker
+							unblockCleanup = sync.OnceFunc(func() { close(blocker.release) })
+							t.Cleanup(unblockCleanup)
+							return func() { session.Close() }
+						}
+					}
+					request := func(secret string) madmin.AddOrUpdateUserReq {
+						return madmin.AddOrUpdateUserReq{SecretKey: secret, Status: madmin.AccountEnabled}
+					}
+					_, err := store.AddUser(ctx, user, request("original-password"))
+					must(err)
+					release := sync.OnceFunc(hold())
+					t.Cleanup(release)
+					writeCtx, cancelWrite := context.WithCancel(ctx)
+					defer cancelWrite()
+					first, second := make(chan error, 1), make(chan error, 1)
+					var writers sync.WaitGroup
+					t.Cleanup(func() {
+						cancelWrite()
+						unblockCleanup()
+						release()
+						writers.Wait()
+					})
+					writers.Go(func() {
+						_, err := store.AddUser(writeCtx, user, request("first-password"))
+						first <- err
+					})
+					select {
+					case <-waiting:
+					case <-time.After(5 * time.Second):
+						t.Fatal("writer did not attempt the held revision lock")
+					}
+					// A second writer must queue without taking the cache's RWMutex:
+					// Go's writer preference would otherwise block every new reader.
+					writers.Go(func() {
+						_, err := store.AddUser(ctx, user, request("second-password"))
+						second <- err
+					})
+					select {
+					case err := <-second:
+						t.Fatalf("second writer bypassed the first: %v", err)
+					case <-time.After(50 * time.Millisecond):
+					}
+					read := make(chan UserIdentity, 1)
+					go func() {
+						u, _ := store.GetUser(user)
+						read <- u
+					}()
+					select {
+					case u := <-read:
+						if u.Credentials.SecretKey != "original-password" {
+							t.Fatal("pending write changed the cached credential")
+						}
+					case <-time.After(time.Second):
+						t.Fatal("distributed lock contention blocked cached authentication")
+					}
+					switch outcome {
+					case "release":
+						release()
+					case "cancel":
+						cancelWrite()
+					}
+					select {
+					case err := <-first:
+						if outcome == "release" {
+							must(err)
+						} else if err == nil {
+							t.Fatal("canceled or timed-out write succeeded")
+						}
+					case <-time.After(5 * time.Second):
+						t.Fatal("lock wait or cancellation cleanup exceeded its deadline")
+					}
+					release()
+					select {
+					case err := <-second:
+						must(err)
+					case <-time.After(5 * time.Second):
+						t.Fatal("queued writer did not recover after the first completed")
+					}
+					cached, ok := store.GetUser(user)
+					if !ok || cached.Credentials.SecretKey != "second-password" {
+						t.Fatal("cached write order was lost")
+					}
+					var persisted UserIdentity
+					must(store.loadIAMConfig(ctx, &persisted, path))
+					if persisted.Credentials.SecretKey != cached.Credentials.SecretKey || !persisted.UpdatedAt.Equal(cached.UpdatedAt) {
+						t.Fatal("persistent and cached revisions differ")
+					}
+				})
+			}
+		})
+	}
+}

cmd/iam-revision.go

@@ -0,0 +1,749 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+var errIAMStaleUpdate = errors.New("IAM update predates a stored revision or revocation")
+
+// The parent is still live; callers must not broadcast a user deletion when
+// only its revocation boundary was retained.
+var errIAMRevocationRetained = errors.New("IAM revocation recorded without deleting the record")
+
+// A revocation advances the boundary even when a newer identity already
+// exists. Keep this operation distinct from replacing/deleting that identity.
+type iamUserRevocation struct {
+	UserIdentity
+	retained bool
+}
+
+type iamGroupRevocation struct {
+	GroupInfo
+	retained     bool
+	requireEmpty bool
+}
+
+// Natural expiration is distinct from revoking a live credential. An expired
+// immutable STS token can be removed; a reusable service-account key retains
+// its revision so an older non-expiring credential cannot return.
+type iamExpireIdentity struct{}
+
+// The authoritative revocation is durable even if dependent cleanup fails.
+// Callers must publish it to sibling caches before returning the error.
+type iamCommittedCleanupError struct {
+	err      error
+	retained bool
+}
+
+func (e *iamCommittedCleanupError) Error() string {
+	return "IAM revocation committed; cleanup failed: " + e.err.Error()
+}
+func (e *iamCommittedCleanupError) Unwrap() error { return e.err }
+
+type iamReplicationTimeKey struct{}
+
+func withIAMReplicationTime(ctx context.Context, at time.Time) context.Context {
+	return context.WithValue(ctx, iamReplicationTimeKey{}, at)
+}
+
+func iamReplicationTime(ctx context.Context) (time.Time, bool) {
+	at, ok := ctx.Value(iamReplicationTimeKey{}).(time.Time)
+	return at, ok
+}
+
+func iamReplicationError(err error) error {
+	if errors.Is(err, errIAMStaleUpdate) {
+		// Retrying an obsolete event cannot change the result.
+		return nil
+	}
+	return wrapSRErr(err)
+}
+
+// Deletions occupy the original IAM config path. They contain no secret or
+// grant and are hidden by the normal loaders, but remain available to heal
+// and to timestamp comparisons after a restart. Do not age them out: a peer
+// can be offline indefinitely.
+type iamRevision struct {
+	UpdatedAt     time.Time        `json:"updatedAt"`
+	UpdateDate    time.Time        `json:"UpdateDate"`
+	Deleted       bool             `json:"deleted"`
+	RevokedBefore time.Time        `json:"revokedBefore"`
+	ExpiresAt     time.Time        `json:"expiresAt,omitempty"`
+	Credentials   auth.Credentials `json:"credentials"`
+}
+
+func (r iamRevision) timestamp() time.Time {
+	if r.UpdateDate.After(r.UpdatedAt) {
+		return r.UpdateDate
+	}
+	return r.UpdatedAt
+}
+
+func loadIAMRevision(ctx context.Context, store IAMStorageAPI, path string) (iamRevision, error) {
+	var r iamRevision
+	err := store.loadIAMConfig(ctx, &r, path)
+	if errors.Is(err, errConfigNotFound) {
+		err = nil
+	}
+	return r, err
+}
+
+func (store *IAMStoreSys) checkIAMRevision(ctx context.Context, path string, deleting bool) error {
+	at, replicated := iamReplicationTime(ctx)
+	if !replicated {
+		return nil
+	}
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		r, err := loadIAMRevision(ctx, store.IAMStorageAPI, path)
+		if err != nil {
+			return err
+		}
+		if r.timestamp().After(at) || (r.Deleted && !deleting && !at.After(r.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		return nil
+	})
+}
+
+// This signed claim records the parent's revocation boundary at issuance.
+// Unlike UpdatedAt, it cannot advance when an offline site edits an old child.
+// It travels in the existing service-account Claims and STS SessionToken fields.
+const iamParentRevocationClaim = "siloParentRevocation"
+
+func setIAMParentRevocationClaim(ctx context.Context, store IAMStorageAPI, parent string, claims map[string]any) error {
+	delete(claims, iamParentRevocationClaim)
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted {
+		return errIAMStaleUpdate
+	}
+	if !r.RevokedBefore.IsZero() {
+		claims[iamParentRevocationClaim] = r.RevokedBefore.Format(time.RFC3339Nano)
+	}
+	return nil
+}
+
+func iamCredentialSurvivesRevocation(cred auth.Credentials, at time.Time) bool {
+	if at.IsZero() {
+		return true
+	}
+	s, _ := cred.Claims[iamParentRevocationClaim].(string)
+	issuedAfter, err := time.Parse(time.RFC3339Nano, s)
+	return err == nil && !issuedAfter.Before(at)
+}
+
+// Parent revocations delete old children even if an offline peer has edited
+// them later. Preserve children that prove issuance after this revocation.
+func iamChildDeletionContext(ctx context.Context, child UserIdentity) (context.Context, bool) {
+	if at, replicated := iamReplicationTime(ctx); replicated {
+		if !at.IsZero() && iamCredentialSurvivesRevocation(child.Credentials, at) {
+			return ctx, false
+		}
+		if child.UpdatedAt.After(at) {
+			ctx = withIAMReplicationTime(ctx, child.UpdatedAt)
+		}
+	}
+	return ctx, true
+}
+
+// A delayed service account or STS event must not outlive deletion of its
+// built-in parent. The caller must populate Claims from the verified token.
+func checkIAMParentRevision(ctx context.Context, store IAMStorageAPI, cred auth.Credentials) error {
+	parent := cred.ParentUser
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted || !iamCredentialSurvivesRevocation(cred, r.RevokedBefore) {
+		return errIAMStaleUpdate
+	}
+	return nil
+}
+
+// Called with the IAM writer mutex and cache lock held. Persistence only
+// touches the caller's record, not the cache. Keep writers serialized while
+// allowing cached authentication reads throughout storage and lock waits.
+func (store *IAMStoreSys) withIAMStorage(ctx context.Context, fn func(context.Context) error) error {
+	store.IAMStorageAPI.unlock()
+	defer store.IAMStorageAPI.lock()
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	return fn(ctx)
+}
+
+func (store *IAMStoreSys) saveIAMRevision(ctx context.Context, path string, item any, opts ...options) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return saveIAMRevision(ctx, store.IAMStorageAPI, path, item, opts...)
+	})
+}
+
+func (store *IAMStoreSys) checkIAMParentRevision(ctx context.Context, cred auth.Credentials) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return checkIAMParentRevision(ctx, store.IAMStorageAPI, cred)
+	})
+}
+
+// Update the caller's record with the persisted revision before it is cached.
+func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item any, opts ...options) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	// Serialize compare-and-write across nodes, as well as goroutines. Use a
+	// separate lock name so saving the config does not reacquire this lock.
+	switch s := store.(type) {
+	case *IAMObjectStore:
+		lock := s.objAPI.NewNSLock(minioMetaBucket, path+".revision-lock")
+		lc, err := lock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		defer lock.Unlock(lc)
+		ctx = lc.Context()
+	case *IAMEtcdStore:
+		// Mutex.Lock also uses Client.Ctx() for cleanup after cancellation.
+		// Borrow the existing services with the operation's bounded context;
+		// never close this facade, which does not own those services.
+		client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(s.client.GetLogger()))
+		client.KV, client.Lease, client.Watcher = s.client.KV, s.client.Lease, s.client.Watcher
+		session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+		if err != nil {
+			return err
+		}
+		defer func() {
+			session.Orphan()
+			// A canceled operation must still release its lease when etcd is
+			// reachable. If it is unavailable, stop waiting and let it expire.
+			cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultContextTimeout)
+			defer cancel()
+			_, _ = s.client.Revoke(cleanupCtx, session.Lease())
+		}()
+		lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+		if err = lock.Lock(ctx); err != nil {
+			return err
+		}
+		// Revoking the session lease releases the lock, including on cancellation.
+	}
+	previous, err := loadIAMRevision(ctx, store, path)
+	if err != nil {
+		return err
+	}
+	if _, expiring := item.(*iamExpireIdentity); expiring {
+		sts := strings.HasPrefix(path, iamConfigSTSPrefix)
+		if previous.Deleted {
+			if sts && !previous.ExpiresAt.IsZero() && UTCNow().After(previous.ExpiresAt) {
+				return expireIAMSTSConfig(ctx, store, path)
+			}
+			return nil
+		}
+		if previous.timestamp().IsZero() || !previous.Credentials.IsExpired() {
+			return nil
+		}
+		if sts {
+			return expireIAMSTSConfig(ctx, store, path)
+		}
+		item = &UserIdentity{Version: 1, Deleted: true}
+		ctx = withIAMReplicationTime(ctx, previous.timestamp())
+	}
+	var revocation *iamUserRevocation
+	if op, ok := item.(*iamUserRevocation); ok {
+		revocation = op
+		op.UserIdentity = UserIdentity{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.UserIdentity = UserIdentity{Version: 1, Credentials: previous.Credentials, UpdatedAt: previous.timestamp(), RevokedBefore: origin}
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.UserIdentity
+	}
+	var groupRevocation *iamGroupRevocation
+	if op, ok := item.(*iamGroupRevocation); ok {
+		groupRevocation = op
+		var group GroupInfo
+		if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+			return err
+		}
+		if op.requireEmpty && !group.Deleted {
+			for _, member := range group.Members {
+				r := store.revisionIndex().get(getUserIdentityPath(member, regUser))
+				at := group.MemberGrants[member]
+				if !r.Deleted && (r.RevokedBefore.IsZero() || at.After(r.RevokedBefore)) && (group.RevokedBefore.IsZero() || at.After(group.RevokedBefore)) {
+					return errGroupNotEmpty
+				}
+			}
+		}
+		op.GroupInfo = GroupInfo{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.GroupInfo = group
+			op.RevokedBefore = origin
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.GroupInfo
+	}
+	var at *time.Time
+	var deleted bool
+	switch v := item.(type) {
+	case *UserIdentity:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *GroupInfo:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if !deleted {
+			var group GroupInfo
+			if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+				return err
+			}
+			mergeIAMGroupMutation(ctx, group, v)
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *MappedPolicy:
+		at, deleted = &v.UpdatedAt, v.Deleted
+	case *PolicyDoc:
+		at, deleted = &v.UpdateDate, v.Deleted
+	default:
+		return errInvalidArgument
+	}
+	if strings.HasPrefix(path, iamConfigSTSPrefix) && previous.Deleted && !deleted {
+		// STS access keys identify immutable tokens, not reusable user names.
+		return errIAMStaleUpdate
+	}
+	if origin, replicated := iamReplicationTime(ctx); replicated {
+		*at = origin
+		if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		if strings.HasPrefix(path, iamConfigServiceAccountsPrefix) && !deleted && previous.Credentials.AccessKey != "" && previous.timestamp().Equal(origin) {
+			// Duplicate service snapshots are acknowledgements, not new creates
+			// or edits. Reload the winner without writing, so even a stale
+			// sibling cache is refreshed by the retry before acknowledging it.
+			return store.loadIAMConfig(ctx, item, path)
+		}
+		if previous.Deleted && deleted && !origin.After(previous.timestamp()) {
+			// An already-applied tombstone needs no further persistent write.
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+	} else {
+		if previous.Deleted && deleted {
+			// A peer notification without an originating revision must not
+			// advance a tombstone past a subsequent deliberate recreation.
+			*at = previous.timestamp()
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+		if at.IsZero() {
+			*at = UTCNow()
+		}
+		if !at.After(previous.timestamp()) {
+			*at = previous.timestamp().Add(time.Nanosecond)
+		}
+	}
+	if v, ok := item.(*UserIdentity); ok {
+		if deleted {
+			// Retain only the parent name for root-account exclusion during heal.
+			v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
+			v.RevokedBefore = *at
+			if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) {
+				// The signed STS token cannot authorize beyond this time, even
+				// if an offline site replays it with a newer event timestamp.
+				v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime)
+				opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
+			}
+		} else {
+			if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
+				claims, err := extractJWTClaims(*v)
+				if err != nil {
+					return err
+				}
+				v.Credentials.Claims = claims.Map()
+			}
+			if err = checkIAMParentRevision(ctx, store, v.Credentials); err != nil {
+				return err
+			}
+		}
+	}
+	if v, ok := item.(*GroupInfo); ok && deleted {
+		v.RevokedBefore = *at
+		v.Members, v.MemberGrants = nil, nil
+	}
+	if _, ok := item.(*MappedPolicy); ok && !deleted {
+		if parentPath := iamMappingParentPath(path); parentPath != "" {
+			parent, err := loadIAMRevision(ctx, store, parentPath)
+			if err != nil {
+				return err
+			}
+			if parent.Deleted {
+				return errIAMStaleUpdate
+			}
+			if !parent.RevokedBefore.IsZero() && !at.After(parent.RevokedBefore) {
+				if _, replicated := iamReplicationTime(ctx); replicated {
+					return errIAMStaleUpdate
+				}
+				*at = parent.RevokedBefore.Add(time.Nanosecond)
+			}
+		}
+	}
+	if err := store.saveIAMConfig(ctx, item, path, opts...); err != nil {
+		return err
+	}
+	if revocation != nil && revocation.retained {
+		return errIAMRevocationRetained
+	}
+	if groupRevocation != nil && groupRevocation.retained {
+		return errIAMRevocationRetained
+	}
+	return nil
+}
+
+func (iamOS *IAMObjectStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var paths []string
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+"/") {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+		paths = append(paths, iamConfigPrefix+"/"+item.Item)
+	}
+	return paths, nil
+}
+
+func (ies *IAMEtcdStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	r, err := ies.client.Get(ctx, iamConfigPrefix+"/", etcd.WithPrefix(), etcd.WithKeysOnly())
+	if err != nil {
+		return nil, err
+	}
+	paths := make([]string, 0, len(r.Kvs))
+	for _, kv := range r.Kvs {
+		paths = append(paths, string(kv.Key))
+	}
+	return paths, nil
+}
+
+func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool) {
+	if (strings.HasPrefix(path, iamConfigUsersPrefix) || strings.HasPrefix(path, iamConfigGroupsPrefix)) && !r.RevokedBefore.IsZero() {
+		// Recreating a parent does not cancel its older revocation of derived
+		// credentials. Replay this boundary even after the parent is live again.
+		r.Deleted = true
+		r.UpdatedAt, r.UpdateDate = r.RevokedBefore, time.Time{}
+	}
+	if !r.Deleted {
+		return item, false
+	}
+	item.UpdatedAt = r.timestamp()
+	switch {
+	case strings.HasPrefix(path, iamConfigUsersPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+		item.Type = madmin.SRIAMItemIAMUser
+		item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
+	case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
+		if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
+			return item, false
+		}
+		item.Type = madmin.SRIAMItemSvcAcc
+		item.SvcAccChange = &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: name}}
+	case strings.HasPrefix(path, iamConfigGroupsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigGroupsPrefix), "/"+iamGroupMembersFile)
+		item.Type = madmin.SRIAMItemGroupInfo
+		item.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: name, IsRemove: true}}
+	case strings.HasPrefix(path, iamConfigPoliciesPrefix):
+		item.Type = madmin.SRIAMItemPolicy
+		item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigPoliciesPrefix), "/"+iamPolicyFile)
+	case strings.HasPrefix(path, iamConfigPolicyDBPrefix):
+		prefix, name, found := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+		if !found {
+			return item, false
+		}
+		typ := regUser
+		switch prefix {
+		case "sts-users":
+			typ = stsUser
+		case "service-accounts":
+			typ = svcUser
+		}
+		item.Type = madmin.SRIAMItemPolicyMapping
+		item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
+	default:
+		// Expired STS credentials are not replayed. Parent revocations and
+		// their retained timestamp reject delayed copies of derived tokens.
+		return item, false
+	}
+	return item, true
+}
+
+func iamDeletionPath(item madmin.SRIAMItem) string {
+	switch item.Type {
+	case madmin.SRIAMItemIAMUser:
+		if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
+			return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
+		}
+	case madmin.SRIAMItemSvcAcc:
+		if item.SvcAccChange != nil && item.SvcAccChange.Delete != nil {
+			return getUserIdentityPath(item.SvcAccChange.Delete.AccessKey, svcUser)
+		}
+	case madmin.SRIAMItemGroupInfo:
+		if item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+			return getGroupInfoPath(item.GroupInfo.UpdateReq.Group)
+		}
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return getPolicyDocPath(item.Name)
+		}
+	case madmin.SRIAMItemPolicyMapping:
+		if p := item.PolicyMapping; p != nil && p.Policy == "" {
+			return getMappedPolicyPath(p.UserOrGroup, IAMUserType(p.UserType), p.IsGroup)
+		}
+	}
+	return ""
+}
+
+func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
+	started := time.Now()
+	defer func() {
+		c.iamRevisionMetrics.healDurationMillis.Store(time.Since(started).Milliseconds())
+		if err != nil {
+			c.iamRevisionMetrics.healFailures.Add(1)
+		} else {
+			c.iamRevisionMetrics.healLastSuccess.Store(time.Now().Unix())
+		}
+	}()
+	c.iamHealMu.Lock()
+	defer c.iamHealMu.Unlock()
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	snapshot := globalIAMSys.store.revisionIndex().snapshot()
+	paths := make([]string, 0, len(snapshot))
+	for path := range snapshot {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	byType := make(map[string][]iamReplicationItem)
+	for _, path := range paths {
+		r := snapshot[path]
+		item, ok := iamDeletionItem(path, r)
+		if !ok {
+			continue
+		}
+		out := iamReplicationItem{SRIAMItem: item}
+		if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
+			out.Type, out.IAMUser = iamUserBoundaryType, nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+		}
+		if item.Type == madmin.SRIAMItemGroupInfo && !r.Deleted {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+		}
+		byType[item.Type] = append(byType[item.Type], out)
+	}
+	var items []iamReplicationItem
+	for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
+		items = append(items, byType[typ]...)
+	}
+	if len(items) == 0 {
+		return nil
+	}
+	if c.iamRevisionProgress == nil {
+		c.iamRevisionProgress = make(map[string]iamRevisionProgress)
+	}
+	for id := range c.iamRevisionProgress {
+		if _, present := c.state.Peers[id]; !present {
+			delete(c.iamRevisionProgress, id)
+		}
+	}
+	var progressMu sync.Mutex
+	cerr := c.concDo(nil, func(id string, p madmin.PeerInfo) error {
+		// Bound each pass, but retain acknowledgements independently of the
+		// pass deadline or unrelated changes at either site.
+		peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+		defer cancel()
+		client, err := c.getAdminClient(peerCtx, id)
+		if err != nil {
+			return err
+		}
+		remote, err := executeIAMRevisionRequest(peerCtx, client, http.MethodGet, nil)
+		if err != nil {
+			return err
+		}
+		progressMu.Lock()
+		progress := c.iamRevisionProgress[id]
+		progressMu.Unlock()
+		progress.observePeer(remote)
+		defer func() {
+			progressMu.Lock()
+			c.iamRevisionProgress[id] = progress
+			progressMu.Unlock()
+		}()
+		var pending []iamReplicationItem
+		for _, item := range items {
+			path, version := iamReplicationMarker(item)
+			if progress.Acknowledged[path] != version {
+				pending = append(pending, item)
+			}
+		}
+		// Acknowledgements are only a replay optimization, never GC proof.
+		for path := range progress.Acknowledged {
+			if _, retained := snapshot[path]; !retained {
+				delete(progress.Acknowledged, path)
+			}
+		}
+		var failures []error
+		for next := 0; next < len(pending); {
+			end := min(next+maxIAMRevisionBatch, len(pending))
+			batch := pending[next:end]
+			remote, err = executeIAMRevisionRequest(peerCtx, client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: batch})
+			if err != nil {
+				var batchErr *iamRevisionBatchError
+				if !errors.As(err, &batchErr) {
+					return errors.Join(append(failures, err)...)
+				}
+				failures = append(failures, err)
+			} else {
+				progress.observePeer(remote)
+				for _, item := range batch {
+					path, version := iamReplicationMarker(item)
+					progress.Acknowledged[path] = version
+				}
+			}
+			next = end
+		}
+		return errors.Join(failures...)
+	}, "IAM revision convergence")
+	return errors.Unwrap(cerr)
+}
+
+func iamReplicationMarker(item iamReplicationItem) (path, version string) {
+	path = iamDeletionPath(item.SRIAMItem)
+	if item.UserRevocation != nil {
+		path = getUserIdentityPath(item.UserRevocation.User, regUser)
+	}
+	if item.GroupRevocation != nil {
+		path = getGroupInfoPath(item.GroupRevocation.Group)
+	}
+	return path, item.Type + ":" + item.UpdatedAt.UTC().Format(time.RFC3339Nano)
+}
+
+func (store *IAMStoreSys) savePolicyDoc(ctx context.Context, policyName string, p *PolicyDoc) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(policyName), p)
+}
+
+func (store *IAMStoreSys) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp *MappedPolicy, opts ...options) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), mp, opts...)
+}
+
+func (store *IAMStoreSys) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u *UserIdentity, opts ...options) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), u, opts...)
+}
+
+func (store *IAMStoreSys) saveGroupInfo(ctx context.Context, name string, gi *GroupInfo) error {
+	return store.saveIAMRevision(ctx, getGroupInfoPath(name), gi)
+}
+
+func (store *IAMStoreSys) deletePolicyDoc(ctx context.Context, name string) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(name), &PolicyDoc{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), &MappedPolicy{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), &UserIdentity{Version: 1, Deleted: true})
+}
+
+// Called under the identity's distributed revision lock, after verifying that
+// its immutable STS token (or early-revocation retention) has expired. Only the
+// old token-key mapping is removed; the reusable parent mapping is unaffected.
+func expireIAMSTSConfig(ctx context.Context, store IAMStorageAPI, path string) error {
+	key := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
+	if err := store.deleteIAMConfig(ctx, getMappedPolicyPath(key, stsUser, false)); err != nil && !errors.Is(err, errConfigNotFound) {
+		return err
+	}
+	return store.deleteIAMConfig(ctx, path)
+}
+
+type (
+	iamExpirationCleanupKey   struct{}
+	iamExpirationCleanupState struct{ failed atomic.Bool }
+)
+
+func withIAMExpirationCleanup(ctx context.Context) context.Context {
+	if _, ok := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState); ok {
+		return ctx
+	}
+	return context.WithValue(ctx, iamExpirationCleanupKey{}, &iamExpirationCleanupState{})
+}
+
+func bestEffortIAMExpiration(ctx context.Context, store IAMStorageAPI, path string) {
+	state, _ := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState)
+	if ctx.Err() != nil || (state != nil && state.failed.Load()) {
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	// Failure leaves the expired record and its existing version intact.
+	// Stop optional reclamation for this load, while still loading healthy
+	// users. Healthy cleanup has no per-scan quota that could build a backlog.
+	if err := saveIAMRevision(ctx, store, path, &iamExpireIdentity{}); err != nil {
+		if state != nil {
+			state.failed.Store(true)
+		}
+		iamLogIf(ctx, err)
+	}
+}

cmd/iam-revision_test.go

@@ -0,0 +1,330 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+)
+
+// Count physical saves: comparing timestamps alone would miss identical
+// tombstones being rewritten on every heal pass.
+type iamRevisionWriteCounter struct {
+	IAMStorageAPI
+	data   []byte
+	writes int
+}
+
+func (s *iamRevisionWriteCounter) loadIAMConfig(_ context.Context, item any, _ string) error {
+	return json.Unmarshal(s.data, item)
+}
+
+func (s *iamRevisionWriteCounter) saveIAMConfig(_ context.Context, item any, _ string, _ ...options) error {
+	data, err := json.Marshal(item)
+	if err == nil {
+		s.data = data
+		s.writes++
+	}
+	return err
+}
+
+func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
+	at := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC)
+	for _, record := range []struct {
+		name string
+		new  func(bool) any
+	}{
+		{"user", func(deleted bool) any { return &UserIdentity{Version: 1, Deleted: deleted} }},
+		{"group", func(deleted bool) any { return &GroupInfo{Version: 1, Deleted: deleted} }},
+		{"policy", func(deleted bool) any { return &PolicyDoc{Version: 1, Deleted: deleted} }},
+		{"mapping", func(deleted bool) any { return &MappedPolicy{Version: 1, Deleted: deleted} }},
+	} {
+		t.Run(record.name, func(t *testing.T) {
+			data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
+			if err != nil {
+				t.Fatal(err)
+			}
+			store := &iamRevisionWriteCounter{data: data}
+			ctx := context.Background()
+			for range 3 {
+				// Site heal carries the original timestamp. Sibling notifications
+				// have no timestamp; both must leave an applied deletion untouched.
+				for _, replay := range []context.Context{withIAMReplicationTime(ctx, at), ctx} {
+					if err := saveIAMRevision(replay, store, record.name, record.new(true)); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			if store.writes != 0 || string(store.data) != string(data) {
+				t.Fatalf("replayed tombstone changed storage: writes=%d, record=%s", store.writes, store.data)
+			}
+			for _, deleted := range []bool{false, true} {
+				err := saveIAMRevision(withIAMReplicationTime(ctx, at.Add(-time.Second)), store, record.name, record.new(deleted))
+				if !errors.Is(err, errIAMStaleUpdate) {
+					t.Fatalf("older event accepted, deleted=%t: %v", deleted, err)
+				}
+			}
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, at), store, record.name, record.new(false)); !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("equal-time recreation accepted: %v", err)
+			}
+			newer := at.Add(time.Minute)
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer), store, record.name, record.new(true)); err != nil {
+				t.Fatal(err)
+			}
+			r, err := loadIAMRevision(ctx, store, record.name)
+			if err != nil || store.writes != 1 || !r.timestamp().Equal(newer) || !r.Deleted {
+				t.Fatalf("newer deletion did not advance storage: writes=%d, revision=%+v, error=%v", store.writes, r, err)
+			}
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer.Add(time.Minute)), store, record.name, record.new(false)); err != nil {
+				t.Fatalf("newer recreation rejected: %v", err)
+			}
+		})
+	}
+}
+
+// Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
+// The object-store case uses the real peer RPC and deletion handler, so a
+// spurious notification actually destroys the parent instead of only counting it.
+func testIAMRevocationReplayAfterRecreation(ctx context.Context, t *testing.T, sys *IAMSys) {
+	t.Helper()
+	peer := &globalSiteReplicationSys
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user := "heal-recreated-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
+	create := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
+	}
+	revoke := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
+	}
+	create(origin)
+	revoke(deleted)
+	create(recreated)
+	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "heal-recreated-child", secretKey: "valid-service-password",
+	})
+	must(err)
+
+	tg, err := grid.SetupTestGrid(2)
+	must(err)
+	t.Cleanup(tg.Cleanup)
+	var deletes atomic.Int32
+	server := &peerRESTServer{}
+	must(deleteUserRPC.Register(tg.Managers[1], func(req *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		deletes.Add(1)
+		return server.DeleteUserHandler(req)
+	}))
+	// Future user updates still use the normal peer reload notification.
+	must(loadUserRPC.Register(tg.Managers[1], server.LoadUserHandler))
+	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+	must(err)
+	previousNotifications := globalNotificationSys
+	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
+		host: host,
+		gridConn: func() *grid.Connection {
+			return tg.Managers[0].Connection(tg.Hosts[1])
+		},
+	}}}
+	t.Cleanup(func() { globalNotificationSys = previousNotifications })
+	assertLive := func(key string) {
+		t.Helper()
+		if _, ok := sys.GetUser(ctx, key); !ok {
+			t.Fatalf("live credential %s lost during deletion replay", key)
+		}
+	}
+	assertNoDelete := func() {
+		t.Helper()
+		if n := deletes.Load(); n != 0 {
+			t.Fatalf("retained revocation sent %d destructive sibling notifications", n)
+		}
+	}
+	for range 3 {
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+		must(err)
+		item, ok := iamDeletionItem(getUserIdentityPath(user, regUser), r)
+		if !ok || item.IAMUser == nil || !item.UpdatedAt.Equal(deleted) {
+			t.Fatal("recreated user lost its durable revocation replay")
+		}
+		must(peer.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt))
+		must(sys.store.LoadIAMCache(ctx, false))
+		assertLive(user)
+		assertLive(child.AccessKey)
+		assertNoDelete()
+	}
+
+	// A divergent site sends a previously unseen revocation between our old
+	// boundary and recreation. Retain it and revoke old children, but never
+	// turn it into an unversioned delete of the recreated parent.
+	delayed := deleted.Add(time.Minute)
+	revoke(delayed)
+	assertNoDelete()
+	must(sys.store.LoadIAMCache(ctx, false))
+	assertLive(user)
+	if _, ok := sys.GetUser(ctx, child.AccessKey); ok {
+		t.Fatal("child from before the delayed revocation remains usable")
+	}
+	r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+	must(err)
+	if r.Deleted || !r.RevokedBefore.Equal(delayed) || !r.timestamp().Equal(recreated) {
+		t.Fatalf("retained revocation damaged the recreated identity: %+v", r)
+	}
+
+	// A genuinely newer deletion must still reach siblings and remove the
+	// parent plus credentials issued under its latest revocation boundary.
+	fresh, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "heal-fresh-child", secretKey: "valid-service-password",
+	})
+	must(err)
+	latest := recreated.Add(time.Minute)
+	revoke(latest)
+	wantDeletes := int32(1)
+	if sys.HasWatcher() {
+		wantDeletes = 0
+	}
+	if n := deletes.Load(); n != wantDeletes {
+		t.Fatalf("new deletion notifications=%d, want %d", n, wantDeletes)
+	}
+	for _, key := range []string{user, fresh.AccessKey} {
+		if _, ok := sys.GetUser(ctx, key); ok {
+			t.Fatalf("newer deletion left credential %s usable", key)
+		}
+	}
+	// Exercise the actual sibling handler again against the already persisted
+	// tombstone. Its context has no revision; it must not re-stamp the record.
+	_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+	if remoteErr != nil {
+		t.Fatal(remoteErr)
+	}
+	r, err = loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+	must(err)
+	if !r.Deleted || !r.timestamp().Equal(latest) {
+		t.Fatalf("sibling re-stamped the tombstone: got %s, want %s", r.timestamp(), latest)
+	}
+	create(latest.Add(time.Minute))
+	_, remoteErr = server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+	if remoteErr != nil {
+		t.Fatal(remoteErr)
+	}
+	assertLive(user)
+	revoke(latest)
+	must(sys.store.LoadIAMCache(ctx, false))
+	assertLive(user)
+}
+
+// Counts what a retained revocation actually sends to sibling nodes.
+func TestIAMRevocationRetainedReloadsSibling(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	initAllSubsystems(ctx)
+	globalIAMSys.Init(ctx, obj, nil, 2*time.Second)
+	defer os.RemoveAll(disks[0])
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+
+	sys, peer := globalIAMSys, &globalSiteReplicationSys
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user := "retained-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
+	create := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
+	}
+	revoke := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
+	}
+	create(origin)
+	revoke(deleted)
+	create(recreated)
+	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "retained-child", secretKey: "valid-service-password",
+	})
+	must(err)
+
+	// A sibling shares persistent state but has an independent IAM cache.
+	sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, sys.usersSysType)}
+	must(sibling.LoadIAMCache(ctx, false))
+	if _, ok := sibling.GetUser(child.AccessKey); !ok {
+		t.Fatal("sibling fixture did not load child")
+	}
+	tg, err := grid.SetupTestGrid(2)
+	must(err)
+	t.Cleanup(tg.Cleanup)
+	var deletes, loads atomic.Int32
+	server := &peerRESTServer{}
+	must(deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		deletes.Add(1)
+		return server.DeleteUserHandler(r)
+	}))
+	must(loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		loads.Add(1)
+		// LoadUserHandler delegates to this same cache reload method.
+		if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), regUser); err != nil {
+			return grid.NoPayload{}, grid.NewRemoteErr(err)
+		}
+		return grid.NoPayload{}, nil
+	}))
+	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+	must(err)
+	prev := globalNotificationSys
+	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
+		host:     host,
+		gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
+	}}}
+	t.Cleanup(func() { globalNotificationSys = prev })
+
+	delayed := deleted.Add(time.Minute)
+	revoke(delayed)
+
+	t.Logf("sibling notifications after a retained revocation: destructive=%d reload=%d", deletes.Load(), loads.Load())
+	if deletes.Load() != 0 {
+		t.Errorf("destructive sibling delete sent: %d", deletes.Load())
+	}
+	if loads.Load() == 0 {
+		t.Errorf("retained revocation did not notify the sibling")
+	}
+	if _, ok := sibling.GetUser(child.AccessKey); ok {
+		t.Error("sibling still resolves revoked child")
+	}
+	if _, ok := sibling.GetUser(user); !ok {
+		t.Error("sibling lost live parent")
+	}
+	if _, ok := sys.store.GetUser(user); !ok {
+		t.Error("live parent lost")
+	}
+	if _, ok := sys.store.GetUser(child.AccessKey); ok {
+		t.Error("revoked child still resolves on the receiving node")
+	}
+}

cmd/iam-revocation_test.go

@@ -0,0 +1,535 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+// Exercise the persisted IAM store and the same peer handler used by site heal.
+// A delete must survive a cache reload and an older create arriving afterwards.
+func TestIAMRevocationRejectsOfflineUser(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "offline-revoked-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "test-password-valid", Status: madmin.AccountEnabled}
+	created, err := globalIAMSys.CreateUser(ctx, user, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = globalIAMSys.DeleteUser(ctx, user, false); err != nil {
+		t.Fatal(err)
+	}
+	if err = globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Fatalf("revoked user restored by old peer event: %v", err)
+	}
+}
+
+func TestIAMRevocationHealingContinuesAfterPeerRejectsDelete(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	if _, err := globalIAMSys.CreateUser(ctx, "heal-sync", req); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.CreateUser(ctx, "heal-deleted", req); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalIAMSys.DeleteUser(ctx, "heal-deleted", false); err != nil {
+		t.Fatal(err)
+	}
+	p, err := globalIAMSys.store.GetPolicy("readwrite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.SetPolicy(ctx, "heal-new-policy", p); err != nil {
+		t.Fatal(err)
+	}
+	var liveUpdates atomic.Int32
+	peer := func(id string, rejectDelete bool) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			switch {
+			case strings.HasSuffix(r.URL.Path, "/metainfo"):
+				_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: id})
+			case r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions":
+				if r.Method == http.MethodGet {
+					_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
+					return
+				}
+				var batch iamRevisionBatch
+				if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+					t.Error(err)
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				for _, item := range batch.Items {
+					if rejectDelete && iamDeletionPath(item.SRIAMItem) != "" {
+						w.WriteHeader(http.StatusForbidden)
+						_, _ = w.Write([]byte(`{"Code":"AccessDenied","Message":"delete rejected"}`))
+						return
+					}
+					if id == "healthy" && item.Type == madmin.SRIAMItemPolicy && item.Name == "heal-new-policy" && len(item.Policy) > 0 {
+						liveUpdates.Add(1)
+					}
+				}
+				_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
+			default:
+				t.Errorf("unexpected peer request %s", r.URL.Path)
+				w.WriteHeader(http.StatusNotFound)
+			}
+		}))
+	}
+	healthy, rejected := peer("healthy", false), peer("rejected", true)
+	defer healthy.Close()
+	defer rejected.Close()
+	c := &SiteReplicationSys{enabled: true, state: srState{
+		ServiceAccountAccessKey: "heal-sync",
+		Peers: map[string]madmin.PeerInfo{
+			globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
+			"healthy":            {Name: "healthy", DeploymentID: "healthy", Endpoint: healthy.URL},
+			"rejected":           {Name: "rejected", DeploymentID: "rejected", Endpoint: rejected.URL},
+		},
+	}}
+	if err := c.healIAMSystem(ctx, obj); err == nil {
+		t.Fatal("deletion failure was not reported")
+	}
+	if liveUpdates.Load() == 0 {
+		t.Fatal("one peer rejecting a deletion blocked unrelated live IAM healing to a healthy peer")
+	}
+}
+
+func TestIAMRevocationLifecycle(t *testing.T) {
+	testIAMRevocationLifecycle(t, nil)
+}
+
+func TestIAMRevocationEtcdLifecycle(t *testing.T) {
+	endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+	if endpoint == "" {
+		t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+	}
+	connection, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer connection.Close()
+	// The facade borrows the connection's services. Close the owning client,
+	// not namespace.Watcher while IAM's canceled watch loop is winding down.
+	ctx, cancel := context.WithCancel(connection.Ctx())
+	defer cancel()
+	client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(connection.GetLogger()))
+	prefix := fmt.Sprintf("/silo-revocation-test/%d/", time.Now().UnixNano())
+	client.KV = namespace.NewKV(connection.KV, prefix)
+	client.Watcher = namespace.NewWatcher(connection.Watcher, prefix)
+	client.Lease = connection.Lease
+	testIAMRevocationLifecycle(t, client)
+}
+
+func testIAMRevocationLifecycle(t *testing.T, client *etcd.Client) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	disk := disks[0]
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err == nil {
+		initAllSubsystems(ctx)
+		globalIAMSys.Init(ctx, obj, client, 2*time.Second)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	sys, peer := globalIAMSys, &globalSiteReplicationSys
+	must := func(t *testing.T, err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	reload := func(t *testing.T) { t.Helper(); must(t, sys.store.LoadIAMCache(ctx, false)) }
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	createUser := func(t *testing.T, name string) {
+		t.Helper()
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, origin))
+	}
+	assertAbsent := func(t *testing.T, name string) {
+		t.Helper()
+		if _, ok := sys.GetUser(ctx, name); ok {
+			t.Fatalf("revoked credential %s is usable", name)
+		}
+	}
+
+	t.Run("replay after recreation", func(t *testing.T) {
+		testIAMRevocationReplayAfterRecreation(ctx, t, sys)
+	})
+
+	t.Run("origin timestamp and recreation", func(t *testing.T) {
+		name := "revocation-recreate"
+		createUser(t, name)
+		ui, ok := sys.store.GetUser(name)
+		if !ok || !ui.UpdatedAt.Equal(origin) {
+			t.Fatalf("origin time changed: %v", ui.UpdatedAt)
+		}
+		must(t, sys.DeleteUser(ctx, name, false))
+		reload(t)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, time.Time{}))
+		assertAbsent(t, name)
+		newTime := UTCNow().Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, newTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, origin.Add(time.Second)))
+		reload(t)
+		ui, ok = sys.store.GetUser(name)
+		if !ok || !ui.UpdatedAt.Equal(newTime) || ui.RevokedBefore.IsZero() {
+			t.Fatalf("newer recreation lost, or deletion boundary missing: present=%v", ok)
+		}
+	})
+
+	t.Run("groups policies and mappings", func(t *testing.T) {
+		user, group, name := "revocation-member", "revocation-group", "revocation-policy"
+		createUser(t, user)
+		p, err := sys.store.GetPolicy("readwrite")
+		must(t, err)
+		must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
+		add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
+		for _, isGroup := range []bool{false, true} {
+			entity := user
+			if isGroup {
+				entity = group
+			}
+			mp := &madmin.SRPolicyMapping{UserOrGroup: entity, Policy: name, UserType: int(regUser), IsGroup: isGroup}
+			must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
+			_, err = sys.PolicyDBSet(ctx, entity, "", regUser, isGroup)
+			must(t, err)
+			must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
+			if _, ok := sys.store.GetMappedPolicy(entity, isGroup); ok {
+				t.Fatal("old grant restored")
+			}
+		}
+		// This receiver never saw the member-removal event preceding deletion.
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, UTCNow()))
+		must(t, sys.DeletePolicy(ctx, name, true))
+		reload(t)
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
+		must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
+		if _, err = sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
+			t.Fatalf("group restored: %v", err)
+		}
+		if _, err = sys.store.GetPolicyDoc(name); !errors.Is(err, errNoSuchPolicy) {
+			t.Fatalf("policy restored: %v", err)
+		}
+		paths, err := sys.store.listIAMConfigPaths(ctx)
+		must(t, err)
+		found := make(map[string]bool)
+		for _, path := range paths {
+			r, err := loadIAMRevision(ctx, sys.store, path)
+			must(t, err)
+			if item, ok := iamDeletionItem(path, r); ok {
+				found[iamDeletionPath(item)] = true
+				if item.UpdatedAt.IsZero() {
+					t.Fatal("undated delete replay")
+				}
+			}
+		}
+		for _, path := range []string{getGroupInfoPath(group), getPolicyDocPath(name), getMappedPolicyPath(user, regUser, false), getMappedPolicyPath(group, regUser, true)} {
+			if !found[path] {
+				t.Errorf("deletion missing from heal: %s", path)
+			}
+		}
+	})
+
+	t.Run("parent revokes service accounts and STS", func(t *testing.T) {
+		parent := "revocation-parent"
+		createUser(t, parent)
+		svc, svcAt, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "revocation-service", secretKey: "valid-service-password"})
+		must(t, err)
+		secret, err := getTokenSigningKey()
+		must(t, err)
+		sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		sts.ParentUser = parent
+		_, err = sys.SetTempUser(withIAMReplicationTime(ctx, origin), sts.AccessKey, sts, "readwrite")
+		must(t, err)
+		must(t, sys.DeleteUser(ctx, parent, false))
+		reload(t)
+		assertAbsent(t, parent)
+		assertAbsent(t, svc.AccessKey)
+		assertAbsent(t, sts.AccessKey)
+		// Recreate the parent, then deliver old child events from the offline site.
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, UTCNow()))
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: svc.AccessKey, SecretKey: svc.SecretKey}}, svcAt))
+		must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken, ParentPolicyMapping: "readwrite"}, origin))
+		reload(t)
+		assertAbsent(t, svc.AccessKey)
+		assertAbsent(t, sts.AccessKey)
+		// A freshly issued credential is still supported after deliberate recreation.
+		_, _, err = sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-service", secretKey: "valid-service-password"})
+		must(t, err)
+		if _, ok := sys.GetUser(ctx, "new-service"); !ok {
+			t.Fatal("fresh service account rejected")
+		}
+	})
+
+	t.Run("delete before first create", func(t *testing.T) {
+		name := "revocation-unseen"
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, UTCNow()))
+		createUser(t, name)
+		assertAbsent(t, name)
+		svc := "unseen-service"
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: svc}}, UTCNow()))
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "revocation-recreate", AccessKey: svc, SecretKey: "valid-service-password"}}, origin))
+		assertAbsent(t, svc)
+	})
+
+	t.Run("recreation arrives before revocation", func(t *testing.T) {
+		parent := "reordered-parent"
+		createUser(t, parent)
+		child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "reordered-child", secretKey: "valid-service-password"})
+		must(t, err)
+		newTime, deleteTime := UTCNow(), origin.Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, newTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+		assertAbsent(t, child.AccessKey)
+		reload(t)
+		assertAbsent(t, child.AccessKey)
+		u, ok := sys.GetUser(ctx, parent)
+		if !ok || !u.UpdatedAt.Equal(newTime) || !u.RevokedBefore.Equal(deleteTime) {
+			t.Fatal("reordered revocation damaged the new parent or lost its boundary")
+		}
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(parent, regUser))
+		must(t, err)
+		item, ok := iamDeletionItem(getUserIdentityPath(parent, regUser), r)
+		if !ok || !item.UpdatedAt.Equal(deleteTime) {
+			t.Fatal("recreation erased deletion replay")
+		}
+	})
+
+	t.Run("user cleanup does not supersede group deletion", func(t *testing.T) {
+		user, group := "cascade-user", "cascade-group"
+		createUser(t, user)
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin))
+		// On the origin site the group was removed before the user, but the
+		// recovering receiver processes those independent events in reverse.
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, origin.Add(time.Minute)))
+		reload(t)
+		if _, err := sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
+			t.Fatalf("deleted group survived reordered cleanup: %v", err)
+		}
+		groups, err := sys.ListGroups(ctx)
+		must(t, err)
+		for _, name := range groups {
+			if name == group {
+				t.Fatal("deleted group listed")
+			}
+		}
+	})
+
+	t.Run("parent revocation covers later updates to existing children", func(t *testing.T) {
+		parent, key := "late-update-parent", "late-update-child"
+		createUser(t, parent)
+		_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
+		must(t, err)
+		// This site missed the deletion and subsequently edited an old child.
+		_, err = sys.UpdateServiceAccount(withIAMReplicationTime(ctx, origin.Add(2*time.Minute)), key, updateServiceAccountOpts{description: "edited while the peer was offline"})
+		must(t, err)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, origin.Add(time.Minute)))
+		assertAbsent(t, key)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(3*time.Minute)))
+		reload(t)
+		assertAbsent(t, key)
+	})
+
+	t.Run("old generation cannot return with a newer event timestamp", func(t *testing.T) {
+		parent := "generation-parent"
+		createUser(t, parent)
+		deleteTime := origin.Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(2*time.Minute)))
+		// Another offline site issued this child under the original parent,
+		// after this site's delete/recreate. Wall-clock ordering cannot identify it.
+		late := origin.Add(3 * time.Minute)
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: "old-gen-service", SecretKey: "valid-service-password"}}, late))
+		secret, err := getTokenSigningKey()
+		must(t, err)
+		sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken}, late))
+		reload(t)
+		assertAbsent(t, "old-gen-service")
+		assertAbsent(t, sts.AccessKey)
+		// A local issuer knows the new boundary and signs it into both kinds
+		// of child. Untrusted inherited claims cannot select that boundary.
+		child, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-gen-service", secretKey: "valid-service-password", claims: map[string]any{iamParentRevocationClaim: "forged"}})
+		must(t, err)
+		newClaims := map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}
+		must(t, setIAMParentRevocationClaim(ctx, sys.store, parent, newClaims))
+		fresh, err := auth.GetNewCredentialsWithMetadata(newClaims, secret)
+		must(t, err)
+		fresh.ParentUser = parent
+		_, err = sys.SetTempUser(ctx, fresh.AccessKey, fresh, "")
+		must(t, err)
+		reload(t)
+		// A periodic reload retains the STS cache. Explicitly clear it to
+		// exercise the cold credential load performed after process restart.
+		cache := sys.store.lock()
+		cache.iamSTSAccountsMap = make(map[string]UserIdentity)
+		sys.store.unlock()
+		for _, key := range []string{child.AccessKey, fresh.AccessKey} {
+			u, ok := sys.GetUser(ctx, key)
+			if !ok || !iamCredentialSurvivesRevocation(u.Credentials, deleteTime) {
+				t.Fatalf("new-generation credential %s rejected", key)
+			}
+		}
+	})
+
+	t.Run("late revocation preserves proven new-generation children", func(t *testing.T) {
+		for _, recreateFirst := range []bool{false, true} {
+			parent := fmt.Sprintf("gen-parent-%t", recreateFirst)
+			createUser(t, parent)
+			deleteTime, createTime := origin.Add(time.Minute), origin.Add(2*time.Minute)
+			if recreateFirst {
+				must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
+			}
+			key := fmt.Sprintf("gen-child-%t", recreateFirst)
+			must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: key, SecretKey: "valid-service-password", Claims: map[string]any{iamParentRevocationClaim: deleteTime.Format(time.RFC3339Nano)}}}, createTime.Add(time.Second)))
+			must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+			if !recreateFirst {
+				assertAbsent(t, key)
+				must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
+			}
+			reload(t)
+			if _, ok := sys.GetUser(ctx, key); !ok {
+				t.Fatal("late revocation deleted a child issued by the recreated parent")
+			}
+		}
+	})
+
+	t.Run("cold loading preserves site-signed STS", func(t *testing.T) {
+		parent := "cold-sts-parent"
+		createUser(t, parent)
+		secret := "site-signing-key-valid"
+		_, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{
+			accessKey: siteReplicatorSvcAcc, secretKey: secret, allowSiteReplicatorAccount: true,
+		})
+		must(t, err)
+		setReplication := func(enabled bool) {
+			globalSiteReplicationSys.Lock()
+			globalSiteReplicationSys.enabled = enabled
+			globalSiteReplicationSys.Unlock()
+			globalSiteReplicatorCred.Set("")
+		}
+		setReplication(true)
+		defer setReplication(false)
+		cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		cred.ParentUser = parent
+		_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
+		must(t, err)
+		// IAM can load before the site replication manager during startup.
+		// A signing key that is not available yet must not delete live tokens.
+		setReplication(false)
+		for range 3 {
+			unverified := make(map[string]UserIdentity)
+			_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, unverified)
+			if _, ok := unverified[cred.AccessKey]; ok {
+				t.Fatal("accepted STS before the signing key became available")
+			}
+		}
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
+		must(t, err)
+		if r.Credentials.SessionToken == "" {
+			t.Fatal("cold IAM load physically deleted a non-expired site-signed STS credential")
+		}
+		setReplication(true)
+		loaded := make(map[string]UserIdentity)
+		must(t, sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded))
+		if _, ok := loaded[cred.AccessKey]; !ok {
+			t.Fatal("STS credential did not recover when the signing key became available")
+		}
+	})
+
+	t.Run("unverifiable STS stay denied and expired STS are removed", func(t *testing.T) {
+		parent := "invalid-sts-parent"
+		createUser(t, parent)
+		// Keep the etcd watcher from cleaning half of the fixture before the
+		// second record is seeded; this subtest exercises the loader directly.
+		sys.store.lock()
+		defer sys.store.unlock()
+		for _, expired := range []bool{false, true} {
+			cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, "unavailable-test-signing-key")
+			must(t, err)
+			cred.ParentUser = parent
+			if expired {
+				cred.Expiration = UTCNow().Add(-time.Minute)
+			}
+			identityPath := getUserIdentityPath(cred.AccessKey, stsUser)
+			mappingPath := getMappedPolicyPath(cred.AccessKey, stsUser, false)
+			// Seed disk directly to exercise loading, including existing records
+			// whose key is unknown. The write API should not accept such tokens.
+			must(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()}, identityPath))
+			must(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Policies: "readwrite"}, mappingPath))
+			loaded := make(map[string]UserIdentity)
+			_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded)
+			if _, ok := loaded[cred.AccessKey]; ok {
+				t.Fatalf("invalid STS accepted, expired=%t", expired)
+			}
+			for _, path := range []string{identityPath, mappingPath} {
+				var record map[string]any
+				err := sys.store.loadIAMConfig(ctx, &record, path)
+				if expired {
+					if !errors.Is(err, errConfigNotFound) {
+						t.Fatalf("expired STS data not cleaned up at %s: %v", path, err)
+					}
+				} else {
+					must(t, err)
+				}
+			}
+		}
+	})
+}

cmd/iam-service-recreation_test.go

@@ -0,0 +1,239 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// The receiver missed a deletion and still has the previous service key.
+// A later full snapshot must replace it, including when the owner changed.
+func TestIAMServiceAccountRecreation(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			origin := UTCNow().Add(-time.Hour)
+			for _, parent := range []string{"old-owner", "new-owner"} {
+				_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), parent, madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+				mustIAM(t, err)
+			}
+			const key = "reusable-service"
+			old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "old-owner", AccessKey: key, SecretKey: "old-service-password"}}
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+			oldIdentity, _ := sys.store.GetUser(key)
+			_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
+			mustIAM(t, err)
+			boundary, newer := origin.Add(time.Minute), origin.Add(2*time.Minute)
+			fresh := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemSvcAcc, UpdatedAt: newer, SvcAccChange: &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "new-owner", AccessKey: key, SecretKey: "new-service-password", Status: auth.AccountOff}}}, RevokedBefore: boundary}
+			mustIAM(t, applyIAMReplicationItem(ctx, fresh))
+			// A sibling may have missed the notification of the committed
+			// replacement. An equal-version retry must refresh that cache too.
+			staleCache := sys.store.lock()
+			staleCache.iamUsersMap[key] = oldIdentity
+			sys.store.unlock()
+			mustIAM(t, applyIAMReplicationItem(ctx, fresh)) // duplicate delivery is acknowledged
+			if current, _ := sys.store.GetUser(key); current.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey {
+				t.Fatal("duplicate snapshot acknowledged without refreshing the stale cache")
+			}
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: key}}, boundary))
+			mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+			u, ok := sys.store.GetUser(key)
+			if !ok || u.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey || u.Credentials.ParentUser != "new-owner" || u.Credentials.Status != auth.AccountOff || !u.UpdatedAt.Equal(newer) || !u.RevokedBefore.Equal(boundary) {
+				t.Fatal("recreation did not retain the new identity, disabled status, source version and revocation")
+			}
+			if _, ok := sys.GetUser(ctx, key); ok {
+				t.Fatal("replicated disabled service can authenticate")
+			}
+			cache := sys.store.rlock()
+			_, mapped := cache.cachedMappedPolicy(key, svcUser, false)
+			sys.store.runlock()
+			if mapped {
+				t.Fatal("recreated service inherited an older mapping")
+			}
+			_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
+			if !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("old service mapping replay was accepted: %v", err)
+			}
+			_, _, err = sys.NewServiceAccount(ctx, "new-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "local-service-password"})
+			if !errors.Is(err, errIAMServiceAccountNotAllowed) {
+				t.Fatalf("local duplicate creation must remain rejected: %v", err)
+			}
+			// Outbound snapshots must carry the retained service boundary too.
+			out, err := globalSiteReplicationSys.replicationItem(ctx, fresh.SRIAMItem)
+			mustIAM(t, err)
+			if !out.RevokedBefore.Equal(boundary) {
+				t.Fatal("outbound service snapshot lost its revocation")
+			}
+		})
+	}
+}
+
+func TestIAMServiceAccountReplicationRejectsOtherCredentialKinds(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "builtin-collision", madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	secret, err := getTokenSigningKey()
+	mustIAM(t, err)
+	token, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "builtin-collision"}, secret)
+	mustIAM(t, err)
+	token.ParentUser = "builtin-collision"
+	_, err = sys.SetTempUser(ctx, token.AccessKey, token, "")
+	mustIAM(t, err)
+	for _, key := range []string{"builtin-collision", token.AccessKey} {
+		_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), "another-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
+		if !errors.Is(err, errIAMServiceAccountNotAllowed) {
+			t.Fatalf("service replication replaced another credential kind: %v", err)
+		}
+	}
+}
+
+// SR configuration can be temporarily unreadable even though a service token
+// is signed with its own valid secret. Do not acknowledge a failed cache load.
+func TestIAMServiceAccountRetryReportsClaimLoadFailure(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	globalSiteReplicatorCred.RLock()
+	previousSigningKey := globalSiteReplicatorCred.secretKey
+	globalSiteReplicatorCred.RUnlock()
+	globalSiteReplicatorCred.Set("")
+	t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSigningKey) })
+	_, err := sys.CreateUser(ctx, "retry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	opts := newServiceAccountOpts{accessKey: "retry-service", secretKey: "valid-service-password"}
+	_, _, err = sys.NewServiceAccount(ctx, "retry-owner", nil, opts)
+	mustIAM(t, err)
+	old, _ := sys.store.GetUser(opts.accessKey)
+	opts.secretKey = "replacement-service-password"
+	at, err := sys.UpdateServiceAccount(ctx, opts.accessKey, updateServiceAccountOpts{secretKey: opts.secretKey})
+	mustIAM(t, err)
+	cache := sys.store.lock()
+	cache.iamUsersMap[opts.accessKey] = old // Missed sibling notification.
+	sys.store.unlock()
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = true // No site-replicator credential is installed.
+	globalSiteReplicationSys.Unlock()
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
+	if err == nil {
+		t.Fatal("acknowledged service retry despite failed claims loading")
+	}
+	if _, ok := sys.store.GetUser(opts.accessKey); ok {
+		t.Fatal("failed cache refresh retained the superseded service secret")
+	}
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = false
+	globalSiteReplicationSys.Unlock()
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
+	mustIAM(t, err)
+}
+
+// A delayed snapshot still has its original absolute expiration. Reapplying
+// the local minimum issuance lifetime would leave the old unexpired key alive.
+func TestIAMServiceAccountReplicationPreservesExpiration(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		for _, action := range []string{"create", "update"} {
+			for _, expired := range []bool{false, true} {
+				name := backend + "/" + action + "/near_expiry"
+				if expired {
+					name = backend + "/" + action + "/expired"
+				}
+				t.Run(name, func(t *testing.T) {
+					ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+					origin := UTCNow().Add(-time.Hour)
+					_, err := sys.CreateUser(ctx, "expiry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+					mustIAM(t, err)
+					old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "old-service-password"}}
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+					expires := UTCNow().Add(time.Minute)
+					if expired {
+						expires = UTCNow().Add(-time.Minute)
+					}
+					change := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
+					if action == "update" {
+						change = &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
+					}
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, change, origin.Add(2*time.Minute)))
+					u, ok := sys.store.GetUser("expiry-service")
+					if !ok || u.Credentials.SecretKey != "new-service-password" || !u.Credentials.Expiration.Equal(expires) {
+						t.Fatal("delayed snapshot lost its new secret or absolute expiration")
+					}
+					_, allowed := sys.GetUser(ctx, "expiry-service")
+					if allowed == expired {
+						t.Fatal("credential validity disagrees with its absolute expiration")
+					}
+					mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+					r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath("expiry-service", svcUser))
+					mustIAM(t, err)
+					if r.Credentials.SecretKey == old.Create.SecretKey || (expired && !r.Deleted) {
+						t.Fatal("old non-expiring credential returned after reload")
+					}
+					_, _, err = sys.NewServiceAccount(ctx, "expiry-owner", nil, newServiceAccountOpts{accessKey: "local-expiry", secretKey: "valid-service-password", expiration: &expires})
+					if !errors.Is(err, errInvalidSvcAcctExpiration) {
+						t.Fatalf("local issuance lifetime check changed: %v", err)
+					}
+				})
+			}
+		}
+	}
+}
+
+// Status-only summaries intentionally omit secrets. Different revisions must
+// still trigger live healing, and disabled identities must be eligible sources.
+func TestIAMServiceAccountHealingNewerSnapshot(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	_, err := sys.CreateUser(ctx, "heal-svc-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), "heal-svc-owner", nil, newServiceAccountOpts{accessKey: "heal-service", secretKey: "valid-service-password"})
+	mustIAM(t, err)
+	at, err := sys.UpdateServiceAccount(ctx, "heal-service", updateServiceAccountOpts{status: auth.AccountOff})
+	mustIAM(t, err)
+	var sent atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			return
+		}
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" || r.Method != http.MethodPut {
+			t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		var batch iamRevisionBatch
+		if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+			t.Error(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		for _, item := range batch.Items {
+			if item.SvcAccChange != nil && item.SvcAccChange.Create != nil && item.SvcAccChange.Create.AccessKey == "heal-service" && item.SvcAccChange.Create.Status == auth.AccountOff && item.UpdatedAt.Equal(at) {
+				sent.Add(1)
+			}
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node", Instance: "boot", Digest: "fixture"}})
+	}))
+	defer server.Close()
+	peers := map[string]madmin.PeerInfo{globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()}, "remote": {Name: "remote", DeploymentID: "remote", Endpoint: server.URL}}
+	c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "heal-svc-owner", Peers: peers}}
+	local := madmin.UserInfo{Status: madmin.AccountStatus(auth.AccountOff), UpdatedAt: at}
+	remote := local
+	remote.UpdatedAt = origin
+	if isUserInfoReplicated(2, 2, []madmin.UserInfo{local, remote}) {
+		t.Fatal("status-only summaries concealed different service revisions")
+	}
+	info := srStatusInfo{Sites: peers, UserStats: map[string]map[string]srUserStatsSummary{"heal-service": {globalDeploymentID(): {userInfo: srUserInfo{UserInfo: local}}, "remote": {SRUserStatsSummary: madmin.SRUserStatsSummary{UserInfoMismatch: true}, userInfo: srUserInfo{UserInfo: remote}}}}}
+	mustIAM(t, c.healUsers(ctx, obj, "heal-service", info))
+	if sent.Load() == 0 {
+		t.Fatal("disabled newer service snapshot was not healed")
+	}
+}

cmd/iam.go

@@ -162,6 +162,15 @@ func (sys *IAMSys) LoadUser(ctx context.Context, objAPI ObjectLayer, accessKey s
 	return sys.store.UserNotificationHandler(ctx, accessKey, userType)
 }
 
+// LoadUserAfterDelete reloads a parent's identity and cached dependents after a
+// sibling committed a deletion. Each record may already have been recreated.
+func (sys *IAMSys) LoadUserAfterDelete(ctx context.Context, accessKey string) error {
+	if !sys.Initialized() {
+		return errServerNotInitialized
+	}
+	return sys.store.UserDeletionNotificationHandler(ctx, accessKey)
+}
+
 // LoadServiceAccount - reloads a specific service account from backend disks or etcd.
 func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) error {
 	if !sys.Initialized() {
@@ -596,10 +605,22 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 		return errServerNotInitialized
 	}
 
-	for _, v := range policy.DefaultPolicies {
-		if v.Name == policyName {
-			if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound {
-				return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+	if _, replicated := iamReplicationTime(ctx); !replicated && notifyPeers {
+		for _, v := range policy.DefaultPolicies {
+			if v.Name == policyName {
+				var err error
+				if objectStore, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
+					err = checkConfig(ctx, objectStore.objAPI, getPolicyDocPath(policyName))
+				} else {
+					var r iamRevision
+					err = sys.store.loadIAMConfig(ctx, &r, getPolicyDocPath(policyName))
+				}
+				if errors.Is(err, errConfigNotFound) {
+					return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+				}
+				if err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -705,20 +726,30 @@ func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers
 		return errServerNotInitialized
 	}
 
-	if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil {
+	err := sys.store.DeleteUser(ctx, accessKey, regUser)
+	var cleanupErr *iamCommittedCleanupError
+	retained := errors.Is(err, errIAMRevocationRetained)
+	if errors.As(err, &cleanupErr) {
+		retained = cleanupErr.retained
+	} else if err != nil && !retained {
 		return err
 	}
-
-	// Notify all other MinIO peers to delete user.
+	// Publish the committed state even when dependent cleanup must be retried.
 	if notifyPeers && !sys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				iamLogIf(ctx, nerr.Err)
+		if retained {
+			sys.notifyForUser(ctx, accessKey, false)
+		} else {
+			for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
+				if nerr.Err != nil {
+					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+					iamLogIf(ctx, nerr.Err)
+				}
 			}
 		}
 	}
-
+	if cleanupErr != nil {
+		return cleanupErr
+	}
 	return nil
 }
 
@@ -1052,6 +1083,7 @@ type newServiceAccountOpts struct {
 	sessionPolicy              *policy.Policy
 	accessKey                  string
 	secretKey                  string
+	status                     string // Used by replication snapshots; local creates default to enabled.
 	name, description          string
 	expiration                 *time.Time
 	allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
@@ -1116,6 +1148,11 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 			m[k] = v
 		}
 	}
+	if _, replicated := iamReplicationTime(ctx); !replicated {
+		if err := setIAMParentRevocationClaim(ctx, sys.store, parentUser, m); err != nil {
+			return auth.Credentials{}, time.Time{}, err
+		}
+	}
 
 	var accessKey, secretKey string
 	var err error
@@ -1134,12 +1171,19 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	cred.ParentUser = parentUser
 	cred.Groups = groups
 	cred.Status = string(auth.AccountOn)
+	switch opts.status {
+	case "", auth.AccountOn, string(madmin.AccountEnabled):
+	case auth.AccountOff, string(madmin.AccountDisabled):
+		cred.Status = auth.AccountOff
+	default:
+		return auth.Credentials{}, time.Time{}, errInvalidArgument
+	}
 	cred.Name = opts.name
 	cred.Description = opts.description
 
 	if opts.expiration != nil {
 		expirationInUTC := opts.expiration.UTC()
-		if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
+		if err := validateSvcExpirationInUTC(ctx, expirationInUTC); err != nil {
 			return auth.Credentials{}, time.Time{}, err
 		}
 		cred.Expiration = expirationInUTC
@@ -1370,7 +1414,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.Credentials.IsServiceAccount() {
+	if _, replicated := iamReplicationTime(ctx); (!ok || !sa.Credentials.IsServiceAccount()) && !replicated {
 		return nil
 	}
 
@@ -1474,8 +1518,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 		}
 	}
 
-	// We ignore any errors
-	_ = sys.store.DeleteUsers(ctx, expiredUsers)
+	// Keep failed revocations visible so the next purge can retry.
+	iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
 }
 
 // purgeExpiredCredentialsForLDAP - validates if local credentials are still
@@ -1503,8 +1547,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 		return
 	}
 
-	// We ignore any errors
-	_ = sys.store.DeleteUsers(ctx, expiredUsers)
+	// Keep failed revocations visible so the next purge can retry.
+	iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
 }
 
 // updateGroupMembershipsForLDAP - updates the list of groups associated with the credential.
@@ -1925,12 +1969,12 @@ func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, membe
 	}
 
 	updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members)
-	if err != nil {
+	var cleanupErr *iamCommittedCleanupError
+	if err != nil && !errors.As(err, &cleanupErr) {
 		return updatedAt, err
 	}
-
 	sys.notifyForGroup(ctx, group)
-	return updatedAt, nil
+	return updatedAt, err
 }
 
 // SetGroupStatus - enable/disabled a group

cmd/ilm-access-move_test.go

@@ -1,529 +0,0 @@
-// Copyright 2026 PGSTY contributors.
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"crypto/md5"
-	"encoding/base64"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io"
-	"maps"
-	"net/http"
-	"net/http/httptest"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/minio/minio/internal/dsync"
-	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
-)
-
-func accessMovePools(t *testing.T) (*erasureServerPools, string) {
-	t.Helper()
-	ctx, cancel := context.WithCancel(t.Context())
-	dirs, err := getRandomDisks(32)
-	if err != nil {
-		t.Fatal(err)
-	}
-	endpoints := mustGetPoolEndpoints(0, dirs[:16]...)
-	endpoints = append(endpoints, mustGetPoolEndpoints(1, dirs[16:]...)...)
-	obj, _, err := initObjectLayer(ctx, endpoints)
-	if err != nil {
-		cancel()
-		removeRoots(dirs)
-		t.Fatal(err)
-	}
-	z := obj.(*erasureServerPools)
-	previous := newObjectLayerFn()
-	setObjectLayer(z)
-	t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
-	bucket := "access-move-test"
-	if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
-		t.Fatal(err)
-	}
-	return z, bucket
-}
-
-func putAccessMoveVersion(t *testing.T, z *erasureServerPools, bucket, object string, pool int, body string, moved bool) ObjectInfo {
-	t.Helper()
-	metadata := map[string]string{"test-value": body}
-	if moved {
-		metadata[accessTierMetadataKey] = accessTierStamp(pool, time.Now().UnixNano())
-	}
-	cs := hash.NewChecksumFromData(hash.ChecksumCRC32C, []byte(body))
-	oi, err := z.serverPools[pool].PutObject(t.Context(), bucket, object,
-		mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""),
-		ObjectOptions{Versioned: true, UserDefined: metadata, WantChecksum: cs})
-	if err != nil {
-		t.Fatal(err)
-	}
-	return oi
-}
-
-func assertAccessMoveVersion(t *testing.T, z *erasureServerPools, bucket, object string, pool int, want ObjectInfo, body string) {
-	t.Helper()
-	gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, http.Header{}, ObjectOptions{VersionID: want.VersionID})
-	if err != nil {
-		t.Errorf("pool %d version %s: %v", pool, want.VersionID, err)
-		return
-	}
-	got, err := io.ReadAll(gr)
-	gr.Close()
-	if err != nil || string(got) != body {
-		t.Errorf("pool %d payload = %q, err = %v, want %q", pool, got, err, body)
-	}
-	if gr.ObjInfo.ETag != want.ETag || !gr.ObjInfo.ModTime.Equal(want.ModTime) {
-		t.Error("move changed ETag or modification time")
-	}
-	if len(want.Checksum) == 0 {
-		t.Fatal("fixture has no checksum")
-	}
-	if !bytes.Equal(gr.ObjInfo.Checksum, want.Checksum) {
-		t.Error("move changed or dropped the stored checksum")
-	}
-	if gr.ObjInfo.UserDefined["test-value"] != body {
-		t.Error("move changed user metadata")
-	}
-}
-
-func TestAccessMoveVersionStack(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "versions"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	dm, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	for _, pair := range [][2]int{{1, 0}, {0, 1}} {
-		if _, err := moveObjectPool(t.Context(), z, bucket, object, pair[0], pair[1], nil); err != nil {
-			t.Fatal(err)
-		}
-		assertAccessMoveVersion(t, z, bucket, object, pair[1], old, "old")
-		assertAccessMoveVersion(t, z, bucket, object, pair[1], latest, "new")
-		versions, err := accessObjectVersions(t.Context(), z, pair[1], bucket, object)
-		if err != nil {
-			t.Fatal(err)
-		}
-		found := false
-		for _, version := range versions {
-			if version.VersionID == dm.VersionID && version.Deleted {
-				found = true
-			}
-		}
-		if !found || len(versions) != 3 {
-			t.Errorf("move lost a version or delete marker: %+v", versions)
-		}
-		if _, err := z.serverPools[pair[0]].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
-			t.Errorf("source remains after completed move: %v", err)
-		}
-	}
-}
-
-func TestAccessMovePreservesNestedObject(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	parent := putAccessMoveVersion(t, z, bucket, "parent", 1, "parent", false)
-	child := putAccessMoveVersion(t, z, bucket, "parent/child", 1, "child", false)
-	if _, err := moveObjectPool(t.Context(), z, bucket, "parent", 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, "parent", 0, parent, "parent")
-	assertAccessMoveVersion(t, z, bucket, "parent/child", 1, child, "child")
-}
-
-func TestAccessMoveNullVersion(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object, body = "null-version", "unversioned"
-	// A null version can remain in a bucket after versioning is enabled.
-	oi, err := z.serverPools[1].PutObject(t.Context(), bucket, object,
-		mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""), ObjectOptions{
-			UserDefined:  map[string]string{"test-value": body},
-			WantChecksum: hash.NewChecksumFromData(hash.ChecksumCRC32C, []byte(body)),
-		})
-	if err != nil || oi.VersionID != "" {
-		t.Fatalf("null version fixture failed: %v %q", err, oi.VersionID)
-	}
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, oi, body)
-	if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
-		t.Fatalf("null version source was not removed: %v", err)
-	}
-}
-
-// Source removal can partially commit before a process or disk fails. The
-// destination can therefore hold the only remaining copy of an older version.
-func TestAccessMoveRetryPreservesDestinationVersions(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "retry"
-	onlyAtDestination := putAccessMoveVersion(t, z, bucket, object, 0, "unique-old", true)
-	source := putAccessMoveVersion(t, z, bucket, object, 1, "source-new", false)
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, onlyAtDestination, "unique-old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, source, "source-new")
-}
-
-type accessMoveFaultDisk struct {
-	StorageAPI
-	failVersion string
-}
-
-func (d accessMoveFaultDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
-	if fi.VersionID == d.failVersion {
-		return RenameDataResp{}, errDiskFull
-	}
-	return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
-}
-
-func TestAccessMoveWriteFailureCanResume(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "write-failure"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	existing := putAccessMoveVersion(t, z, bucket, object, 0, "unique-destination", true)
-	set := z.serverPools[0].getHashedSet(object)
-	getDisks := set.getDisks
-	disks := getDisks()
-	faulty := make([]StorageAPI, len(disks))
-	for i, disk := range disks {
-		faulty[i] = accessMoveFaultDisk{StorageAPI: disk, failVersion: latest.VersionID}
-	}
-	set.getDisks = func() []StorageAPI { return faulty }
-	_, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil)
-	set.getDisks = getDisks
-	if err == nil {
-		t.Fatal("injected destination write failure was ignored")
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 1, latest, "new")
-	assertAccessMoveVersion(t, z, bucket, object, 0, existing, "unique-destination")
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-	assertAccessMoveVersion(t, z, bucket, object, 0, existing, "unique-destination")
-}
-
-func TestAccessMoveExcludesSourceWriter(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "concurrent"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	_, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, func(ObjectInfo, uint64) error {
-		ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
-		defer cancel()
-		_, err := z.serverPools[1].PutObject(ctx, bucket, object,
-			mustGetPutObjReader(t, bytes.NewBufferString("racing-write"), 12, "", ""), ObjectOptions{Versioned: true})
-		if err == nil {
-			t.Error("source writer committed while move was in progress")
-		}
-		if ctx.Err() == nil {
-			return errors.New("writer did not wait for the move lock")
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-}
-
-func TestAccessMoveResumesPartialCopy(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "partial-copy"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	gr, err := z.serverPools[1].GetObjectNInfo(t.Context(), bucket, object, nil, http.Header{}, ObjectOptions{VersionID: old.VersionID, NoDecryption: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Model a process stopping after the first copy commits, before cleanup.
-	if err := moveAccessTierVersion(t.Context(), z, 1, 0, bucket, gr, time.Now().UnixNano()); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-}
-
-func TestAccessMoveRefusesConflictingVersion(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "conflicting-version"
-	source := putAccessMoveVersion(t, z, bucket, object, 1, "source", false)
-	destination, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
-		mustGetPutObjReader(t, bytes.NewBufferString("target"), 6, "", ""), ObjectOptions{
-			VersionID: source.VersionID, MTime: source.ModTime,
-			UserDefined:  map[string]string{"test-value": "target"},
-			WantChecksum: hash.NewChecksumFromData(hash.ChecksumCRC32C, []byte("target")),
-		})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); !errors.Is(err, errAccessTierNotEligible) {
-		t.Fatalf("conflicting version should be left intact: %v", err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, source, "source")
-	assertAccessMoveVersion(t, z, bucket, object, 0, destination, "target")
-}
-
-type accessMoveDeleteFaultDisk struct {
-	StorageAPI
-	bucket, object, version string
-}
-
-func (d accessMoveDeleteFaultDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
-	if volume == d.bucket && path == d.object && fi.VersionID == d.version {
-		return errDiskFull
-	}
-	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
-}
-
-func TestAccessMoveSourceDeleteFailureCanResume(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "source-delete-failure"
-	old := putAccessMoveVersion(t, z, bucket, object, 1, "old", false)
-	latest := putAccessMoveVersion(t, z, bucket, object, 1, "new", false)
-	set := z.serverPools[1].getHashedSet(object)
-	getDisks := set.getDisks
-	faulty := append([]StorageAPI(nil), getDisks()...)
-	// Commit removal of the old version, then reject the latest version on
-	// every disk. This fixes the retry boundary without relying on how a
-	// partially deleted erasure quorum is subsequently resolved or healed.
-	for i := range faulty {
-		faulty[i] = accessMoveDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: latest.VersionID}
-	}
-	set.getDisks = func() []StorageAPI { return faulty }
-	_, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil)
-	set.getDisks = getDisks
-	if err == nil {
-		t.Fatal("injected partial source purge was ignored")
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-	if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: old.VersionID}); !isErrVersionNotFound(err) {
-		t.Fatalf("old source version should already be removed: %v", err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, latest, "new")
-	if _, err := moveObjectPool(t.Context(), z, bucket, object, 1, 0, nil); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, old, "old")
-	assertAccessMoveVersion(t, z, bucket, object, 0, latest, "new")
-}
-
-// Exercise the public multipart API and compare whole-object and part reads
-// across both directions of a real pool move, including raw SSE-C ciphertext.
-func TestAccessMoveMultipartChecksums(t *testing.T) {
-	z, _ := accessMovePools(t)
-	ctx, cancel := context.WithCancel(t.Context())
-	defer cancel()
-	bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
-	if err != nil {
-		t.Fatal(err)
-	}
-	previousTLS := globalIsTLS
-	globalIsTLS = true
-	defer func() { globalIsTLS = previousTLS }()
-	partData, full := multipartChecksumTestData()
-	for _, tc := range []struct {
-		typ  hash.ChecksumType
-		ssec bool
-	}{{hash.ChecksumCRC32, false}, {hash.ChecksumCRC32C, false}, {hash.ChecksumCRC64NVME, false}, {hash.ChecksumCRC32C, true}} {
-		t.Run(fmt.Sprintf("%s/ssec=%t", tc.typ, tc.ssec), func(t *testing.T) {
-			object := getRandomObjectName()
-			headers := map[string]string{}
-			if tc.ssec {
-				key := bytes.Repeat([]byte{0x42}, 32)
-				keyMD5 := md5.Sum(key)
-				headers[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
-				headers[xhttp.AmzServerSideEncryptionCustomerKey] = base64.StdEncoding.EncodeToString(key)
-				headers[xhttp.AmzServerSideEncryptionCustomerKeyMD5] = base64.StdEncoding.EncodeToString(keyMD5[:])
-			}
-			do := func(method, url string, body []byte, extra map[string]string) *httptest.ResponseRecorder {
-				t.Helper()
-				h := maps.Clone(headers)
-				maps.Copy(h, extra)
-				req, err := newTestSignedRequestV4(method, url, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, h)
-				if err != nil {
-					t.Fatal(err)
-				}
-				rec := httptest.NewRecorder()
-				router.ServeHTTP(rec, req)
-				if rec.Code != http.StatusOK && rec.Code != http.StatusPartialContent {
-					t.Fatalf("%s %s: %d %s", method, url, rec.Code, rec.Body.String())
-				}
-				return rec
-			}
-			init := do(http.MethodPost, getNewMultipartURL("", bucket, object), nil, map[string]string{
-				xhttp.AmzChecksumAlgo: tc.typ.String(), xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
-			})
-			var upload InitiateMultipartUploadResponse
-			if err := xml.Unmarshal(init.Body.Bytes(), &upload); err != nil {
-				t.Fatal(err)
-			}
-			parts := make([]CompletePart, len(partData))
-			for i, data := range partData {
-				rec := do(http.MethodPut, getPutObjectPartURL("", bucket, object, upload.UploadID, fmt.Sprint(i+1)), data,
-					map[string]string{tc.typ.Key(): mustChecksum(t, tc.typ, data)})
-				parts[i] = CompletePart{PartNumber: i + 1, ETag: canonicalizeETag(rec.Header()[xhttp.ETag][0])}
-			}
-			body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
-			if err != nil {
-				t.Fatal(err)
-			}
-			do(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, upload.UploadID), body,
-				map[string]string{tc.typ.Key(): mustChecksum(t, tc.typ, full), xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject})
-			source, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
-			if err != nil || len(source.Checksum) == 0 {
-				t.Fatalf("source checksum missing: %v", err)
-			}
-			src := 0
-			if _, err := z.serverPools[src].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); isErrObjectNotFound(err) {
-				src = 1
-			}
-			for i := 0; i < 2; i++ {
-				dst := 1 - src
-				if _, err := moveObjectPool(t.Context(), z, bucket, object, src, dst, nil); err != nil {
-					t.Fatal(err)
-				}
-				got, err := z.serverPools[dst].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
-				if err != nil || !bytes.Equal(got.Checksum, source.Checksum) || got.ETag != source.ETag || got.VersionID != source.VersionID || !got.ModTime.Equal(source.ModTime) {
-					t.Fatalf("move changed version metadata: %v checksumEqual=%t ETag=%q/%q version=%q/%q modTimeEqual=%t", err, bytes.Equal(got.Checksum, source.Checksum), got.ETag, source.ETag, got.VersionID, source.VersionID, got.ModTime.Equal(source.ModTime))
-				}
-				rec := do(http.MethodGet, getGetObjectURL("", bucket, object), nil, map[string]string{xhttp.AmzChecksumMode: "ENABLED"})
-				if !bytes.Equal(rec.Body.Bytes(), full) || rec.Header().Get(tc.typ.Key()) != mustChecksum(t, tc.typ, full) {
-					t.Fatal("GET payload or checksum changed after move")
-				}
-				for j, data := range partData {
-					rec := do(http.MethodGet, getGetObjectURL("", bucket, object)+fmt.Sprintf("?partNumber=%d", j+1), nil, nil)
-					if !bytes.Equal(rec.Body.Bytes(), data) {
-						t.Fatalf("part %d changed after move", j+1)
-					}
-				}
-				src = dst
-			}
-		})
-	}
-}
-
-type accessMoveNamedLocker struct {
-	*localLocker
-	address string
-}
-
-func (l accessMoveNamedLocker) String() string { return l.address }
-
-func TestAccessMoveSharedDistributedLockers(t *testing.T) {
-	// Three overlapping sets of real dsync lock servers. Taking independent
-	// quorum locks for the same object would contend with our own earlier lock.
-	peers := make([]dsync.NetLocker, 5)
-	for i := range peers {
-		peers[i] = accessMoveNamedLocker{localLocker: newLocker(), address: fmt.Sprint(i)}
-	}
-	z := &erasureServerPools{}
-	for i := 0; i < 3; i++ {
-		setPeers := peers[i : i+3]
-		set := &erasureObjects{nsMutex: &nsLockMap{isDistErasure: true}, getLockers: func() ([]dsync.NetLocker, string) {
-			return setPeers, "access-move-fixture"
-		}}
-		z.serverPools = append(z.serverPools, &erasureSets{sets: []*erasureObjects{set}, distributionAlgo: formatErasureVersionV3DistributionAlgoV3})
-	}
-	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
-	defer cancel()
-	_, unlock, err := lockAccessTierObject(ctx, z, "bucket", "object", 1, 2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	release := sync.OnceFunc(unlock)
-	defer release()
-	for i, pool := range z.serverPools {
-		ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
-		lock := pool.NewNSLock("bucket", "object")
-		lc, err := lock.GetLock(ctx, globalOperationTimeout)
-		cancel()
-		if err == nil {
-			lock.Unlock(lc)
-			t.Fatalf("pool %d writer acquired its quorum during the move", i)
-		}
-	}
-	release()
-	// Every acquired peer lock is released, so ordinary writes can resume.
-	for _, peer := range peers {
-		// Distributed Unlock sends releases asynchronously.
-		lock := (&nsLockMap{isDistErasure: true}).NewNSLock(func() ([]dsync.NetLocker, string) {
-			return []dsync.NetLocker{peer}, "access-move-fixture"
-		}, "bucket", "object")
-		ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
-		lc, err := lock.GetLock(ctx, globalOperationTimeout)
-		cancel()
-		if err != nil {
-			t.Fatalf("peer %s leaked a lock: %v", peer.String(), err)
-		}
-		lock.Unlock(lc)
-	}
-}
-
-func TestAccessMoveConfiguredPromotionAndDemotion(t *testing.T) {
-	z, bucket := accessMovePools(t)
-	const object = "configured-move"
-	version := putAccessMoveVersion(t, z, bucket, object, 1, "configured", false)
-	oldCfg, oldTracker := globalILMConfig.accessCfg(), globalAccessTracker
-	defer func() { globalILMConfig.update(oldCfg); globalAccessTracker = oldTracker }()
-	cfg := oldCfg
-	cfg.AccessTiering, cfg.AccessPools = true, []int{0, 1}
-	cfg.AccessMinResidency, cfg.AccessPromoteWatermark = 0, 99
-	globalILMConfig.update(cfg)
-	globalAccessTracker = newAccessTracker()
-	lc := accessLifecycleForTest(t)
-	data, err := xml.Marshal(lc)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, data); err != nil {
-		t.Fatal(err)
-	}
-	state := newAccessTierState(t.Context())
-	state.z, state.objAPI = z, z
-	task := accessTierTask{ctx: t.Context(), bucket: bucket, object: object, src: 1, dst: 0, direction: accessTierPromote, bytes: uint64(version.Size)}
-	// A queued move is rechecked when configuration is disabled dynamically.
-	cfg.AccessTiering = false
-	globalILMConfig.update(cfg)
-	if err := state.processTask(task); !errors.Is(err, errAccessTierNotEligible) {
-		t.Fatalf("disabled mover = %v", err)
-	}
-	cfg.AccessTiering = true
-	globalILMConfig.update(cfg)
-	globalAccessTracker.merged.Store(&mergedAccess{binWidth: 60, entries: map[string]accessEntry{
-		accessKey(bucket, object): {Bins: []uint32{100}, LastAt: time.Now().Unix()},
-	}})
-	if reason, ok := state.reservePromotion(task, cfg, 0); !ok {
-		t.Fatalf("promotion reservation failed: %s", reason)
-	}
-	if err := state.processTask(task); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 0, version, "configured")
-	// Advance the counter view beyond the rule's idle period.
-	globalAccessTracker.merged.Store(&mergedAccess{binWidth: 60, entries: map[string]accessEntry{
-		accessKey(bucket, object): {Bins: []uint32{0}, LastAt: time.Now().Add(-2 * time.Hour).Unix()},
-	}})
-	task.src, task.dst, task.direction, task.bytes = 0, 1, accessTierDemote, 0
-	if err := state.processTask(task); err != nil {
-		t.Fatal(err)
-	}
-	assertAccessMoveVersion(t, z, bucket, object, 1, version, "configured")
-	if state.promotions.Load() != 1 || state.demotions.Load() != 1 || state.bytesMoved.Load() != 2*uint64(version.Size) || len(state.pending) != 0 || len(state.reserved) != 0 {
-		t.Fatal("promotion/demotion metrics or reservation cleanup are incorrect")
-	}
-}

cmd/ilm-access-tier_test.go

@@ -1,263 +0,0 @@
-// Copyright (c) 2015-2026 MinIO, Inc.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/klauspost/compress/zstd"
-	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/config/ilm"
-	"github.com/tinylib/msgp/msgp"
-)
-
-func accessLifecycleForTest(t *testing.T) *lifecycle.Lifecycle {
-	t.Helper()
-	xml := "<LifecycleConfiguration>" +
-		"<Rule><ID>access</ID><Status>Enabled</Status>" +
-		"<AccessTransition><Window>10m</Window><PromoteAfterAccesses>100</PromoteAfterAccesses>" +
-		"<DemoteAfterAccesses>5</DemoteAfterAccesses><DemoteAfterIdle>1h</DemoteAfterIdle></AccessTransition>" +
-		"</Rule></LifecycleConfiguration>"
-	lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml))
-	if err != nil {
-		t.Fatal(err)
-	}
-	return lc
-}
-
-func TestAccessTierStampAndDemoteEligibility(t *testing.T) {
-	oldTracker := globalAccessTracker
-	globalAccessTracker = newAccessTracker()
-	t.Cleanup(func() { globalAccessTracker = oldTracker })
-
-	now := time.Now()
-	cfg := ilm.Config{
-		AccessTiering: true, AccessPools: []int{0, 1},
-		AccessBinWidth: time.Minute, AccessBins: 12,
-		AccessMinResidency: 30 * time.Minute,
-	}
-	oi := ObjectInfo{
-		Bucket: "bucket", Name: "object", Size: 10, IsLatest: true,
-		UserDefined: map[string]string{
-			accessTierMetadataKey: "0:" + strconv.FormatInt(now.Add(-2*time.Hour).UnixNano(), 10),
-		},
-	}
-	if !accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
-		t.Fatal("cold, resident object should be demotion eligible")
-	}
-	oi.UserDefined[accessTierMetadataKey] = "0:" + strconv.FormatInt(now.Add(-time.Minute).UnixNano(), 10)
-	if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
-		t.Fatal("object inside minimum residency was eligible")
-	}
-	oi.UserDefined[accessTierMetadataKey] = "broken"
-	if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
-		t.Fatal("object with malformed marker was eligible")
-	}
-}
-
-func TestAccessTierReservationsEnforceCaps(t *testing.T) {
-	state := newAccessTierState(t.Context())
-	state.usageReady = true
-	state.baseUsage["a"] = 80
-	state.baseTotal = 180
-
-	cfg := ilm.Config{AccessMaxSize: 200}
-	task := accessTierTask{ctx: t.Context(), bucket: "a", object: "one", bytes: 30}
-	if reason, ok := state.reservePromotion(task, cfg, 0); ok || reason != "max-size" {
-		t.Fatalf("max-size reserve = %q/%v", reason, ok)
-	}
-
-	cfg.AccessMaxSize = 0
-	if reason, ok := state.reservePromotion(task, cfg, 100); ok || reason != "quota" {
-		t.Fatalf("quota reserve = %q/%v", reason, ok)
-	}
-
-	task.bytes = 20
-	if reason, ok := state.reservePromotion(task, cfg, 100); !ok || reason != "" {
-		t.Fatalf("valid reserve = %q/%v", reason, ok)
-	}
-	if got := state.bucketUsageLocked("a"); got != 100 {
-		t.Fatalf("reserved bucket usage = %d, want 100", got)
-	}
-	state.releasePending(task, true)
-}
-
-func TestDataMovementDestinationIsGated(t *testing.T) {
-	dst := 0
-	if got := dataMovementDstPool(ObjectOptions{DstPoolIdx: &dst}); got != nil {
-		t.Fatal("destination honored without DataMovement")
-	}
-	if got := dataMovementDstPool(ObjectOptions{DataMovement: true, DstPoolIdx: &dst}); got == nil || *got != 0 {
-		t.Fatalf("destination = %v", got)
-	}
-}
-
-func TestForcedDestinationPoolSelection(t *testing.T) {
-	z := &erasureServerPools{serverPools: make([]*erasureSets, 2)}
-	dst := 1
-	if got, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err != nil || got != dst {
-		t.Fatalf("destination = %d, err = %v", got, err)
-	}
-	bad := 2
-	if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &bad); !errors.Is(err, errInvalidArgument) {
-		t.Fatalf("out-of-range error = %v", err)
-	}
-	z.poolMeta.Pools = make([]PoolStatus, 2)
-	z.poolMeta.Pools[1].Decommission = &PoolDecommissionInfo{}
-	if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err == nil {
-		t.Fatal("suspended destination was accepted")
-	}
-}
-
-func TestHotTierAccounting(t *testing.T) {
-	entry := dataUsageEntry{}
-	entry.addSizes(sizeSummary{totalSize: 100, hotTierSize: 60, versions: 1})
-	entry.merge(dataUsageEntry{Size: 50, HotTierSize: 25})
-	if entry.Size != 150 || entry.HotTierSize != 85 {
-		t.Fatalf("usage = size:%d hot:%d", entry.Size, entry.HotTierSize)
-	}
-}
-
-func TestScannerCyclesApart(t *testing.T) {
-	tests := []struct {
-		current, previous uint32
-		want              uint32
-	}{
-		{current: 12, previous: 10, want: 2},
-		{current: 0, previous: ^uint32(0), want: 1},
-		// A cycle counter reset is not evidence that a complete pass covered
-		// recent moves, so it must not release conservative deltas.
-		{current: 1, previous: 100, want: 0},
-	}
-	for _, tt := range tests {
-		if got := scannerCyclesApart(tt.current, tt.previous); got != tt.want {
-			t.Fatalf("scannerCyclesApart(%d, %d) = %d, want %d", tt.current, tt.previous, got, tt.want)
-		}
-	}
-}
-
-func TestDataUsageCacheV8Migration(t *testing.T) {
-	var encoded bytes.Buffer
-	if err := encoded.WriteByte(dataUsageCacheVerV8); err != nil {
-		t.Fatal(err)
-	}
-	zw, err := zstd.NewWriter(&encoded)
-	if err != nil {
-		t.Fatal(err)
-	}
-	mw := msgp.NewWriter(zw)
-	if err = mw.WriteMapHeader(2); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("Info"); err != nil {
-		t.Fatal(err)
-	}
-	info := dataUsageCacheInfo{Name: dataUsageRoot, NextCycle: 17, LastUpdate: time.Now().UTC()}
-	if err = info.EncodeMsg(mw); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("Cache"); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteMapHeader(1); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("entry"); err != nil {
-		t.Fatal(err)
-	}
-	// A v8 entry has no hts field. Encoding only populated fields also
-	// verifies that its map decoder retains normal msgpack compatibility.
-	if err = mw.WriteMapHeader(2); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("sz"); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteInt64(123); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteString("os"); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.WriteUint64(2); err != nil {
-		t.Fatal(err)
-	}
-	if err = mw.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if err = zw.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	var migrated dataUsageCache
-	if err = migrated.deserialize(bytes.NewReader(encoded.Bytes())); err != nil {
-		t.Fatal(err)
-	}
-	entry := migrated.Cache["entry"]
-	if entry.Size != 123 || entry.Objects != 2 || entry.HotTierSize != 0 {
-		t.Fatalf("migrated entry = size:%d objects:%d hot:%d", entry.Size, entry.Objects, entry.HotTierSize)
-	}
-	if migrated.Info.NextCycle != 17 || !migrated.Info.LastUpdate.Equal(info.LastUpdate) {
-		t.Fatalf("migrated cache info = %+v", migrated.Info)
-	}
-}
-
-// The stamp written on every moved version and the stamp the rollback path
-// matches against must agree, otherwise rollback silently skips its own work.
-func TestAccessTierStampRoundTrip(t *testing.T) {
-	movedAt := time.Now().UnixNano()
-	stamp := accessTierStamp(3, movedAt)
-
-	pool, at, ok := parseAccessTierStamp(map[string]string{accessTierMetadataKey: stamp})
-	if !ok {
-		t.Fatalf("stamp %q did not parse", stamp)
-	}
-	if pool != 3 {
-		t.Fatalf("pool = %d, want 3", pool)
-	}
-	if at.UnixNano() != movedAt {
-		t.Fatalf("movedAt = %d, want %d", at.UnixNano(), movedAt)
-	}
-	// A different move of the same object must not match, so a concurrent
-	// client overwrite is never mistaken for our own copy.
-	if stamp == accessTierStamp(3, movedAt+1) {
-		t.Fatal("stamps from distinct moves collided")
-	}
-	if stamp == accessTierStamp(4, movedAt) {
-		t.Fatal("stamps from distinct pools collided")
-	}
-}
-
-func TestAccessHitsMightPromote(t *testing.T) {
-	lc := accessLifecycleForTest(t)
-	hot := accessEntry{Bins: []uint32{100}, HeadAt: 0}
-	cold := accessEntry{Bins: []uint32{1}, HeadAt: 0}
-	if !accessHitsMightPromote(lc, "object", hot, 60) {
-		t.Fatal("object above promote threshold was rejected")
-	}
-	if accessHitsMightPromote(lc, "object", cold, 60) {
-		t.Fatal("object below promote threshold was accepted")
-	}
-
-	xml := "<LifecycleConfiguration><Rule><ID>logs</ID><Status>Enabled</Status>" +
-		"<Filter><Prefix>logs/</Prefix></Filter>" +
-		"<AccessTransition><Window>10m</Window><PromoteAfterAccesses>100</PromoteAfterAccesses>" +
-		"<DemoteAfterAccesses>5</DemoteAfterAccesses><DemoteAfterIdle>1h</DemoteAfterIdle></AccessTransition>" +
-		"</Rule></LifecycleConfiguration>"
-	prefixed, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if accessHitsMightPromote(prefixed, "data/object", hot, 60) {
-		t.Fatal("object outside the rule prefix was accepted")
-	}
-	if !accessHitsMightPromote(prefixed, "logs/object", hot, 60) {
-		t.Fatal("object inside the rule prefix was rejected")
-	}
-}

cmd/ilm-access-tracker.go

@@ -1,598 +0,0 @@
-// Copyright (c) 2015-2026 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"cmp"
-	"context"
-	"fmt"
-	"slices"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/minio/minio/internal/config/ilm"
-	"github.com/zeebo/xxh3"
-)
-
-//go:generate msgp -file=$GOFILE -unexported
-//msgp:ignore accessTracker mergedAccess
-
-const (
-	// accessTrackerPrefix is where each node publishes its own counters.
-	// One object per node, merged by every node on a timer.
-	accessTrackerPrefix = minioConfigPrefix + "/ilm/access"
-
-	// accessQueueSize bounds the GET -> tracker handoff. Overflow drops
-	// samples rather than slowing down reads.
-	accessQueueSize = 100000
-
-	// accessMaxDemoteCandidates bounds what the scanner may hand over in a
-	// single flush interval.
-	accessMaxDemoteCandidates = 100000
-
-	// accessShardStaleFactor multiplies the flush interval to decide when a
-	// peer's counters are too old to trust, e.g. after a node is removed.
-	accessShardStaleFactor = 5
-)
-
-func accessShardFresh(now int64, shard accessShard, stale int64) bool {
-	if shard.UpdatedAt <= 0 {
-		return false
-	}
-	if stale <= 0 {
-		return true
-	}
-	age := now - shard.UpdatedAt
-	return age >= -stale && age <= stale
-}
-
-// accessEntry is a rolling hit counter for one object. Bins[0] is the current
-// bin and each subsequent bin is one bin-width older, so a rule asking for
-// "100 hits in 10 minutes" sums the newest ceil(10m/binWidth) bins.
-//
-// A fixed window is used rather than an exponentially decayed score because
-// the rule is stated to operators in exactly those terms.
-type accessEntry struct {
-	Bins   []uint32 `msg:"b"`
-	HeadAt int64    `msg:"h"` // unix seconds at the start of Bins[0]
-	LastAt int64    `msg:"l"` // unix seconds of the most recent hit
-}
-
-// demoteCandidate is an object the scanner found sitting on a fast pool with
-// no recent reads. Candidates ride the node's own counter shard so they reach
-// the leader without a new peer RPC.
-type demoteCandidate struct {
-	Bucket string `msg:"b"`
-	Object string `msg:"o"`
-	Pool   int    `msg:"p"`
-}
-
-// accessShard is what one node publishes. BinWidth is carried so a peer that
-// has not yet picked up a configuration change is ignored rather than merged
-// with mismatched bins.
-type accessShard struct {
-	UpdatedAt int64                  `msg:"u"`
-	BinWidth  int64                  `msg:"bw"`
-	Entries   map[string]accessEntry `msg:"e"`
-	Demote    []demoteCandidate      `msg:"d"`
-}
-
-// binStart truncates a unix timestamp to the start of its bin.
-func binStart(now, binWidth int64) int64 {
-	if binWidth <= 0 {
-		return now
-	}
-	return now - now%binWidth
-}
-
-// rollTo advances the counter to now, zeroing the bins that elapsed since the
-// last update and resizing if the configured bin count changed.
-func (e *accessEntry) rollTo(now, binWidth int64, nbins int) {
-	if nbins <= 0 || binWidth <= 0 {
-		return
-	}
-	if len(e.Bins) != nbins {
-		resized := make([]uint32, nbins)
-		copy(resized, e.Bins)
-		e.Bins = resized
-	}
-	head := binStart(now, binWidth)
-	if e.HeadAt == 0 {
-		e.HeadAt = head
-		return
-	}
-	steps := (head - e.HeadAt) / binWidth
-	if steps <= 0 {
-		return
-	}
-	if steps >= int64(nbins) {
-		clear(e.Bins)
-	} else {
-		copy(e.Bins[steps:], e.Bins[:nbins-int(steps)])
-		clear(e.Bins[:steps])
-	}
-	e.HeadAt = head
-}
-
-// hits returns the number of accesses recorded over the newest bins covering
-// window. A window longer than the configured history is clamped to it.
-//
-// The newest bin is partial, so the covered span is between window-binWidth
-// and window. Operators tune resolution with ilm access_bin_width.
-func (e accessEntry) hits(window time.Duration, binWidth int64) uint64 {
-	if binWidth <= 0 || len(e.Bins) == 0 {
-		return 0
-	}
-	n := int((int64(window/time.Second) + binWidth - 1) / binWidth)
-	if n < 1 {
-		n = 1
-	}
-	if n > len(e.Bins) {
-		n = len(e.Bins)
-	}
-	var total uint64
-	for _, v := range e.Bins[:n] {
-		total += uint64(v)
-	}
-	return total
-}
-
-// total is the whole retained history, used to decide what to evict.
-func (e accessEntry) total() uint64 {
-	var t uint64
-	for _, v := range e.Bins {
-		t += uint64(v)
-	}
-	return t
-}
-
-// mergeFrom adds another node's counters for the same object. Both sides must
-// already be rolled to the same head.
-func (e *accessEntry) mergeFrom(o accessEntry) {
-	for i := range e.Bins {
-		if i < len(o.Bins) {
-			total := uint64(e.Bins[i]) + uint64(o.Bins[i])
-			if total > uint64(^uint32(0)) {
-				total = uint64(^uint32(0))
-			}
-			e.Bins[i] = uint32(total)
-		}
-	}
-	if o.LastAt > e.LastAt {
-		e.LastAt = o.LastAt
-	}
-}
-
-// mergedAccess is an immutable cluster-wide snapshot. Readers take it from an
-// atomic pointer, so the hot scanner and sweep paths never take a lock.
-type mergedAccess struct {
-	entries  map[string]accessEntry
-	binWidth int64
-	at       int64
-}
-
-func (m *mergedAccess) hits(key string, window time.Duration) uint64 {
-	if m == nil {
-		return 0
-	}
-	e, ok := m.entries[key]
-	if !ok {
-		return 0
-	}
-	return e.hits(window, m.binWidth)
-}
-
-func (m *mergedAccess) lastAccess(key string) int64 {
-	if m == nil {
-		return 0
-	}
-	return m.entries[key].LastAt
-}
-
-// accessTracker records how often each object is read.
-//
-// Ownership is deliberately narrow: the live counter map is touched only by
-// run()'s goroutine, so it needs no lock. Everything read from elsewhere goes
-// through the immutable merged snapshot.
-type accessTracker struct {
-	ch      chan string
-	enabled atomic.Bool
-	merged  atomic.Pointer[mergedAccess]
-
-	// Demote candidates arrive from scanner goroutines, so this one does
-	// need a lock. It is small: only objects we previously promoted.
-	demoteMu sync.Mutex
-	demote   map[string]demoteCandidate
-
-	dropped atomic.Uint64
-}
-
-var globalAccessTracker = newAccessTracker()
-
-func newAccessTracker() *accessTracker {
-	return &accessTracker{
-		ch:     make(chan string, accessQueueSize),
-		demote: make(map[string]demoteCandidate),
-	}
-}
-
-// accessKey is the tracker's map key. Bucket names cannot contain '/', so the
-// join is unambiguous.
-func accessKey(bucket, object string) string {
-	return bucket + "/" + object
-}
-
-func splitAccessKey(key string) (bucket, object string, ok bool) {
-	bucket, object, ok = strings.Cut(key, "/")
-	if !ok || bucket == "" || object == "" {
-		return "", "", false
-	}
-	return bucket, object, true
-}
-
-// note records one read. It is called from the GET path and must never block
-// or allocate meaningfully: on a full queue the sample is dropped.
-func (t *accessTracker) note(bucket, object string) {
-	if t == nil || !t.enabled.Load() {
-		return
-	}
-	select {
-	case t.ch <- accessKey(bucket, object):
-	default:
-		t.dropped.Add(1)
-	}
-}
-
-// noteDemoteCandidate is called by the scanner for an object it found on a
-// fast pool that has gone quiet. The leader picks these up on the next merge.
-func (t *accessTracker) noteDemoteCandidate(bucket, object string, pool int) {
-	if t == nil || !t.enabled.Load() {
-		return
-	}
-	t.demoteMu.Lock()
-	defer t.demoteMu.Unlock()
-	if len(t.demote) >= accessMaxDemoteCandidates {
-		return
-	}
-	t.demote[accessKey(bucket, object)] = demoteCandidate{Bucket: bucket, Object: object, Pool: pool}
-}
-
-// hits reports cluster-wide accesses to an object over window.
-func (t *accessTracker) hits(bucket, object string, window time.Duration) uint64 {
-	if t == nil {
-		return 0
-	}
-	return t.merged.Load().hits(accessKey(bucket, object), window)
-}
-
-// lastAccess reports the cluster-wide time an object was last read. A zero
-// time means "no read on record", which for demotion purposes is idle.
-func (t *accessTracker) lastAccess(bucket, object string) time.Time {
-	if t == nil {
-		return time.Time{}
-	}
-	sec := t.merged.Load().lastAccess(accessKey(bucket, object))
-	if sec == 0 {
-		return time.Time{}
-	}
-	return time.Unix(sec, 0)
-}
-
-// snapshot returns the current merged view, or nil if none has been published.
-func (t *accessTracker) snapshot() *mergedAccess {
-	if t == nil {
-		return nil
-	}
-	return t.merged.Load()
-}
-
-// takeDemoteCandidates drains and returns the pending candidates.
-func (t *accessTracker) takeDemoteCandidates() []demoteCandidate {
-	t.demoteMu.Lock()
-	defer t.demoteMu.Unlock()
-	if len(t.demote) == 0 {
-		return nil
-	}
-	out := make([]demoteCandidate, 0, len(t.demote))
-	for _, c := range t.demote {
-		out = append(out, c)
-	}
-	t.demote = make(map[string]demoteCandidate)
-	return out
-}
-
-// restoreDemoteCandidates puts candidates back when the publisher is still
-// busy. Dropping access samples is acceptable; dropping the only scanner
-// discovery of an idle promoted object would delay demotion by a full scan.
-func (t *accessTracker) restoreDemoteCandidates(candidates []demoteCandidate) {
-	if len(candidates) == 0 {
-		return
-	}
-	t.demoteMu.Lock()
-	defer t.demoteMu.Unlock()
-	for _, c := range candidates {
-		if len(t.demote) >= accessMaxDemoteCandidates {
-			return
-		}
-		t.demote[accessKey(c.Bucket, c.Object)] = c
-	}
-}
-
-// shardName is this node's counter object. The node name is hashed so that
-// host:port never has to be escaped into an object key.
-func (t *accessTracker) shardName() string {
-	return fmt.Sprintf("%s/%016x.bin", accessTrackerPrefix, xxh3.HashString(globalLocalNodeName))
-}
-
-// run owns the live counter map. It drains reads, and on every flush interval
-// hands a marshaled shard to a background publisher.
-//
-// Everything here is best effort: this is accounting for a background data
-// movement decision, not a durability path.
-func (t *accessTracker) run(ctx context.Context, objAPI ObjectLayer) {
-	cfg := globalILMConfig.accessCfg()
-	t.enabled.Store(cfg.AccessTiering)
-
-	live := make(map[string]accessEntry)
-	if cfg.AccessTiering {
-		if buf, err := readConfig(ctx, objAPI, t.shardName()); err == nil {
-			var previous accessShard
-			if _, err = previous.UnmarshalMsg(buf); err == nil && previous.BinWidth == int64(cfg.AccessBinWidth/time.Second) {
-				now := time.Now().Unix()
-				for key, entry := range previous.Entries {
-					entry.rollTo(now, previous.BinWidth, cfg.AccessBins)
-					if entry.total() != 0 {
-						live[key] = entry
-					}
-				}
-			}
-		}
-	}
-	ticker := time.NewTicker(cfg.AccessFlush)
-	defer ticker.Stop()
-
-	// One publisher goroutine keeps object-layer I/O off the drain loop.
-	type pub struct {
-		shard accessShard
-		cfg   ilm.Config
-	}
-	pubCh := make(chan pub, 1)
-	go func() {
-		for p := range pubCh {
-			t.publish(ctx, objAPI, p.shard, p.cfg)
-		}
-	}()
-	defer close(pubCh)
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-
-		case key := <-t.ch:
-			now := time.Now().Unix()
-			e := live[key]
-			e.rollTo(now, int64(cfg.AccessBinWidth/time.Second), cfg.AccessBins)
-			if len(e.Bins) > 0 {
-				if e.Bins[0] != ^uint32(0) {
-					e.Bins[0]++
-				}
-			}
-			e.LastAt = now
-			live[key] = e
-
-		case <-ticker.C:
-			newCfg := globalILMConfig.accessCfg()
-			t.enabled.Store(newCfg.AccessTiering)
-			if newCfg.AccessFlush != cfg.AccessFlush && newCfg.AccessFlush > 0 {
-				ticker.Reset(newCfg.AccessFlush)
-			}
-			cfg = newCfg
-			if !cfg.AccessTiering {
-				// Feature turned off: release the counters rather than
-				// holding a stale working set for the process lifetime.
-				clear(live)
-				t.merged.Store(nil)
-				continue
-			}
-
-			now := time.Now().Unix()
-			binWidth := int64(cfg.AccessBinWidth / time.Second)
-			t.evict(live, now, binWidth, cfg)
-
-			shard := accessShard{
-				UpdatedAt: now,
-				BinWidth:  binWidth,
-				Entries:   make(map[string]accessEntry, len(live)),
-				Demote:    t.takeDemoteCandidates(),
-			}
-			for k, e := range live {
-				e.Bins = slices.Clone(e.Bins)
-				shard.Entries[k] = e
-			}
-			select {
-			case pubCh <- pub{shard: shard, cfg: cfg}:
-			default:
-				// Previous publish still running; skip this round
-				// rather than queueing work we cannot keep up with.
-				t.restoreDemoteCandidates(shard.Demote)
-			}
-		}
-	}
-}
-
-// evict rolls every counter forward and drops the ones with no hits left in
-// the retained history, then enforces the tracked-object cap.
-//
-// ponytail: amortized O(n) sweep on the flush tick; a heap would only pay off
-// past ~10M tracked keys.
-func (t *accessTracker) evict(live map[string]accessEntry, now, binWidth int64, cfg ilm.Config) {
-	for k, e := range live {
-		e.rollTo(now, binWidth, cfg.AccessBins)
-		if e.total() == 0 {
-			delete(live, k)
-			continue
-		}
-		live[k] = e
-	}
-	if len(live) <= cfg.AccessMaxTracked {
-		return
-	}
-	// Over the cap: keep the hottest and drop the rest. Objects that fall
-	// out are cold by construction, which is exactly what the demotion path
-	// already assumes about anything missing from the map.
-	type kt struct {
-		key   string
-		total uint64
-	}
-	all := make([]kt, 0, len(live))
-	for k, e := range live {
-		all = append(all, kt{k, e.total()})
-	}
-	slices.SortFunc(all, func(a, b kt) int { return cmp.Compare(b.total, a.total) })
-	for _, x := range all[cfg.AccessMaxTracked:] {
-		delete(live, x.key)
-	}
-}
-
-// publish writes this node's shard and rebuilds the merged snapshot from every
-// node's shard. Both halves are best effort.
-func (t *accessTracker) publish(ctx context.Context, objAPI ObjectLayer, shard accessShard, cfg ilm.Config) {
-	buf, err := shard.MarshalMsg(nil)
-	if err != nil {
-		ilmLogIf(ctx, err)
-		return
-	}
-	if err := saveConfig(ctx, objAPI, t.shardName(), buf); err != nil {
-		ilmLogIf(ctx, err)
-		// Still rebuild the snapshot below: our own counters are already
-		// in hand and a stale peer view beats no view.
-	}
-	t.merged.Store(t.mergeShards(ctx, objAPI, shard, cfg))
-}
-
-// mergeShards sums every live node's counters, including our own in-memory
-// shard so this node's most recent reads are never a flush behind.
-func (t *accessTracker) mergeShards(ctx context.Context, objAPI ObjectLayer, own accessShard, cfg ilm.Config) *mergedAccess {
-	now := time.Now().Unix()
-	binWidth := int64(cfg.AccessBinWidth / time.Second)
-	out := &mergedAccess{
-		entries:  make(map[string]accessEntry, len(own.Entries)),
-		binWidth: binWidth,
-		at:       now,
-	}
-
-	add := func(s accessShard) {
-		if s.BinWidth != binWidth {
-			// A peer has not yet picked up a bin-width change; merging
-			// its bins would silently mis-scale the counts.
-			return
-		}
-		for k, e := range s.Entries {
-			e.rollTo(now, binWidth, cfg.AccessBins)
-			cur, ok := out.entries[k]
-			if !ok {
-				cur = accessEntry{Bins: make([]uint32, cfg.AccessBins)}
-			}
-			cur.mergeFrom(e)
-			out.entries[k] = cur
-		}
-	}
-
-	add(own)
-
-	ownName := t.shardName()
-	stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor
-	for _, name := range t.listShards(ctx, objAPI) {
-		if name == ownName {
-			continue
-		}
-		buf, err := readConfig(ctx, objAPI, name)
-		if err != nil {
-			continue
-		}
-		var s accessShard
-		if _, err := s.UnmarshalMsg(buf); err != nil {
-			continue
-		}
-		if !accessShardFresh(now, s, stale) {
-			continue // node is gone or wedged
-		}
-		add(s)
-	}
-	return out
-}
-
-func (t *accessTracker) listShards(ctx context.Context, objAPI ObjectLayer) []string {
-	res, err := objAPI.ListObjects(ctx, minioMetaBucket, accessTrackerPrefix+"/", "", "", maxObjectList)
-	if err != nil {
-		return nil
-	}
-	names := make([]string, 0, len(res.Objects))
-	for _, o := range res.Objects {
-		if strings.HasSuffix(o.Name, ".bin") {
-			names = append(names, o.Name)
-		}
-	}
-	return names
-}
-
-// collectDemoteCandidates returns every node's pending demote candidates. Only
-// the leader calls this, right before running a demotion pass.
-//
-// Our own shard is read back rather than skipped: run() drains the local map
-// into the published shard, so anything found since the last leader pass lives
-// there, not in memory. The local map is still drained here to pick up
-// candidates recorded since that publish.
-func (t *accessTracker) collectDemoteCandidates(ctx context.Context, objAPI ObjectLayer, cfg ilm.Config) []demoteCandidate {
-	seen := make(map[string]struct{})
-	var out []demoteCandidate
-
-	for _, c := range t.takeDemoteCandidates() {
-		key := accessKey(c.Bucket, c.Object)
-		seen[key] = struct{}{}
-		out = append(out, c)
-	}
-
-	now := time.Now().Unix()
-	stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor
-	for _, name := range t.listShards(ctx, objAPI) {
-		buf, err := readConfig(ctx, objAPI, name)
-		if err != nil {
-			continue
-		}
-		var s accessShard
-		if _, err := s.UnmarshalMsg(buf); err != nil {
-			continue
-		}
-		if !accessShardFresh(now, s, stale) {
-			continue
-		}
-		for _, c := range s.Demote {
-			key := accessKey(c.Bucket, c.Object)
-			if _, dup := seen[key]; dup {
-				continue
-			}
-			seen[key] = struct{}{}
-			out = append(out, c)
-		}
-	}
-	return out
-}

cmd/ilm-access-tracker_gen.go

@@ -1,742 +0,0 @@
-// Code generated by github.com/tinylib/msgp DO NOT EDIT.
-
-package cmd
-
-import (
-	"github.com/tinylib/msgp/msgp"
-)
-
-// DecodeMsg implements msgp.Decodable
-func (z *accessEntry) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Bins")
-				return
-			}
-			if cap(z.Bins) >= int(zb0002) {
-				z.Bins = (z.Bins)[:zb0002]
-			} else {
-				z.Bins = make([]uint32, zb0002)
-			}
-			for za0001 := range z.Bins {
-				z.Bins[za0001], err = dc.ReadUint32()
-				if err != nil {
-					err = msgp.WrapError(err, "Bins", za0001)
-					return
-				}
-			}
-		case "h":
-			z.HeadAt, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "HeadAt")
-				return
-			}
-		case "l":
-			z.LastAt, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "LastAt")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *accessEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
-	// write "b"
-	err = en.Append(0x83, 0xa1, 0x62)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.Bins)))
-	if err != nil {
-		err = msgp.WrapError(err, "Bins")
-		return
-	}
-	for za0001 := range z.Bins {
-		err = en.WriteUint32(z.Bins[za0001])
-		if err != nil {
-			err = msgp.WrapError(err, "Bins", za0001)
-			return
-		}
-	}
-	// write "h"
-	err = en.Append(0xa1, 0x68)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.HeadAt)
-	if err != nil {
-		err = msgp.WrapError(err, "HeadAt")
-		return
-	}
-	// write "l"
-	err = en.Append(0xa1, 0x6c)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.LastAt)
-	if err != nil {
-		err = msgp.WrapError(err, "LastAt")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *accessEntry) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
-	// string "b"
-	o = append(o, 0x83, 0xa1, 0x62)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.Bins)))
-	for za0001 := range z.Bins {
-		o = msgp.AppendUint32(o, z.Bins[za0001])
-	}
-	// string "h"
-	o = append(o, 0xa1, 0x68)
-	o = msgp.AppendInt64(o, z.HeadAt)
-	// string "l"
-	o = append(o, 0xa1, 0x6c)
-	o = msgp.AppendInt64(o, z.LastAt)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *accessEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Bins")
-				return
-			}
-			if cap(z.Bins) >= int(zb0002) {
-				z.Bins = (z.Bins)[:zb0002]
-			} else {
-				z.Bins = make([]uint32, zb0002)
-			}
-			for za0001 := range z.Bins {
-				z.Bins[za0001], bts, err = msgp.ReadUint32Bytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Bins", za0001)
-					return
-				}
-			}
-		case "h":
-			z.HeadAt, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "HeadAt")
-				return
-			}
-		case "l":
-			z.LastAt, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "LastAt")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *accessEntry) Msgsize() (s int) {
-	s = 1 + 2 + msgp.ArrayHeaderSize + (len(z.Bins) * (msgp.Uint32Size)) + 2 + msgp.Int64Size + 2 + msgp.Int64Size
-	return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *accessShard) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "u":
-			z.UpdatedAt, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "UpdatedAt")
-				return
-			}
-		case "bw":
-			z.BinWidth, err = dc.ReadInt64()
-			if err != nil {
-				err = msgp.WrapError(err, "BinWidth")
-				return
-			}
-		case "e":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Entries")
-				return
-			}
-			if z.Entries == nil {
-				z.Entries = make(map[string]accessEntry, zb0002)
-			} else if len(z.Entries) > 0 {
-				clear(z.Entries)
-			}
-			for zb0002 > 0 {
-				zb0002--
-				var za0001 string
-				za0001, err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Entries")
-					return
-				}
-				var za0002 accessEntry
-				err = za0002.DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "Entries", za0001)
-					return
-				}
-				z.Entries[za0001] = za0002
-			}
-		case "d":
-			var zb0003 uint32
-			zb0003, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Demote")
-				return
-			}
-			if cap(z.Demote) >= int(zb0003) {
-				z.Demote = (z.Demote)[:zb0003]
-			} else {
-				z.Demote = make([]demoteCandidate, zb0003)
-			}
-			for za0003 := range z.Demote {
-				var zb0004 uint32
-				zb0004, err = dc.ReadMapHeader()
-				if err != nil {
-					err = msgp.WrapError(err, "Demote", za0003)
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "Demote", za0003)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "b":
-						z.Demote[za0003].Bucket, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Bucket")
-							return
-						}
-					case "o":
-						z.Demote[za0003].Object, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Object")
-							return
-						}
-					case "p":
-						z.Demote[za0003].Pool, err = dc.ReadInt()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Pool")
-							return
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003)
-							return
-						}
-					}
-				}
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *accessShard) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 4
-	// write "u"
-	err = en.Append(0x84, 0xa1, 0x75)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.UpdatedAt)
-	if err != nil {
-		err = msgp.WrapError(err, "UpdatedAt")
-		return
-	}
-	// write "bw"
-	err = en.Append(0xa2, 0x62, 0x77)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.BinWidth)
-	if err != nil {
-		err = msgp.WrapError(err, "BinWidth")
-		return
-	}
-	// write "e"
-	err = en.Append(0xa1, 0x65)
-	if err != nil {
-		return
-	}
-	err = en.WriteMapHeader(uint32(len(z.Entries)))
-	if err != nil {
-		err = msgp.WrapError(err, "Entries")
-		return
-	}
-	for za0001, za0002 := range z.Entries {
-		err = en.WriteString(za0001)
-		if err != nil {
-			err = msgp.WrapError(err, "Entries")
-			return
-		}
-		err = za0002.EncodeMsg(en)
-		if err != nil {
-			err = msgp.WrapError(err, "Entries", za0001)
-			return
-		}
-	}
-	// write "d"
-	err = en.Append(0xa1, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.Demote)))
-	if err != nil {
-		err = msgp.WrapError(err, "Demote")
-		return
-	}
-	for za0003 := range z.Demote {
-		// map header, size 3
-		// write "b"
-		err = en.Append(0x83, 0xa1, 0x62)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Demote[za0003].Bucket)
-		if err != nil {
-			err = msgp.WrapError(err, "Demote", za0003, "Bucket")
-			return
-		}
-		// write "o"
-		err = en.Append(0xa1, 0x6f)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Demote[za0003].Object)
-		if err != nil {
-			err = msgp.WrapError(err, "Demote", za0003, "Object")
-			return
-		}
-		// write "p"
-		err = en.Append(0xa1, 0x70)
-		if err != nil {
-			return
-		}
-		err = en.WriteInt(z.Demote[za0003].Pool)
-		if err != nil {
-			err = msgp.WrapError(err, "Demote", za0003, "Pool")
-			return
-		}
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *accessShard) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
-	// string "u"
-	o = append(o, 0x84, 0xa1, 0x75)
-	o = msgp.AppendInt64(o, z.UpdatedAt)
-	// string "bw"
-	o = append(o, 0xa2, 0x62, 0x77)
-	o = msgp.AppendInt64(o, z.BinWidth)
-	// string "e"
-	o = append(o, 0xa1, 0x65)
-	o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
-	for za0001, za0002 := range z.Entries {
-		o = msgp.AppendString(o, za0001)
-		o, err = za0002.MarshalMsg(o)
-		if err != nil {
-			err = msgp.WrapError(err, "Entries", za0001)
-			return
-		}
-	}
-	// string "d"
-	o = append(o, 0xa1, 0x64)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.Demote)))
-	for za0003 := range z.Demote {
-		// map header, size 3
-		// string "b"
-		o = append(o, 0x83, 0xa1, 0x62)
-		o = msgp.AppendString(o, z.Demote[za0003].Bucket)
-		// string "o"
-		o = append(o, 0xa1, 0x6f)
-		o = msgp.AppendString(o, z.Demote[za0003].Object)
-		// string "p"
-		o = append(o, 0xa1, 0x70)
-		o = msgp.AppendInt(o, z.Demote[za0003].Pool)
-	}
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *accessShard) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "u":
-			z.UpdatedAt, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "UpdatedAt")
-				return
-			}
-		case "bw":
-			z.BinWidth, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "BinWidth")
-				return
-			}
-		case "e":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Entries")
-				return
-			}
-			if z.Entries == nil {
-				z.Entries = make(map[string]accessEntry, zb0002)
-			} else if len(z.Entries) > 0 {
-				clear(z.Entries)
-			}
-			for zb0002 > 0 {
-				var za0002 accessEntry
-				zb0002--
-				var za0001 string
-				za0001, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Entries")
-					return
-				}
-				bts, err = za0002.UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Entries", za0001)
-					return
-				}
-				z.Entries[za0001] = za0002
-			}
-		case "d":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Demote")
-				return
-			}
-			if cap(z.Demote) >= int(zb0003) {
-				z.Demote = (z.Demote)[:zb0003]
-			} else {
-				z.Demote = make([]demoteCandidate, zb0003)
-			}
-			for za0003 := range z.Demote {
-				var zb0004 uint32
-				zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Demote", za0003)
-					return
-				}
-				for zb0004 > 0 {
-					zb0004--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Demote", za0003)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "b":
-						z.Demote[za0003].Bucket, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Bucket")
-							return
-						}
-					case "o":
-						z.Demote[za0003].Object, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Object")
-							return
-						}
-					case "p":
-						z.Demote[za0003].Pool, bts, err = msgp.ReadIntBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003, "Pool")
-							return
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Demote", za0003)
-							return
-						}
-					}
-				}
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *accessShard) Msgsize() (s int) {
-	s = 1 + 2 + msgp.Int64Size + 3 + msgp.Int64Size + 2 + msgp.MapHeaderSize
-	if z.Entries != nil {
-		for za0001, za0002 := range z.Entries {
-			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
-		}
-	}
-	s += 2 + msgp.ArrayHeaderSize
-	for za0003 := range z.Demote {
-		s += 1 + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Bucket) + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Object) + 2 + msgp.IntSize
-	}
-	return
-}
-
-// DecodeMsg implements msgp.Decodable
-func (z *demoteCandidate) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			z.Bucket, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Bucket")
-				return
-			}
-		case "o":
-			z.Object, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Object")
-				return
-			}
-		case "p":
-			z.Pool, err = dc.ReadInt()
-			if err != nil {
-				err = msgp.WrapError(err, "Pool")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z demoteCandidate) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
-	// write "b"
-	err = en.Append(0x83, 0xa1, 0x62)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Bucket)
-	if err != nil {
-		err = msgp.WrapError(err, "Bucket")
-		return
-	}
-	// write "o"
-	err = en.Append(0xa1, 0x6f)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Object)
-	if err != nil {
-		err = msgp.WrapError(err, "Object")
-		return
-	}
-	// write "p"
-	err = en.Append(0xa1, 0x70)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.Pool)
-	if err != nil {
-		err = msgp.WrapError(err, "Pool")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z demoteCandidate) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
-	// string "b"
-	o = append(o, 0x83, 0xa1, 0x62)
-	o = msgp.AppendString(o, z.Bucket)
-	// string "o"
-	o = append(o, 0xa1, 0x6f)
-	o = msgp.AppendString(o, z.Object)
-	// string "p"
-	o = append(o, 0xa1, 0x70)
-	o = msgp.AppendInt(o, z.Pool)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *demoteCandidate) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "b":
-			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Bucket")
-				return
-			}
-		case "o":
-			z.Object, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Object")
-				return
-			}
-		case "p":
-			z.Pool, bts, err = msgp.ReadIntBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Pool")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z demoteCandidate) Msgsize() (s int) {
-	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 2 + msgp.IntSize
-	return
-}

cmd/ilm-access-tracker_gen_test.go

@@ -1,349 +0,0 @@
-// Code generated by github.com/tinylib/msgp DO NOT EDIT.
-
-package cmd
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/tinylib/msgp/msgp"
-)
-
-func TestMarshalUnmarshalaccessEntry(t *testing.T) {
-	v := accessEntry{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeaccessEntry(t *testing.T) {
-	v := accessEntry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeaccessEntry Msgsize() is inaccurate")
-	}
-
-	vn := accessEntry{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeaccessEntry(b *testing.B) {
-	v := accessEntry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalaccessShard(t *testing.T) {
-	v := accessShard{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgaccessShard(b *testing.B) {
-	v := accessShard{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgaccessShard(b *testing.B) {
-	v := accessShard{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalaccessShard(b *testing.B) {
-	v := accessShard{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeaccessShard(t *testing.T) {
-	v := accessShard{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeaccessShard Msgsize() is inaccurate")
-	}
-
-	vn := accessShard{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeaccessShard(b *testing.B) {
-	v := accessShard{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeaccessShard(b *testing.B) {
-	v := accessShard{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshaldemoteCandidate(t *testing.T) {
-	v := demoteCandidate{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgdemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgdemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshaldemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodedemoteCandidate(t *testing.T) {
-	v := demoteCandidate{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodedemoteCandidate Msgsize() is inaccurate")
-	}
-
-	vn := demoteCandidate{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodedemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodedemoteCandidate(b *testing.B) {
-	v := demoteCandidate{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

cmd/ilm-access-tracker_test.go

@@ -1,95 +0,0 @@
-// Copyright (c) 2015-2026 MinIO, Inc.
-
-package cmd
-
-import (
-	"math"
-	"testing"
-	"time"
-
-	"github.com/minio/minio/internal/config/ilm"
-)
-
-func TestAccessEntryRollAndHits(t *testing.T) {
-	entry := accessEntry{Bins: []uint32{3, 2, 1}, HeadAt: 100, LastAt: 107}
-	entry.rollTo(120, 10, 3)
-	want := []uint32{0, 0, 3}
-	for i := range want {
-		if entry.Bins[i] != want[i] {
-			t.Fatalf("bins = %v, want %v", entry.Bins, want)
-		}
-	}
-	if entry.HeadAt != 120 || entry.LastAt != 107 {
-		t.Fatalf("head/last = %d/%d", entry.HeadAt, entry.LastAt)
-	}
-
-	entry = accessEntry{Bins: []uint32{10, 20, 30}, HeadAt: 120}
-	if got := entry.hits(20*time.Second, 10); got != 30 {
-		t.Fatalf("20s hits = %d, want 30", got)
-	}
-	if got := entry.hits(21*time.Second, 10); got != 60 {
-		t.Fatalf("21s hits = %d, want 60", got)
-	}
-	entry.rollTo(200, 10, 3)
-	if got := entry.total(); got != 0 {
-		t.Fatalf("expired total = %d, want 0", got)
-	}
-}
-
-func TestAccessEntryMergeSaturates(t *testing.T) {
-	entry := accessEntry{Bins: []uint32{math.MaxUint32 - 1}}
-	entry.mergeFrom(accessEntry{Bins: []uint32{10}, LastAt: 50})
-	if entry.Bins[0] != math.MaxUint32 || entry.LastAt != 50 {
-		t.Fatalf("merged entry = %+v", entry)
-	}
-}
-
-func TestAccessTrackerEvictsColdest(t *testing.T) {
-	tracker := newAccessTracker()
-	live := map[string]accessEntry{
-		"a": {Bins: []uint32{1}, HeadAt: 100},
-		"b": {Bins: []uint32{5}, HeadAt: 100},
-		"c": {Bins: []uint32{3}, HeadAt: 100},
-	}
-	tracker.evict(live, 100, 10, ilm.Config{AccessBins: 1, AccessMaxTracked: 2})
-	if len(live) != 2 {
-		t.Fatalf("len = %d, want 2", len(live))
-	}
-	if _, ok := live["a"]; ok {
-		t.Fatal("coldest entry was retained")
-	}
-}
-
-func TestAccessKeyRoundTrip(t *testing.T) {
-	key := accessKey("bucket", "a/b/c")
-	bucket, object, ok := splitAccessKey(key)
-	if !ok || bucket != "bucket" || object != "a/b/c" {
-		t.Fatalf("split %q = %q/%q/%v", key, bucket, object, ok)
-	}
-}
-
-func TestAccessShardFresh(t *testing.T) {
-	const now = int64(1000)
-	if !accessShardFresh(now, accessShard{UpdatedAt: 950}, 100) {
-		t.Fatal("fresh shard rejected")
-	}
-	if accessShardFresh(now, accessShard{UpdatedAt: 899}, 100) {
-		t.Fatal("stale shard accepted")
-	}
-	if accessShardFresh(now, accessShard{UpdatedAt: 1101}, 100) {
-		t.Fatal("far-future shard accepted")
-	}
-	if accessShardFresh(now, accessShard{}, 100) {
-		t.Fatal("zero timestamp accepted")
-	}
-}
-
-func TestAccessTrackerRestoresDemoteCandidates(t *testing.T) {
-	tracker := newAccessTracker()
-	candidate := demoteCandidate{Bucket: "bucket", Object: "object", Pool: 1}
-	tracker.restoreDemoteCandidates([]demoteCandidate{candidate})
-	got := tracker.takeDemoteCandidates()
-	if len(got) != 1 || got[0] != candidate {
-		t.Fatalf("restored candidates = %+v, want %+v", got, candidate)
-	}
-}

cmd/ilm-config.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"sync"
-	"time"
 
 	"github.com/minio/minio/internal/config/ilm"
 )
@@ -28,16 +27,6 @@ var globalILMConfig = ilmConfig{
 	cfg: ilm.Config{
 		ExpirationWorkers: 100,
 		TransitionWorkers: 100,
-		// Access tiering stays off until configured, but the counter
-		// geometry must be sane from the start: the tracker divides by
-		// AccessBinWidth before any config is loaded.
-		AccessPromoteWatermark: 85,
-		AccessBinWidth:         time.Minute,
-		AccessBins:             12,
-		AccessFlush:            time.Minute,
-		AccessMinResidency:     24 * time.Hour,
-		AccessWorkers:          10,
-		AccessMaxTracked:       1000000,
 	},
 }
 
@@ -60,25 +49,6 @@ func (c *ilmConfig) getTransitionWorkers() int {
 	return c.cfg.TransitionWorkers
 }
 
-// accessCfg returns a copy of the access tiering settings. Callers take the
-// whole struct rather than one getter per field because the promotion and
-// demotion paths need a consistent view of several knobs at once.
-func (c *ilmConfig) accessCfg() ilm.Config {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	return c.cfg
-}
-
-// accessTieringEnabled is the cheap gate used on the scanner path.
-func (c *ilmConfig) accessTieringEnabled() bool {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	_, ok := c.cfg.HotPool()
-	return ok
-}
-
 func (c *ilmConfig) update(cfg ilm.Config) {
 	c.mu.Lock()
 	defer c.mu.Unlock()

cmd/lceventsrc_string.go

@@ -19,12 +19,11 @@ func _() {
 	_ = x[lcEventSrc_s3PutObject-8]
 	_ = x[lcEventSrc_s3CopyObject-9]
 	_ = x[lcEventSrc_s3CompleteMultipartUpload-10]
-	_ = x[lcEventSrc_AccessTier-11]
 }
 
-const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUploadAccessTier"
+const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUpload"
 
-var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109, 119}
+var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109}
 
 func (i lcEventSrc) String() string {
 	idx := int(i) - 0

cmd/metrics-v3-cluster-iam.go

@@ -34,6 +34,10 @@ const (
 	sinceLastSyncMillis                    = "since_last_sync_millis"
 	syncFailures                           = "sync_failures"
 	syncSuccesses                          = "sync_successes"
+	revocationRecords                      = "revocation_records"
+	revocationHealFailures                 = "revocation_heal_failures"
+	revocationHealDurationMillis           = "revocation_heal_duration_millis"
+	revocationHealLastSuccess              = "revocation_heal_last_success_timestamp_seconds"
 )
 
 var (
@@ -47,10 +51,20 @@ var (
 	sinceLastSyncMillisMD                    = NewCounterMD(sinceLastSyncMillis, "Time (in milliseconds) since last successful IAM data sync.")
 	syncFailuresMD                           = NewCounterMD(syncFailures, "Number of failed IAM data syncs since server start.")
 	syncSuccessesMD                          = NewCounterMD(syncSuccesses, "Number of successful IAM data syncs since server start.")
+	revocationRecordsMD                      = NewGaugeMD(revocationRecords, "Retained IAM deletion records and revocation boundaries in this node's index.")
+	revocationHealFailuresMD                 = NewCounterMD(revocationHealFailures, "Failed IAM revocation convergence passes since server start.")
+	revocationHealDurationMillisMD           = NewGaugeMD(revocationHealDurationMillis, "Duration of the last IAM revocation convergence pass in milliseconds.")
+	revocationHealLastSuccessMD              = NewGaugeMD(revocationHealLastSuccess, "Unix timestamp of the last successful IAM revocation convergence pass.")
 )
 
 // loadClusterIAMMetrics - `MetricsLoaderFn` for cluster IAM metrics.
 func loadClusterIAMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
+	if globalIAMSys.Initialized() {
+		m.Set(revocationRecords, float64(globalIAMSys.store.revisionIndex().count()))
+	}
+	m.Set(revocationHealFailures, float64(globalSiteReplicationSys.iamRevisionMetrics.healFailures.Load()))
+	m.Set(revocationHealDurationMillis, float64(globalSiteReplicationSys.iamRevisionMetrics.healDurationMillis.Load()))
+	m.Set(revocationHealLastSuccess, float64(globalSiteReplicationSys.iamRevisionMetrics.healLastSuccess.Load()))
 	m.Set(lastSyncDurationMillis, float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)))
 	pluginAuthNMetrics := globalAuthNPlugin.Metrics()
 	m.Set(pluginAuthnServiceFailedRequestsMinute, float64(pluginAuthNMetrics.FailedRequests))

cmd/metrics-v3-ilm.go

@@ -26,17 +26,6 @@ const (
 	transitionActiveTasks          = "transition_active_tasks"
 	transitionPendingTasks         = "transition_pending_tasks"
 	transitionMissedImmediateTasks = "transition_missed_immediate_tasks"
-	accessTierActiveTasks          = "access_tier_active_tasks"
-	accessTierPendingTasks         = "access_tier_pending_tasks"
-	accessTierPromotionsTotal      = "access_tier_promotions_total"
-	accessTierDemotionsTotal       = "access_tier_demotions_total"
-	accessTierBytesMovedTotal      = "access_tier_bytes_moved_total"
-	accessTierFailuresTotal        = "access_tier_failures_total"
-	accessTierSkippedWatermark     = "access_tier_skipped_watermark_total"
-	accessTierSkippedMaxSize       = "access_tier_skipped_max_size_total"
-	accessTierSkippedQuota         = "access_tier_skipped_bucket_quota_total"
-	accessTierHotBytes             = "access_tier_hot_bytes"
-	accessTierSamplesDropped       = "access_tier_samples_dropped_total"
 	versionsScanned                = "versions_scanned"
 )
 
@@ -45,17 +34,6 @@ var (
 	ilmTransitionActiveTasksMD          = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks")
 	ilmTransitionPendingTasksMD         = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue")
 	ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks")
-	ilmAccessTierActiveTasksMD          = NewGaugeMD(accessTierActiveTasks, "Number of active access-tier pool moves")
-	ilmAccessTierPendingTasksMD         = NewGaugeMD(accessTierPendingTasks, "Number of pending access-tier pool moves")
-	ilmAccessTierPromotionsTotalMD      = NewCounterMD(accessTierPromotionsTotal, "Total objects promoted by access-tier ILM")
-	ilmAccessTierDemotionsTotalMD       = NewCounterMD(accessTierDemotionsTotal, "Total objects demoted by access-tier ILM")
-	ilmAccessTierBytesMovedTotalMD      = NewCounterMD(accessTierBytesMovedTotal, "Total logical bytes moved by access-tier ILM")
-	ilmAccessTierFailuresTotalMD        = NewCounterMD(accessTierFailuresTotal, "Total failed access-tier ILM moves")
-	ilmAccessTierSkippedWatermarkMD     = NewCounterMD(accessTierSkippedWatermark, "Promotions skipped because the hot pool reached its watermark")
-	ilmAccessTierSkippedMaxSizeMD       = NewCounterMD(accessTierSkippedMaxSize, "Promotions skipped because the cluster hot-tier size cap was reached")
-	ilmAccessTierSkippedQuotaMD         = NewCounterMD(accessTierSkippedQuota, "Promotions skipped because the bucket hot-tier quota was reached")
-	ilmAccessTierHotBytesMD             = NewGaugeMD(accessTierHotBytes, "Logical bytes currently accounted to the hot tier", "bucket")
-	ilmAccessTierSamplesDroppedMD       = NewCounterMD(accessTierSamplesDropped, "GET samples dropped because the access tracker queue was full")
 	ilmVersionsScannedMD                = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start")
 )
 
@@ -69,21 +47,6 @@ func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
 		m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks()))
 		m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks()))
 	}
-	if globalAccessTierState != nil {
-		m.Set(accessTierActiveTasks, float64(globalAccessTierState.ActiveTasks()))
-		m.Set(accessTierPendingTasks, float64(globalAccessTierState.PendingTasks()))
-		m.Set(accessTierPromotionsTotal, float64(globalAccessTierState.promotions.Load()))
-		m.Set(accessTierDemotionsTotal, float64(globalAccessTierState.demotions.Load()))
-		m.Set(accessTierBytesMovedTotal, float64(globalAccessTierState.bytesMoved.Load()))
-		m.Set(accessTierFailuresTotal, float64(globalAccessTierState.failures.Load()))
-		m.Set(accessTierSkippedWatermark, float64(globalAccessTierState.skippedWatermark.Load()))
-		m.Set(accessTierSkippedMaxSize, float64(globalAccessTierState.skippedMaxSize.Load()))
-		m.Set(accessTierSkippedQuota, float64(globalAccessTierState.skippedQuota.Load()))
-		for bucket, bytes := range globalAccessTierState.hotUsageSnapshot() {
-			m.Set(accessTierHotBytes, float64(bytes), "bucket", bucket)
-		}
-	}
-	m.Set(accessTierSamplesDropped, float64(globalAccessTracker.dropped.Load()))
 	m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM)))
 
 	return nil

cmd/metrics-v3.go

@@ -323,6 +323,10 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			sinceLastSyncMillisMD,
 			syncFailuresMD,
 			syncSuccessesMD,
+			revocationRecordsMD,
+			revocationHealFailuresMD,
+			revocationHealDurationMillisMD,
+			revocationHealLastSuccessMD,
 		},
 		loadClusterIAMMetrics,
 	)
@@ -392,17 +396,6 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			ilmTransitionActiveTasksMD,
 			ilmTransitionPendingTasksMD,
 			ilmTransitionMissedImmediateTasksMD,
-			ilmAccessTierActiveTasksMD,
-			ilmAccessTierPendingTasksMD,
-			ilmAccessTierPromotionsTotalMD,
-			ilmAccessTierDemotionsTotalMD,
-			ilmAccessTierBytesMovedTotalMD,
-			ilmAccessTierFailuresTotalMD,
-			ilmAccessTierSkippedWatermarkMD,
-			ilmAccessTierSkippedMaxSizeMD,
-			ilmAccessTierSkippedQuotaMD,
-			ilmAccessTierHotBytesMD,
-			ilmAccessTierSamplesDroppedMD,
 			ilmVersionsScannedMD,
 		},
 		loadILMMetrics,

cmd/object-api-interface.go

@@ -122,10 +122,6 @@ type ObjectOptions struct {
 	SkipRebalancing bool
 
 	SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same
-	// DstPoolIdx forces a data-movement write onto a specific server pool.
-	// It is ignored unless DataMovement is true; a pointer keeps pool zero
-	// distinguishable from the unset value.
-	DstPoolIdx *int
 
 	DataMovement bool // indicates an going decommisionning or rebalacing
 

cmd/object-handlers.go

@@ -576,8 +576,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		return
 	}
 
-	globalAccessTracker.note(bucket, object)
-
 	// Notify object accessed via a GET request.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectAccessedGet,

cmd/peer-rest-server.go

@@ -204,7 +204,9 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(mss *grid.MSS) (np grid.NoP
 		return np, grid.NewRemoteErr(errors.New("service account name is missing"))
 	}
 
-	if err := globalIAMSys.DeleteServiceAccount(context.Background(), accessKey, false); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err := globalIAMSys.LoadServiceAccount(ctx, accessKey); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 
@@ -230,7 +232,8 @@ func (s *peerRESTServer) LoadServiceAccountHandler(mss *grid.MSS) (np grid.NoPay
 	return np, nerr
 }
 
-// DeleteUserHandler - deletes a user on the server.
+// DeleteUserHandler reloads the state committed by another node. A delayed
+// notification must not delete an identity recreated since that commit.
 func (s *peerRESTServer) DeleteUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
@@ -242,7 +245,9 @@ func (s *peerRESTServer) DeleteUserHandler(mss *grid.MSS) (np grid.NoPayload, ne
 		return np, grid.NewRemoteErr(errors.New("username is missing"))
 	}
 
-	if err := globalIAMSys.DeleteUser(context.Background(), accessKey, false); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err := globalIAMSys.LoadUserAfterDelete(ctx, accessKey); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 
@@ -271,7 +276,9 @@ func (s *peerRESTServer) LoadUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr
 		userType = stsUser
 	}
 
-	if err = globalIAMSys.LoadUser(context.Background(), objAPI, accessKey, userType); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err = globalIAMSys.LoadUser(ctx, objAPI, accessKey, userType); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 

cmd/server-main.go

@@ -501,7 +501,6 @@ func initAllSubsystems(ctx context.Context) {
 	globalTierConfigMgr = NewTierConfigMgr()
 
 	globalTransitionState = newTransitionState(GlobalContext)
-	globalAccessTierState = newAccessTierState(GlobalContext)
 	globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext)
 }
 
@@ -1070,10 +1069,6 @@ func serverMain(ctx *cli.Context) {
 		bootstrapTrace("globalTransitionState.Init", func() {
 			globalTransitionState.Init(newObject)
 		})
-		bootstrapTrace("globalAccessTierState.Init", func() {
-			globalAccessTierState.Init(newObject)
-			go globalAccessTracker.run(GlobalContext, newObject)
-		})
 
 		go func() {
 			// Initialize transition tier configuration manager

cmd/site-replication-metadata-gate_test.go

@@ -0,0 +1,341 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+func TestBucketMetadataTombstoneExportAndInitialSync(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			recordBucketConfigPeer(t, cred)
+			old := globalSiteReplicationMetadataTombstones
+			defer func() { globalSiteReplicationMetadataTombstones = old }()
+			created := UTCNow().Add(-time.Hour)
+			deletedAt := created.Add(time.Minute)
+			for _, enabled := range []bool{false, true} {
+				globalSiteReplicationMetadataTombstones = enabled
+				meta := newBucketMetadata(bucket)
+				meta.Created = created
+				meta.defaultTimestamps()
+				for _, file := range []string{bucketPolicyConfig, bucketTaggingConfig, bucketSSEConfig, bucketQuotaConfigFile} {
+					_, at := replicatedBucketConfig(&meta, file)
+					*at = deletedAt
+				}
+				if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					t.Fatal(err)
+				}
+				// Force a disk reload instead of accepting the just-published cache.
+				globalBucketMetadataSys.Remove(bucket)
+				info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(t.Context(), obj, madmin.SRStatusOptions{Buckets: true})
+				if err != nil {
+					t.Fatal(err)
+				}
+				exported := info.Buckets[bucket]
+				if !exported.PolicyUpdatedAt.Equal(deletedAt) {
+					t.Fatal("Policy tombstone hidden by gate")
+				}
+				for _, at := range []time.Time{exported.TagConfigUpdatedAt, exported.SSEConfigUpdatedAt, exported.QuotaConfigUpdatedAt} {
+					if enabled && !at.Equal(deletedAt) || !enabled && !at.IsZero() {
+						t.Fatalf("gate=%v timestamp=%v", enabled, at)
+					}
+				}
+				for file, data := range bucketConfigTestData(bucket) {
+					event, send, err := initialBucketConfigReplicationEvent(meta, file)
+					wantSend := enabled && !bucketConfigUpdateOnly(file)
+					if err != nil || send != wantSend || send && !event.UpdatedAt.Equal(deletedAt) {
+						t.Fatalf("%s initial gate=%v: %+v %v %v", file, enabled, event, send, err)
+					}
+					baseline := newBucketMetadata(bucket)
+					baseline.Created = created
+					baseline.defaultTimestamps()
+					if _, send, err := initialBucketConfigReplicationEvent(baseline, file); err != nil || send {
+						t.Fatalf("empty baseline sent: %s", file)
+					}
+					value, _ := replicatedBucketConfig(&baseline, file)
+					*value = data
+					event, send, err = initialBucketConfigReplicationEvent(baseline, file)
+					if err != nil || !send || !event.UpdatedAt.Equal(created) {
+						t.Fatalf("historical baseline-live omitted: %s %v", file, err)
+					}
+				}
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataLegacyAndGeneration(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			old := globalSiteReplicationMetadataTombstones
+			defer func() { globalSiteReplicationMetadataTombstones = old }()
+			created := UTCNow().Add(-time.Hour)
+			for _, enabled := range []bool{false, true} {
+				globalSiteReplicationMetadataTombstones = enabled
+				for file, data := range bucketConfigTestData(bucket) {
+					meta := newBucketMetadata(bucket)
+					meta.Created = created
+					meta.defaultTimestamps()
+					if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+						t.Fatal(err)
+					}
+					counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+					setObjectLayer(counter)
+					event := newBucketConfigReplicationEvent(bucket, file, bucketConfigState{data: data, at: created.Add(-time.Second)})
+					rec := applySRBucketMetaViaAdmin(t, cred, event)
+					if rec.Code != http.StatusOK || counter.writes.Load() != 0 {
+						t.Fatalf("pre-creation apply: %s %d writes=%d", file, rec.Code, counter.writes.Load())
+					}
+					// A live baseline may initialize. The same-time nil cannot delete it.
+					event.UpdatedAt = created
+					rec = applySRBucketMetaViaAdmin(t, cred, event)
+					if rec.Code != http.StatusOK {
+						t.Fatalf("baseline apply: %s %s", file, rec.Body.String())
+					}
+					before := counter.writes.Load()
+					rec = applySRBucketMetaViaAdmin(t, cred, madmin.SRBucketMeta{Type: event.Type, Bucket: bucket, UpdatedAt: created})
+					if rec.Code != http.StatusOK || counter.writes.Load() != before {
+						t.Fatalf("nil baseline cleared configuration: %s", file)
+					}
+					event.UpdatedAt = time.Time{}
+					for range 2 {
+						rec = applySRBucketMetaViaAdmin(t, cred, event)
+						if rec.Code != http.StatusOK {
+							t.Fatalf("legacy-zero rejected: %s %s", file, rec.Body.String())
+						}
+					}
+					meta, err := loadBucketMetadata(t.Context(), obj, bucket)
+					if err != nil {
+						t.Fatal(err)
+					}
+					value, at := replicatedBucketConfig(&meta, file)
+					if len(*value) == 0 || !at.After(created) {
+						t.Fatalf("legacy-zero not reclocked: %s %v", file, *at)
+					}
+					setObjectLayer(obj)
+				}
+			}
+		})
+	}})
+}
+
+type bucketMetadataCreatedObjectLayer struct {
+	ObjectLayer
+	missing bool
+}
+
+func (o bucketMetadataCreatedObjectLayer) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
+	if opts.NoMetadata {
+		if o.missing {
+			return BucketInfo{}, BucketNotFound{Bucket: bucket}
+		}
+		// A physical bucket that reports no creation time either.
+		return BucketInfo{Name: bucket}, nil
+	}
+	return o.ObjectLayer.GetBucketInfo(ctx, bucket, opts)
+}
+
+func TestPeerBucketMetadataUnknownCreated(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			defer setObjectLayer(obj)
+			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
+			for _, mode := range []string{"unknown", "missing"} {
+				t.Run(mode, func(t *testing.T) {
+					setObjectLayer(obj)
+					if err := globalBucketMetadataSys.save(t.Context(), newBucketMetadata(bucket)); err != nil {
+						t.Fatal(err)
+					}
+					counter := &bucketConfigWriteCounter{ObjectLayer: bucketMetadataCreatedObjectLayer{ObjectLayer: obj, missing: mode == "missing"}}
+					setObjectLayer(counter)
+					stamp := UTCNow()
+					_, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketTaggingConfig, data, false, false, &stamp)
+					if err == nil || counter.writes.Load() != 0 {
+						t.Fatalf("unknown generation was invented: %v writes=%d", err, counter.writes.Load())
+					}
+				})
+			}
+		})
+	}})
+}
+
+// setPhysicalBucketCreated stamps the bucket directory on every local drive,
+// which is what StatVol reports as the physical creation time.
+func setPhysicalBucketCreated(t *testing.T, bucket string, at time.Time) {
+	t.Helper()
+	globalLocalDrivesMu.RLock()
+	drives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+	if len(drives) == 0 {
+		t.Fatal("no local drives registered")
+	}
+	for _, drive := range drives {
+		if err := os.Chtimes(pathJoin(drive.Endpoint().Path, bucket), at, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// Recovery has to run against the real ObjectLayer: a stub GetBucketInfo
+// returning the expected time would hide the cached zero creation time
+// overwriting it, which is what a bucket that never held a configuration has.
+func TestBucketMetadataPhysicalCreatedRecovery(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		// One known time on every drive, so the recovered value can be neither
+		// confused with UTCNow() nor dependent on which drive answers first.
+		physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
+		for _, missing := range []bool{false, true} {
+			for _, file := range replicatedBucketConfigs {
+				t.Run(backend+"/"+file+"/missing="+strconv.FormatBool(missing), func(t *testing.T) {
+					ctx := t.Context()
+					setPhysicalBucketCreated(t, bucket, physical)
+					if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
+						t.Fatal(err)
+					}
+					if missing {
+						if err := deleteConfig(ctx, obj, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile)); err != nil {
+							t.Fatal(err)
+						}
+					}
+					data := bucketConfigTestData(bucket)[file]
+					at, err := globalBucketMetadataSys.Update(ctx, bucket, file, data)
+					if err != nil {
+						t.Fatalf("bucket without a recorded creation time cannot update %s: %v", file, err)
+					}
+					got, err := readBucketMetadata(ctx, obj, bucket)
+					if err != nil || !got.Created.Equal(physical) || !at.After(physical) {
+						t.Fatalf("physical creation not persisted: created=%v physical=%v updated=%v err=%v", got.Created, physical, at, err)
+					}
+				})
+			}
+		}
+	}})
+}
+
+func TestBucketMetadataInitialSyncPhysicalCreated(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, cred.AccessKey, nil, newServiceAccountOpts{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { globalIAMSys.DeleteServiceAccount(context.Background(), serviceCred.AccessKey, false) })
+			physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
+			setPhysicalBucketCreated(t, bucket, physical)
+			meta := newBucketMetadata(bucket)
+			meta.TaggingConfigXML = bucketConfigTestData(bucket)[bucketTaggingConfig]
+			if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+				t.Fatal(err)
+			}
+			var mu sync.Mutex
+			var createdAt string
+			var events []madmin.SRBucketMeta
+			peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mu.Lock()
+				defer mu.Unlock()
+				if r.URL.Query().Get("operation") == string(madmin.MakeWithVersioningBktOp) {
+					createdAt = r.URL.Query().Get("createdAt")
+				}
+				if strings.HasSuffix(r.URL.Path, "/bucket-meta") {
+					var event madmin.SRBucketMeta
+					if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
+						t.Error(err)
+					}
+					events = append(events, event)
+				}
+				if r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions" {
+					_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "initial-peer", Instance: "initial-boot", Digest: "ack"}})
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer peer.Close()
+			// Exercise the complete outgoing sync sequence with real source
+			// storage. This peer acknowledges RPCs; it is not a second ObjectLayer.
+			c := &SiteReplicationSys{enabled: true, state: srState{
+				ServiceAccountAccessKey: serviceCred.AccessKey,
+				Peers:                   map[string]madmin.PeerInfo{"initial-peer": {DeploymentID: "initial-peer", Endpoint: peer.URL}},
+			}}
+			if err := c.syncToAllPeers(ctx, madmin.SRAddOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if createdAt != physical.Format(time.RFC3339Nano) {
+				t.Fatalf("peer creation time %q, want physical time %s", createdAt, physical)
+			}
+			for _, event := range events {
+				if event.Bucket == bucket && event.Type == madmin.SRBucketMetaTypeTags {
+					if event.Tags == nil || *event.Tags != base64.StdEncoding.EncodeToString(meta.TaggingConfigXML) || !event.UpdatedAt.Equal(physical) {
+						t.Fatalf("historical tags lost baseline: %+v", event)
+					}
+					return
+				}
+			}
+			t.Fatal("initial sync silently skipped historical tags")
+		})
+	}})
+}
+
+func TestPeerBucketMetadataPhysicalCreatedBoundary(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			physical := UTCNow().Truncate(time.Second)
+			setPhysicalBucketCreated(t, bucket, physical)
+			if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
+				t.Fatal(err)
+			}
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
+			at := physical.Add(-time.Hour)
+			_, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketTaggingConfig, data, false, false, &at)
+			if err != nil || counter.writes.Load() != 0 {
+				t.Fatalf("event before recovered creation must be skipped: err=%v writes=%d", err, counter.writes.Load())
+			}
+			// Physical mtime is only an approximation. A local correction can
+			// establish it; an earlier peer event cannot lower the bucket identity.
+			at, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := readBucketMetadata(ctx, obj, bucket)
+			if err != nil || !got.Created.Equal(physical) || !at.After(physical) || !bytes.Equal(got.TaggingConfigXML, data) {
+				t.Fatalf("local correction did not establish physical creation: %+v %v", got, err)
+			}
+		})
+	}})
+}

cmd/site-replication-metadata-heal_test.go

@@ -0,0 +1,323 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/target/testlogger"
+)
+
+func bucketConfigTestData(bucket string) map[string][]byte {
+	return map[string][]byte{
+		bucketPolicyConfig:     []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)),
+		bucketTaggingConfig:    []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`),
+		bucketSSEConfig:        []byte(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`),
+		bucketQuotaConfigFile:  []byte(`{"quota":1024,"quotatype":"hard"}`),
+		bucketVersioningConfig: enabledBucketVersioningConfig,
+		objectLockConfig:       enabledBucketObjectLockConfig,
+	}
+}
+
+type bucketConfigLogCapture struct {
+	testing.TB
+	mu    sync.Mutex
+	lines []string
+}
+
+func (c *bucketConfigLogCapture) Logf(format string, args ...any) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.lines = append(c.lines, fmt.Sprintf(format, args...))
+}
+
+func TestHealBucketConfigDiagnostics(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			recordBucketConfigPeer(t, cred)
+			capture := &bucketConfigLogCapture{TB: t}
+			defer testlogger.T.SetLogTB(capture)()
+			disabled := logger.DisableLog
+			logger.DisableLog = false
+			defer func() { logger.DisableLog = disabled }()
+			created := UTCNow().Add(-time.Hour)
+			local := globalDeploymentID()
+			bs := map[string]srBucketStatsSummary{
+				local:            bucketConfigTestInfo(bucket, bucketTaggingConfig, nil, created, created),
+				"missing-bucket": {},
+				"broken-rpc":     bucketConfigTestInfo(bucket, bucketTaggingConfig, nil, created, created),
+			}
+			info := srStatusInfo{Sites: map[string]madmin.PeerInfo{local: {}, "missing-bucket": {}, "unreachable": {}, "broken-rpc": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{bucket: bs}}
+			if err := globalSiteReplicationSys.healBucketConfig(t.Context(), bucket, bucketTaggingConfig, info); err != nil {
+				t.Fatal(err)
+			}
+			if len(capture.lines) != 0 {
+				t.Fatal("empty baselines produced diagnostics")
+			}
+			bs[local] = bucketConfigTestInfo(bucket, bucketTaggingConfig, bucketConfigTestData(bucket)[bucketTaggingConfig], created.Add(time.Minute), created)
+			for range 2 {
+				if err := globalSiteReplicationSys.healBucketConfig(t.Context(), bucket, bucketTaggingConfig, info); err != nil {
+					t.Fatal(err)
+				}
+			}
+			capture.mu.Lock()
+			defer capture.mu.Unlock()
+			var unreachable, peerError int
+			for _, line := range capture.lines {
+				switch {
+				case strings.Contains(line, "bucket metadata replication: unreachable"):
+					unreachable++
+				case strings.Contains(line, "bucket metadata replication: peer-error"):
+					peerError++
+				default:
+					t.Fatalf("unexpected diagnostic: %s", line)
+				}
+				if !strings.HasPrefix(line, "WARNING:") {
+					t.Fatalf("diagnostic is not a warning: %s", line)
+				}
+			}
+			if unreachable != 1 || peerError != 1 {
+				t.Fatalf("distinct reasons were lost or not deduplicated: unreachable=%d peer-error=%d", unreachable, peerError)
+			}
+		})
+	}})
+}
+
+func TestHealBucketConfigWithoutSourceDiagnostics(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			events := recordBucketConfigPeer(t, cred)
+			disabled := logger.DisableLog
+			logger.DisableLog = false
+			defer func() { logger.DisableLog = disabled }()
+			created := UTCNow().Add(-time.Hour)
+			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
+			for _, tc := range []struct {
+				name    string
+				data    []byte
+				at      time.Time
+				created time.Time
+				wantLog bool
+			}{
+				{"unknown-created", data, created.Add(time.Minute), time.Time{}, true},
+				{"malformed", []byte("<Tagging>"), created.Add(time.Minute), created, true},
+				{"before-created", data, created.Add(-time.Minute), created, true},
+				{"empty-baseline", nil, created, created, false},
+			} {
+				t.Run(tc.name, func(t *testing.T) {
+					capture := &bucketConfigLogCapture{TB: t}
+					defer testlogger.T.SetLogTB(capture)()
+					// Each case has its own diagnostic key; no source can be selected
+					// and neither local storage nor the recording peer may be written.
+					name := bucket + "-" + tc.name
+					local := globalDeploymentID()
+					info := srStatusInfo{
+						Sites: map[string]madmin.PeerInfo{local: {}, "metadata-peer": {}, "unreachable": {}},
+						BucketStats: map[string]map[string]srBucketStatsSummary{name: {
+							local:           bucketConfigTestInfo(name, bucketTaggingConfig, tc.data, tc.at, tc.created),
+							"metadata-peer": {},
+						}},
+					}
+					for range 2 {
+						if err := globalSiteReplicationSys.healBucketConfig(t.Context(), name, bucketTaggingConfig, info); err != nil {
+							t.Fatal(err)
+						}
+					}
+					capture.mu.Lock()
+					defer capture.mu.Unlock()
+					want := 0
+					if tc.wantLog {
+						want = 1
+					}
+					if len(capture.lines) != want {
+						t.Fatalf("got %d diagnostics, want %d: %v", len(capture.lines), want, capture.lines)
+					}
+					for _, line := range capture.lines {
+						if !strings.HasPrefix(line, "WARNING:") || !strings.Contains(line, "bucket metadata replication: indeterminate") {
+							t.Fatalf("unexpected diagnostic: %s", line)
+						}
+					}
+					if len(events()) != 0 {
+						t.Fatal("healing without a source sent a metadata RPC")
+					}
+				})
+			}
+		})
+	}})
+}
+
+// Construct independent wire fixtures, including timestamps hidden by legacy
+// exporters, without going through the production event or state helpers.
+func bucketConfigTestInfo(bucket, file string, data []byte, at, created time.Time) srBucketStatsSummary {
+	info := madmin.SRBucketInfo{Bucket: bucket, CreatedAt: created}
+	var payload *string
+	if len(data) != 0 {
+		encoded := base64.StdEncoding.EncodeToString(data)
+		payload = &encoded
+	}
+	switch file {
+	case bucketPolicyConfig:
+		info.Policy, info.PolicyUpdatedAt = data, at
+	case bucketTaggingConfig:
+		info.Tags, info.TagConfigUpdatedAt = payload, at
+	case bucketSSEConfig:
+		info.SSEConfig, info.SSEConfigUpdatedAt = payload, at
+	case bucketQuotaConfigFile:
+		info.QuotaConfig, info.QuotaConfigUpdatedAt = payload, at
+	case bucketVersioningConfig:
+		info.Versioning, info.VersioningConfigUpdatedAt = payload, at
+	case objectLockConfig:
+		info.ObjectLockConfig, info.ObjectLockConfigUpdatedAt = payload, at
+	}
+	return srBucketStatsSummary{meta: srBucketMetaInfo{SRBucketInfo: info}}
+}
+
+func TestLatestBucketConfigCandidates(t *testing.T) {
+	created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	for file, data := range bucketConfigTestData("bucket") {
+		t.Run(file, func(t *testing.T) {
+			info := srStatusInfo{Sites: map[string]madmin.PeerInfo{"a": {}, "b": {}, "c": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{"bucket": {}}}
+			bs := info.BucketStats["bucket"]
+			baseline := bucketConfigTestInfo("bucket", file, nil, created, created)
+			bs["a"], bs["b"], bs["c"] = baseline, baseline, baseline
+			if _, found := latestBucketConfig("bucket", file, info); found {
+				t.Fatal("empty baseline chosen as source")
+			}
+			// Unknown and unavailable IDs cannot contribute, even with future clocks.
+			bs[""], bs["unknown"] = bucketConfigTestInfo("bucket", file, data, created.Add(10*time.Hour), created), bucketConfigTestInfo("bucket", file, data, created.Add(20*time.Hour), created)
+			if _, found := latestBucketConfig("bucket", file, info); found {
+				t.Fatal("unknown source chosen")
+			}
+			liveBaseline := bucketConfigTestInfo("bucket", file, data, created, created)
+			modified := bucketConfigTestInfo("bucket", file, data, created.Add(time.Hour), created)
+			oldGeneration := bucketConfigTestInfo("bucket", file, data, created, created.Add(time.Second))
+			states := []srBucketStatsSummary{liveBaseline, modified, oldGeneration}
+			for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} {
+				bs["a"], bs["b"], bs["c"] = states[order[0]], states[order[1]], states[order[2]]
+				winner, found := latestBucketConfig("bucket", file, info)
+				if !found || !winner.at.Equal(created.Add(time.Hour)) || !bytes.Equal(winner.data, data) {
+					t.Fatalf("permutation %v chose %+v found=%v", order, winner, found)
+				}
+			}
+			bs["a"], bs["b"], bs["c"] = baseline, liveBaseline, baseline
+			if winner, found := latestBucketConfig("bucket", file, info); !found || !bytes.Equal(winner.data, data) || winner.real {
+				t.Fatal("historical baseline-live cannot initialize")
+			}
+			if !bucketConfigUpdateOnly(file) {
+				// A late-created empty baseline cannot beat a real earlier write.
+				bs["a"], bs["b"], bs["c"] = modified, bucketConfigTestInfo("bucket", file, nil, created.Add(2*time.Hour), created.Add(2*time.Hour)), baseline
+				if winner, found := latestBucketConfig("bucket", file, info); !found || len(winner.data) == 0 {
+					t.Fatal("default became a deletion")
+				}
+				bs["c"] = bucketConfigTestInfo("bucket", file, nil, created.Add(time.Hour), created)
+				if winner, found := latestBucketConfig("bucket", file, info); !found || len(winner.data) != 0 || !winner.real {
+					t.Fatal("real tombstone lost equal-time tie")
+				}
+			}
+		})
+	}
+}
+
+func TestHealBucketConfigSourceAndQuiescence(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			events := recordBucketConfigPeer(t, cred)
+			local := globalDeploymentID()
+			created := UTCNow().Add(-time.Hour)
+			oldAt, newAt := created.Add(time.Minute), created.Add(2*time.Minute)
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			for file, data := range bucketConfigTestData(bucket) {
+				t.Run(file, func(t *testing.T) {
+					meta := newBucketMetadata(bucket)
+					meta.Created = created
+					meta.defaultTimestamps()
+					value, at := replicatedBucketConfig(&meta, file)
+					*value, *at = data, newAt
+					if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+						t.Fatal(err)
+					}
+					bs := map[string]srBucketStatsSummary{
+						local:           bucketConfigTestInfo(bucket, file, data, newAt, created),
+						"metadata-peer": bucketConfigTestInfo(bucket, file, data, oldAt, created),
+						"":              {}, "unknown": bucketConfigTestInfo(bucket, file, nil, newAt.Add(time.Hour), created),
+						// Known but absent from c.state.Peers, forcing getAdminClient failure.
+						"broken": bucketConfigTestInfo(bucket, file, nil, created, created),
+					}
+					info := srStatusInfo{Sites: map[string]madmin.PeerInfo{local: {}, "metadata-peer": {}, "broken": {}}, BucketStats: map[string]map[string]srBucketStatsSummary{bucket: bs}}
+					before := len(events())
+					writes := counter.writes.Load()
+					if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
+						t.Fatal(err)
+					}
+					got := events()[before:]
+					if len(got) != 1 || !got[0].UpdatedAt.Equal(newAt) {
+						t.Fatalf("same-payload timestamp heal did not reach healthy peer: %+v", got)
+					}
+					if file == bucketTaggingConfig && got[0].Tags == nil {
+						t.Fatal("tag event missing payload")
+					}
+					if counter.writes.Load() != writes {
+						t.Fatal("matching local state was rewritten")
+					}
+					bs["metadata-peer"], bs["broken"] = bs[local], bs[local]
+					if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
+						t.Fatal(err)
+					}
+					if len(events()) != before+1 || counter.writes.Load() != writes {
+						t.Fatal("stable second heal wrote or broadcast")
+					}
+					if bucketConfigUpdateOnly(file) {
+						return
+					}
+					bs["metadata-peer"] = bucketConfigTestInfo(bucket, file, nil, newAt.Add(time.Minute), created)
+					delete(bs, "broken")
+					if err := globalSiteReplicationSys.healBucketConfig(ctx, bucket, file, info); err != nil {
+						t.Fatal(err)
+					}
+					after, err := loadBucketMetadata(ctx, obj, bucket)
+					if err != nil {
+						t.Fatal(err)
+					}
+					value, at = replicatedBucketConfig(&after, file)
+					if len(*value) != 0 || !at.Equal(newAt.Add(time.Minute)) {
+						t.Fatalf("local heal lost deletion/source time: %s %v", *value, *at)
+					}
+					if file == bucketQuotaConfigFile {
+						q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
+						if err != nil || q.Quota != 0 {
+							t.Fatalf("quota cache not cleared: %+v %v", q, err)
+						}
+					}
+				})
+			}
+		})
+	}})
+}

cmd/site-replication-metadata.go

@@ -0,0 +1,192 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/logger"
+)
+
+// Read once at startup. Enable only after every participating node is fixed.
+var globalSiteReplicationMetadataTombstones bool
+
+func logBucketConfigReplication(ctx context.Context, bucket, file, reason string, at, created time.Time, detail string) {
+	// LogOnceIf compares error text as well as its key. Keep both stable; changing
+	// times and peer errors belong in ReqInfo, not in the error's message.
+	req := &logger.ReqInfo{API: "SiteReplicationMetadata", BucketName: bucket}
+	req.AppendTags("field", file)
+	req.AppendTags("sourceTime", at.UTC().Format(time.RFC3339Nano))
+	req.AppendTags("created", created.UTC().Format(time.RFC3339Nano))
+	req.AppendTags("detail", detail)
+	replLogOnceIf(logger.SetReqInfo(ctx, req), errors.New("bucket metadata replication: "+reason),
+		"bucket-metadata/"+bucket+"/"+file+"/"+reason, logger.WarningKind)
+}
+
+func initialBucketConfigReplicationEvent(meta BucketMetadata, file string) (madmin.SRBucketMeta, bool, error) {
+	data, at := replicatedBucketConfig(&meta, file)
+	state, err := newBucketConfigState(meta.Name, file, *data, *at, meta.Created, len(meta.ObjectLockConfigXML) != 0)
+	if err != nil {
+		return madmin.SRBucketMeta{}, false, err
+	}
+	if !state.candidate() || (len(state.data) == 0 && !globalSiteReplicationMetadataTombstones) {
+		return madmin.SRBucketMeta{}, false, nil
+	}
+	return newBucketConfigReplicationEvent(meta.Name, file, state), true, nil
+}
+
+func bucketConfigStateFromInfo(bucket, file string, meta madmin.SRBucketInfo) (bucketConfigState, error) {
+	var payload *string
+	var at time.Time
+	switch file {
+	case bucketPolicyConfig:
+		return newBucketConfigState(bucket, file, meta.Policy, meta.PolicyUpdatedAt, meta.CreatedAt, false)
+	case bucketTaggingConfig:
+		payload, at = meta.Tags, meta.TagConfigUpdatedAt
+	case bucketSSEConfig:
+		payload, at = meta.SSEConfig, meta.SSEConfigUpdatedAt
+	case bucketQuotaConfigFile:
+		payload, at = meta.QuotaConfig, meta.QuotaConfigUpdatedAt
+	case bucketVersioningConfig:
+		payload, at = meta.Versioning, meta.VersioningConfigUpdatedAt
+	case objectLockConfig:
+		payload, at = meta.ObjectLockConfig, meta.ObjectLockConfigUpdatedAt
+	}
+	var data []byte
+	if payload != nil {
+		var err error
+		data, err = base64.StdEncoding.DecodeString(*payload)
+		if err != nil {
+			return bucketConfigState{}, err
+		}
+	}
+	lockEnabled := meta.ObjectLockConfig != nil && len(*meta.ObjectLockConfig) != 0
+	return newBucketConfigState(bucket, file, data, at, meta.CreatedAt, lockEnabled)
+}
+
+func newBucketConfigReplicationEvent(bucket, file string, state bucketConfigState) madmin.SRBucketMeta {
+	event := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: state.at}
+	var payload *string
+	if len(state.data) != 0 {
+		encoded := base64.StdEncoding.EncodeToString(state.data)
+		payload = &encoded
+	}
+	switch file {
+	case bucketPolicyConfig:
+		event.Type, event.Policy = madmin.SRBucketMetaTypePolicy, state.data
+	case bucketTaggingConfig:
+		event.Type, event.Tags = madmin.SRBucketMetaTypeTags, payload
+	case bucketSSEConfig:
+		event.Type, event.SSEConfig = madmin.SRBucketMetaTypeSSEConfig, payload
+	case bucketQuotaConfigFile:
+		event.Type, event.Quota = madmin.SRBucketMetaTypeQuotaConfig, state.data
+	case bucketVersioningConfig:
+		event.Type, event.Versioning = madmin.SRBucketMetaTypeVersionConfig, payload
+	case objectLockConfig:
+		event.Type, event.ObjectLockConfig = madmin.SRBucketMetaTypeObjectLockConfig, payload
+	}
+	return event
+}
+
+func latestBucketConfig(bucket, file string, info srStatusInfo) (bucketConfigState, bool) {
+	var latest bucketConfigState
+	found := false
+	for id, status := range info.BucketStats[bucket] {
+		if _, known := info.Sites[id]; !known || id == "" {
+			continue
+		}
+		state, err := bucketConfigStateFromInfo(bucket, file, status.meta.SRBucketInfo)
+		if err != nil || !state.candidate() {
+			continue
+		}
+		if !found || compareBucketConfigStates(state, latest) > 0 {
+			latest, found = state, true
+		}
+	}
+	return latest, found
+}
+
+func (c *SiteReplicationSys) healBucketConfig(ctx context.Context, bucket, file string, info srStatusInfo) error {
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	latest, found := latestBucketConfig(bucket, file, info)
+	// Every reason keeps its own log key, so a site that did not report cannot
+	// deduplicate away an unusable peer state or a real heal RPC failure for
+	// the same bucket and field.
+	if found {
+		for id := range info.Sites {
+			if _, present := info.BucketStats[bucket][id]; !present {
+				logBucketConfigReplication(ctx, bucket, file, "unreachable", latest.at, time.Time{}, "peer "+id+" did not report")
+			}
+		}
+	}
+	for id, status := range info.BucketStats[bucket] {
+		if _, known := info.Sites[id]; !known || id == "" {
+			continue
+		}
+		target := status.meta.SRBucketInfo
+		current, currentErr := bucketConfigStateFromInfo(bucket, file, target)
+		// A peer without the bucket reports neither a field nor a creation
+		// time; bucket healing covers that normal transient. Report only a
+		// state that exists and still cannot be ordered.
+		if currentErr != nil || (!current.valid && (len(current.data) != 0 || !current.at.IsZero())) {
+			logBucketConfigReplication(ctx, bucket, file, "indeterminate", current.at, target.CreatedAt, "unusable peer "+id)
+		}
+		// Invalid existing state still needs a diagnosis when no source can
+		// be selected. Only propagation depends on having a valid source.
+		if !found || target.CreatedAt.IsZero() {
+			continue
+		}
+		if latest.at.Before(target.CreatedAt) {
+			logBucketConfigReplication(ctx, bucket, file, "before-created", latest.at, target.CreatedAt, "peer "+id)
+			continue
+		}
+		// Versioning can be normalized differently until Object Lock itself has
+		// converged. Compare what this target would actually persist.
+		incoming, err := newBucketConfigState(bucket, file, latest.data, latest.at, target.CreatedAt,
+			target.ObjectLockConfig != nil && len(*target.ObjectLockConfig) != 0)
+		if err != nil || !incoming.candidate() {
+			continue
+		}
+		if currentErr == nil && compareBucketConfigStates(incoming, current) <= 0 {
+			continue
+		}
+		if id == globalDeploymentID() {
+			_, err = globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, file, latest.data, false, false, &latest.at)
+		} else {
+			var client *madmin.AdminClient
+			client, err = c.getAdminClient(ctx, id)
+			if err == nil {
+				err = client.SRPeerReplicateBucketMeta(ctx, newBucketConfigReplicationEvent(bucket, file, incoming))
+			}
+		}
+		if err != nil {
+			// A missing credential or unreachable peer must not abandon the other
+			// targets simply because it happened to be visited first in this map.
+			logBucketConfigReplication(ctx, bucket, file, "peer-error", latest.at, target.CreatedAt, "peer "+id+": "+err.Error())
+		}
+	}
+	return nil
+}

cmd/site-replication-metadata_test.go

@@ -0,0 +1,779 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func TestPeerBucketMetadataSourceTimeAndDeletion(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPeerBucketMetadataSourceTimeAndDeletion})
+}
+
+func testPeerBucketMetadataSourceTimeAndDeletion(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+	ctx := t.Context()
+	created := UTCNow().Add(-time.Hour)
+	putAt := created.Add(10 * time.Minute)
+	delAt := putAt.Add(time.Minute)
+	enc := func(s string) *string { v := base64.StdEncoding.EncodeToString([]byte(s)); return &v }
+	base := newBucketMetadata(bucket)
+	base.SetCreatedAt(created)
+	base.defaultTimestamps()
+	quotaJSON, err := json.Marshal(madmin.BucketQuota{Quota: 1024, Type: madmin.HardQuota})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cases := []struct {
+		name, file string
+		put        madmin.SRBucketMeta
+		value      func(BucketMetadata) []byte
+		stamp      func(BucketMetadata) time.Time
+		deletable  bool
+	}{
+		{"policy", bucketPolicyConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypePolicy, Policy: []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))}, func(m BucketMetadata) []byte { return m.PolicyConfigJSON }, func(m BucketMetadata) time.Time { return m.PolicyConfigUpdatedAt }, true},
+		{"tags", bucketTaggingConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeTags, Tags: enc(`<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>`)}, func(m BucketMetadata) []byte { return m.TaggingConfigXML }, func(m BucketMetadata) time.Time { return m.TaggingConfigUpdatedAt }, true},
+		{"sse", bucketSSEConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeSSEConfig, SSEConfig: enc(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)}, func(m BucketMetadata) []byte { return m.EncryptionConfigXML }, func(m BucketMetadata) time.Time { return m.EncryptionConfigUpdatedAt }, true},
+		{"quota", bucketQuotaConfigFile, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeQuotaConfig, Quota: quotaJSON}, func(m BucketMetadata) []byte { return m.QuotaConfigJSON }, func(m BucketMetadata) time.Time { return m.QuotaConfigUpdatedAt }, true},
+		{"versioning", bucketVersioningConfig, madmin.SRBucketMeta{Type: madmin.SRBucketMetaTypeVersionConfig, Versioning: enc(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)}, func(m BucketMetadata) []byte { return m.VersioningConfigXML }, func(m BucketMetadata) time.Time { return m.VersioningConfigUpdatedAt }, false},
+		{"objectlock", objectLockConfig, newSRBucketObjectLockMeta(bucket, enc(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`), putAt), func(m BucketMetadata) []byte { return m.ObjectLockConfigXML }, func(m BucketMetadata) time.Time { return m.ObjectLockConfigUpdatedAt }, false},
+	}
+	for _, tc := range cases {
+		t.Run(backend+"/"+tc.name, func(t *testing.T) {
+			if err := globalBucketMetadataSys.save(ctx, base); err != nil {
+				t.Fatal(err)
+			}
+			item := tc.put
+			item.Bucket, item.UpdatedAt = bucket, putAt
+			apply := func(item madmin.SRBucketMeta) {
+				t.Helper()
+				rec := applySRBucketMetaViaAdmin(t, cred, item)
+				if rec.Code != http.StatusOK {
+					t.Fatalf("admin apply returned %d: %s", rec.Code, rec.Body.String())
+				}
+			}
+			read := func() BucketMetadata {
+				t.Helper()
+				m, err := loadBucketMetadata(ctx, obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				return m
+			}
+			apply(item)
+			put := read()
+			if len(tc.value(put)) == 0 {
+				t.Fatal("PUT did not establish live config")
+			}
+			if !tc.stamp(put).Equal(putAt) {
+				t.Errorf("SOURCE_TIME: persisted %s, want source %s", tc.stamp(put), putAt)
+			}
+			apply(madmin.SRBucketMeta{Type: item.Type, Bucket: bucket, UpdatedAt: delAt})
+			after := read()
+			if !tc.deletable {
+				if !bytes.Equal(tc.value(put), tc.value(after)) || !tc.stamp(put).Equal(tc.stamp(after)) {
+					t.Error("NIL_NOOP: update-only config changed")
+				}
+				t.Log("nil payload is correctly a no-op")
+				return
+			}
+			if len(tc.value(after)) != 0 {
+				t.Error("NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE")
+			}
+			if _, err := globalBucketMetadataSys.Delete(ctx, bucket, tc.file); err != nil {
+				t.Fatal(err)
+			}
+			tombstone := read()
+			if len(tc.value(tombstone)) != 0 {
+				t.Fatal("local DELETE failed to establish tombstone")
+			}
+			apply(item)
+			after = read()
+			if len(tc.value(after)) != 0 {
+				t.Error("STALE_RESURRECTION: older source PUT resurrected a locally deleted config")
+			} else {
+				t.Log("older source PUT did not resurrect config")
+			}
+		})
+	}
+}
+
+func TestPeerBucketMetadataBulkOrdering(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			newAt := meta.Created.Add(time.Hour)
+			newXML := `<Tagging><TagSet><Tag><Key>key</Key><Value>new</Value></Tag></TagSet></Tagging>`
+			meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = []byte(newXML), newAt
+			if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+				t.Fatal(err)
+			}
+			oldXML := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>`))
+			item := madmin.SRBucketMeta{Bucket: bucket, Tags: &oldXML, UpdatedAt: newAt.Add(-time.Minute)}
+			rec := applySRBucketMetaViaAdmin(t, cred, item)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("bulk apply returned %d: %s", rec.Code, rec.Body.String())
+			}
+			after, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(after.TaggingConfigXML, []byte(newXML)) || !after.TaggingConfigUpdatedAt.Equal(newAt) {
+				t.Errorf("BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value=%q time=%s", after.TaggingConfigXML, after.TaggingConfigUpdatedAt)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataOrderingUnderLock(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			oldAt, newAt := meta.Created.Add(time.Hour), meta.Created.Add(2*time.Hour)
+			oldXML := base64.StdEncoding.EncodeToString([]byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>`))
+			newXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>newest</Value></Tag></TagSet></Tagging>`)
+			lockCtx, unlock, err := lockBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			locked := true
+			defer func() {
+				if locked {
+					unlock()
+				}
+			}()
+			ready := make(chan struct{}, 1)
+			hook := func(name string) {
+				if name == bucket {
+					select {
+					case ready <- struct{}{}:
+					default:
+					}
+				}
+			}
+			lockBucketMetadataAcquireHook.Store(&hook)
+			defer lockBucketMetadataAcquireHook.Store(nil)
+			done := make(chan error, 1)
+			go func() { done <- globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, bucket, &oldXML, oldAt) }()
+			select {
+			case <-ready:
+			case <-time.After(5 * time.Second):
+				t.Fatal("older event never reached metadata lock")
+			}
+			// A newer writer commits while holding the existing metadata lock.
+			// The old peer event has already checked the pre-commit cache.
+			meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = newXML, newAt
+			if err := globalBucketMetadataSys.saveMetadata(lockCtx, obj, &meta); err != nil {
+				t.Fatal(err)
+			}
+			unlock()
+			locked = false
+			select {
+			case err := <-done:
+				if err != nil {
+					t.Fatal(err)
+				}
+			case <-time.After(5 * time.Second):
+				t.Fatal("older event did not finish")
+			}
+			after, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(after.TaggingConfigXML) != string(newXML) {
+				t.Errorf("CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with %q", after.TaggingConfigXML)
+			}
+		})
+	}})
+}
+
+// Count actual metadata persistence, including writes that would be invisible
+// in a value-only assertion after a duplicate or rejected event.
+type bucketConfigWriteCounter struct {
+	ObjectLayer
+	writes atomic.Int64
+}
+
+func (o *bucketConfigWriteCounter) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
+	if bucket == minioMetaBucket && path.Base(object) == bucketMetadataFile {
+		o.writes.Add(1)
+	}
+	return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
+}
+
+func TestBucketConfigStateOrdering(t *testing.T) {
+	created := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	live := []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>a</Value></Tag></TagSet></Tagging>`)
+	other := bytes.ReplaceAll(live, []byte("<Value>a"), []byte("<Value>b"))
+	state := func(data []byte, at, birth time.Time) bucketConfigState {
+		t.Helper()
+		s, err := newBucketConfigState("bucket", bucketTaggingConfig, data, at, birth, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return s
+	}
+	baseline := state(nil, created, created)
+	baselineLive := state(live, created, created)
+	put := state(live, created.Add(time.Second), created)
+	deleted := state(nil, put.at, created)
+	lateBaseline := state(other, created.Add(time.Hour), created.Add(time.Hour))
+	tests := []struct {
+		name          string
+		lower, higher bucketConfigState
+	}{
+		{"baseline initialization", baseline, baselineLive},
+		{"real before later baseline", lateBaseline, put},
+		{"delete wins tie", put, deleted},
+		{"live key tie", put, state(other, put.at, created)},
+		{"new PUT after deletion", deleted, state(live, put.at.Add(time.Second), created)},
+		{"baseline key ignores creation time", state(live, created.Add(time.Hour), created.Add(time.Hour)), state(other, created, created)},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if compareBucketConfigStates(tc.lower, tc.higher) >= 0 || compareBucketConfigStates(tc.higher, tc.lower) <= 0 {
+				t.Fatal("ordering is not antisymmetric or chose the wrong winner")
+			}
+		})
+	}
+	for _, s := range []bucketConfigState{baseline, state(live, created.Add(-time.Second), created), state(live, created, time.Time{})} {
+		if s.candidate() {
+			t.Fatal("default, pre-creation, or unknown-generation state became a source")
+		}
+	}
+	for _, file := range []string{bucketVersioningConfig, objectLockConfig} {
+		s, err := newBucketConfigState("bucket", file, nil, put.at, created, false)
+		if err != nil || s.candidate() {
+			t.Fatalf("empty update-only candidate: %+v %v", s, err)
+		}
+	}
+}
+
+func TestBucketPolicyReplicationKey(t *testing.T) {
+	// Permute independent set arrays and parse separately, as two sites would.
+	a := []byte(`{"Version":"2012-10-17","Statement":[{"Sid":"one","Effect":"Allow","Principal":{"AWS":["b","a"]},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::bucket/b*","arn:aws:s3:::bucket/a*"],"Condition":{"StringLike":{"s3:prefix":["b*","a*"]}}},{"Sid":"two","Effect":"Deny","Principal":"*","NotAction":["s3:GetObject","s3:PutObject"],"NotResource":["arn:aws:s3:::bucket/d*","arn:aws:s3:::bucket/c*"]}]}`)
+	// Use a condition valid for object actions.
+	a = bytes.ReplaceAll(a, []byte("s3:prefix"), []byte("aws:UserAgent"))
+	b := []byte(`{"Statement":[{"NotResource":["arn:aws:s3:::bucket/c*","arn:aws:s3:::bucket/d*"],"NotAction":["s3:PutObject","s3:GetObject"],"Principal":"*","Effect":"Deny","Sid":"two"},{"Condition":{"StringLike":{"aws:UserAgent":["a*","b*"]}},"Resource":["arn:aws:s3:::bucket/a*","arn:aws:s3:::bucket/b*"],"Action":["s3:PutObject","s3:GetObject"],"Principal":{"AWS":["a","b"]},"Effect":"Allow","Sid":"one"}],"Version":"2012-10-17"}`)
+	_, key, err := bucketConfigPayload("bucket", bucketPolicyConfig, a, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range 30 {
+		_, next, err := bucketConfigPayload("bucket", bucketPolicyConfig, b, false)
+		if err != nil || !bytes.Equal(key, next) {
+			t.Fatalf("independent parse %d: %s != %s (%v)", i, key, next, err)
+		}
+	}
+	_, sid, err := bucketConfigPayload("bucket", bucketPolicyConfig, bytes.ReplaceAll(a, []byte(`"one"`), []byte(`"other"`)), false)
+	if err != nil || bytes.Equal(key, sid) {
+		t.Fatalf("Sid difference lost: %v", err)
+	}
+	n, err := canonicalBucketPolicyJSON([]byte(`{"n":[9007199254740993,9007199254740992]}`))
+	if err != nil || string(n) != `{"n":[9007199254740992,9007199254740993]}` {
+		t.Fatalf("integer precision lost: %s %v", n, err)
+	}
+}
+
+func TestBucketPolicyReplicationStatusLegacyOrder(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			// Statement order here is the reverse of the canonical encoder's,
+			// which is what an upgraded peer stores: the permutation must not
+			// be reported as a permanent mismatch.
+			legacy := []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"allow","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"},{"Sid":"deny","Effect":"Deny","Principal":"*","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket, bucket))
+			source, err := policy.ParseBucketPolicyConfig(bytes.NewReader(legacy), bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			meta := newBucketMetadata(bucket)
+			meta.Created = UTCNow().Add(-time.Hour)
+			meta.PolicyConfigJSON = legacy
+			meta.PolicyConfigUpdatedAt = meta.Created.Add(time.Minute)
+			event, send, err := initialBucketConfigReplicationEvent(meta, bucketPolicyConfig)
+			if err != nil || !send {
+				t.Fatalf("legacy initial event: %v send=%v", err, send)
+			}
+			target := newBucketMetadata(bucket)
+			target.Created = meta.Created
+			if err := globalBucketMetadataSys.save(ctx, target); err != nil {
+				t.Fatal(err)
+			}
+			if rec := applySRBucketMetaViaAdmin(t, cred, event); rec.Code != http.StatusOK {
+				t.Fatalf("peer apply: %d %s", rec.Code, rec.Body.String())
+			}
+			received, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !isBktPolicyReplicated(2, []*policy.BucketPolicy{source, received}) {
+				t.Fatal("equivalent legacy and received policy reported as permanently mismatched")
+			}
+			changed, err := policy.ParseBucketPolicyConfig(bytes.NewReader(legacy), bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			changed.Statements[0].SID = "different"
+			if isBktPolicyReplicated(2, []*policy.BucketPolicy{source, changed}) {
+				t.Fatal("distinct policy state reported as replicated")
+			}
+			if isBktPolicyReplicated(2, []*policy.BucketPolicy{source, nil}) || !isBktPolicyReplicated(2, []*policy.BucketPolicy{nil, nil}) {
+				t.Fatal("per-site presence accounting changed")
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataWireAtomicity(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			stamp := meta.Created.Add(time.Hour)
+			policyData := []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))
+			meta.PolicyConfigJSON, meta.PolicyConfigUpdatedAt = policyData, stamp
+			meta.QuotaConfigJSON, meta.QuotaConfigUpdatedAt = []byte(`{"quota":1024,"quotatype":"hard"}`), stamp
+			meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt = []byte(`<Tagging><TagSet><Tag><Key>k</Key><Value>a</Value></Tag></TagSet></Tagging>`), stamp
+			if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+				t.Fatal(err)
+			}
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			read := func() BucketMetadata {
+				t.Helper()
+				m, err := loadBucketMetadata(ctx, obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				return m
+			}
+			apply := func(fragment string, at time.Time, ok bool) {
+				t.Helper()
+				raw := fmt.Sprintf(`{"bucket":%q,"updatedAt":%q,%s}`, bucket, at.Format(time.RFC3339Nano), fragment)
+				var item madmin.SRBucketMeta
+				if err := json.Unmarshal([]byte(raw), &item); err != nil {
+					t.Fatal(err)
+				}
+				rec := applySRBucketMetaViaAdmin(t, cred, item)
+				if (rec.Code == http.StatusOK) != ok {
+					t.Fatalf("%s: %d %s", raw, rec.Code, rec.Body.String())
+				}
+			}
+			// Omitted/null *string and empty update-only payloads are all no-ops.
+			apply(`"policy":null,"quota":null,"tags":null,"versioningConfig":"","objectLockConfig":""`, stamp.Add(time.Minute), true)
+			current := read()
+			if len(current.PolicyConfigJSON) != 0 || len(current.QuotaConfigJSON) == 0 {
+				t.Fatal("explicit null semantics lost")
+			}
+			if q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err != nil || q.Quota != 0 {
+				t.Fatalf("zero quota cache: %+v %v", q, err)
+			}
+			before := counter.writes.Load()
+			apply(`"tags":null,"versioningConfig":"","objectLockConfig":""`, stamp.Add(2*time.Minute), true)
+			if counter.writes.Load() != before {
+				t.Fatal("omitted fields or empty update-only wrote metadata")
+			}
+			// An invalid second field cannot persist the valid deletion in this bulk.
+			apply(`"tags":"","quota":{"quota":1,"quotatype":"invalid"}`, stamp.Add(2*time.Minute), false)
+			if counter.writes.Load() != before || len(read().TaggingConfigXML) == 0 {
+				t.Fatal("invalid bulk partially committed")
+			}
+			apply(`"tags":""`, stamp.Add(2*time.Minute), true)
+			before = counter.writes.Load()
+			apply(`"tags":""`, stamp.Add(2*time.Minute), true)
+			apply(`"tags":""`, stamp.Add(time.Minute), true)
+			if counter.writes.Load() != before {
+				t.Fatal("duplicate or old event wrote metadata")
+			}
+			localAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, []byte(`{}`))
+			if err != nil || !localAt.After(current.QuotaConfigUpdatedAt) {
+				t.Fatalf("local future monotonic time: %v %v", localAt, err)
+			}
+			deletedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketQuotaConfigFile)
+			if err != nil || !deletedAt.After(localAt) {
+				t.Fatalf("adjacent local time: %v %v", deletedAt, err)
+			}
+			if q, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err != nil || q.Quota != 0 {
+				t.Fatalf("deleted quota cache: %+v %v", q, err)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketAdoptionRebasesOnlyDefaults(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		for _, shift := range []time.Duration{-time.Hour, 0, time.Hour, 3 * time.Hour} {
+			t.Run(fmt.Sprintf("%s/%s", backend, shift), func(t *testing.T) {
+				created := UTCNow().Add(-3 * time.Hour)
+				meta := newBucketMetadata(bucket)
+				meta.Created = created
+				meta.defaultTimestamps()
+				meta.QuotaConfigUpdatedAt = time.Time{}
+				meta.TaggingConfigUpdatedAt = created.Add(2 * time.Hour) // real deletion
+				meta.EncryptionConfigXML = []byte(`<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
+				meta.EncryptionConfigUpdatedAt = created.Add(2 * time.Hour)
+				if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					t.Fatal(err)
+				}
+				if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucket, MakeBucketOptions{CreatedAt: created.Add(shift)}); err != nil {
+					t.Fatal(err)
+				}
+				got, err := readBucketMetadata(t.Context(), obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !got.PolicyConfigUpdatedAt.Equal(got.Created) || !got.QuotaConfigUpdatedAt.Equal(got.Created) {
+					t.Fatal("default turned into tombstone or invalid source")
+				}
+				if !got.TaggingConfigUpdatedAt.Equal(meta.TaggingConfigUpdatedAt) || !got.EncryptionConfigUpdatedAt.Equal(meta.EncryptionConfigUpdatedAt) || !bytes.Equal(got.EncryptionConfigXML, meta.EncryptionConfigXML) {
+					t.Fatal("actual state changed during adoption")
+				}
+				if shift > 2*time.Hour {
+					// Preserve history, but do not promote state from an earlier
+					// bucket generation to a new valid source by retimestamping it.
+					for _, file := range []string{bucketTaggingConfig, bucketSSEConfig} {
+						data, at := replicatedBucketConfig(&got, file)
+						state, err := newBucketConfigState(bucket, file, *data, *at, got.Created, false)
+						if err != nil || state.candidate() {
+							t.Fatalf("pre-generation history became a source: %s %v", file, err)
+						}
+						// As a target, that invalid history must yield to a valid
+						// state from the adopted generation, including a live tag
+						// replacing the preserved earlier-generation deletion.
+						incoming := bucketConfigTestData(bucket)[file]
+						incomingAt := got.Created.Add(time.Minute)
+						changed, err := applyBucketConfig(&got, file, incoming, incomingAt)
+						if err != nil || !changed || !at.Equal(incomingAt) || !bytes.Equal(*data, incoming) {
+							t.Fatalf("adopted generation did not replace invalid target: %s %v", file, err)
+						}
+					}
+				}
+			})
+		}
+	}})
+}
+
+func recordBucketConfigPeer(t *testing.T, cred auth.Credentials) func() []madmin.SRBucketMeta {
+	t.Helper()
+	var mu sync.Mutex
+	var events []madmin.SRBucketMeta
+	remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var event madmin.SRBucketMeta
+		if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
+			t.Error(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		mu.Lock()
+		events = append(events, event)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(remote.Close)
+	serviceCred, err := auth.CreateCredentials("metadata-source-time-svc", "metadata-source-time-service-secret")
+	if err != nil {
+		t.Fatal(err)
+	}
+	serviceCred.ParentUser = cred.AccessKey
+	globalSiteReplicatorCred.Set(serviceCred.SecretKey)
+	t.Cleanup(func() { globalSiteReplicatorCred.Set("") })
+	if _, err = globalIAMSys.store.AddServiceAccount(t.Context(), serviceCred); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { globalIAMSys.DeleteServiceAccount(context.Background(), serviceCred.AccessKey, false) })
+	globalSiteReplicationSys.Lock()
+	oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
+	globalSiteReplicationSys.enabled = true
+	globalSiteReplicationSys.state = srState{Name: "metadata-source-test", ServiceAccountAccessKey: serviceCred.AccessKey, Peers: map[string]madmin.PeerInfo{
+		globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
+		"metadata-peer":      {Name: "remote", DeploymentID: "metadata-peer", Endpoint: remote.URL},
+	}}
+	globalSiteReplicationSys.Unlock()
+	t.Cleanup(func() {
+		globalSiteReplicationSys.Lock()
+		globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
+		globalSiteReplicationSys.Unlock()
+	})
+	return func() []madmin.SRBucketMeta {
+		mu.Lock()
+		defer mu.Unlock()
+		return append([]madmin.SRBucketMeta(nil), events...)
+	}
+}
+
+// The final metadata lock is taken after reading the ZIP. Commit another
+// writer immediately before that acquisition, using the existing lock wrapper.
+type importInterleavingObjectLayer struct {
+	ObjectLayer
+	once  atomic.Bool
+	write func()
+}
+
+func (o *importInterleavingObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket != minioMetaBucket || len(objects) != 1 || path.Base(objects[0]) != "metadata.lock" {
+		return lock
+	}
+	return metadataObservedRWLocker{RWLocker: lock, onLock: func(context.Context) {
+		if o.once.CompareAndSwap(false, true) {
+			o.write()
+		}
+	}}
+}
+
+func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"PutBucketPolicy", "GetBucketPolicy", "PutBucketVersioning"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			ctx := t.Context()
+			events := recordBucketConfigPeer(t, cred)
+			putPolicy := func(data []byte) {
+				t.Helper()
+				req, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucket), int64(len(data)), bytes.NewReader(data), cred.AccessKey, cred.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				rec := httptest.NewRecorder()
+				router.ServeHTTP(rec, req)
+				if rec.Code != http.StatusNoContent {
+					t.Fatalf("empty policy PUT: %d %s", rec.Code, rec.Body.String())
+				}
+			}
+			putPolicy([]byte(`{"Version":"2012-10-17","Statement":[]}`))
+			meta, err := loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := events()
+			if len(got) != 1 || got[0].Type != madmin.SRBucketMetaTypePolicy || len(got[0].Policy) != 0 || len(meta.PolicyConfigJSON) != 0 || !meta.PolicyConfigUpdatedAt.Equal(got[0].UpdatedAt) {
+				t.Fatalf("empty policy event/disk mismatch: %+v", got)
+			}
+			req, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", bucket), 0, nil, cred.AccessKey, cred.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := httptest.NewRecorder()
+			router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusNotFound {
+				t.Fatalf("empty policy GET: %d %s", rec.Code, rec.Body.String())
+			}
+			// Supported negative sets must survive the same serializer on PUT,
+			// peer apply, GET and export instead of failing on empty Action.
+			putPolicy([]byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","NotAction":["s3:DeleteObject"],"NotResource":["arn:aws:s3:::%s/private/*"]}]}`, bucket)))
+			req, err = newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", bucket), 0, nil, cred.AccessKey, cred.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec = httptest.NewRecorder()
+			router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusOK || !bytes.Contains(rec.Body.Bytes(), []byte(`"NotAction"`)) || !bytes.Contains(rec.Body.Bytes(), []byte(`"NotResource"`)) {
+				t.Fatalf("negative policy set GET: %d %s", rec.Code, rec.Body.String())
+			}
+
+			corsAdminRequest(t, cred, http.MethodPut, "/set-bucket-quota?bucket="+bucket, []byte(`{}`))
+			meta, err = loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got = events()
+			last := got[len(got)-1]
+			if last.Type != madmin.SRBucketMetaTypeQuotaConfig || len(last.Quota) == 0 || !last.UpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+				t.Fatalf("zero quota event: %+v", last)
+			}
+
+			injectedAt := UTCNow().Add(2 * time.Hour)
+			interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() {
+				data := []byte(`{"quota":2048,"quotatype":"hard"}`)
+				_, err := globalBucketMetadataSys.updateAndParseMetadata(ctx, bucket, bucketQuotaConfigFile, data, false, false, &injectedAt)
+				if err != nil {
+					t.Error(err)
+				}
+			}}
+			setObjectLayer(interleaving)
+			defer setObjectLayer(obj)
+			before := len(events())
+			rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
+				bucket + "/" + bucketPolicyConfig:     []byte(`{"Version":"2012-10-17","Statement":[]}`),
+				bucket + "/quota.json":                []byte(`{}`),
+				bucket + "/" + bucketTaggingConfig:    []byte(`<Tagging><TagSet><Tag><Key>imported</Key><Value>yes</Value></Tag></TagSet></Tagging>`),
+				bucket + "/" + objectLockConfig:       enabledBucketObjectLockConfig,
+				bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
+			}))
+			report := corsImportReport(t, rec).Buckets[bucket]
+			if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" {
+				t.Fatalf("import: %+v", report)
+			}
+			meta, err = loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !meta.QuotaConfigUpdatedAt.After(injectedAt) || !meta.TaggingConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) || !meta.PolicyConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+				t.Fatalf("import time %v must exceed intervening %v; policy=%v tags=%v", meta.QuotaConfigUpdatedAt, injectedAt, meta.PolicyConfigUpdatedAt, meta.TaggingConfigUpdatedAt)
+			}
+			if !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || !meta.VersioningConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) || !meta.ObjectLockConfigUpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+				t.Fatal("import normalization or common source time lost")
+			}
+			got = events()[before:]
+			if len(got) != 2 {
+				t.Fatalf("import events: %+v", got)
+			}
+			for _, event := range got {
+				if !event.UpdatedAt.Equal(meta.QuotaConfigUpdatedAt) {
+					t.Fatal("import hook used ZIP start time")
+				}
+				if event.Type == madmin.SRBucketMetaTypePolicy {
+					if len(event.Policy) != 0 {
+						t.Fatal("empty policy import is not deletion")
+					}
+				} else if event.Tags == nil || len(event.Quota) == 0 {
+					t.Fatalf("bulk omitted imported state: %+v", event)
+				}
+				if event.Type == "" {
+					if event.Versioning == nil || event.ObjectLockConfig == nil {
+						t.Fatal("normalized import fields omitted")
+					}
+					payload, err := base64.StdEncoding.DecodeString(*event.Versioning)
+					if err != nil || !bytes.Equal(payload, enabledBucketVersioningConfig) {
+						t.Fatal("import sent pre-normalization versioning")
+					}
+				}
+			}
+			beforeMeta, beforeEvents := meta, len(events())
+			rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
+				bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>only</Key><Value>tags</Value></Tag></TagSet></Tagging>`),
+			}))
+			if status := corsImportReport(t, rec).Buckets[bucket]; status.Err != "" || status.Tagging.Err != "" {
+				t.Fatalf("tags-only import: %+v", status)
+			}
+			meta, err = loadBucketMetadata(ctx, obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(meta.PolicyConfigJSON, beforeMeta.PolicyConfigJSON) || !meta.PolicyConfigUpdatedAt.Equal(beforeMeta.PolicyConfigUpdatedAt) || !bytes.Equal(meta.QuotaConfigJSON, beforeMeta.QuotaConfigJSON) || !meta.QuotaConfigUpdatedAt.Equal(beforeMeta.QuotaConfigUpdatedAt) {
+				t.Fatal("tags-only import changed Policy or Quota")
+			}
+			got = events()[beforeEvents:]
+			if len(got) != 1 || got[0].Tags == nil || got[0].Policy != nil || got[0].Quota != nil || got[0].Versioning != nil || got[0].ObjectLockConfig != nil || !got[0].UpdatedAt.Equal(meta.TaggingConfigUpdatedAt) {
+				t.Fatalf("tags-only import hook leaked unspecified fields: %+v", got)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataNormalizesBeforeComparison(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			meta, err := loadBucketMetadata(t.Context(), obj, bucket)
+			if err != nil {
+				t.Fatal(err)
+			}
+			versioning := base64.StdEncoding.EncodeToString([]byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>skip/</Prefix></ExcludedPrefixes><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`))
+			lock := base64.StdEncoding.EncodeToString(enabledBucketObjectLockConfig)
+			item := madmin.SRBucketMeta{Bucket: bucket, ObjectLockConfig: &lock, Versioning: &versioning, UpdatedAt: meta.Created.Add(time.Hour)}
+			counter := &bucketConfigWriteCounter{ObjectLayer: obj}
+			setObjectLayer(counter)
+			defer setObjectLayer(obj)
+			for range 2 {
+				rec := applySRBucketMetaViaAdmin(t, cred, item)
+				if rec.Code != http.StatusOK {
+					t.Fatalf("bulk: %d %s", rec.Code, rec.Body.String())
+				}
+			}
+			if counter.writes.Load() != 1 {
+				t.Fatalf("normalized duplicate persisted %d times", counter.writes.Load())
+			}
+			meta, err = loadBucketMetadata(t.Context(), obj, bucket)
+			if err != nil || !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || !meta.VersioningConfigUpdatedAt.Equal(item.UpdatedAt) {
+				t.Fatalf("normalization: %s %v", meta.VersioningConfigXML, err)
+			}
+			// Ordinary local updates use the identical effective document in their
+			// commit result, even if Object Lock changed since the handler's precheck.
+			data, _ := base64.StdEncoding.DecodeString(versioning)
+			result, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketVersioningConfig, data, false, false, nil)
+			if err != nil || !bytes.Equal(result.meta.VersioningConfigXML, enabledBucketVersioningConfig) {
+				t.Fatalf("local commit snapshot: %s %v", result.meta.VersioningConfigXML, err)
+			}
+		})
+	}})
+}
+
+func TestPeerBucketMetadataEqualTimeArrivalOrders(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
+		t.Run(backend, func(t *testing.T) {
+			created := UTCNow().Add(-time.Hour)
+			a := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>a</Value></Tag></TagSet></Tagging>`)
+			b := bytes.ReplaceAll(a, []byte("<Value>a"), []byte("<Value>b"))
+			for _, pair := range [][2][]byte{{a, b}, {b, a}, {a, nil}, {nil, a}} {
+				meta := newBucketMetadata(bucket)
+				meta.Created = created
+				meta.defaultTimestamps()
+				if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					t.Fatal(err)
+				}
+				for _, data := range pair {
+					var payload *string
+					if data != nil {
+						encoded := base64.StdEncoding.EncodeToString(data)
+						payload = &encoded
+					}
+					rec := applySRBucketMetaViaAdmin(t, cred, madmin.SRBucketMeta{Bucket: bucket, Type: madmin.SRBucketMetaTypeTags, Tags: payload, UpdatedAt: created.Add(time.Minute)})
+					if rec.Code != http.StatusOK {
+						t.Fatalf("equal time apply: %d %s", rec.Code, rec.Body.String())
+					}
+				}
+				got, err := readBucketMetadata(t.Context(), obj, bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				want := b
+				if pair[0] == nil || pair[1] == nil {
+					want = nil
+				}
+				if !bytes.Equal(got.TaggingConfigXML, want) || !got.TaggingConfigUpdatedAt.Equal(created.Add(time.Minute)) {
+					t.Fatalf("arrival order changed winner: %q at %v", got.TaggingConfigXML, got.TaggingConfigUpdatedAt)
+				}
+			}
+		})
+	}})
+}

cmd/site-replication_test.go

@@ -133,28 +133,3 @@ func TestSRBucketMetaCorsRoundTrip(t *testing.T) {
 		t.Fatalf("expected nil Cors for deletion, got %q", *gotDel.Cors)
 	}
 }
-
-// TestIsBucketMetadataEqualCors covers the pointer-comparison helper used by
-// the CORS heal path to decide whether a peer already holds the latest config.
-func TestIsBucketMetadataEqualCors(t *testing.T) {
-	a := base64.StdEncoding.EncodeToString([]byte("config-a"))
-	b := base64.StdEncoding.EncodeToString([]byte("config-b"))
-
-	cases := []struct {
-		name string
-		one  *string
-		two  *string
-		want bool
-	}{
-		{"both nil", nil, nil, true},
-		{"one nil", &a, nil, false},
-		{"other nil", nil, &b, false},
-		{"equal", &a, &a, true},
-		{"different", &a, &b, false},
-	}
-	for _, tc := range cases {
-		if got := isBucketMetadataEqual(tc.one, tc.two); got != tc.want {
-			t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
-		}
-	}
-}

cmd/sts-handlers.go

@@ -624,6 +624,10 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 
 	claims[expClaim] = UTCNow().Add(duration).Unix()
 	claims[parentClaim] = user.AccessKey
+	if err := setIAMParentRevocationClaim(ctx, globalIAMSys.store, user.AccessKey, claims); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
+		return
+	}
 
 	tokenRevokeType := r.Form.Get(stsRevokeTokenType)
 	if tokenRevokeType != "" {

cmd/testdata/data-usage-v9/README.md

@@ -0,0 +1,27 @@
+# Data-usage v9 retirement fixture
+
+Generated using unmodified `scanDataFolder` and `dataUsageCache.serializeTo` from
+Server commit `89637554d60c27cfc51d2281d0a4fe15e415f06d` (Go 1.27.1, darwin/arm64).
+The scanner visits three real local files; its size callback supplies synthetic
+version/delete-marker/remote-tier summaries, following `TestDataUsageCacheSerialize`.
+It is a scanner/cache compatibility fixture, not a distributed object-store test.
+
+The old scanner counts 74,962 bytes, 3 objects, 5 versions and 2 delete markers.
+Its nonzero retired `hts` totals 9,426 bytes. Remote tier `COLD` contains 65,536
+bytes, one version and one object. The cache includes nested children, both
+histograms, a fixed timestamp and scanner cycle 42. The JSON is the old scanner's
+complete expected cache (including `HotTierSize`, which the new reader ignores).
+
+Binary SHA-256: `4c9c7e94cea7758fe9b498b19f639188764b2787582783e6cc950f2c44d52a8b`.
+
+To regenerate, create a detached worktree at that exact source commit, copy
+`generate.go.txt` to `cmd/retirement-fixture_test.go`, and run:
+
+```sh
+SILO_RETIRE_FIXTURE_DIR=/absolute/output/directory go test ./cmd -run '^TestGenerateAccessRetirementV9Fixture$' -count=1 -v
+```
+
+The historical production writer adds the version byte, compresses with zstd
+and encodes msgp, including the nonzero `hts` field. Do not regenerate using the
+retired implementation or by changing the header of a v8 payload. Map order may
+change serialized bytes across regeneration; compare the decoded full cache.

cmd/testdata/data-usage-v9/data-usage-v9.json

@@ -0,0 +1,118 @@
+{
+  "Info": {
+    "Name": "/",
+    "NextCycle": 42,
+    "LastUpdate": "2026-09-15T00:00:00Z",
+    "SkipHealing": true
+  },
+  "Cache": {
+    "/": {
+      "Children": {
+        "v9-bucket": {}
+      },
+      "Size": 0,
+      "HotTierSize": 0,
+      "Objects": 0,
+      "Versions": 0,
+      "DeleteMarkers": 0,
+      "ObjSizes": [
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "ObjVersions": [
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "AllTierStats": null,
+      "Compacted": false
+    },
+    "v9-bucket": {
+      "Children": {
+        "v9-bucket/nested": {}
+      },
+      "Size": 1234,
+      "HotTierSize": 1234,
+      "Objects": 1,
+      "Versions": 1,
+      "DeleteMarkers": 0,
+      "ObjSizes": [
+        0,
+        1,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "ObjVersions": [
+        0,
+        1,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "AllTierStats": null,
+      "Compacted": false
+    },
+    "v9-bucket/nested": {
+      "Children": null,
+      "Size": 73728,
+      "HotTierSize": 8192,
+      "Objects": 2,
+      "Versions": 4,
+      "DeleteMarkers": 2,
+      "ObjSizes": [
+        0,
+        1,
+        1,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "ObjVersions": [
+        0,
+        1,
+        1,
+        0,
+        0,
+        0,
+        0
+      ],
+      "AllTierStats": {
+        "Tiers": {
+          "COLD": {
+            "TotalSize": 65536,
+            "NumVersions": 1,
+            "NumObjects": 1
+          }
+        }
+      },
+      "Compacted": true
+    }
+  }
+}

cmd/testdata/data-usage-v9/generate.go.txt

@@ -0,0 +1,82 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/cachevalue"
+)
+
+// Run only on 89637554d, before access-tiering is removed. This uses the real
+// folder scanner and v9 serializer with synthetic file-size/tier summaries,
+// following TestDataUsageCacheSerialize; it does not relabel a v8 payload.
+func TestGenerateAccessRetirementV9Fixture(t *testing.T) {
+ if dataUsageCacheVerCurrent != 9 { t.Fatal("requires the historical v9 writer") }
+ base := t.TempDir()
+ const bucket = "v9-bucket"
+ createUsageTestFiles(t, base, bucket, []usageTestFile{
+  {name: "root", size: 1234},
+  {name: "nested/versions", size: 8192},
+  {name: "nested/remote", size: 65536},
+ })
+ getSize := func(item scannerItem) (s sizeSummary, err error) {
+  if item.Typ&os.ModeDir != 0 { return s, nil }
+  info, err := os.Stat(item.Path)
+  if err != nil { return s, err }
+  s.totalSize, s.hotTierSize, s.versions = info.Size(), info.Size(), 1
+  if filepath.Base(item.Path) == "versions" { s.versions, s.deleteMarkers = 3, 2 }
+  if filepath.Base(item.Path) == "remote" {
+   s.hotTierSize = 0
+   s.tiers = map[string]tierStats{"COLD": {TotalSize: uint64(info.Size()), NumVersions: 1, NumObjects: 1}}
+  }
+  return s, nil
+ }
+ xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()}
+ xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(context.Context) (DiskInfo,error) {
+  return DiskInfo{Total: 1<<40, Free: 1<<40}, nil
+ })
+ cache, err := scanDataFolder(t.Context(), nil, &xls, dataUsageCache{Info:dataUsageCacheInfo{Name:bucket, SkipHealing:true}}, getSize, 0, func()bool{return false})
+ if err != nil { t.Fatal(err) }
+ root := *cache.find(bucket)
+ cache.replace(dataUsageRoot, "", dataUsageEntry{})
+ cache.replace(bucket, dataUsageRoot, root)
+ cache.Info.Name = dataUsageRoot
+ cache.Info.LastUpdate = time.Date(2026, 9, 15, 0, 0, 0, 0, time.UTC)
+ cache.Info.NextCycle = 42
+ flat := cache.flatten(*cache.root())
+ if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 || flat.HotTierSize != 9426 {
+  t.Fatalf("unexpected scanner fixture: %+v", flat)
+ }
+ var buf bytes.Buffer
+ if err := cache.serializeTo(&buf); err != nil { t.Fatal(err) }
+ dir := os.Getenv("SILO_RETIRE_FIXTURE_DIR")
+ if dir == "" { t.Fatal("SILO_RETIRE_FIXTURE_DIR required") }
+ if err := os.MkdirAll(dir, 0755); err != nil { t.Fatal(err) }
+ if err := os.WriteFile(filepath.Join(dir,"data-usage-v9.bin"),buf.Bytes(),0644);err != nil{t.Fatal(err)}
+ expected, err := json.MarshalIndent(cache,"","  ")
+ if err != nil { t.Fatal(err) }
+ if err := os.WriteFile(filepath.Join(dir,"data-usage-v9.json"),append(expected,'\n'),0644);err != nil{t.Fatal(err)}
+ t.Logf("old scanner/v9 writer: bytes=%d size=%d objects=%d versions=%d markers=%d hts=%d",buf.Len(),flat.Size,flat.Objects,flat.Versions,flat.DeleteMarkers,flat.HotTierSize)
+}

cmd/xl-storage.go

@@ -583,7 +583,6 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 	}
 
 	poolIdx, setIdx, _ := s.GetDiskLoc()
-	hotPool, hotPoolOK := globalILMConfig.accessCfg().HotPool()
 
 	disks, err := objAPI.GetDisks(poolIdx, setIdx)
 	if err != nil {
@@ -593,7 +592,6 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 	cache.Info.updates = updates
 
 	dataUsageInfo, err := scanDataFolder(ctx, disks, s, cache, func(item scannerItem) (sizeSummary, error) {
-		item.poolIdx = poolIdx
 		// Look for `xl.meta/xl.json' at the leaf.
 		if !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFile) &&
 			!strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFileV1) {
@@ -657,9 +655,6 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 				sizeS.versions++
 			}
 			sizeS.totalSize += sz
-			if hotPoolOK && poolIdx == hotPool {
-				sizeS.hotTierSize += sz
-			}
 
 			// Skip tier accounting if object version is a delete-marker or a free-version
 			// tracking deleted transitioned objects

dockerscripts/build-static-curl.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+
+# Build both release architectures from the same verified upstream source.
+# The previous binary supplier stopped publishing current ARM64 builds.
+CURL_VERSION=8.22.0
+CURL_SHA256=f7ef3ae8a22e521f289803fe93543eb64c329b58aa73a9e224dfd915a2a5f4f7
+
+case ${TARGETARCH:?TARGETARCH is required} in
+amd64) expected_arch=x86_64 ;;
+arm64) expected_arch=aarch64 ;;
+*) echo "Unsupported cURL architecture: ${TARGETARCH}" >&2; exit 1 ;;
+esac
+[ "$(apk --print-arch)" = "$expected_arch" ] || {
+    echo "cURL must be built natively for ${TARGETARCH}" >&2
+    exit 1
+}
+
+apk add --no-cache --upgrade build-base curl ca-certificates \
+    openssl-libs-static nghttp2-static libpsl-static libidn2-static \
+    libunistring-static zlib-static brotli-static zstd-static libssh2-static
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+curl --fail --location --silent --show-error --retry 3 \
+    --connect-timeout 15 --max-time 180 \
+    "https://curl.se/download/curl-${CURL_VERSION}.tar.xz" -o "$work/curl.tar.xz"
+printf '%s  %s\n' "$CURL_SHA256" "$work/curl.tar.xz" | sha256sum -c -
+tar -xJf "$work/curl.tar.xz" -C "$work"
+cd "$work/curl-${CURL_VERSION}"
+PKG_CONFIG="pkg-config --static" LDFLAGS="-static" ./configure \
+    --disable-shared --enable-static --with-openssl --with-libssh2 \
+    --with-nghttp2 --with-brotli --with-zstd --with-libidn2 --with-libpsl \
+    --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-path \
+    --disable-ldap --disable-ldaps
+# libtool's -static only embeds libcurl; -all-static also embeds its dependencies.
+make -j"${CURL_BUILD_JOBS:-4}" LDFLAGS="-all-static"
+strip src/curl
+if readelf -l src/curl | grep -q INTERP || readelf -d src/curl | grep -q NEEDED; then
+    echo "cURL still requires a dynamic loader or shared libraries" >&2
+    exit 1
+fi
+src/curl --version
+mkdir -p /go/bin /go/share/curl
+cp src/curl /go/bin/curl
+cp COPYING /go/share/curl/COPYING
+apk info -v | sort > /go/share/curl/build-packages.txt

dockerscripts/download-static-curl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Pin to v8.11.0 – the last release that includes curl-aarch64.
-# v8.17.0 (current latest) dropped the aarch64 build.
-STATIC_CURL_VERSION="v8.11.0"
-
-case ${TARGETARCH:?TARGETARCH is required} in
-amd64)
-	asset=curl-amd64
-	expected=d18aa1f4e03b50b649491ca2c401cd8c5e89e72be91ff758952ad2ab5a83135d
-	;;
-arm64)
-	asset=curl-aarch64
-	expected=1b050abd1669f9a2ac29b34eb022cdeafb271dce5a4fb57d8ef8fadff6d7be1f
-	;;
-*)
-	echo "Unsupported static cURL architecture: ${TARGETARCH}" >&2
-	exit 1
-	;;
-esac
-
-curl --fail --location --silent --show-error --retry 3 \
-	"https://github.com/moparisthebest/static-curl/releases/download/${STATIC_CURL_VERSION}/${asset}" \
-	--output /go/bin/curl
-printf '%s  %s\n' "${expected}" /go/bin/curl | sha256sum -c
-chmod +x /go/bin/curl

docs/bucket/lifecycle/README.md

@@ -226,149 +226,11 @@ aws s3api restore-object --bucket srcbucket \
 
 Note that transition event notification is a Silo extension.
 
-## 5. Access-based tiering between server pools
-
-Silo can move frequently read objects to a faster server pool and return them
-to a slower pool after they become idle. This is different from remote ILM
-transition: the object remains a native local object and all versions move
-together.
-
-Access tiering requires at least two server pools. Pool indices follow the
-order on the server command line:
-
-~~~sh
-silo server /srv/nvme{1...4} /srv/hdd{1...8}
-#             pool 0 (fast)   pool 1 (slow)
-~~~
-
-Server pools are erasure-coding expansion units, not individual drives. Each
-pool should consist of internally homogeneous media.
-
-The feature is disabled by default. Configure the topology and safety limits
-with `mc admin config set`; ILM is a dynamic subsystem, so this applies without
-a restart. Environment variables (`MINIO_ILM_ACCESS_TIERING`,
-`MINIO_ILM_ACCESS_POOLS`, `MINIO_ILM_ACCESS_MAX_SIZE`,
-`MINIO_ILM_ACCESS_PROMOTE_WATERMARK`, `MINIO_ILM_ACCESS_BIN_WIDTH`,
-`MINIO_ILM_ACCESS_BINS`, `MINIO_ILM_ACCESS_FLUSH`,
-`MINIO_ILM_ACCESS_MIN_RESIDENCY`, `MINIO_ILM_ACCESS_WORKERS`,
-`MINIO_ILM_ACCESS_MAX_TRACKED`) are read at process start and override the
-stored config.
-
-~~~sh
-mc admin config set local ilm \
-  access_tiering=on \
-  access_pools="0,1" \
-  access_max_size="2TiB" \
-  access_promote_watermark=85 \
-  access_bin_width=1m \
-  access_bins=12 \
-  access_flush=1m \
-  access_min_residency=24h \
-  access_workers=10 \
-  access_max_tracked=1000000
-~~~
-
-The pool list is ordered hottest to coldest. With three or more pools,
-promotion always targets the first index and demotion always targets the last;
-intermediate pools are not hop targets. An access_max_size value of zero means
-no cluster-wide logical-byte cap. Promotion also stops when the hottest pool
-reaches access_promote_watermark.
-
-New PUTs still land via the usual free-space pool picker; they are not steered
-onto the cold pool. Size the capacity pool larger than the hot pool so new
-objects tend to land there.
-
-Add an AccessTransition to the bucket lifecycle XML:
-
-~~~xml
-<LifecycleConfiguration>
-  <AccessTierQuota>500GiB</AccessTierQuota>
-  <Rule>
-    <ID>hot-logs</ID>
-    <Status>Enabled</Status>
-    <Filter>
-      <And>
-        <Prefix>logs/</Prefix>
-        <ObjectSizeGreaterThan>65536</ObjectSizeGreaterThan>
-      </And>
-    </Filter>
-    <AccessTransition>
-      <Window>10m</Window>
-      <PromoteAfterAccesses>100</PromoteAfterAccesses>
-      <DemoteAfterAccesses>5</DemoteAfterAccesses>
-      <DemoteAfterIdle>24h</DemoteAfterIdle>
-    </AccessTransition>
-  </Rule>
-</LifecycleConfiguration>
-~~~
-
-This promotes a matching object after 100 successful GETs in 10 minutes. An
-object becomes eligible to return to the coldest configured pool only after
-access tiering has already moved it (the `x-minio-internal-ilm-atier` stamp),
-it has stayed put for the server-wide minimum residency, it has been idle at
-least 24 hours, and it has no more than 5 GETs in the window. Objects that
-landed on the hot pool via a normal PUT never demote. Prefix, tag, and
-object-size lifecycle filters are honored.
-
-Access-based moves are a parallel path: they are not lifecycle `Eval` actions
-and do not appear in S3 prediction headers. If the same object is also due
-for age-based remote `Transition` or expiry, that scanner action wins and
-demotion discovery is skipped for that pass; promotions still run from the
-GET tracker. Site replication copies expiry rules only, same as remote
-Transition, so AccessTransition stays local to the cluster.
-
-AccessTierQuota is an optional bucket-wide cap. Promotion checks, in order:
-
-1. hot-pool used percentage;
-2. cluster-wide access_max_size;
-3. bucket AccessTierQuota.
-
-Demotion is not blocked by these caps and is processed before promotion.
-Access moves pause during rebalance or decommission, never target a suspended
-pool, skip remotely transitioned objects and objects with excessive version
-counts, and recheck eligibility while holding the object namespace lock.
-Moves also lock the source and destination write locations. All lock servers
-for those locations must be reachable; otherwise the background move is
-deferred. Ordinary S3 requests keep their existing quorum requirements.
-
-Each move preserves version IDs, delete markers, ETags, checksums, encryption
-and user metadata. A retry copies only missing versions and retains versions
-already at the destination. If the same version ID has conflicting metadata
-in the two pools, the move is skipped and both copies are left intact. Source
-data is removed only after the complete version stack exists at the destination.
-
-The hit counter is intentionally best effort. Only successfully served GET
-requests count; HEAD requests do not. Counters are merged across nodes and
-bounded by access_max_tracked. A rule window longer than
-access_bin_width multiplied by access_bins is clamped to retained history.
-
-AccessTransition and AccessTierQuota are Silo lifecycle extensions. A stock
-AWS SDK that reads and rewrites the lifecycle configuration may discard
-unknown fields. Use a raw signed S3 PUT lifecycle request, such as
-[setup_ilm_access_tiering.sh](setup_ilm_access_tiering.sh), when installing
-the rule. Save the XML above as `rule.xml`, then run:
-
-~~~sh
-AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin \
-  ./setup_ilm_access_tiering.sh http://127.0.0.1:9000 testbucket us-east-1 rule.xml
-~~~
-
-Access-tier activity is exposed under /minio/metrics/v3/ilm, including move
-counts, moved bytes, queue depth, hot bytes per bucket, failed moves, dropped
-GET samples, and separate skip counters for each capacity limit.
-
-To stop scheduling moves, set `access_tiering=off` (and remove any environment
-override). Objects already moved remain in their current pools and stay
-accessible; disabling the feature does not move them back. Before downgrading
-to a release without access tiering, disable it and let active moves finish.
-The object storage format is unchanged. The data-usage cache advances from
-v8 to v9: this release reads both, but an older binary discards v9 caches and
-rebuilds usage statistics through the scanner. Usage and quota statistics can
-therefore take time to repopulate after a downgrade. Keep a copy of lifecycle
-XML containing Silo extensions, since an older binary may omit those fields
-when rewriting a lifecycle rule.
-
 ## Explore Further
 
 - [MinIO Go client API reference (S3-compatible SDK)](https://pkg.go.dev/github.com/minio/minio-go/v7)
 - [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)
+
+## Access-frequency tiering removal
+
+The opt-in cross-pool access-frequency extension has been removed. For upgrades from a main/snapshot build that included it, see [the migration notes](access-tiering-removal.md). Ordinary lifecycle expiration and remote-tier transitions remain supported.

docs/bucket/lifecycle/access-tiering-removal.md

@@ -0,0 +1,36 @@
+# Removing access-frequency pool tiering
+
+PR #60 introduced an opt-in scheduler that moved objects between local server pools according to GET frequency. It and its feature-specific fixes have been removed. This does not remove ordinary lifecycle expiration, transitions to remote tiers, rebalance, decommission, or the general multi-pool correctness fixes from PR #178.
+
+The [introduction and rollback record](../../investigations/access-tiering-revert.md) documents the commit history, scope decision, review corrections and unresolved validation findings.
+
+The published Server 20260903 predates this feature. These instructions concern main/snapshot deployments that included #60; upgrading from the published version does not require access-tier configuration cleanup.
+
+## Before upgrading a build with access tiering
+
+1. Save a copy of the server ILM configuration and each affected bucket's lifecycle XML. Use an API client that preserves the nonstandard XML; do not rely on a client model that silently omits unknown elements.
+2. On the old server, set `ilm access_tiering=off` and remove or disable any access-tier environment overrides. Allow in-progress moves to finish before replacing nodes. This reduces movement intermediate states; the removal itself changes no storage RPC protocol.
+3. Remove top-level `AccessTierQuota` and rule-level `AccessTransition` elements. **Delete rules whose only action was `AccessTransition`**. For a mixed rule, retain its filter, status, ID and ordinary expiration/transition actions. An access-only rule loads harmlessly after upgrade, but becomes an actionless rule and fails validation on the next lifecycle edit. If no rules remain, delete the lifecycle configuration through the S3 API.
+4. Use a coordinated maintenance window: stop the deployment, install the same new binary on every node, then restart all nodes. The existing bootstrap check compares binary checksums; in a four-node test the first new node could not finish starting among three old nodes. Do not assume that an unchanged RPC protocol permits replacing one node at a time and waiting for it to become ready. This removal does not relax that check. Apply the same environment changes on every node: bootstrap also compares server environment settings, so removing an old override on only some nodes can block startup even with matching binaries. The check runs only during startup and is not a safety guarantee for nodes already running different binaries.
+5. After restarting, verify object reads, bucket listing, ILM worker settings and a lifecycle edit. Check storage access from every request-serving node to each pool's drives: successful reads or bucket listing establish less than complete drive reachability, and admin disk summaries aggregate server-local state. Retirement acceptance used unique probes and storage trace to confirm every node-to-drive path before and after version deletion. Startup connection times varied, so a fixed sleep is insufficient. Complete distributed upgrade acceptance for the exact binaries before production rollout.
+
+## What happens to stored state
+
+| State | Behavior after removal |
+| --- | --- |
+| Ten old ILM keys | `access_tiering`, `access_pools`, `access_max_size`, `access_promote_watermark`, `access_bin_width`, `access_bins`, `access_flush`, `access_min_residency`, `access_workers`, `access_max_tracked` are accepted but ignored. Existing transition/expiration worker settings are preserved. |
+| Admin configuration | Deprecated keys may still appear in `mcli admin config get ilm`; setting them may succeed but has no effect, even with `access_tiering=on`. Remove obsolete environment settings from deployment manifests. |
+| Lifecycle XML | `AccessTierQuota` and `AccessTransition` are ignored when read and omitted when re-encoded. The same parser handles new PUT requests, so these extensions are also silently discarded there; access-only rules still fail action validation. |
+| Data-usage cache | Both v8 and v9 caches are read, preserving ordinary counts, sizes, histograms and remote-tier statistics. The retired hot-tier byte count is discarded; subsequent writes use v8. No feature-driven full statistics rebuild is required. |
+| Objects already moved | Remain in their current pools with the same versions and timestamps. There is no bulk move-back or object metadata rewrite. |
+| Internal leftovers | `x-minio-internal-ilm-atier` and `.minio.sys/config/ilm/access/` counter objects may remain unused. They do not require a cleanup service or an object scan. |
+
+Interrupted rebalance/decommission can leave the same version in more than one pool independently of access tiering. Removing the scheduler does not remove such existing copies. General Object Lock, conditional-delete, metadata reconciliation and shared remote-tier reference protections remain in place.
+
+## Version deletion scope
+
+Ordinary single-object `DELETE ?versionId=...` reconciles the addressed UUID, null version or delete marker across pools. Unqualified DELETE of a directory marker (a key ending in `/`) also addresses its null version and uses this path. A successful request applies the deletion to every resolved pool copy under the existing per-pool quorum rules; other version IDs remain. If outbound delete replication is pending, copies retain `VersionPurgePending` until the existing replication worker completes the purge. Success does not guarantee immediate physical removal from every drive.
+
+If a pool is unreadable, these requests can return 503 even when another pool has a readable copy. Insufficient read quorum returns `503 SlowDownRead`; other failures retain their corresponding error codes. This extends an existing failure surface: previously the result could depend on whether the unreadable pool preceded the successful pool in traversal order; it now fails consistently. Retry after recovery. Cleanup failures also return an error. Ordinary unqualified DELETE retains its existing semantics. Batch `DeleteObjects` already fans out across pools.
+
+Incoming replicated deletes, lifecycle expiration, free-version cleanup and movement-internal calls retain their existing contracts. In particular, an incoming replicated version delete can leave movement duplicates in other pools; this change does not solve that separate case. Expiration scanners process their own pools and may remove duplicate expired copies in later cycles; free-version cleanup remains local to a pool. Do not treat the ordinary DELETE repair as a guarantee for every source of deletion.

docs/bucket/lifecycle/setup_ilm_access_tiering.sh

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-# Install a lifecycle XML document without an SDK normalizing away Silo's
-# AccessTransition and AccessTierQuota extension elements.
-set -eu
-
-if [ "$#" -ne 4 ]; then
-	echo "usage: AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... $0 ENDPOINT BUCKET REGION LIFECYCLE_XML" >&2
-	exit 2
-fi
-
-endpoint=$1
-bucket=$2
-region=$3
-lifecycle_file=$4
-
-: "$AWS_ACCESS_KEY_ID"
-: "$AWS_SECRET_ACCESS_KEY"
-
-if [ ! -r "$lifecycle_file" ]; then
-	echo "cannot read lifecycle document: $lifecycle_file" >&2
-	exit 2
-fi
-
-content_md5=$(openssl dgst -md5 -binary "$lifecycle_file" | openssl base64)
-endpoint=$(printf '%s' "$endpoint" | sed 's:/*$::')
-
-curl --fail-with-body --silent --show-error \
-	--request PUT \
-	--aws-sigv4 "aws:amz:$region:s3" \
-	--user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" \
-	--header "Content-MD5: $content_md5" \
-	--header "Content-Type: application/xml" \
-	--data-binary "@$lifecycle_file" \
-	"$endpoint/$bucket?lifecycle"
-
-echo "installed lifecycle configuration on $bucket"

docs/iam/password-permissions.md

@@ -1,5 +1,11 @@
 # Password and user-management permissions
 
+> **Release boundary, 2026-09-13:** this guide describes Server main, paired with
+> silo-pkg v3.14.0 and Console source `417559bb2c97` or its accepted successor.
+> The latest published Server 20260903 and Console v2.4.0 do not contain this
+> split. The pkg v3.14.0 and mcli 20260913 releases alone do not change an old
+> Server's authorization. See [the component matrix](https://silo.pgsty.com/compatibility/versions/).
+
 **Breaking change: the password-permission split changes the meaning of
 existing IAM policies.** The same stored policy can authorize a request after
 this update that it denied before, or deny a request it previously authorized.

docs/investigations/access-tiering-revert.md

@@ -0,0 +1,193 @@
+# 访问频率分层：引入、修复与回退记录
+
+记录日期：2026-09-15。本记录说明 PR #60 的设计、合入后的取舍，以及此次回退为什么同时保留并补齐通用多池正确性修复。操作步骤见[退役迁移说明](../bucket/lifecycle/access-tiering-removal.md)。合并、正式发布和生产部署是不同状态；本记录随回退变更交付，不代表已经发布。
+
+初稿审查时（2026-09-15 03:20 UTC），最终候选尚未移植到远端基线、尚未冻结 PR head，合并前全量检查与三次有效 Linux 运行尚未执行，本变更尚未合并。后续执行状态以承载本记录的 PR 及其绑定提交的验收记录为准；下文的历史实验不替代这些检查。
+
+**当前验收状态：** 回退、普通版本 DELETE 调和、扫描复用及文档由 [PR #188](https://github.com/pgsty/silo/pull/188) 交付。本地与 CI 检查通过；首次 Linux 验收因 DELETE204 后的 HEAD/GET503 停止。随后完成可控机制实验、匹配写入负载对照，并修正准备检查；独立的新轮次 R-Upgrade-2 三次完整升级验收均通过。旧失败没有改判，原单次请求的逐盘状态仍不可追溯。最终合并状态以 PR 为准，正式发布与部署另行验收。详情见第 9 至 11 节。
+
+## 1. 引入的目标和实际范围
+
+[@mrjavadseydi](https://github.com/mrjavadseydi) 在 [PR #60](https://github.com/pgsty/silo/pull/60) 提出了基于 GET 频率的本地池间分层。成功 GET 更新有界滚动计数，后台调度器把热对象提升到配置中的首个池，把已经迁移且变冷的对象降到末个池。功能默认关闭，至少需要两个池；它与普通生命周期过期、远端对象存储 transition、rebalance 和 decommission 是不同机制。
+
+实现不只是一个后台任务：它增加了 GET 计数入口、leader 调度、跨池版本栈复制、来源清理及失败恢复、热池配额、生命周期 XML 扩展、配置项、指标和扫描统计。访问热度统计使 data-usage cache 从 v8 升到 v9。对象本体的存储格式没有因此改变。
+
+搬移要保持完整版本历史、删除标记、null version、时间戳、ETag、校验和和加密元数据；同时需要处理并发写入、目的端已有版本、来源部分删除和远端 tier 引用。合入前的修补和测试针对并覆盖了这些边界，不应把回退解释成贡献无效。贡献者署名继续保留，独立的其他贡献也不回退。
+
+## 2. 可追溯时间线
+
+下表的 PR/Issue 时间使用 UTC；提交链接对应具体代码，不把“报告时间”当作“缺陷首次出现时间”。
+
+| 时间 | 事件 | 本次处理 |
+| --- | --- | --- |
+| 2026-08-15 10:15 | #60 创建；原始实现 [`7a060cab1`](https://github.com/pgsty/silo/commit/7a060cab1edd5bbc17da7f703bbd1ab7415b6f7c) | 随特性撤销 |
+| 2026-09-06 00:09 | [#133](https://github.com/pgsty/silo/issues/133) 报告多池副本写不能权威调和 Object Lock 状态 | 保留解决它的通用修复 |
+| 2026-09-06 15:12 | [#144](https://github.com/pgsty/silo/issues/144) 报告条件 DELETE 原子性仅限单个纠删码集合 | 保留解决它的通用修复 |
+| 2026-09-08 05:18 | [`9a6e1477f`](https://github.com/pgsty/silo/commit/9a6e1477f45067559def8423d431ee177795134f) 补访问分层兼容标识清单 | 删除功能专属标识，保留有依据的退役兼容 |
+| 2026-09-08 07:08 | [`374de0fa3`](https://github.com/pgsty/silo/commit/374de0fa32aa1d6eda57dbba6ac522ebf793b6be) 修复搬移的版本保全、写隔离和删除范围 | 随专属搬移器撤销 |
+| 2026-09-08 07:28 | [`5ac33e158`](https://github.com/pgsty/silo/commit/5ac33e1583e838ade3f56c7d60e80e7a854f9a88) 确定性覆盖搬移失败恢复 | 随已删除搬移器的专属测试撤销 |
+| 2026-09-08 07:42 | #60 以 [`a3df317ae`](https://github.com/pgsty/silo/commit/a3df317ae0725eb650e4d3e21551154f69be6229) 合入 | 以该 merge 的第一父差异确定功能边界 |
+| 2026-09-11 12:34 | [#178](https://github.com/pgsty/silo/pull/178) 合入通用多池写入、元数据与条件删除调和 | 保留，解除测试对访问搬移器的依赖 |
+| 2026-09-13 | [`2dd1e00da`](https://github.com/pgsty/silo/commit/2dd1e00da49faf995f2db29807fc6211b8376d7d) 的 CHANGELOG 同时汇总访问分层和通用多池修复 | 拆开表述，不整条删除独立修复历史 |
+| 2026-09-15 | 维护者决定收缩访问分层；完成来源分析、三种候选反证、退役兼容、普通版本 DELETE 修补与外部评审 | 形成此次选择性回退 |
+
+#178 中的 [`e59a3d938`](https://github.com/pgsty/silo/commit/e59a3d938ed25c1bcd51efbb4ad6955073d195f7) 提供池级串行化、字段调和和条件删除；[`ccb676e60`](https://github.com/pgsty/silo/commit/ccb676e60cb7441ee65ff7c35f3b7828979101fd) 保护仍被其他副本使用的远端 tier 引用；[`51d41345f`](https://github.com/pgsty/silo/commit/51d41345f7ac532f9c6fea2b7dba5f29da930b8f) 保存 Linux 重启及 OIDC 验收记录。这三项不属于仅为访问频率调度而存在的代码。
+
+截至回退评估时，公开 Server `RELEASE.2026-09-03T13-18-01Z` 早于 #60 合入。退役迁移主要针对运行过后续 main、自行构建或快照版本的实例；不能据此声称正式 release 用户普遍启用过此特性。
+
+## 3. 为什么回退，为什么不能整批撤销后续修复
+
+维护者的取舍是：默认关闭的可选调度能力，对配置、生命周期、缓存、统计和核心多池写入路径带来了过大的维护面。此次移除的是该能力及专属实现，没有测量并宣称吞吐量提升、延迟降低或固定减少一次分布式锁往返。
+
+“后来改过同一文件”不等于“由 #60 引发”。#133/#144 的报告早于 #60 合入；更关键的是，原有 rebalance、decommission 和复制写入也会使同一版本暂时存在于多个池。访问分层消失后，这些状态仍然合法存在。移除调和与锁纪律会重新允许旧副本遮蔽新元数据、条件删除选错版本、清理错误被吞掉等问题。
+
+评估在隔离工作树中实际比较了三条路线：
+
+| 候选 | 通用多池回归 | 普通指定版本 DELETE |
+| --- | --- | --- |
+| A：撤销 #60，保留 #178 | 原有 13 组通过 | 仍能成功返回后留下可读副本 |
+| B：同时撤销 #60 和 #178 的存储修补 | 相同 13 组中 10 组失败 | 问题仍在 |
+| C：A 加普通版本 DELETE 调和 | 13 组原有及当时新增的 8 组通过 | 同一复现通过 |
+
+这些是 2026-09-15 的历史对照结果，不是最终 PR head 的发布验收。后续补上目录标记、真实 rebalance 中断等覆盖后，通用多池测试达到 23 组。测试通过不能替代来源分析，来源分析也不能替代最终候选的运行验证。
+
+## 4. 最终保留与删除的边界
+
+- 删除访问 tracker、调度/搬移器、GET 和 scanner 钩子、热池配额、专属配置帮助、生命周期动作、指标及专属测试。
+- 保留普通过期、远端 transition、rebalance/decommission、复制写入，以及 #178 的 Object Lock、标签、条件删除、元数据调和与远端引用保护。
+- 原有 data-usage 生成解码器恢复到 #60 前的实现；允许读取 v8/v9，利用字段编码跳过已退役热度字段，继续写 v8。普通字段保真由历史真实 v9 样本测试覆盖。
+- 仅容忍准确的十个退役 ILM 键；读取生命周期时丢弃退役扩展。纯访问动作的规则需先清理才能再次编辑；混合规则保留普通动作。
+- 已经搬移的对象留在当前池；没有全量搬回、自动删除所有重复版本、后台清理服务或对象元数据重写。
+
+曾对本地审查提交 `d06f1c614` 做过声明来源复核：消失的 357 个声明均不在 #60 之前，删除的十个文件均由 #60 引入；#60 原先删换的 41 行旧文本按忽略空白比较有 40 行恢复，剩下一行保留 #178 在已持有池锁时调用 `getWritePoolIdx(..., true)` 的修正，避免对同一对象再次取锁。生成缓存解码器与功能前逐字节一致，原有 13 组通用测试没有删除。这是该审查版本的保全证据，不能把数字脱离 SHA 当作未来所有版本的保证。
+
+## 5. 普通版本 DELETE 是独立补洞
+
+功能移除不会自动消除历史重复副本。普通单对象指定版本 DELETE 因而复用既有调和路径：在池级对象锁内读取每个池的目标版本，计算一次条件及回调，先删除非权威副本，再处理权威副本；任何不可读池或清理错误都不能当作成功。
+
+范围包括 UUID、null version、delete marker，以及原先就被解析为 null version 的未指定版本目录标记 DELETE。入站复制、搬移内部调用、生命周期过期和 free-version 清理保留各自语义；批量 `DeleteObjects` 原本就会向池并发扇出，不是此次遗漏。
+
+删除标记需要向 retention/metadata 回调传入与 set 层相同的 `MethodNotAllowed` 或 `ObjectNotFound` 语义。直接复用拒绝 marker 的元数据更新入口会错误地拒绝合法版本删除。回调从所有副本合并独立更新的 Object Lock 和标签，不能随意只采用一个池的状态。
+
+存在两项明确的成功/失败边界：
+
+1. 读法定多数不足时返回 `503 SlowDownRead`，即使另一个池有可读副本。旧路径的结果会受池遍历顺序影响；新路径把失败语义统一。它是正确性与可用性的取舍，需要恢复后重试。
+2. 出站删除复制尚未完成时，成功响应可以表示各副本进入 `VersionPurgePending`，由既有 worker 完成清理。原有每池 quorum 规则也继续适用；不能把成功响应等同于每一块盘立即物理删除。
+
+## 6. 审查如何改变了方案
+
+本地先形成三个线性审查提交：`8fdfdabd9` 移除特性，`d06f1c614` 补普通 DELETE，`6e3fdca97` 去除重复扫描。它们记录审查演进，最终 PR 在独立远端基线上重放，提交 ID 会改变；不应把线性演进误认为三份同时维护的实现。
+
+Claude Code Opus 5 / max 的五轮实现评审要求补齐退役兼容、说明协调停机及环境一致性、验证真实池故障，并纠正运行证据措辞。随后 Claude 与 ZCode 的独立复核再次确认了回退边界和普通 DELETE 语义；最终两轮计划商榷收束了交付流程。
+
+| 意见 | 裁定与处理 |
+| --- | --- |
+| 指定版本 DELETE 重复扫描所有池 | 接受。第一次扫描已在同一锁内得到目标副本，直接合并其结果。16 盘池在回调前的读取计数从 32 降为 16；这不等于总 I/O 或延迟减半。 |
+| N 个副本产生 N 条 DELETE 审计 | 反驳。底层调用只追加上下文标签，HTTP 层向每个配置审计目标发送一次请求事件。成功完成调和时池标签最终指向 primary；不增加额外 NoAuditLog 修改。 |
+| retention/metadata 顺序与 set 层不同 | 差异存在，但已有明确注释。保留 retention 优先，拒绝后不删除、不调度删除复制、不 Sweep；不宣称任意自定义回调都能交换。 |
+| 把优化 amend 进旧提交并直接丢弃 main 脏修改 | 不 amend 已审历史。先保存完整文件、补丁、哈希及可达恢复引用，核对覆盖后受控恢复。八组新增通用测试承接，访问搬移专用测试由真实 rebalance 中断覆盖替代。 |
+| 从当前本地分支直接开 PR | 调整。其祖先含另一个任务的 IAM/超时提交 `ebc9937d9`；从远端 `89637554d` 仅移植本次变更，不夹带或删除独立工作。 |
+| 合并之后再跑全量与 Linux 验收 | 不接受。先完成文档和移植，冻结 PR head，再验收；不能把旧 SHA 的测试直接提升为新基线的通过记录。 |
+| 不同基线 diff 必须逐字节一致 | 改为每提交 stable patch-id、路径与完整树等价性核验。blob hash 和行号随基线变化，不应成为错误的拒绝依据。 |
+
+`objectPoolInfos` 的并行查询作为独立性能跟进，本次不增加并发实现。Contributor 署名、#132 配额指标、#77 桶元数据、federated COPY、IAM、超时及其他独立修复不因本次取舍被整体撤销。
+
+## 7. 历史运行证据与未定案事项
+
+以下是移植前 `d06f1c614` 的历史实验，不是最终 PR head 的验收替身。
+
+| 运行 | 实际观察 | 不应推导的结论 |
+| --- | --- | --- |
+| `f590538f`，四节点双池 | 旧版真实 rebalance 中断留下 9 个重复 UUID；协调停机换新后对象四节点可读，旧配置与普通规则可编辑，真实缓存头 v9→v8；删除一个 addressed UUID 后四节点 HEAD/GET 404，另一个版本仍可读 | 没有验证全部 9 个不同 UUID 收敛；物理元数据仅每池抽查一盘；没有整份运行时统计守恒测量 |
+| `fd93cd37`，三节点双池 | 停掉一个池所在节点，保留 namespace 锁的 2/3 法定多数；DELETE 返回 503 SlowDownRead，源全部四盘保留目标版本；恢复后 DELETE204，各节点 HEAD/GET404 | 不能推广为跨主机网络、持久盘及压力验收 |
+| 被弃用的四节点停池拓扑 | 同时丢失 namespace 锁法定多数，发生客户端超时，记录脚本还遇到 NoneType 错误 | 不是“池读取返回503”的证明 |
+| `83676ca2` | 升级后配置/生命周期编辑之后一次 HeadObject 返回503；artifact 没有记录 DELETE 自身响应 | “DELETE之后”仅来自脚本顺序，不能写成已证实 DELETE204 后异常，也不能归类为已修复、既有问题或暂态 |
+
+**开放项：`83676ca2` 的 HEAD503 仍未定案。** 可直接比较的目标阶段历史运行是一失败、一成功；一次未复现不足以关闭问题。仅凭 `SlowDownWrite` 等错误名字也不能给其他失败确定容量或环境根因。
+
+最终候选的合并前复核采用有界规则：要求三次有效升级后 DELETE/HEAD 运行，最多五次总尝试；每次记录 DELETE 码/耗时、失败 HEAD 的节点与版本、GET 错误码、两池全盘元数据、固定间隔重试时序和旧版同拓扑对照。旧版可能保留副本返回200，不要求它满足新增跨池删除契约。
+
+有证据证明在目标操作前失败的 harness 尝试才能不计入有效运行，但仍计入总尝试。任何目标阶段的新503或数据不变量失败都不能通过补跑抹掉，必须暂停合并并定位。三次通过也只满足这项工程检查，不证明历史异常已消失；开放项在合并和正式发布评估时仍须可见。
+
+## 8. 交付与恢复纪律
+
+最终候选使用独立分支；main 的五个旧修改先保存完整内容、二进制补丁、SHA-256 和具名 Git 恢复引用，再核对原 HEAD/哈希及测试覆盖，只恢复这五个文件。禁止用整树 reset 或 clean 代替受控归一；若用户已有新增编辑，应保留并重新核对。
+
+全量 cmd/internal、相关 race、构建、vet、lint、生成文件和兼容检查，以及上述 Linux 验收，绑定最终候选的实际 SHA。若纳入新的远端提交，重新记录基线与 head，复核变更并重跑受影响验收。文档与原始测试记录各自保留其对应版本，不篡改旧失败、不用新通过覆盖旧记录。
+
+逐节点滚动升级未通过既有二进制校验检查，采用[协调停机方案](../bucket/lifecycle/access-tiering-removal.md#before-upgrading-a-build-with-access-tiering)。实验使用单个 Docker Linux VM 和 tmpfs；正式 tag、包、镜像、跨主机及生产部署仍是独立交付。未验证全部重复 UUID 或运行时统计守恒应如实披露，不能反向引入自动搬回/清理需求或无关重构。
+
+## 9. 执行后记：最终基线、恢复窗口与验收
+
+本次实际交付由 [PR #188](https://github.com/pgsty/silo/pull/188) 承载。选择的远端基线为 `89637554d60c27cfc51d2281d0a4fe15e415f06d`，移植没有包含本地独立 IAM/超时提交 `ebc9937d9`。前三项实现和历史文档逐提交通过 stable patch-id 对照；虚拟补回独立 IAM 差异后，完整树与原审查分支一致。
+
+首次本地 lint 发现新增回调选择分支触发 `gocritic/ifElseChain`，因此追加等价的无表达式 `switch` 改写，保持 marker、指定版本和普通元数据查找的条件顺序及分支体。重新固定的代码候选为 [`41aa84609`](https://github.com/pgsty/silo/commit/41aa84609754769cfb1861d7fd060c2e84182b98)。这一提交上，全量 cmd/internal 得到 6,428 个测试及子测试通过、166 个跳过，50 个有测试的包通过；相关 race 得到 283 个测试及子测试通过。`make build`、全包构建、vet、lint、生成文件及 rebrand/compat 检查均通过。[Go CI](https://github.com/pgsty/silo/actions/runs/34925534139)、[DCO](https://github.com/pgsty/silo/actions/runs/34925534110)、[VulnCheck](https://github.com/pgsty/silo/actions/runs/34925534142) 和[发布流水线的测试运行](https://github.com/pgsty/silo/actions/runs/34925534198)共 11 项检查通过；后者没有发布正式制品。
+
+第一次最终候选停池实验 `dcd5c2e5` 在源版本保全断言后失败：停掉另一池返回 `503 SlowDownRead`，源四盘版本保留；恢复后 DELETE 成功，节点 0/2 的 HEAD/GET 返回 404，节点 1 返回 503。DELETE 成功由脚本已通过的 204 断言确定，原输出没有单独保存该次 DELETE 响应。此次失败如实保留，不能把后来的成功写回原记录。
+
+复核发现，`ListBuckets` 以及位于源池的现存版本 GET，不能证明每个协调节点对另一池的读取连接已经恢复。随后进行了旧基线与候选的同拓扑对照，探测的是从未写入过的随机 UUID，并且在这些探测之前没有执行任何 DELETE：
+
+| 高时间分辨率对照 | 桶列表和现存版本 | 从未写入版本的 HEAD/GET | 观察到的恢复窗口 |
+| --- | --- | --- | --- |
+| `20502a2f`，旧基线 `89637554d` | 三节点均为 200 | 节点 0/2 为 404，节点 1 为 503 SlowDownRead，连续 16 组 | 从重启后的观察循环起算约 1.60–2.50 秒 |
+| `246aaf77`，候选 `41aa84609` | 三节点均为 200 | 同样是节点 1 的 503，连续 11 组 | 约 1.67–2.30 秒 |
+
+两边在缺失版本全节点连续三轮返回 404 后执行 DELETE，均得到 204、全节点 HEAD/GET 404、目标 UUID 在八盘均不存在且其他版本可读。另有两次较低时间分辨率诊断未捕捉到窗口，同样保留；这四次诊断不计入三次升级验收。
+
+这给出了基线在零 DELETE 下的正向复现，证明原恢复条件不足。`getLatestObjectInfoWithIdx` 的读选择函数与基线文本相同：现存副本可以遮蔽另一池的读错误；缺失版本则必须确认所有池，不可读时返回 503。Claude 复核后同意修正实验准备条件并继续验收，明确反对把这一路径的 503 改成 404。最初失败没有瞬时 RPC 全貌，不能逐请求追溯每条连接；这些对照也不能给 `83676ca2` 归因。
+
+修订后的验收在升级/恢复后逐节点探测从未写入的 UUID，记录首次全 404 时刻，要求连续三轮全 404；并列保存各节点 `admin info` 的全盘状态。最多等待 30 秒，超时仍失败，DELETE 之后仍严格要求 204/404，不把 503 纳入通过条件。后续失败保留实验资源供即时取证，再受控清理。
+
+修订准备条件后的独立停池验收 `40a59b3b` 通过：离线 DELETE `503 SlowDownRead`，源四盘版本保留；恢复门约 1.48 秒完成，DELETE `204`，三节点 HEAD/GET `404`，目标在八盘均不存在，另一版本仍可读。恢复早期 `admin info` 中也记录到了各节点不同的离线盘视图，最后恢复为全盘正常。
+
+完整升级验收随后实际进行了三次尝试：
+
+| 尝试 | 运行 | 结果与证据边界 |
+| --- | --- | --- |
+| 1 | `83f88c59` | 准备失败，未启动候选。旧版 rebalance 报 Completed、搬移版本数为 0；源池占用约 6.5%，到平均空闲目标的差值约 3.13%，落入代码既有 5% 容差。增加造数从 64 到 192 个 2 MiB 版本后再试；本次仍计入五次总尝试上限。 |
+| 2，第一轮有效运行 | `ea58d0c5` | 通过。实际 rebalance 中断产生跨八盘的重复版本；停机复制同一数据供旧版对照。旧版 DELETE204 后仍可读；候选 DELETE204 后四节点立即及后续固定间隔 HEAD/GET 均404，八盘目标清除、另一版本四节点可读；旧 ILM/生命周期编辑和真实缓存 v9→v8 通过。 |
+| 3，第二轮有效运行 | `2059bc6b` | **候选失败，阻断合并。** 两阶段逐节点缺失版本连续三轮404、admin info全盘ok之后，DELETE明确204（约12.98ms）；节点2随后的HEAD503/GET503 SlowDownRead，另外三节点404；八盘快照均无目标，首次重试及后续采样全404，其他版本四节点可读。首次失败保留，不因重试恢复改判。 |
+
+第三次尝试发生后停止剩余验收，保留原容器、卷、元数据与响应时序进行诊断。不能把四节点顺序探测中的“节点2异常”直接解释为永久节点故障：它也可能与采样时间有关。随后在保留环境中，对三个额外重复版本做并发、不同顺序的“刚删 UUID / 从未写入 UUID”对照，候选稳定期均为204/404，未复现；旧版克隆数据的双次 DELETE 对照则遇到 `SlowDownWrite`，没有完成其全部断言。这些是诊断结果，不补入有效升级通过计数，也不证明第三次尝试已解释。
+
+为观察逐盘返回，另在独立临时工作树编译仅增加日志的诊断二进制，**没有进入 PR**。它证明了第二层准备检查盲点：`getObjectFileInfo` 的四个响应信号中，可以只有两个实际 `file version not found`，其余两个是被跳过的盘所保留的 `errDiskOngoingReq`；`objectQuorumFromMeta` 的预期读 quorum 为2，因而仍可返回404。同期 `admin info` 汇集各服务器本地盘态为ok，不能证明请求节点到各盘的路径都可用。一次诊断 DELETE 的逐盘返回为 `[nil, nil, drive not found, drive not found]`，达不到写 quorum 3，故返回 `SlowDownWrite`。
+
+Claude 撤回了此前“逐节点三轮404已是最强全池准备条件”的表述，同意这只能证明读 quorum，不能证明全盘可达或写 quorum。诊断给出了候选机制，但**第三次尝试失败瞬间没有逐请求逐盘日志，仍不足以确定其具体原因**；不能把后来稳定期的成功、诊断中的配额不足，或对错误名的解释当成该次故障的直接证据。
+
+## 10. 首次执行的停止位置与恢复资料
+
+首次执行停止时，代码为 `41aa84609`；后续提交只记录执行，不改变已测试的生产代码。当时有效升级运行是一通过、一失败，未满足三次有效运行全部通过的约定；总尝试3次，没有通过继续补跑消耗剩余次数来冲淡失败。`2059bc6b` 和先前的 `83676ca2` 均保持 **OPEN**。Claude 与 Codex 当时的裁定是 **NO-GO for merge**，没有把证据不足升级为“已修复”“既有问题”或“暂态”。后续收尾见第 11 节；实际合并状态以 [#188](https://github.com/pgsty/silo/pull/188) 为准。
+
+主工作区原五文件的完整内容、二进制补丁和 SHA-256 已归档；另有可达 Git 引用 `refs/archive/access-tiering-main-five-files-20260915`，指向快照 `c7fbfc6ada0f0f2abcbe8c0a9681f07e12dafe52`。逐文件核验快照与原已审内容一致。首次停止时因验收未通过，没有执行五文件恢复，也没有移动或改写独立 IAM 提交 `ebc9937d9`。当时唯一待交付候选在 PR 分支，旧变体冻结等待验收裁定。
+
+本机执行资料归档在 `~/.codex/outputs/silo-access-revert-assessment-20260915/final-execution/`：保存了每次尝试、旧/新基线对照、二进制 SHA-256、准备条件、源/目的全盘元数据、诊断补丁、独立评审意见，以及受控清理记录。原始失败记录不覆写；保存的诊断卷内容用于继续调查，不是生产数据或发布制品。
+
+继续推进需要一次能区分机制的取证：在条件可控的升级实验中，于失败请求当时记录每池每盘的真实应答和错误类型，并同时读刚删版本与从未写入版本；明确区分盘面不同步、请求节点的不可用路径与其他原因。若四盘均真实应答仍返回503，应沿错误归约/元数据路径定位；若盘不可用，应查明连接或初始化状态，并验证准备条件。后续成功本身不能关闭本次失败，更不能通过把不可判定的503改成404来满足验收。正式发布、制品和生产部署继续作为独立交付。
+
+## 11. 收尾复核：准备检查、可控机制与独立新轮次
+
+后续收尾没有继续修改生产代码。旧基线 `89637554d` 与候选 `41aa84609` 使用各自的独立诊断构建，仅对测试桶记录逐盘应答；这些日志补丁没有进入 PR。第一轮完整诊断 `53ebe161` 通过但未复现503，仅作为一个样本保留。第二轮 `c951f762` 得到了不同的、可直接解释的失败：两个池的删除调用均记录 `quorum=3 errs=[nil,nil,nil,drive not found]`，DELETE204、所有读样本404，但八盘快照的盘3、7仍保留目标版本。它符合既有写 quorum 契约，并正向证明旧准备门会放行尚未完成挂盘的协调节点；它没有复现或解释 `2059bc6b` 那一次请求。
+
+审查还纠正了两项推断：后续诊断进程在04:19的重连日志不能用于解释03:56的原失败；错误归约按具体错误值计数，不能把“最高同值计数不足”简单等同于“实际应答盘数不足”。原失败之后的全盘快照也不是失败瞬间的原子快照。这些界限继续保留。
+
+准备检查改用服务器已有的 storage trace：从每个 S3 节点，对各自唯一、从未写入的对象执行 `GetObjectTagging`，该路径等待所有盘；按唯一对象名和后端节点、盘路径匹配真实 `storage.ReadVersion` 应答。四节点双池需要每轮32条实际缺失应答，连续三轮完成才满足准备条件；单纯404或 `admin info` 的本地盘状态不够。探针不创建对象，不改变服务器读写语义。trace 输出是多行 JSON 对象流，按流解析；缺失 trace 证据会使检查失败，不能据此假定对应盘健康。
+
+为了比较相同的跨池写入负载，停止真实 rebalance 夹具后复制两份相同数据。旧基线使用 #178 已有的 `If-Match` 条件 DELETE，候选使用普通指定版本 DELETE，二者都处理同一目标的两个池副本。两臂分别完成96次“已删版本／从未写入版本”的 HEAD/GET 对照，均404，目标八盘均清除。准备检查分别耗时约0.60秒和17.08秒。这是一对匹配样本，没有观察到候选专属差异，不是吞吐比较，也不能排除所有可能的故障机制。
+
+`f55967b7` 另用明确制造的重复副本夹具完成因果实验，而非冒充真实 rebalance：pool0 的四盘由一个节点承载并保留 namespace 锁；pool1 的四盘分布到另外四个节点。先停止 pool1 一盘，DELETE204 后直接确认只有该盘残留目标版本。重新接入这份副本，再停止 pool1 两块已删除该版本的盘，保留“一盘返回版本、一盘返回缺失”的读视图。三个被测协调节点的已删版本均返回 `503 SlowDownRead`，从未写入版本均返回404；同一失败请求的逐盘记录明确为 `[drive not found, drive not found, file version not found, nil]`，没有足够的同值应答达到读 quorum。整个实验没有丢失 pool0 的 namespace 锁，也没有将不确定状态改为404。
+
+该因果实验的部分节点重启未在35秒内恢复全部40条访问路径，原实验因此仍记 FAIL；正向机制观察与这个恢复失败分开记录。随后协调重启全部五节点，独立恢复检查通过，五节点 HEAD/GET 均404。这不是滚动恢复已通过的声明，也不追溯原 `2059bc6b` 的逐盘状态。可控复现确定的是故障机制类别，原单次实例继续 **OPEN**。
+
+Claude 复核上述证据后同意显式开启 **R-Upgrade-2**：原轮次的一通过、一失败和总尝试3次保持原样，不合并计数，不静默重置。新轮使用未经诊断修改的 `41aa84609` 二进制，要求三次有效运行全部通过、最多五次总尝试；任何新503或数据不变量失败仍须停止。每次 DELETE 前后均检查32条路径，响应后的即时 HEAD/GET 先于后置准备检查执行，避免等待掩盖短暂错误。
+
+| 新轮次运行 | 完整运行耗时 | 升级后32路径准备耗时 | 验收结果 |
+| --- | --- | --- | --- |
+| `8eb016db` | 90.20秒 | 17.04秒 | PASS |
+| `2a2b7b64` | 80.83秒 | 0.62秒 | PASS |
+| `371e7b9f` | 96.28秒 | 0.60秒 | PASS |
+
+三次均实际走到候选阶段：DELETE204，所有即时及后续 HEAD/GET 样本404，目标在八盘均不存在，其他版本从四节点读回；旧配置、普通生命周期规则编辑及真实缓存 v9→v8 均通过。删除前后各三轮32路径检查也全部通过。准备时间有明显波动，应验证访问路径，不能用固定等待秒数代替检查。这些结果满足修正准备条件后的有界验收；不把有限样本写成“历史503已消失”，不把不同阶段的诊断通过计入新轮次。
+
+新轮仍绑定生产代码 `41aa84609`（Linux 二进制 SHA-256 `28e1339d630a22fa5a0e4659b6182224e81f6cd7cd4079856534390e40a697b1`），后续仅更新文档。合并前须核对源码等价性和最终 CI，按第8节的恢复纪律处理旧五文件，保留独立 IAM 提交。原 `2059bc6b`／`83676ca2`、部分重启恢复边界、单 VM/tmpfs、未验证全部不同重复 UUID 和整份运行时统计守恒继续可见；不为此新增自动搬回、清理服务或读错误降级。
+
+本轮详细资料位于原归档的 `final-execution/closure-20260915/`，包括匹配对照、同请求逐盘日志、`qualification-summary.json`、独立 R-Upgrade-2 账本、诊断补丁和完整卷归档。原失败现场、后续对照及恢复后的卷分别标注时点。临时实验资源在归档验证后清理；这些资料与正式发布制品区分管理。

docs/investigations/go127-stack.md

@@ -1,5 +1,11 @@
 # SILO stack: Go 1.27 compatibility audit
 
+> This is a dated investigation, with the source and runtime boundaries recorded
+> below. It does not establish the current dependency pins or a later release.
+> See [the current changelog](../../CHANGELOG.md) and
+> [component matrix](https://silo.pgsty.com/compatibility/versions/).
+
+
 2026-09-09. Scope: the maintained Server, silo-pkg, mcli, and Console. This extends
 the [OIDC #154 investigation](issue-154.md) to other paths using the same TLS
 configuration and to adjacent standard-library changes. It records local

docs/investigations/issue-154.md

@@ -1,5 +1,11 @@
 # SILO #154：OIDC discovery 连接重置调查
 
+> This is a dated investigation, with the source and runtime boundaries recorded
+> below. It does not establish the current dependency pins or a later release.
+> See [the current changelog](../../CHANGELOG.md) and
+> [component matrix](https://silo.pgsty.com/compatibility/versions/).
+
+
 前两轮调查时间：2026-09-09；公开 issue 最后核对于 07:53 UTC。第二轮补充同源码、同依赖、不同 Go 工具链的 Linux 完整 Server 对照和候选补丁认证链路验证。
 
 **后续更新：用户已授权扩展至整个 SILO 技术栈并修复。现已确认 Server 其他 TLS 路径也存在同类覆盖问题，并在产品工作区完成统一使用 Go 默认曲线的修复。当前实现、验证和交付状态见 [全栈调查](go127-stack.md)。下文保留前两轮的诊断与当时的 OIDC 局部候选；“未修改产品”和“不要扩大范围”等表述仅适用于当时的调查阶段，局部候选已被后续全路径修复取代。**

docs/investigations/issue-77-current.md

@@ -0,0 +1,103 @@
+# Issue #77 当前核验与最小修复建议
+
+> 历史快照（2026-09-12，实施前）。其中的 Issue 状态、待办和方案约束只描述当时情况；最终实现、修正与验收以 [归档总记录](issue-77.md) 为准。
+
+核验日期：2026-09-12。对象：[SILO #77](https://github.com/pgsty/silo/issues/77)。代码基准：远端 `main` 的 `5c576581631561c446f30ae5b566f0aa793adc1c`，在独立 detached worktree 中运行测试。用户工作目录仍位于 `12f631b50`；两者差异为 #179 的 federation 修复，不涉及本次复制元数据代码。
+
+**结论：问题真实，当前主干仍未修完，应保持打开。原方案的 A/B 核心必要，但不能照搬 8 月的实施清单；已有基础设施可以复用，同时必须补上批量复制入口和相同内容的时间戳同步。**
+
+本次是分析与复现，没有修改产品代码、提交、发布或修改 GitHub Issue。下述测试是本地 ObjectLayer 与进程内 HTTP/RPC 复现，不是线上多站点验收。
+
+## 1. 当前状态
+
+GitHub API 实时返回 #77 为 `OPEN`，未分配负责人或 milestone，最后更新为 `2026-09-08T14:45:10Z`。核验时仓库没有打开的 PR。
+
+| 范围 | 当前事实 | 是否仍属遗留项 |
+| --- | --- | --- |
+| #77-C，各站点统计 | [#91](https://github.com/pgsty/silo/pull/91) 于 8 月 29 日合并；本次现有统计回归测试通过 | 否，不应重复实现 |
+| #77-A，heal 选源与输入 | 仍先用 map 首项初始化，再跳过创建默认值；Tag 远端 heal 仍不携带时间戳 | 是 |
+| #77-B，源时间与删除状态 | 六类专用处理函数仍调用到达时间写入路径；部分删除状态不导出 | 是 |
+| #77-D，持续不一致诊断 | 已有损坏配置日志；旧事件静默跳过、默认状态无法选源等诊断未完成 | 是 |
+| Object Lock 错误 wire 字段、接管桶覆盖已有配置 | [#76](https://github.com/pgsty/silo/issues/76)、[#78](https://github.com/pgsty/silo/issues/78) 已关闭，对应 #89、#90 已合并；相关测试本次通过 | 否 |
+| 元数据整记录并发写、删除后重建 | [#103](https://github.com/pgsty/silo/pull/103)、[#156](https://github.com/pgsty/silo/pull/156) 已提供桶锁和保存前物理桶检查；删除后排队写测试本次通过 | 基础已具备，但不能代替同字段时间排序 |
+| 评论提及的 MRF 丢弃观测、delete-marker purge | [#152](https://github.com/pgsty/silo/issues/152)、[#153](https://github.com/pgsty/silo/issues/153) 已于 9 月 9 日随 [#162](https://github.com/pgsty/silo/pull/162) 关闭 | 不再列为 #77 的待实现项 |
+
+#162 明确保留“405 表示 marker 仍存在”的语义，没有采用报告人建议的“405 一律判定永久删除完成”；它也未声称复现外部报告的请求量或持续 405 风暴。本次只核验其合并/关闭状态与提交说明，没有重跑那组故障实验。
+
+最新公开 Release 仍是 `RELEASE.2026-09-03T13-18-01Z`。#77 的状态不能用“已有统计修复”或“已有桶锁”推导为已修复，也不能用较晚 PR 的合并推导为已发布。
+
+## 2. 已复现的真实问题
+
+最直接的例子：源端 10:00 PUT 配置，10:01 DELETE；目标端积压到 10:05 才接收 PUT，并把配置时间写成 10:05。随后接收源时间为 10:01 的 DELETE，判断它比本地旧，返回 HTTP 200，却保留配置。这个例子不需要机器时钟偏差，只需要事件延迟。
+
+根因在 [updateAndParse](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/bucket-metadata-sys.go#L129)：它在锁内统一使用 `UTCNow()`；专用 peer handler 在调用它之前，用 getter 返回的时间做比较。
+
+| 配置类型 | peer PUT 保留源时间 | 较新源 DELETE | 本地删除后旧 PUT | 删除时间在元数据导出中可见 |
+| --- | --- | --- | --- | --- |
+| Policy | 否 | HTTP 200，但未删除 | 会复活 | 是 |
+| Tags | 否 | HTTP 200，但未删除 | 会复活 | 否 |
+| SSE | 否 | HTTP 200，但未删除 | 会复活 | 否 |
+| Quota | 否 | HTTP 200，但未删除 | 本次未复活，getter 保留删除时间 | 否 |
+| Versioning | 否 | nil 正确地不修改配置 | 不适用 | 不应套用可删除配置规则 |
+| Object Lock | 否 | nil 正确地不修改配置 | 不适用 | 不应套用可删除配置规则 |
+
+矩阵每项均在 `ErasureSD` 与 16 盘 `Erasure` 两个本地后端执行。六类专用事件通过实际签名 admin 路由调用，检查落盘状态；导出调用实际 `SiteReplicationMetaInfo`。不能把 Quota 与 Policy/Tags/SSE 的 getter 行为写成完全相同。
+
+另外复现了以下路径：
+
+1. **检查与写入不在同一临界区。** 阻塞旧 Tag 事件的锁获取，在锁内提交较新 Tag，再释放旧事件；旧事件仍覆盖新内容。#103 解决跨字段丢更新，没有解决 handler 的 getter → Update 竞态。
+2. **批量元数据入口绕过逐字段排序。** 已持久化较新 Tag 后，发送较旧 bulk 事件，实际 admin 路由返回 HTTP 200，Tag 内容和时间都倒退。该入口被 [桶元数据导入](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/admin-bucket-handlers.go#L1106) 使用；[PeerBucketMetadataUpdateHandler](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/site-replication.go#L1608) 只检查桶创建时间，非 CORS 字段缺少与已有字段时间的比较。这是旧实施清单漏掉的入口。
+3. **默认值能够成为错误的 heal 来源。** 两站点状态中，一个有真实 Policy，另一个只有更晚的 `CreatedAt == PolicyUpdatedAt` 默认值。最后一次复现分别有 5/32、2/32 轮错误删除有效 Policy，取决于 Go map 遍历顺序。该计数只是复现样本，不能推断生产发生率。
+4. **远端 Tag heal 丢失源时间。** 实际 HTTP 捕获的 `UpdatedAt` 为零，见 [发送字段](https://github.com/pgsty/silo/blob/5c576581631561c446f30ae5b566f0aa793adc1c/cmd/site-replication.go#L5011)。
+5. **同内容、不同时间戳没有同步。** 即便强制 `TagMismatch=true`，内容比较仍使 heal 跳过，较旧排序时间保持不变。两站点看似内容相同，之后却可能对同一个延迟事件做出不同决定。旧方案只统一写入时间、保留全部 heal 跳过条件，仍不完整。
+6. **Quota 的 nil heal 留下缓存。** 向现有 heal 传入显式较新删除状态，磁盘 `QuotaConfigJSON` 被清空，但 `GetQuotaConfig` 仍返回 1024 字节的旧硬配额。这里显式构造了删除状态，因为当前 exporter 正在隐藏它；这是新增 tombstone 导出不能直接交给旧 heal 的具体证据。
+
+从当前 `git blame` 和基线比对看，核心错误继承自 MinIO：heal 初始化逻辑来自 2022 年 `3a64580663`，专用 handler 的旧事件检查来自 2022 年 `7cc9286e0f`，bulk 入口来自 2023 年 `0cde37be50`；0806 基线已包含相关逻辑。此次没有发现其由 SILO 最近的 CORS 或统计修改引入。
+
+对于使用 site replication 的部署，建议按 P1 正确性问题处理：撤销的桶策略可能保留或复活，默认加密和配额也可能与源端不一致。没有启用站点复制的正常单站点请求不触发这些复制路径；本次没有证明对象数据本体丢失。
+
+## 3. 最小且必要的修复范围
+
+**第一部分：收束 A 的选源和输入修复。** 提取一个小的内部选择函数，在初始化候选之前排除零时间和创建默认值；无可信候选时不写入。六类 heal 使用它，补齐 Tag 的源 `UpdatedAt`。Versioning/Object Lock 遇到 nil 保持 no-op，Quota 删除使用正确的删除路径。不能重新做 #76/#78 或把 Lifecycle、CORS 一并重构。
+
+**第二部分：收束 B 的状态更新，包含本次发现的入口遗漏。** 在既有 `metadata.lock` 内完成读取原始字段时间、判旧、写入源时间和保存；普通本地写入保留现有契约。复用 `updateAndParse` 的类型分支及现有 `saveMetadata`，不再从公开 getter 获取删除时间，也不改变 getter 的 S3 错误语义。peer 更新和删除保留各自解析规则，内部返回是否应用，重复且相同的状态不重复保存。
+
+本地 heal 必须使用同一排序路径，不能选好源后调用普通 `Update/Delete` 再生成到达时间。批量复制入口也必须在其现有整记录锁内逐字段判旧；bulk 的 nil 继续表示“未提供此字段”，不能改成批量删除。heal 需要同步较新的时间戳，即便内容已经一致；这不要求重写 #91 的计数或增加公开 API。
+
+当前 `saveMetadata` 已在锁内检查物理桶存在，桶删除也使用同一把锁。8 月方案里另加一套 `peer-require-existing` 防幽灵桶机制已没有必要，应保留并复用现有保护。确有 `Created == 0` 的历史桶如何补齐创建时间仍应覆盖，但不能退化当前所有写入都执行的存在性检查。
+
+**第三部分：单独启用删除时间导出，补足轻量诊断。** Policy 已导出时间；实际新增的是 Tags/SSE/Quota 的 nil payload 时间。先使接收与 heal 能正确处理删除，再开放这些状态。混合旧版 SILO 的风险已经有 Quota 缓存复现支撑；这是产品自身的滚动升级问题，不是要求兼容未经修改的上游 MinIO。
+
+可以使用仅针对这项导出的明确 opt-in：默认关闭新增导出，全站点升级到具备 A/B 修复的版本后启用。若项目选择自动能力协商，可后续单独实现；不必为了它阻塞没有新增 wire 字段的选源与接收端修复，也不需要为本 Issue 建一个通用能力框架。具体开关尚未实现，不应把这里的建议当成现有配置。
+
+诊断只需解释“过旧事件”“早于桶创建”“只有默认/未知候选”等跳过原因，使用现有有界去重/限频设施，保持 RPC 的既有成功语义。不需要另建重试队列、每轮日志或大型监控系统。
+
+有三项边界必须在实现中明确：
+
+- 旧版 Tag heal 会发送零时间。新实现不能无说明地全部拒绝；可保留明确的兼容降级路径，在旧节点存在时不承诺完整排序收敛。
+- 相同时间戳、不同内容的冲突：只给 heal 增加 deployment ID 平局规则，并不能让到达顺序不同的 peer apply 本身确定。若承诺此类冲突也收敛，apply/heal 应共用一个类型内比较规则；可参考 CORS 的删除优先和载荷排序，不需要新增持久化源站 ID。若不纳入本轮，必须列为剩余边界，不能称全量收敛审计完成。
+- 旧版本已经写坏的到达时间无法从现有记录还原。升级不会自动恢复历史操作顺序；应由操作者确认权威状态，并在升级和时钟检查后重新提交相关配置/删除，再验证各站点。
+
+因此，建议交付为有限的选源修复、统一源时间更新、删除导出与诊断三个可审核部分。已有锁、物理桶保护和 C 的计数修复直接复用。预期产品修改集中在 Server 的 site-replication 与 bucket-metadata 路径，无须为核心修复升级 Console/mcli/silo-pkg、改存储格式或重写复制架构。
+
+## 4. 验证证据与关闭条件
+
+本次最后一轮执行：
+
+```text
+GOWORK=off go test -tags kqueue,dev ./cmd \
+  -run 'TestIssue77Current|TestSiteReplicationStatusAccountsPerSiteAndSurvivesMalformedConfig|TestPeerBucketObjectLockMetadata|TestPeerBucketAdoption|TestQueuedMetadataUpdateAfterDelete' \
+  -count=1 -timeout=5m -v
+```
+
+四组审计复现测试包含 24 个后端/场景子测试，均暴露预期的现存缺陷；八个现有回归测试通过。总命令退出码为 1，原因是上述针对期望正确行为的断言失败，不代表修复验收通过。未运行全量测试、race suite、真实多站点中断/重启或滚动升级实验。
+
+保留的证据：
+
+- [完整运行日志](issue-77/current-tests.log)
+- [源时间、删除和 bulk 审计测试](issue-77/issue77_review_test.go.txt)
+- [heal 与锁竞态审计测试](issue-77/issue77_heal_review_test.go.txt)
+
+测试以 `.go.txt` 保存，避免将故意失败的审计用例加入正常 Go 测试集。可在上述 SHA 的独立 worktree 中复制为 `cmd/issue77_review_test.go` 和 `cmd/issue77_heal_review_test.go` 后重跑。创建默认值测试使用重复 map 遍历来观测缺陷，正式回归应在抽出选择函数后改成确定性用例。
+
+关闭 #77 前需要：上述失败场景转为正确行为；分别覆盖六类配置、落盘重载、漏发/重复/乱序、同字段竞态、同内容新时间、批量导入入口、legacy 桶、删除后的桶及混合版本。全站点升级后的实际断线重连/重启验证应保留独立证据。当前结论是“缺陷与实施范围已核实”，不是“修复完成”。

docs/investigations/issue-77-plan.md

@@ -0,0 +1,123 @@
+# Issue #77 最小充分修复计划
+
+> 历史快照（2026-09-12，实施前）。其中的 Issue 状态、待办和方案约束只描述当时情况；最终实现、修正与验收以 [归档总记录](issue-77.md) 为准。
+
+版本：v4 定稿，2026-09-12。状态：作者复核完成，Claude Code `claude-opus-5 --effort max` 四轮实际评审后，最终结论为 **GO_WITH_NONBLOCKING_NOTES，实施前阻断 0**。非阻断说明已落实到本计划，见 [最终评审汇总](issue-77/review/final-review.md)。
+
+基准：`pgsty/silo` main `5c576581631561c446f30ae5b566f0aa793adc1c`。本轮交付是方案与评审，不表示已经实现、合并或发布。问题证据见 [当前核验](issue-77-current.md)，意见处置见 [首轮处置](issue-77/review/decisions-v2.md)、[第二轮处置](issue-77/review/decisions-v3.md)、[接管边界补查](issue-77/review/decisions-v4.md)。
+
+## 目标与边界
+
+修复 Policy、Tags、SSE、Quota、Versioning、Object Lock 六类桶配置的源时间丢失、锁外判旧、heal 错选源和删除传播不完整。覆盖普通本地写入、专用 peer 事件、bulk/import、local/remote heal、元数据导出与初次同步。
+
+在修复 PGSTY 栈、同一已知桶世代、合法带源时间事件的范围内，使重复、乱序和漏发后的状态能够确定性收敛。开关关闭期间新增删除信息不可见，不承诺完整删除收敛；无时间旧事件、旧版污染时间和桶创建世代冲突需要单独解释，不能自动推断历史真相。
+
+不重做 #91 的计数、#76 的 Object Lock wire 修复、#78 的桶接管、#103/#156 的锁与删除保护。不改变 CORS、Lifecycle/expiry、notification、对象复制、MRF、resync、IAM 的语义；不改存储 schema、SDK、Console、mcli 或 silo-pkg，不新建能力协商、复制框架、锁或重试系统。
+
+## 行为契约
+
+### 1. 明确事件、缺省值与删除
+
+| 类型 | 专用事件 | bulk 字段未提供 | bulk 字段明确提供 |
+| --- | --- | --- | --- |
+| Policy | nil 为删除；沿用现有解析器 `IsEmpty()` 为删除 | 保留 | 非空 RawMessage 按现有解析器处理；语义空策略归一为删除 |
+| Quota | nil 为删除；非 nil 按现有 quota 解析器处理 | 保留 | 非空 RawMessage 按现有解析器处理，零 quota 仍是 live 文档 |
+| Tags / SSE | nil 或 base64 解码后空内容为删除 | 保留 | 空字符串为删除，其他内容按原规则解码、校验 |
+| Versioning / Object Lock | nil/空内容为 no-op | 保留 | nil/空内容仍为 no-op，不能清空配置 |
+
+“未提供”必须依据真实 wire 类型判定：Policy/Quota 的 `json.RawMessage` 为 nil 或空切片时经 `omitempty` 省略；**显式 JSON `null` 解码后是非 nil 的 `[]byte("null")`**，不能与缺省混淆。按现有解析器，Policy `null` 是语义空策略，Quota `null` 是零值 quota 文档。`*string` 类型的 JSON `null` 则解码为 nil。不得使用统一的 `len(payload)==0 => 删除` 来处理 bulk。
+
+合法 `{}`/零值 quota 保持 live；取消普通 quota PUT 出站时“零配额改写为 nil”的逻辑，保存与发送同一个语义状态。Policy 保留现有专用 peer 的“空策略=删除”解释，本地 PUT、导入和 bulk 统一归一为同样的删除状态；这是需要写入兼容说明的小范围变化：空策略本地立即按删除处理，GET 返回既有 NotFound 行为，不再先保留空文档、等复制后才被清除。
+
+**同一次操作的落盘状态与出站事件，经同一归一规则后，必须具有相同的 `(kind, payload key, source time)`。** JSON 的无意义编码次序不要求字节相同；Object Lock 改写后的有效 Versioning 文档必须来自提交结果。
+
+### 2. 统一状态与排序
+
+内部仅需一个小状态表示：baseline / live / tombstone，以及比较键和字段源时间。baseline 可携带历史有效配置，但没有真实修改时间；它绝不能被当成删除。复用 CORS 已有设计思路，不改 CORS 本身或扩展为通用框架。
+
+- 使用 `BucketMetadata` 原始字段时间；字段时间为零时在比较视图中补为 Created，与 `defaultTimestamps()` 一致，不借用会隐藏墓碑时间的 getter 或整记录 `lastUpdate()`。
+- 在已知 Created 下，零字段时间回退后等于 Created 的状态是 baseline：有有效非空配置的 baseline-live 可作为初始化候选；空 baseline 只是缺省值，不能作为删除或 heal 来源。真正严格早于自身 Created 的字段不是候选；Versioning/Object Lock 的空候选无论时间如何都不参与选源。真实 live/tombstone 的时间必须大于 Created。
+- 专用 peer 的零时间保留兼容例外：锁内按本地新操作分配时间并限频记录 legacy-zero；不受删除传播开关影响，不在源时间排序保证之内。bulk 零时间仍按现状拒绝。
+- 非零事件**严格早于**目标桶 Created 才成功/no-op 并记录 before-created，heal 对这种目标跳过。等于目标 Created 的 live 事件可更新仍是 baseline 的目标字段，时间仍保存为源 Created，不盖上到达时间；同时间的空/nil 只算空 baseline，不能删除配置。不同桶世代不能自动合并，需要运维处理，不纳入收敛承诺。
+- 排序先比较是否为真实状态：任何真实 live/tombstone 都胜 baseline，不能让较晚创建的默认值压过较早的真实修改。真实状态之间再比较源时间，同时间 tombstone 胜 live、live/live 的**稳定比较键字节序较大者胜**。baseline-live 只胜空 baseline，或在 baseline-live 之间按同一稳定键较大者胜；永远不能覆盖真实 live/tombstone。所有候选都是 baseline-live 时仍可确定性初始化并收敛；全部为空 baseline 才安静 found=false。键、状态级别和源时间相同为 no-op。deployment ID 不参与上述比较，也不作为新字段保存。
+- Quota 使用现有解析结果的 JSON 编码作为比较键。Policy 在 Server 内生成确定性比较表示：既有解析器校验/去重后，对其**完整 JSON 树**的对象键和集合数组递归排序，统一覆盖 Statement、Action/NotAction、Resource/NotResource、Principal、Condition；保留数字类型和精度。比较键必须是已解析策略的纯函数，同一配置从两个站点分别解析也必须得到相同键。普通 `json.Marshal(BucketPolicy)` 不稳定，不能直接作键；不增加依赖不支持的 NotPrincipal 语法，不改 wire/schema。新状态比较仅使用这一套键：不把忽略 Sid 且对 Statement 顺序敏感的 `BucketPolicy.Equals` 再叠加为另一套判等规则，既有公开统计对 Equals 的使用保持不变。
+- XML 使用有效文档字节，保留大小写和实际内容；Versioning 先应用下述现有 Object Lock 约束。比较器不能依靠字节序方向来补偿保存阶段的隐式改写。
+
+### 3. 先得到有效状态，再比较与提交
+
+原始读取、类型处理、比较、修改及保存均在现有 `metadata.lock` 内。内部入口必须传递现有 lock context，避免 legacy migration 再次取锁。
+
+- 提取并复用 `parseAllConfigs` 已有的 Object Lock→Enabled Versioning 归一规则，使比较视图和真正保存一致；不得扩大 suspend、prefix exclusion、retention 的限制。bulk 先确定实际接受的 Object Lock，再比较该约束下的 Versioning，并在最终提交前应用同一规则。
+- 以归一化后的有效 `(kind, key, time)` 判定变化；更新了时间也算变化。完全重复不保存、不通知；一次 bulk 校验失败不保存部分结果；成功至多保存一次，解锁后通知。
+- 可删除字段清空必须基于现有 parse=false 的新加载对象，不能在已经解析且仍持有旧 quota 的对象上执行 Update(nil)。bulk 保留原始读取再解析保存的方式。不要顺手修改 Quota getter 或所有 `parseAllConfigs` 空分支。
+- 保存函数必须让需要发送 hook 的调用方拿到**本次提交的最终快照**，不能先解锁再读取“最新”状态拼接旧时间。最小做法是让内部 `saveMetadata` 接收元数据指针并回写 Save 的归一化结果，机械更新现有少量调用；对外 `Update/Delete` 签名不变，新增内部提交结果仅供需要该快照的本地 handler/import 使用。不得原地修改已发布到缓存的引用字段。
+- 保留现有物理桶存在检查、删除锁序、迁移、后台通知上下文。真实历史桶 Created 为零时仅走少见的物理桶 Created 补齐路径；物理桶缺失返回现有错误；创建时间仍未知则不伪造到达时间，报告 indeterminate。
+
+## 三个实现提交
+
+### 提交 1：原子 apply、发送一致性与本地时间
+
+主要文件：`cmd/bucket-metadata-sys.go`、`cmd/bucket-metadata.go`、`cmd/site-replication.go`、`cmd/admin-bucket-handlers.go`，以及实际需要提交快照的本地配置 handler。
+
+1. 在现有 update/delete 内部路径增加源时间、状态比较和提交结果；公开签名不变。六个 peer handler 移除锁外 getter 判旧，锁内持久化原始源时间。保留 Object Lock 的 legacy Tags 字段载荷回退。
+2. bulk 对明确提供的六类字段在已有锁内逐字段比较，再原子保存；未提供字段不动，不能循环调用会重入锁的公开 handler。保留 CORS 独立分支与既有行为。
+3. 六类本地实际写入在锁内分配 `max(UTCNow(), Created+1ns, 当前字段时间+1ns)`。其他类型不变。`enablePeerBucketVersioning` 的实际变更也使用它，只有缺失配置的创建 bootstrap 继续 Created 默认值。
+   接管已有桶时保留原 Created，再执行现有 `SetCreatedAt`；如果 Created 改变，仅将这六类中原本为零或等于原 Created 的默认字段时间调整到新 Created，随后再做既有 versioning/lock bootstrap。原本晚于旧 Created 的真实修改/删除时间及其配置保持不变。不能仅凭 payload 为 nil 判断默认值。该小分支防止 Created 前移时默认值变成假墓碑、后移时历史初值被误判为无效，复用已有接管锁，不重做 #78 的配置保护。代码注释明确限定六类的原因；本轮不改新比较器未读取的其它配置时间。
+4. quota 本地 PUT 保留零值文档并原样表示该语义；Policy 空策略本地与 peer 一致走删除。需要归一化的本地 handler 从本次提交快照生成 hook；其他内容不发生归一变化的路径可保留既有编码，但必须满足三元组一致性。
+5. 导入在每桶最终提交锁内，为本次涉及的六类字段生成共同 commitAt，严格大于 Created 和这些字段当前时间且不早于锁内现在。该时间同时用于落盘和 bulk hook，不能沿用 ZIP 开始时间。bulk hook 从最终提交快照构建；若导入的空 Policy 已归一成删除，另外使用现有专用 Policy nil 事件表达它，不能因 `omitempty` 漏发。未导入字段不改，Object Lock 的既有派生 Versioning 修正保留原时间语义；CORS 继续独立时间/事件，其他字段不参与该上界。
+
+完成条件：六类源时间落盘；旧事件不能越过锁覆盖新状态；四类删除不会被旧 PUT 复活；真实 wire、落盘和出站状态一致；重复无写入；bulk 与 import 没有绕过排序或静默遗漏删除。
+
+### 提交 2：heal 选源与应用同规则
+
+主要文件：`cmd/site-replication.go`。
+
+1. 先过滤空 baseline、无效来源、严格早于自身 Created 的字段和 update-only 空配置，再按上述强弱排序选最大状态；无候选必须显式返回 found=false。保留历史 baseline-live 的初次同步和 heal 能力，消除六处“先 seed map 首项，再过滤默认值”的写法。
+2. 选源和目标遍历都跳过 `info.Sites` 中不存在的 deployment ID，包括不可达站点的空 ID 占位项；单一 peer 失败记录后继续其它目标，不因 map 顺序放弃健康站点。不改变状态计数或新建重试机制。
+3. 本地 heal 使用提交 1 的源时间 update/delete；远端仍用原有逐类型 RPC，全部携带源时间，补齐 Tag 的 UpdatedAt。
+4. 比较完整有效状态，去掉公开 mismatch/payload-only 对写入的门控；同内容较旧时间也同步。对已归一且相同的目标不写入、不发 RPC。Versioning 比较使用与该站点 Object Lock 一致的有效文档；全站点 Lock 状态补齐后不再因旧原始文档产生空转。
+5. 保留 #91 的计数与公开字段；创建世代冲突、无可用来源通过有限诊断解释。已知 baseline 且各站点无实质差异时安静 no-op。
+
+完成条件：map 顺序不影响结果；默认空值不再删真配置；历史 baseline-live 可以初始化新站点、不能覆盖真实状态；同内容不同时间、同时间冲突最终一致；Tag 时间完整；Quota 删除后磁盘、缓存、重载一致；不可达占位项不阻断健康目标；完整状态已可见且稳定时，第二轮 heal 无写入/广播。
+
+### 提交 3：新增删除传播与有界诊断
+
+只增加一个启动开关，暂定 `MINIO_SITE_REPLICATION_METADATA_TOMBSTONES=off/on`，默认 off；实现沿用现有 env 开关写法，不做能力协商。
+
+| 行为 | off：升级阶段 | on：所有参与节点修复后 |
+| --- | --- | --- |
+| 带时间 peer apply、锁内排序与 heal | 使用提交 1/2 | 同左 |
+| 专用事件零时间 | 兼容应用并记录 legacy-zero | 同左，不新增协议拒绝 |
+| Tags/SSE/Quota nil payload 时间导出 | 保留旧版条件导出 | 导出 `time > Created` 的真实删除时间 |
+| Policy 时间导出 | 保留已有行为 | 保留已有行为 |
+| 初次同步的真实删除 | 保留已有行为 | Policy/Tags/SSE/Quota 都发送专用 nil + source time 事件 |
+
+开关只控制**新增**删除信息的导出/初次发送，普通本地删除事件照常复制。**off 不等于禁止删除传播：Policy 墓碑原本已导出，修复后在 off 下也照常参与 heal；Tags/SSE/Quota 的新增墓碑信息才被门控。** off 期间这些字段的隐藏墓碑会使 heal 继续尝试过时 RPC，由接收端排序拒绝；这是状态不可见时的已知代价，不承诺第二轮零 RPC，也不为这种正常拒绝增加每轮日志。
+
+开关不检测或证明远端能力。启用条件是所有参与站点的全部节点已经修复，同一站点配置一致，旧请求排空；旧节点仍在线时保持 off。此隔离有实证依据：旧版接到新增 Quota heal 墓碑会留下已解析缓存残留。
+
+日志仅保留三个实际原因：legacy-zero、before-created、indeterminate（未知创建时间、缺失/不可达来源或有实际差异却无可用候选）。精确重复、正常旧事件和成功裁决的同时间冲突不记警告。复用 `LogOnceIf`，以稳定的桶/字段/原因作为 key，**错误正文也必须稳定**；变化的时间与 peer 详情放入日志 ReqInfo，沿用现有每小时清理，不新增限流框架、不输出完整策略。
+
+Server 文档解释启用顺序和回滚：降级前所有修复节点先关开关，然后滚动降级；旧软件缺陷会恢复。点名旧版 Tag heal 无 UpdatedAt 的来源。旧版到达时间污染、legacy-zero 产生的新本地时间以及创建世代分歧无法自动还原；操作者查看状态后在权威站点重新提交需要纠正的配置/删除。历史世代冲突先处理桶身份，不能靠任意站点强刷绕过创建保护。
+
+## 最小验收矩阵
+
+| 组 | 必须覆盖 | 证据方式 |
+| --- | --- | --- |
+| T1 | 六类 PUT 源时间；四类 DELETE；旧事件不回退；重复无写入；零 quota 三元组一致；带合法 Version 的空 Policy PUT 成功、GET NotFound、专用删除事件与落盘一致 | 真实 admin/S3 路由、ErasureSD/Erasure16、磁盘重载、RPC 捕获 |
+| T2 | 同时间两种到达顺序结果相同、删除优先；Policy 多集合及 NotAction/NotResource/Condition 反复编码与排列后，两站点独立解析得到相同键；Sid 差异及大整数不被错误合并 | 确定性比较器测试与代表性真实 handler |
+| T3 | Versioning/Object Lock nil/空 no-op；legacy Tags 载荷回退；Object Lock + prefix exclusion/ExcludeFolders 在普通写、peer、bulk/import 后有效状态一致，第二轮 heal 无额外写入 | 原有 #76/#78 回归加针对性用例 |
+| T4 | 锁前旧事件排队、较新写先提交后旧事件不得覆盖；不同字段并发均保留 | 复用已有 `lockBucketMetadataAcquireHook` / RMW 屏障、两种 ObjectLayer、目标 race |
+| T5 | bulk 新/旧/缺省字段混合；真实 JSON 编解码的 nil、空 RawMessage、显式 null、空字符串、空策略、零 quota；非法字段不部分保存；只导入 tags 不修改 Policy/Quota | 真实 bulk 路由、缓存与磁盘 |
+| T6 | 本地时间胜过已有未来时间；相邻提交不倒序；ZIP 导入期间插入写入，最终落盘和发出事件的状态与时间一致，空 Policy 删除不漏发 | 本地 API、import 路由与 RPC 捕获 |
+| T7 | map 排列、空 baseline/全无候选；baseline-live 初始化与同级冲突收敛，但不能覆盖真实 live/tombstone；nil@Created 不删除配置；空 update-only；Tag 时间、Quota 清缓存、空 ID、世代冲突 | 确定性 heal 本地/远端用例 |
+| T8 | 墓碑经保存/缓存失效/重启仍有效；缺桶、排队写入、零 Created；历史字段时间等于 Created 的桶经初次同步、一轮 heal 后一致，第二轮无写入/广播；接管 Created 前移/后移/不变时默认时间仍是 baseline，真实 PUT/DELETE 时间不变；on/off 与 legacy-zero，off 下 Policy 仍 heal、其它隐藏墓碑允许被拒 RPC 但无每轮警告 | ObjectLayer 与进程内旧版 wire/SRInfo 模拟；复用既有删除/迁移回归 |
+| T9 | 修复版双站点短暂断线、漏发/重复/乱序后六类合法配置收敛；删除传播启用后第二轮稳定无写入；日志确实有界 | 隔离双站点实验，不把未知桶世代或零时间事件算成通过 |
+
+固定旧版与修复版混合进程仅作一次性升级冒烟，不新增为长期提交门槛。将已有审计用例改成正式确定性回归，不能把遍历 map 的概率复现直接提交。先记录未修复失败，再验证通过；复用原有 Object Lock、adoption、metadata-lock、计数、CORS 回归，并在实现后运行目标 race、仓库必需检查和完整 Go CI。方案审查、局部测试、双站点结果、合并与发布是不同证据。
+
+## 作者复核结论
+
+三个提交分别处理写入正确性、heal 收敛和新增删除信息的升级边界；每项对应已证实缺陷或本次修复直接触及的实际路径。缩减了拒绝零时间的新协议行为、混合版本长期测试门槛和新锁屏障；保留已有持久化、通知与缓存语义。
+
+Opus 第二轮已确认首轮 R1/R2/R3 实质关闭，第三轮确认历史桶初始化边界，第四轮确认接管默认时间修正必要、充分且未扩大范围；最后两轮均为 0 阻断。四轮原文、版本快照、逐项处置与模型调用证据均保留。方案可以进入实现；当前只完成方案和诊断，仍须通过上述实现期验证，不能据此认定问题已修复或可关闭。

docs/investigations/issue-77.md

@@ -0,0 +1,85 @@
+# #77：桶配置复制修复与验收归档
+
+归档日期：2026-09-12。对象为 [SILO #77](https://github.com/pgsty/silo/issues/77)。问题真实；来源时间、删除状态和 heal 选源共同决定是否收敛，不能只补一个删除分支。最终实际 Claude Code Opus 5 Max 实现审查结论为 **GO_WITH_NONBLOCKING_NOTES，阻断 0**，完整 cmd 和最终 lint 随后通过。
+
+本文固定研究与验收时的事实。合并状态以 Issue 关联 PR 为准；主干包含代码不等于镜像、软件包或生产部署已经更新。研究叙述见伴生站的[中文设计记录](https://github.com/pgsty/silo.pgsty.com/blob/main/content/blog/design/bucket-metadata-convergence.zh.md)和[英文设计记录](https://github.com/pgsty/silo.pgsty.com/blob/main/content/blog/design/bucket-metadata-convergence.md)，操作契约见 [Server site-replication README](../site-replication/README.md)。
+
+## 实现为什么最小、必要且足够
+
+源端 10:00 PUT、10:01 DELETE，目标到 10:05 才收到 PUT。旧实现把字段时间写成 10:05，随后把源时间为 10:01 的删除当作旧事件跳过，返回成功但保留配置。删除后没有导出时间的字段，在漏发后还会被对端旧值恢复。另有锁外判旧、批量入口绕过排序、默认值抢占来源、Tags heal 丢失时间和同内容不更新时间等独立入口。
+
+| 必要改动 | 少了它会发生什么 | 复用的边界 |
+| --- | --- | --- |
+| 在现有整桶锁内读取、比较并保存来源状态 | 锁外判旧仍可覆盖并发的新状态；到达时间继续污染排序 | 既有 `.metadata.bin`、`metadata.lock` 和物理桶存在检查 |
+| Policy、Tags、SSE、Quota、Versioning、Object Lock 共用确定性比较 | 等时冲突仍依赖到达顺序；heal 与接收端可能选出不同结果 | 既有载荷、字段时间及 Created，不增加 wire 或持久化字段 |
+| 专用事件、bulk、导入、本地写和 heal 都使用提交后的状态 | 只改一个 handler 会留下旁路；归一化后的 Versioning/Quota 与出站事件可能不同 | 既有解析、保存与复制钩子，bulk 仍是一次原子保存 |
+| 删除导出开关默认关闭 | 直接对旧版节点导出新增删除状态有已复现的 Quota 缓存风险 | 全部参与节点升级后统一启用，不引入能力协商框架 |
+| 物理 Created 恢复与初次同步传播 | 历史无创建时间的真实桶会失去六类配置写入能力 | 现有物理探测；目录 mtime 只是近似值，不推断真实桶世代 |
+| Policy 状态键与 heal 一致；异常按原因去重 | 永久假 mismatch 无法修复；正常空基线噪声或无来源异常静默 | 既有每站点统计与 logger，不增加后台协调系统 |
+
+真实状态优先于创建基线；真实状态按来源时间、等时删除优先、规范内容键排序。历史 `live@Created` 可以初始化空目标，但空基线不能删除真实配置。本地纠正时间在锁内取 `max(now, Created+1ns, fieldTime+1ns)`；带时间 peer 事件保留来源时间。Versioning/Object Lock 的 nil 保持 no-op，Quota `null`/`{}` 保持零配额配置，空 Policy 归一为删除。
+
+Policy 编码器保留在 GET/export/peer 是必要的：既有解析器可以保存的负集合策略必须能读回和复制。PUT/import 继续使用同一编码器，避免额外表示分支。没有为了本轮重写 CORS、Lifecycle、对象复制、IAM 或 #91 已完成的计数，也没有修改 SDK、模块依赖或升级协议。
+
+充分性限定在同一桶世代、有效且可排序的来源状态、全部参与节点升级并开启删除导出的范围内。历史时间污染、零时间兼容事件和桶身份冲突不属于自动恢复承诺。
+
+## 计划与实际对抗审查
+
+- [实施前核验与失败复现](issue-77-current.md)、[最终 v4 计划](issue-77-plan.md)。这些文件保留历史状态，不能当成当前待办。
+- [四轮计划审查与意见处置](issue-77/review/final-review.md)：前两轮 NO_GO，后两轮零阻断；保留各版计划、完整最终意见、调用元数据和必要探针。
+- 三轮实现审查均使用实际 Claude Code `claude-opus-5 --effort max`。模型身份取自实际 assistant 消息，effort 取自显式调用参数。源码在独立快照中审查，审查者读代码和执行结果，没有代为运行这些验收。
+
+| 实现审查 | 固定源码 | 结论与处置 |
+| --- | --- | --- |
+| [首轮完整意见](issue-77/implementation-review/opus5-max-review.md) / [调用记录](issue-77/implementation-review/session.json) | `4089113e3` | GO_WITH_NONBLOCKING_NOTES，条件性阻断 F1；物理 Created、Policy 假 mismatch、诊断和证据缺口随后修复 |
+| [第二轮意见](issue-77/implementation-review/round-2/opus5-max-review.md) / [调用记录](issue-77/implementation-review/round-2/session.json) | `62cf066ff` | 条件性和无条件阻断均为 0；修正首轮对 Policy 编码器的过宽质疑，进一步补齐无来源诊断和初次同步回归 |
+| [最终定向意见](issue-77/implementation-review/round-3/opus5-max-review.md) / [调用记录](issue-77/implementation-review/round-3/session.json) | `fcbb93e89` | 阻断 0，确认后续 `461e9a721` 的测试改写等价；读取时尚在运行的 cmd/lint 后续均退出 0 |
+
+最初实现审查 SHA `4089113e3` 补签 DCO 后对应 `1ee64a8d8`，两者树完全相同。生产代码最终固定在 `fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025`；验收源码 HEAD 为 `461e9a721047c63e1a95f54ad4b533a6b89def30`，只在两个测试文件有格式和等价条件改写，见 [tree 对照](issue-77/implementation-review/round-3/final-tree-equivalence.json)与 [patch](issue-77/implementation-review/round-3/test-style.diff)。本次归档不改变生产代码或正式回归测试。
+
+审查原文保留当时的判断，不将后续作者修正倒写成评审者已经观察到的结果。首轮误读 README 中既有的空 Policy / 零 Quota 说明，第二轮修正编码器必要性的判断；最终处置以本文和伴生站的逐项表为准。
+
+## 验收证据
+
+| 检查 | 执行对象与结果 |
+| --- | --- |
+| [完整 cmd](issue-77/implementation-review/round-3/final-cmd.log) / [命令及退出码](issue-77/implementation-review/round-3/final-cmd.json) | `fcbb93e89`，CGO=0，全部通过，包测试 492.776 秒 |
+| [最终目标 race](issue-77/implementation-review/round-3/final-target-race.log) / [命令](issue-77/implementation-review/round-3/final-target-race.json) | `fcbb93e89`，真实创建时间、诊断、初次同步、接管、Policy 状态和全部 CORS 命名用例通过 |
+| [测试改写后的 race](issue-77/implementation-review/round-3/final-style-target-race.log) | `461e9a721`，受影响用例通过 |
+| [build](issue-77/implementation-review/round-3/final-build.json)、[vet](issue-77/implementation-review/round-3/final-vet.json)、[lint](issue-77/implementation-review/round-3/final-lint.json) | 最终生产树 build/vet 通过，`461e9a721` lint 零问题；可选 typos 未安装，按 Makefile 跳过 |
+| internal、S3 Select race、生成文件、兼容检查、六平台编译 | `62cf066ff` 的 `ci-*-after.log` 作为较早阶段的补充证据，不冒充最终树在全部平台运行通过；归档中的空生成日志本身不证明退出状态 |
+| [两个真实站点进程](issue-77/implementation-review/round-3/final-runtime.log) / [执行元数据](issue-77/implementation-review/round-3/final-runtime.json) | 干净 `fcbb93e89` 构建：六类历史配置、真实漏发恢复、乱序、四类删除跨重启、两次各 65 秒零 metadata RPC、异常去重、gate=off 与固定旧版的 PUT/DELETE 冒烟均通过 |
+| [伴生站构建](issue-77/implementation-review/round-3/docs-check.log) | 提交为 `9fa6248` 的文章内容通过 Hugo 严格构建和站内链接检查，EN 1207 / ZH 1219 页；后续合并状态文案单独检查 |
+
+[归档清单](issue-77/archive-manifest.json)保存每份材料的原始和归档后 SHA-256，以及全部改动源码的 SHA-256。[二进制身份](issue-77/implementation-review/round-3/binary-identity.json)保留实际版本、Go 构建身份和摘要：最终运行二进制来自干净 `fcbb93e89`，SHA-256 为 `4825a801ce0ac48d636d9428ce4cd5c17a20b6cfec0b569de70a124c9c005049`；旧版来自干净基线 `5c5765816`。第三轮审查的 cmd/lint 条件由上述已完成的 JSON 关闭。
+
+### 反向复现与排除的证据
+
+- [confirmed-before.log](issue-77/implementation-review/round-2/confirmed-before.log)：正式测试叠加 `1ee64a8d8` 的 `erasure-server-pool.go` / `site-replication.go`，真实 ObjectLayer 创建时间恢复及旧 Policy 顺序失败；[修复后](issue-77/implementation-review/round-2/confirmed-after.log)和 [race](issue-77/implementation-review/round-2/confirmed-after-race.log)通过。
+- [confirmed-diagnostics-before.log](issue-77/implementation-review/round-2/confirmed-diagnostics-before.log)：叠加旧诊断路径，普通空基线误报。第一处断言已经终止测试，不能声称它同时证明后面的 Warning 级别和去重断言在旧码失败。
+- [no-source-before.log](issue-77/implementation-review/round-2/no-source-before.log)：直接运行 `62cf066ff`，三种已有无效状态在没有有效来源时缺少诊断；这一次不是旧码 overlay。
+- [initial-sync-before.log](issue-77/implementation-review/round-2/initial-sync-before.log)：保留物理恢复修复，仅叠加 `1ee64a8d8` 的 `site-replication.go`，首次同步出站仍传零 Created。该单测的 peer 只确认 RPC，不证明真实对端或本地 peer 分支已经落盘。
+
+`findings-before.log` / `findings-after-1.log` 是早期夹具失败，不能证明产品缺陷；过渡阶段的旧 helper 遗留引用和测试格式失败也不能当成通过记录。这些草稿不进入本归档的验收证据。另一次 Codex 调用因使用限额失败，不计入完成审查次数。
+
+### 重跑真实双站点验收
+
+[独立 Go 驱动](issue-77/runtime/main.go)与原执行内容相同，仅添加 `ignore` 构建标记及说明，避免进入普通包测试。它只启动回环地址上的一次性实验实例，每站四个数据目录，使用文件内声明的专用实验凭据。给它一个全新目录和自行从固定提交构建的两个二进制：
+
+```sh
+go run docs/investigations/issue-77/runtime/main.go \
+  /absolute/new-lab-directory /absolute/silo-fixed /absolute/silo-before
+```
+
+固定构建分别为 `fcbb93e89` 和 `5c5765816`，可在独立干净 worktree 使用 `CGO_ENABLED=0 go build -trimpath`；记录 `--version`、`go version -m` 和 SHA-256。省略旧版二进制参数会跳过混合版本冒烟，不能将其报告为执行过。最终删除收敛快照见 [converged.json](issue-77/runtime/converged.json)，数组顺序由驱动的 `states` 函数定义。
+
+原执行环境为 `go1.27.1 darwin/arm64`。因宿主盘可用空间比例触发存储保留阈值，完整 cmd 和进程实验使用独立 16 GiB APFS 测试卷，没有降低生产阈值。临时卷、审查 worktree 和实例进程均已清理。上述是本地可复核的执行证据，不替代 GitHub Actions、Linux 多节点集群、发布制品或生产验证。
+
+## 剩余边界
+
+1. `MINIO_SITE_REPLICATION_METADATA_TOMBSTONES=off` 是默认值。全部站点全部节点升级、配置一致并排空旧请求后，统一开启并重启，才具有新增 Tags/SSE/Quota 删除的漏发自愈能力。普通删除事件与原有 Policy 删除导出仍保留。
+2. 到达时间污染、零时间旧事件和桶世代冲突无法凭现有数据重建。物理 Created 只是目录 mtime 的近似值；较早事件仍跳过，由操作者核对权威状态后重新提交纠正。
+3. 日志按每桶/字段/原因去重，仍可能随桶和有值字段数量增长。本轮不增加全站日志调度。
+4. 最终审查保留三项非阻断改进：解码失败诊断中的时间可能为零；初次同步单测没有证明本地 peer 分支落盘；未来并发后台日志可能需要更强的测试隔离。生产路径已复核，本轮不为这些建议新增 helper 或 hook。
+
+归档保留最终报告与调用身份，不发布原始模型推理流、二进制、实验数据卷或无效夹具日志。工作站路径、历史文档链接与非必要 JSON 字段经过整理；哈希清单区分原始产物与归档副本，不能将整理后的报告哈希冒充原文件哈希。

docs/investigations/issue-77/archive-manifest.json

@@ -0,0 +1,437 @@
+{
+  "archived_at": "2026-09-12",
+  "issue": "https://github.com/pgsty/silo/issues/77",
+  "base": "5c576581631561c446f30ae5b566f0aa793adc1c",
+  "tested_source_head": "461e9a721047c63e1a95f54ad4b533a6b89def30",
+  "reviewed_production_head": "fcbb93e8957275bfa7ad4e6154e93f4d7b0a7025",
+  "source_sha256": {
+    "buildscripts/rebrand-guard/compat-baseline.json": "7ea6485008dd998373144ebded82483fe2c33ae60712abcf2fba973a7506b3e1",
+    "cmd/admin-bucket-handlers.go": "3aa8fc80cb8f2c4725acb8fbec026d47cd71a60ad58d06f905aa013540b30db7",
+    "cmd/bucket-cors-site-replication_test.go": "2cff52d3857e912526aaae40736b077e6b8bd640e098934b8af0ec76354f3747",
+    "cmd/bucket-metadata-replication.go": "07730f6a19b78a9b51dd9d56696812299b3aedce1a779b23178643e2684ba686",
+    "cmd/bucket-metadata-sys.go": "6d0936f537f104bdae1c460554c5ef6f34e6f057d8eb16569a23f012a9498d60",
+    "cmd/bucket-metadata.go": "2b692e42df36373fb98a74792098e1251ae886f5e2759add91a00909b596d04c",
+    "cmd/bucket-policy-handlers.go": "1ac487ef97f63ab19358212f1d582659b4d6fbd17888f5bb21cd5e5028e783e9",
+    "cmd/bucket-versioning-handler.go": "23ec85d45bacf687ada30ea6a06cb5e6d5d5dd32a81865a24aa52a030759e3ce",
+    "cmd/common-main.go": "c06df6c3051ceb264f7ddbdddea33ebe38a59e0c113b5c4c44906c1c2e08a01b",
+    "cmd/erasure-server-pool.go": "1ebb616ac971a22bd736d1b381845c0011cbc95b18dc273e4a946eec8244c80d",
+    "cmd/site-replication-metadata-gate_test.go": "1f170b84d2d25d67a4386451ce26597a752f96c9215155f3156ac1fd140ead86",
+    "cmd/site-replication-metadata-heal_test.go": "37191261f643dae88fbe5be44da8f08b2c52455245d5c61a2df564ecb7c8bffd",
+    "cmd/site-replication-metadata.go": "5756a5495723a2d9456b1c3ed7875cec789298eefb8230ea57126f9de83c4287",
+    "cmd/site-replication-metadata_test.go": "2898c88d48d9693d1c974ee7fd6846b86fe354f378caa630eac2ad0d9b9edfe8",
+    "cmd/site-replication.go": "5015550411bd4d82119cdb96b7f663fd082e4906baee259b30aedbaa616354c8",
+    "cmd/site-replication_test.go": "d85dd2aa991a6cce1fccfa17205e4a478f4ced00e0585fbe2aee2dbf97884b8a",
+    "docs/site-replication/README.md": "abe39529bf5a864dc7c945fd368b647f3fa2121fb324cea1cf63373eff9683a7"
+  },
+  "original_review_head": "4089113e3edbd21a29be8e6af74662462bcc22af",
+  "equivalent_signed_commit": "1ee64a8d895f0876580d5e27d0ce6facf41c347e",
+  "reviewed_tree": "0d18c940c6dec380dffc977cb5720b4e2384e46a",
+  "environment": "go version go1.27.1 darwin/arm64",
+  "scope": "Historical analysis and plan, final review reports and invocation metadata, selected executed test logs, source/binary identity, and rerunnable loopback lab. Excludes model reasoning streams, binaries, temporary volumes and invalid fixture logs. Paths, historical document links and trailing console whitespace were normalized; JSON billing fields and build-info dependency lists were omitted. Original and archived hashes distinguish these editorial transformations.",
+  "final_verdict": "GO_WITH_NONBLOCKING_NOTES",
+  "blocking_findings": 0,
+  "records": [
+    {
+      "file": "issue-77-current.md",
+      "source_record": "issue-77-current.md",
+      "original_sha256": "702f9fe93703c8d4b79a3e2c4709e7f49614dc20211ed5221f3adbd5e3809e2d",
+      "archived_sha256": "b22b50e584de73cfe2f101d35bb912d314feb7f0e0ce48018e1b4a48dc4cf5c5"
+    },
+    {
+      "file": "issue-77-plan.md",
+      "source_record": "issue-77-plan.md",
+      "original_sha256": "2ee214694f16e5949ed58290364770149aee3748faee3614122fee189ef83add",
+      "archived_sha256": "36ae4f99bbc22827cc9a1a437b413713e61f442a568347eeef836b4245c67081"
+    },
+    {
+      "file": "issue-77/current-tests.log",
+      "source_record": "issue-77/current-tests.log",
+      "original_sha256": "d6ed0ef373a37f8254d387a0757a431c2785b6b9a30c4e06d0034688ca3a41e8",
+      "archived_sha256": "d6ed0ef373a37f8254d387a0757a431c2785b6b9a30c4e06d0034688ca3a41e8"
+    },
+    {
+      "file": "issue-77/issue77_heal_review_test.go.txt",
+      "source_record": "issue-77/issue77_heal_review_test.go.txt",
+      "original_sha256": "f1b164afff783282253660b8fa5c131b0de946a078d370cdedd95c338fa73810",
+      "archived_sha256": "f1b164afff783282253660b8fa5c131b0de946a078d370cdedd95c338fa73810"
+    },
+    {
+      "file": "issue-77/issue77_review_test.go.txt",
+      "source_record": "issue-77/issue77_review_test.go.txt",
+      "original_sha256": "77e557bd098fad1345085b6bca27ff8a733984092c68c6b26d0dc74e5fca8caa",
+      "archived_sha256": "77e557bd098fad1345085b6bca27ff8a733984092c68c6b26d0dc74e5fca8caa"
+    },
+    {
+      "file": "issue-77/review/adoption-baseline-probe.go.txt",
+      "source_record": "issue-77/review/adoption-baseline-probe.go.txt",
+      "original_sha256": "4cf7aae6763f1ae736af087244dae01e32af1c546923fa782891759967981574",
+      "archived_sha256": "4cf7aae6763f1ae736af087244dae01e32af1c546923fa782891759967981574"
+    },
+    {
+      "file": "issue-77/review/adoption-baseline-probe.log",
+      "source_record": "issue-77/review/adoption-baseline-probe.log",
+      "original_sha256": "d5ad1bfe7d4faafc7bfc6f55800dc47a7d60b7ec7124cc31ec9ed11b0154bde7",
+      "archived_sha256": "d5ad1bfe7d4faafc7bfc6f55800dc47a7d60b7ec7124cc31ec9ed11b0154bde7"
+    },
+    {
+      "file": "issue-77/review/adoption-baseline-probe.metadata.json",
+      "source_record": "issue-77/review/adoption-baseline-probe.metadata.json",
+      "original_sha256": "618e81cb5852b08fbc3029d812647b9d2dbe3560349ee2e44b860356e12cffa9",
+      "archived_sha256": "a462ad6b10cb7e2737330eed5d5e0a5f98ac0d664aa6fcf40a1f5f2f073709f6"
+    },
+    {
+      "file": "issue-77/review/author-followup-findings.md",
+      "source_record": "issue-77/review/author-followup-findings.md",
+      "original_sha256": "b6d60d813702db300313430325046aa89e45071d3cdc20083d82f061b75ab2c4",
+      "archived_sha256": "b6d60d813702db300313430325046aa89e45071d3cdc20083d82f061b75ab2c4"
+    },
+    {
+      "file": "issue-77/review/decisions-v2.md",
+      "source_record": "issue-77/review/decisions-v2.md",
+      "original_sha256": "52c85f0b45073471e96abe86aa5a182858ff00a5dfe00f290f570f2aad86c5a1",
+      "archived_sha256": "52c85f0b45073471e96abe86aa5a182858ff00a5dfe00f290f570f2aad86c5a1"
+    },
+    {
+      "file": "issue-77/review/decisions-v3.md",
+      "source_record": "issue-77/review/decisions-v3.md",
+      "original_sha256": "7325864c0a3e93689989612da0427b88f890ef8baf197f609c570636b16a2b85",
+      "archived_sha256": "7325864c0a3e93689989612da0427b88f890ef8baf197f609c570636b16a2b85"
+    },
+    {
+      "file": "issue-77/review/decisions-v4.md",
+      "source_record": "issue-77/review/decisions-v4.md",
+      "original_sha256": "982b2e802aed883f2b5e211f0e1677388571ccde8e32af878a35b4192c1f4d16",
+      "archived_sha256": "982b2e802aed883f2b5e211f0e1677388571ccde8e32af878a35b4192c1f4d16"
+    },
+    {
+      "file": "issue-77/review/final-review.md",
+      "source_record": "issue-77/review/final-review.md",
+      "original_sha256": "2410f7b92e2c74b235d54bc155b822675c6b5f9ab0b7226fccca42dc38b0a7ea",
+      "archived_sha256": "2410f7b92e2c74b235d54bc155b822675c6b5f9ab0b7226fccca42dc38b0a7ea"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v1.md",
+      "source_record": "issue-77/review/opus5-max-v1.md",
+      "original_sha256": "8a3b750a261ef520acbe658395d3303cea0835b07184b96022781a2a5d0d4dcf",
+      "archived_sha256": "8a3b750a261ef520acbe658395d3303cea0835b07184b96022781a2a5d0d4dcf"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v1.metadata.json",
+      "source_record": "issue-77/review/opus5-max-v1.metadata.json",
+      "original_sha256": "959a9eac3a8449ef27027fbc65aca55a32b1f46063175d7187922296ecd0a957",
+      "archived_sha256": "d5e3be5c5377adeb88fdcb444ad5b6ee338af91d095027e1c9502298eda51d6d"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v2.md",
+      "source_record": "issue-77/review/opus5-max-v2.md",
+      "original_sha256": "848a476f18ce88208b72d8bf6cb49a2316b2b14dcba64ad66e636fe0a3d70595",
+      "archived_sha256": "848a476f18ce88208b72d8bf6cb49a2316b2b14dcba64ad66e636fe0a3d70595"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v2.metadata.json",
+      "source_record": "issue-77/review/opus5-max-v2.metadata.json",
+      "original_sha256": "ff63843a993830ea48e1a44987390b94613dea99a796453bc91bebca2f4e119f",
+      "archived_sha256": "3cd83d43f461fb1e13bbc4f4163769fa6daf4d151d8442d8615a3e813da38853"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v3.md",
+      "source_record": "issue-77/review/opus5-max-v3.md",
+      "original_sha256": "557ef70288a9a3939842a2f76d63694e2c103a8bfcaa99b7c161ee89e1f58061",
+      "archived_sha256": "557ef70288a9a3939842a2f76d63694e2c103a8bfcaa99b7c161ee89e1f58061"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v3.metadata.json",
+      "source_record": "issue-77/review/opus5-max-v3.metadata.json",
+      "original_sha256": "ad488156f42e33e47ed5d0a267ee69ee82efff69881d0e6400e88cf88a09ef60",
+      "archived_sha256": "1c84da615bccc2f1d7a2aa917ef7af751b0943cc7e413b39f59164a65acc2c98"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v4.md",
+      "source_record": "issue-77/review/opus5-max-v4.md",
+      "original_sha256": "cf10b9d7963ff2b7d4c0fbb1fe744a0e853549eaf7ce675a94cc000f74fe6682",
+      "archived_sha256": "cf10b9d7963ff2b7d4c0fbb1fe744a0e853549eaf7ce675a94cc000f74fe6682"
+    },
+    {
+      "file": "issue-77/review/opus5-max-v4.metadata.json",
+      "source_record": "issue-77/review/opus5-max-v4.metadata.json",
+      "original_sha256": "95bae4427322fc42974a9d94ed5dd6d272a46b6eb669f95669a2fe1ef6ee4d42",
+      "archived_sha256": "60d68b2db11557ee09d60b67748844a56d7b1f323803163ef03d420ad1cc8c42"
+    },
+    {
+      "file": "issue-77/review/plan-v1.md",
+      "source_record": "issue-77/review/plan-v1.md",
+      "original_sha256": "0218a2ca59750acd6a24dc5ece1d55a19762a863f1a2de8edc6ffc9b4ab072e0",
+      "archived_sha256": "97d88bbd010cbd19744c2e2fee0fed8d3713ca553b5d0f06161299772ea8104b"
+    },
+    {
+      "file": "issue-77/review/plan-v2.md",
+      "source_record": "issue-77/review/plan-v2.md",
+      "original_sha256": "2b2c712b17f757ca1a8501714c6c2472bd442a6be50b29d92f2d87e23c412ba8",
+      "archived_sha256": "b193ee6e47b24576f3f4d06608f1bcd0f7f2b0c27ef782ea4f74ee3706f7aa4d"
+    },
+    {
+      "file": "issue-77/review/plan-v3.md",
+      "source_record": "issue-77/review/plan-v3.md",
+      "original_sha256": "8c4ab51a397246a07bb1cc967df3377d26ac084b7928191eb8fecacf6e7c3aed",
+      "archived_sha256": "fb3b37ab9dd2f4ab12b8050293603ed589645a730f1e23c81e2f7162ca46f7d2"
+    },
+    {
+      "file": "issue-77/review/plan-v4.md",
+      "source_record": "issue-77/review/plan-v4.md",
+      "original_sha256": "fbfd37498ab32a589d06254f32182869d5d49fc97722416c46c10b41e5f77816",
+      "archived_sha256": "8624e9b86d3cb6daa497d43ac8cb50318ebabdd7eb4bb7c605e0720522a73bd9"
+    },
+    {
+      "file": "issue-77/review/policy-encoding-probe.go.txt",
+      "source_record": "issue-77/review/policy-encoding-probe.go.txt",
+      "original_sha256": "4e07cba70bdc329a0003ca0b6711886a6de22323293e8de3625cdc838ccaf537",
+      "archived_sha256": "4e07cba70bdc329a0003ca0b6711886a6de22323293e8de3625cdc838ccaf537"
+    },
+    {
+      "file": "issue-77/review/wire-state-probe.go.txt",
+      "source_record": "issue-77/review/wire-state-probe.go.txt",
+      "original_sha256": "cafc6e2d482cc8e178ae3e17651410e3070049cd60009f0edd30fbac29e1309c",
+      "archived_sha256": "cafc6e2d482cc8e178ae3e17651410e3070049cd60009f0edd30fbac29e1309c"
+    },
+    {
+      "file": "issue-77/review/wire-state-probe.log",
+      "source_record": "issue-77/review/wire-state-probe.log",
+      "original_sha256": "5077cd264a50cc05fe9dd628a6e98a3b0aaf94e04b65b712563191e11e4ca9f1",
+      "archived_sha256": "5077cd264a50cc05fe9dd628a6e98a3b0aaf94e04b65b712563191e11e4ca9f1"
+    },
+    {
+      "file": "issue-77/implementation-review/opus5-max-review.md",
+      "source_record": "implementation-review/opus5-max-review.md",
+      "original_sha256": "e40dad6e1969898e35c82b7fd24517a5ee714d603a54fe3598b9e69c20922118",
+      "archived_sha256": "e40dad6e1969898e35c82b7fd24517a5ee714d603a54fe3598b9e69c20922118"
+    },
+    {
+      "file": "issue-77/implementation-review/session.json",
+      "source_record": "implementation-review/session.json",
+      "original_sha256": "c347259a0a501242eb787319d43fe16c117d021ea4e418ce74b09b6c9c33ad08",
+      "archived_sha256": "d26e9c3a7f3d0ba3d34de54e95c64e5939b8d5da1d190bb94d80423b6ca79205"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/opus5-max-review.md",
+      "source_record": "implementation-review/round-2/opus5-max-review.md",
+      "original_sha256": "306a856010f7f9bd1cbec39aba83236978f5a811c98f7fc3f8a08217f72de111",
+      "archived_sha256": "306a856010f7f9bd1cbec39aba83236978f5a811c98f7fc3f8a08217f72de111"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/session.json",
+      "source_record": "implementation-review/round-2/session.json",
+      "original_sha256": "64113634f4e8d0525ed17dd74d092c74419d172aa94b501187daae8ebc27e279",
+      "archived_sha256": "4f4dfe9c50b43b64f15fad830c39c116a296c302655d2c718475249521a68f19"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/opus5-max-review.md",
+      "source_record": "implementation-review/round-3/opus5-max-review.md",
+      "original_sha256": "9106938a7a34cfbc6a11d372dd5ffb71ae7417a77a56d10bbed178f539de8fba",
+      "archived_sha256": "9106938a7a34cfbc6a11d372dd5ffb71ae7417a77a56d10bbed178f539de8fba"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/session.json",
+      "source_record": "implementation-review/round-3/session.json",
+      "original_sha256": "28944efc7a7b2c5451beedf0a05202ffc23b29fad2d9bd04cc89fad3d72184b9",
+      "archived_sha256": "bf31d2dbd8009f0a966c0e820dfdbd5a2820e8b36e71e615f783932fd4ecc9de"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/confirmed-before.log",
+      "source_record": "implementation-review/round-2/confirmed-before.log",
+      "original_sha256": "af2df1c296c3377edb6aae45caa9fafd5e620ffb5a09438c15ca50e3d2cb1c10",
+      "archived_sha256": "af2df1c296c3377edb6aae45caa9fafd5e620ffb5a09438c15ca50e3d2cb1c10"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/confirmed-after.log",
+      "source_record": "implementation-review/round-2/confirmed-after.log",
+      "original_sha256": "0c8be7325817049afe2bd2d3ed668e61cbde2887a443b988b8defbf12304b5bc",
+      "archived_sha256": "0c8be7325817049afe2bd2d3ed668e61cbde2887a443b988b8defbf12304b5bc"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/confirmed-after-race.log",
+      "source_record": "implementation-review/round-2/confirmed-after-race.log",
+      "original_sha256": "da2725133b6c1b41a50a92700ff6adaf6dae7e9ac9d6b5812844715fd2b70e55",
+      "archived_sha256": "da2725133b6c1b41a50a92700ff6adaf6dae7e9ac9d6b5812844715fd2b70e55"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/confirmed-diagnostics-before.log",
+      "source_record": "implementation-review/round-2/confirmed-diagnostics-before.log",
+      "original_sha256": "3d2c1325f1612c9ce79172b148d1ae02cf5443087270e9caeb46046fec1ea2c8",
+      "archived_sha256": "3d2c1325f1612c9ce79172b148d1ae02cf5443087270e9caeb46046fec1ea2c8"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/no-source-before.log",
+      "source_record": "implementation-review/round-2/no-source-before.log",
+      "original_sha256": "e6bac0d117eb195b0ff67ef55eaff302c5f7ee36abf236fdcb69486330b31185",
+      "archived_sha256": "e6bac0d117eb195b0ff67ef55eaff302c5f7ee36abf236fdcb69486330b31185"
+    },
+    {
+      "file": "issue-77/implementation-review/round-2/initial-sync-before.log",
+      "source_record": "implementation-review/round-2/initial-sync-before.log",
+      "original_sha256": "f200f9b961baf20e48f22b50cebd82525dd4647ea42203179d140b5474a28427",
+      "archived_sha256": "f200f9b961baf20e48f22b50cebd82525dd4647ea42203179d140b5474a28427"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/docs-check.json",
+      "source_record": "implementation-review/round-3/docs-check.json",
+      "original_sha256": "76747f4d197d250e6bd3aaa1eba5a7013e56b655f10d143b984faed9d22b503b",
+      "archived_sha256": "322f206ca48816c415ba29f9c67935c0eba038823eddfac8fb7e298ff50bda4a"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/docs-check.log",
+      "source_record": "implementation-review/round-3/docs-check.log",
+      "original_sha256": "af8752f3d8a072954dc6de2b8c72a4054cffbe3583aeae225a8d6623fc95e9fa",
+      "archived_sha256": "a1d67bf392e45da04af9d819a5d76cec98006d1fc78c1a4cce486eb305bdc3ac"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-build.json",
+      "source_record": "implementation-review/round-3/final-build.json",
+      "original_sha256": "924df2a034af81acfe8961489664fa213791096db637e962ced6b78ba7ea8388",
+      "archived_sha256": "c5de13eceee16b23886290fbe351a5269735a549723ed400ddb3e5f0bf7c88e5"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-build.log",
+      "source_record": "implementation-review/round-3/final-build.log",
+      "original_sha256": "d5d78e4d1a5b0dc5bc6042ae94ad64aaf2c500ff3e9091ea1dc9dbdfff04b263",
+      "archived_sha256": "b212590ec72fc2489ea00c36458b3ce900b5b23543c7b141d211ac7b4b06f07b"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-cmd.json",
+      "source_record": "implementation-review/round-3/final-cmd.json",
+      "original_sha256": "431f663b748e81b9c311fa1b931483fe0b1ed09dec796b0894bb5d4a37f5f6b9",
+      "archived_sha256": "39a02d590c9615721fbded8e95cc3b04ebacb32ba5c943d8e9a6f55d912c80c7"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-cmd.log",
+      "source_record": "implementation-review/round-3/final-cmd.log",
+      "original_sha256": "681576adac6d627140a791a22085fc520b61f4a6e04172bc7fb5d0f3967e9c40",
+      "archived_sha256": "67723bedd3472d22a2e3f043e48fdab9bc5e8a212d932c513d44d61cc13eb09d"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-lint.json",
+      "source_record": "implementation-review/round-3/final-lint.json",
+      "original_sha256": "1dc2b38de9c4f37983e45e11ad5f416810f0c96e2cfeeff64b25846544918751",
+      "archived_sha256": "35229aa6e418c54072abf141b4296be491a43cdba87c2f56f853076d78d66559"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-lint.log",
+      "source_record": "implementation-review/round-3/final-lint.log",
+      "original_sha256": "40eb5ec64ae64b73489fc7b6e145aabfea6554e0c73fa22be7b4a8201bb63d18",
+      "archived_sha256": "d4b9455691ff7bcce76a0d886f603eda2179e7fa4e408b8e3bb987d693987c3a"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-runtime.json",
+      "source_record": "implementation-review/round-3/final-runtime.json",
+      "original_sha256": "6f76d83dbf9749f9dde0d6386f0b8a6b52cda6677c17ef9d739f05ea9379c52f",
+      "archived_sha256": "b57c593e1023a0226e7950c142d5bd9a231832a9e7aefc7d9d584075b70b9aca"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-runtime.log",
+      "source_record": "implementation-review/round-3/final-runtime.log",
+      "original_sha256": "4c0137bbe554426026e1d4c8baf48e28368e2450e886c506e0055e0591cd5e5e",
+      "archived_sha256": "2b6228f3c6003e896193be17a7c94970992c1fbcbaba013b1984348bf772bc1f"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-style-target-race.json",
+      "source_record": "implementation-review/round-3/final-style-target-race.json",
+      "original_sha256": "09b77aeb0d720de59ea5b1c31f017e8bf32e89c8dc0d4ecc3f35704f21bc0863",
+      "archived_sha256": "075c8d7393164e5b43364d25c163f5da8675f5b69c67e7523a579943a710aaca"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-style-target-race.log",
+      "source_record": "implementation-review/round-3/final-style-target-race.log",
+      "original_sha256": "3089ad9a1c7283a23853a137311728598a7e1ae3bdbbf16e5f000e1149a7e12f",
+      "archived_sha256": "a2609e9a889bcc68828e58c15967bb04820a92462c3f94de86ea584e33a6660a"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-target-race.json",
+      "source_record": "implementation-review/round-3/final-target-race.json",
+      "original_sha256": "68dfcad5fc63d5981065f86fb00482cc466c46f275c4c340095c26ad719f079c",
+      "archived_sha256": "5fb25fa150e66478b2d170faeed3374d71325311f7e625996fe1653cfc957b66"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-target-race.log",
+      "source_record": "implementation-review/round-3/final-target-race.log",
+      "original_sha256": "b437b206ebd12fff304f9cdb59b39f25fd0ecf54e9f510aca98c444b0b516c89",
+      "archived_sha256": "a5b6b6337ebbcf0eca02c2ca15bb1fa562d39ab6fe29f41cd304dd2979976f59"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-tree-equivalence.json",
+      "source_record": "implementation-review/round-3/final-tree-equivalence.json",
+      "original_sha256": "f7d1563d2f675ae282b4bec0bcedcc9809b0a0c985332d862c61ff4b8c75a59f",
+      "archived_sha256": "f7d1563d2f675ae282b4bec0bcedcc9809b0a0c985332d862c61ff4b8c75a59f"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-vet.json",
+      "source_record": "implementation-review/round-3/final-vet.json",
+      "original_sha256": "934ae2812aef8b8635517f838a07cc81dc4780603d621770f9a150bf00d43561",
+      "archived_sha256": "be1b653811da4f6503f329f7e7c8b321b3426107548f0079c8560b177bc0c37f"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/final-vet.log",
+      "source_record": "implementation-review/round-3/final-vet.log",
+      "original_sha256": "f8166088e218c7c8afa25988c02cee77ec58b552f531121a07959b31bd97ebbb",
+      "archived_sha256": "012d45737551ed5ffdccc3b3535ce0be152a9e9f869f50779d5da04e965a9936"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/test-style.diff",
+      "source_record": "implementation-review/round-3/test-style.diff",
+      "original_sha256": "69a7f7efbb17d8dcdd2a0d7f91098569a48d7f1aeeecb6d2aefe04b6047a9025",
+      "archived_sha256": "69a7f7efbb17d8dcdd2a0d7f91098569a48d7f1aeeecb6d2aefe04b6047a9025"
+    },
+    {
+      "file": "issue-77/implementation-review/ci-crosscompile-after.log",
+      "source_record": "implementation-review/ci-crosscompile-after.log",
+      "original_sha256": "68ef16136fabb16babd8fa26d2e3fcf6ffddac30002f874f7b9dfdd61961e82f",
+      "archived_sha256": "68ef16136fabb16babd8fa26d2e3fcf6ffddac30002f874f7b9dfdd61961e82f"
+    },
+    {
+      "file": "issue-77/implementation-review/ci-gen-after.log",
+      "source_record": "implementation-review/ci-gen-after.log",
+      "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "archived_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    {
+      "file": "issue-77/implementation-review/ci-internal-after.log",
+      "source_record": "implementation-review/ci-internal-after.log",
+      "original_sha256": "775e5b2b35fda828ffd69f47902e065f58aa2e2ecd915ace9e50f4085aa2d0c4",
+      "archived_sha256": "775e5b2b35fda828ffd69f47902e065f58aa2e2ecd915ace9e50f4085aa2d0c4"
+    },
+    {
+      "file": "issue-77/implementation-review/ci-lint-after.log",
+      "source_record": "implementation-review/ci-lint-after.log",
+      "original_sha256": "cd29668fa61e94079dda874e10920623279083c491798fa9cbdb903ebeee4d82",
+      "archived_sha256": "cd29668fa61e94079dda874e10920623279083c491798fa9cbdb903ebeee4d82"
+    },
+    {
+      "file": "issue-77/implementation-review/ci-s3select-after.log",
+      "source_record": "implementation-review/ci-s3select-after.log",
+      "original_sha256": "30fc4888ba394ef0daa6c76fa497c28086b7307ec30d195fe4f2b725badc59c9",
+      "archived_sha256": "30fc4888ba394ef0daa6c76fa497c28086b7307ec30d195fe4f2b725badc59c9"
+    },
+    {
+      "file": "issue-77/implementation-review/ci-verify-after.log",
+      "source_record": "implementation-review/ci-verify-after.log",
+      "original_sha256": "2b6860f20bdfd453bcc299f8b57a117cd698bab77d6a474913e31794c3937bd3",
+      "archived_sha256": "2b6860f20bdfd453bcc299f8b57a117cd698bab77d6a474913e31794c3937bd3"
+    },
+    {
+      "file": "issue-77/implementation-review/round-3/binary-identity.json",
+      "source_record": "implementation-review/round-3/binary-identity.json",
+      "original_sha256": "ffdb768a1931388f174eaca0980cc840015a8e0995b5509ef68718d10044f5e5",
+      "archived_sha256": "d4ea7abc1f419e834746820b5b2d2f1b3264e42114161e7c6803f4e952fcf17d"
+    },
+    {
+      "file": "issue-77/runtime/converged.json",
+      "source_record": "runtime-reviewed/converged.json",
+      "original_sha256": "d6d97bc01a12fd624af37432e9558356255b5cd9d4333eaa616e39998c2ce94f",
+      "archived_sha256": "d6d97bc01a12fd624af37432e9558356255b5cd9d4333eaa616e39998c2ce94f"
+    },
+    {
+      "file": "issue-77/runtime/main.go",
+      "source_record": "twosite/main.go",
+      "original_sha256": "ec63a5e2553907a826727af416145057215adabfb9fa2839861f0c05b7cc2a8f",
+      "archived_sha256": "023459a3873753d5a373c2ba6c078d690c9a2a18c31efa98ecd5f4976ecaba4a"
+    }
+  ]
+}

docs/investigations/issue-77/current-tests.log

@@ -0,0 +1,129 @@
+=== RUN   TestQueuedMetadataUpdateAfterDelete
+=== RUN   TestQueuedMetadataUpdateAfterDelete/expiry=false
+=== RUN   TestQueuedMetadataUpdateAfterDelete/expiry=true
+--- PASS: TestQueuedMetadataUpdateAfterDelete (0.43s)
+    --- PASS: TestQueuedMetadataUpdateAfterDelete/expiry=false (0.22s)
+    --- PASS: TestQueuedMetadataUpdateAfterDelete/expiry=true (0.21s)
+=== RUN   TestIssue77CurrentHealInputs
+=== RUN   TestIssue77CurrentHealInputs/ErasureSD/quota-tombstone-cache
+    issue77_heal_review_test.go:58: QUOTA_CACHE: disk has no quota, cache still enforces 1024
+=== RUN   TestIssue77CurrentHealInputs/ErasureSD/same-payload-time-barrier
+    issue77_heal_review_test.go:79: BARRIER_NOT_HEALED: same payload remains at 2026-09-12 06:59:55.50105 +0800 CST, latest is 2026-09-11 23:00:55.50105 +0000 UTC
+=== RUN   TestIssue77CurrentHealInputs/ErasureSD/creation-default-selection
+    issue77_heal_review_test.go:105: DEFAULT_SELECTED: creation-default erased valid policy in 5/32 heal rounds
+=== RUN   TestIssue77CurrentHealInputs/ErasureSD/tag-remote-source-time
+    issue77_heal_review_test.go:144: TAG_WIRE_TIME: sent 0001-01-01 00:00:00 +0000 UTC, want 2026-09-11 23:00:55.50105 +0000 UTC
+=== RUN   TestIssue77CurrentHealInputs/Erasure/quota-tombstone-cache
+    issue77_heal_review_test.go:58: QUOTA_CACHE: disk has no quota, cache still enforces 1024
+=== RUN   TestIssue77CurrentHealInputs/Erasure/same-payload-time-barrier
+    issue77_heal_review_test.go:79: BARRIER_NOT_HEALED: same payload remains at 2026-09-12 06:59:55.71847 +0800 CST, latest is 2026-09-11 23:00:55.71847 +0000 UTC
+=== RUN   TestIssue77CurrentHealInputs/Erasure/creation-default-selection
+    issue77_heal_review_test.go:105: DEFAULT_SELECTED: creation-default erased valid policy in 2/32 heal rounds
+=== RUN   TestIssue77CurrentHealInputs/Erasure/tag-remote-source-time
+    issue77_heal_review_test.go:144: TAG_WIRE_TIME: sent 0001-01-01 00:00:00 +0000 UTC, want 2026-09-11 23:00:55.71847 +0000 UTC
+--- FAIL: TestIssue77CurrentHealInputs (1.21s)
+    --- FAIL: TestIssue77CurrentHealInputs/ErasureSD/quota-tombstone-cache (0.01s)
+    --- FAIL: TestIssue77CurrentHealInputs/ErasureSD/same-payload-time-barrier (0.00s)
+    --- FAIL: TestIssue77CurrentHealInputs/ErasureSD/creation-default-selection (0.08s)
+    --- FAIL: TestIssue77CurrentHealInputs/ErasureSD/tag-remote-source-time (0.00s)
+    --- FAIL: TestIssue77CurrentHealInputs/Erasure/quota-tombstone-cache (0.04s)
+    --- FAIL: TestIssue77CurrentHealInputs/Erasure/same-payload-time-barrier (0.02s)
+    --- FAIL: TestIssue77CurrentHealInputs/Erasure/creation-default-selection (0.83s)
+    --- FAIL: TestIssue77CurrentHealInputs/Erasure/tag-remote-source-time (0.02s)
+=== RUN   TestIssue77CurrentPeerCheckBeforeLock
+=== RUN   TestIssue77CurrentPeerCheckBeforeLock/ErasureSD
+    issue77_heal_review_test.go:209: CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with "<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>"
+=== RUN   TestIssue77CurrentPeerCheckBeforeLock/Erasure
+    issue77_heal_review_test.go:209: CHECK_OUTSIDE_LOCK: queued older event replaced newer committed tags with "<Tagging><TagSet><Tag><Key>key</Key><Value>older</Value></Tag></TagSet></Tagging>"
+--- FAIL: TestIssue77CurrentPeerCheckBeforeLock (0.30s)
+    --- FAIL: TestIssue77CurrentPeerCheckBeforeLock/ErasureSD (0.01s)
+    --- FAIL: TestIssue77CurrentPeerCheckBeforeLock/Erasure (0.07s)
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/policy
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.017159 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/tags
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.029759 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.03401 +0800 CST
+    issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/sse
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.04395 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.048028 +0800 CST
+    issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/quota
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.057746 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.061453 +0800 CST
+    issue77_review_test.go:112: older source PUT did not resurrect config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/versioning
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.069105 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
+    issue77_review_test.go:87: nil payload is correctly a no-op
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/objectlock
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.077601 +0800 CST, want source 2026-09-11 22:59:57.012479 +0000 UTC
+    issue77_review_test.go:87: nil payload is correctly a no-op
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/Erasure/policy
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.247526 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/Erasure/tags
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.369557 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.394147 +0800 CST
+    issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/Erasure/sse
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.466111 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.489771 +0800 CST
+    issue77_review_test.go:110: STALE_RESURRECTION: older source PUT resurrected a locally deleted config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/Erasure/quota
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.562629 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
+    issue77_review_test.go:91: NEWER_DELETE: HTTP 200, but live config remained after newer source DELETE
+    issue77_review_test.go:105: TOMBSTONE_EXPORT: got 0001-01-01 00:00:00 +0000 UTC, want 2026-09-12 07:49:57.590096 +0800 CST
+    issue77_review_test.go:112: older source PUT did not resurrect config
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/Erasure/versioning
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.640941 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
+    issue77_review_test.go:87: nil payload is correctly a no-op
+=== RUN   TestIssue77CurrentSourceTimeAndDeletion/Erasure/objectlock
+    issue77_review_test.go:79: SOURCE_TIME: persisted 2026-09-12 07:49:57.690024 +0800 CST, want source 2026-09-11 22:59:57.221624 +0000 UTC
+    issue77_review_test.go:87: nil payload is correctly a no-op
+--- FAIL: TestIssue77CurrentSourceTimeAndDeletion (0.78s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/policy (0.01s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/tags (0.01s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/sse (0.01s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/quota (0.01s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/versioning (0.01s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/ErasureSD/objectlock (0.01s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/policy (0.12s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/tags (0.10s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/sse (0.10s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/quota (0.08s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/versioning (0.05s)
+    --- FAIL: TestIssue77CurrentSourceTimeAndDeletion/Erasure/objectlock (0.05s)
+=== RUN   TestIssue77CurrentBulkApplyOrdering
+=== RUN   TestIssue77CurrentBulkApplyOrdering/ErasureSD
+    issue77_review_test.go:143: BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value="<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>" time=2026-09-12 08:48:57.792608 +0800 CST
+=== RUN   TestIssue77CurrentBulkApplyOrdering/Erasure
+    issue77_review_test.go:143: BULK_STALE_OVERWRITE: older bulk event overwrote newer config, value="<Tagging><TagSet><Tag><Key>key</Key><Value>old</Value></Tag></TagSet></Tagging>" time=2026-09-12 08:48:57.880379 +0800 CST
+--- FAIL: TestIssue77CurrentBulkApplyOrdering (0.27s)
+    --- FAIL: TestIssue77CurrentBulkApplyOrdering/ErasureSD (0.01s)
+    --- FAIL: TestIssue77CurrentBulkApplyOrdering/Erasure (0.05s)
+=== RUN   TestPeerBucketAdoptionPreservesLockAndVersioningConfigs
+--- PASS: TestPeerBucketAdoptionPreservesLockAndVersioningConfigs (0.30s)
+=== RUN   TestPeerBucketAdoptionBootstrapsMissingConfigs
+--- PASS: TestPeerBucketAdoptionBootstrapsMissingConfigs (0.24s)
+=== RUN   TestPeerBucketAdoptionNormalizesVersioningWhenEnablingLock
+--- PASS: TestPeerBucketAdoptionNormalizesVersioningWhenEnablingLock (0.28s)
+=== RUN   TestPeerBucketAdoptionEnablesSuspendedVersioning
+--- PASS: TestPeerBucketAdoptionEnablesSuspendedVersioning (0.27s)
+=== RUN   TestPeerBucketObjectLockMetadataCurrentAndLegacyPayloads
+--- PASS: TestPeerBucketObjectLockMetadataCurrentAndLegacyPayloads (0.30s)
+=== RUN   TestPeerBucketObjectLockMetadataWithoutLockEnabled
+--- PASS: TestPeerBucketObjectLockMetadataWithoutLockEnabled (0.24s)
+=== RUN   TestSiteReplicationStatusAccountsPerSiteAndSurvivesMalformedConfig
+--- PASS: TestSiteReplicationStatusAccountsPerSiteAndSurvivesMalformedConfig (0.52s)
+FAIL
+FAIL	github.com/minio/minio/cmd	7.167s
+FAIL