Merge main after R4 validation into R7 candidate

Signed-off-by: Feng Ruohang <rh@vonng.com>

buildscripts/rebrand-guard/compat-baseline.json

@@ -829,6 +829,7 @@
     "/site-replication/peer/bucket-ops",
     "/site-replication/peer/edit",
     "/site-replication/peer/iam-item",
+    "/site-replication/peer/iam-revisions",
     "/site-replication/peer/idp-settings",
     "/site-replication/peer/join",
     "/site-replication/peer/remove",
@@ -877,6 +878,7 @@
     "/v2/metrics/cluster",
     "/v2/metrics/node",
     "/v2/metrics/resource",
+    "/v3/site-replication/peer/iam-revisions",
     "/var/vcap/bosh",
     "/verifybinary",
     "/version",

cmd/admin-router.go

@@ -388,6 +388,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
+		adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))

cmd/handler-utils.go

@@ -246,16 +246,6 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
 
 // extractMetadata extracts metadata from map values.
 func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
-	return extractMetadataFromMimeWithReplication(ctx, v, m, false)
-}
-
-// extractReplicationMetadataFromMime restores replication-only metadata after the
-// caller has validated that the request is a trusted replication write.
-func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
-	return extractMetadataFromMimeWithReplication(ctx, v, m, true)
-}
-
-func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIMEHeader, m map[string]string, allowReplication bool) error {
 	if v == nil {
 		bugLogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
@@ -267,18 +257,14 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
 		nv[http.CanonicalHeaderKey(k)] = kv
 	}
 
-	// Save all supported headers.
+	// Save ordinary object metadata. Replication-only headers are restored only
+	// after the request has been validated as a trusted replication write.
 	for _, supportedHeader := range supportedHeaders {
-		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
-		if ok {
-			if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
-				if !allowReplication {
-					continue
-				}
-				m[v] = strings.Join(value, ",")
-			} else {
-				m[supportedHeader] = strings.Join(value, ",")
-			}
+		if _, ok := replicationToInternalHeaders[supportedHeader]; ok {
+			continue
+		}
+		if value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]; ok {
+			m[supportedHeader] = strings.Join(value, ",")
 		}
 	}
 
@@ -287,8 +273,7 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
 			if !stringsHasPrefixFold(key, prefix) {
 				continue
 			}
-			value, ok := nv[http.CanonicalHeaderKey(key)]
-			if ok {
+			if value, ok := nv[http.CanonicalHeaderKey(key)]; ok {
 				m[key] = strings.Join(value, ",")
 				break
 			}
@@ -297,6 +282,31 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
 	return nil
 }
 
+// extractReplicationMetadataFromMime restores replication-only metadata after the
+// caller has validated that the request is a trusted replication write.
+func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
+	if v == nil {
+		bugLogIf(ctx, errInvalidArgument)
+		return errInvalidArgument
+	}
+
+	nv := make(textproto.MIMEHeader, len(v))
+	for k, kv := range v {
+		// Canonicalize all headers, to remove any duplicates.
+		nv[http.CanonicalHeaderKey(k)] = kv
+	}
+
+	// Ordinary object metadata belongs to the caller. Re-extracting it would
+	// undo normalization (such as removing aws-chunked) or copy an outer
+	// Snowball archive's metadata onto its individual entries.
+	for header, internalHeader := range replicationToInternalHeaders {
+		if value, ok := nv[http.CanonicalHeaderKey(header)]; ok {
+			m[internalHeader] = strings.Join(value, ",")
+		}
+	}
+	return nil
+}
+
 // Returns access credentials in the request Authorization header.
 func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
 	cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)

cmd/handler-utils_test.go

@@ -254,6 +254,9 @@ func TestExtractMetadataFromRequestKeepsQueryCompatibility(t *testing.T) {
 
 func TestExtractReplicationMetadataHeaders(t *testing.T) {
 	header := http.Header{
+		"Content-Type":      []string{"application/wasm"},
+		"Content-Encoding":  []string{"aws-chunked"},
+		"X-Amz-Meta-Source": []string{"client"},
 		"X-Minio-Replication-Server-Side-Encryption-Sealed-Key":     []string{"sealed-key"},
 		"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
 		"X-Minio-Replication-Server-Side-Encryption-Iv":             []string{"iv"},
@@ -262,12 +265,17 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
 		ReplicationSsecChecksumHeader:                               []string{"checksum"},
 	}
 
-	metadata := make(map[string]string)
+	metadata := map[string]string{
+		"content-type":      "application/wasm",
+		"x-amz-meta-source": "client",
+	}
 	if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
 		t.Fatalf("failed to extract replication metadata: %v", err)
 	}
 
 	expected := map[string]string{
+		"content-type":      "application/wasm",
+		"x-amz-meta-source": "client",
 		"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "sealed-key",
 		"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
 		"X-Minio-Internal-Server-Side-Encryption-Iv":             "iv",
@@ -279,6 +287,9 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
 	if !reflect.DeepEqual(metadata, expected) {
 		t.Fatalf("unexpected replication metadata: expected %#v, got %#v", expected, metadata)
 	}
+	if _, ok := metadata["content-encoding"]; ok {
+		t.Fatalf("replication metadata restored transport content-encoding: %#v", metadata)
+	}
 }
 
 func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {

cmd/iam-credential-retention_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+func TestIAMCredentialRetention(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			secret, err := getTokenSigningKey()
+			mustIAM(t, err)
+			parent := "external-idp-parent"
+			credential := func(exp time.Time) auth.Credentials {
+				cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": exp.Unix(), parentClaim: parent}, secret)
+				mustIAM(t, err)
+				cred.ParentUser = parent
+				return cred
+			}
+			// Disablement of an external identity must include cached STS,
+			// which are kept separately from regular and service accounts.
+			cred := credential(UTCNow().Add(time.Hour))
+			_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
+			mustIAM(t, err)
+			mustIAM(t, sys.store.DeleteUsers(ctx, []string{parent}))
+			r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
+			mustIAM(t, err)
+			if !r.Deleted || !r.ExpiresAt.Equal(cred.Expiration.Add(globalMaxSkewTime)) || r.Credentials.SessionToken != "" || r.Credentials.SecretKey != "" {
+				t.Fatal("early STS revocation lost its retention boundary or retained a secret")
+			}
+			if _, ok := sys.store.GetUser(cred.AccessKey); ok {
+				t.Fatal("external disablement left the STS cache live")
+			}
+			_, err = sys.SetTempUser(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), cred.AccessKey, cred, "")
+			if !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("same revoked token was reissued by replay: %v", err)
+			}
+			var mp MappedPolicy
+			err = sys.store.loadIAMConfig(ctx, &mp, getMappedPolicyPath(cred.AccessKey, stsUser, false))
+			if !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("random STS key produced a permanent mapping: %v", err)
+			}
+
+			// Seed genuinely expired immutable tokens, as an ordinary startup
+			// loader sees them. Natural expiry leaves no permanent tombstone.
+			expired := credential(UTCNow().Add(-time.Hour))
+			path := getUserIdentityPath(expired.AccessKey, stsUser)
+			mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: expired, UpdatedAt: UTCNow().Add(-2 * time.Hour)}, path))
+			_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
+			var u UserIdentity
+			if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("natural expiration retained a random key: %v", err)
+			}
+			// A retained early-revocation record is collectable only after the
+			// immutable token's expiration plus the skew allowance.
+			tomb := UserIdentity{Version: 1, Deleted: true, UpdatedAt: UTCNow().Add(-2 * time.Hour), ExpiresAt: expired.Expiration.Add(globalMaxSkewTime)}
+			mustIAM(t, sys.store.saveIAMConfig(ctx, &tomb, path))
+			_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
+			if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
+				t.Fatalf("expired STS revocation not collected: %v", err)
+			}
+			if _, ok := sys.store.revisionIndex().snapshot()[path]; ok {
+				t.Fatal("expired STS retained an index entry")
+			}
+		})
+	}
+}
+
+func TestIAMPolicyDeletionRemainsExplicit(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			mustIAM(t, sys.DeletePolicy(ctx, "misspelled-policy", true))
+			r, err := loadIAMRevision(ctx, sys.store, getPolicyDocPath("misspelled-policy"))
+			mustIAM(t, err)
+			if r.Deleted {
+				t.Fatal("local nonexistent policy created a tombstone")
+			}
+			p, err := sys.store.GetPolicy("readwrite")
+			mustIAM(t, err)
+			if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil {
+				t.Fatal("local pristine builtin policy became deletable")
+			}
+			_, err = sys.SetPolicy(ctx, "readwrite", p)
+			mustIAM(t, err)
+			mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true))
+			mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+			if _, err := sys.store.GetPolicy("readwrite"); !errors.Is(err, errNoSuchPolicy) {
+				t.Fatalf("reload restored an explicitly deleted override: %v", err)
+			}
+			_, err = sys.SetPolicy(ctx, "readwrite", p)
+			mustIAM(t, err)
+			if _, err := sys.store.GetPolicy("readwrite"); err != nil {
+				t.Fatal("explicit policy recreation failed", err)
+			}
+			mustIAM(t, globalSiteReplicationSys.PeerAddPolicyHandler(ctx, "remote-unknown-policy", nil, UTCNow()))
+			r, err = loadIAMRevision(ctx, sys.store, getPolicyDocPath("remote-unknown-policy"))
+			mustIAM(t, err)
+			if !r.Deleted {
+				t.Fatal("replicated unknown deletion lost its version")
+			}
+		})
+	}
+}

cmd/iam-etcd-store.go

@@ -26,7 +26,6 @@ import (
 	"sync"
 	"time"
 
-	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
@@ -62,6 +61,7 @@ type IAMEtcdStore struct {
 	sync.RWMutex
 
 	*iamCache
+	index iamRevisionIndex
 
 	usersSysType UsersSysType
 
@@ -69,13 +69,17 @@ type IAMEtcdStore struct {
 }
 
 func newIAMEtcdStore(client *etcd.Client, usersSysType UsersSysType) *IAMEtcdStore {
-	return &IAMEtcdStore{
+	store := &IAMEtcdStore{
 		iamCache:     newIamCache(),
 		client:       client,
 		usersSysType: usersSysType,
 	}
+	store.revisions = &store.index
+	return store
 }
 
+func (ies *IAMEtcdStore) revisionIndex() *iamRevisionIndex { return &ies.index }
+
 func (ies *IAMEtcdStore) rlock() *iamCache {
 	ies.RLock()
 	return ies.iamCache
@@ -103,6 +107,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
 	if err != nil {
 		return err
 	}
+	plain := data
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, itemPath),
@@ -111,24 +116,28 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
 			return err
 		}
 	}
-	return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
+	if err := saveKeyEtcd(ctx, ies.client, itemPath, data, opts...); err != nil {
+		return err
+	}
+	ies.index.observe(itemPath, plain)
+	return nil
 }
 
-func getIAMConfig(item any, data []byte, itemPath string) error {
-	data, err := decryptData(data, itemPath)
+func (ies *IAMEtcdStore) decodeIAMConfig(item any, data []byte, path string) error {
+	data, err := decryptData(data, path)
 	if err != nil {
 		return err
 	}
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	ies.index.observe(path, data)
 	return json.Unmarshal(data, item)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item any, path string) error {
-	data, err := readKeyEtcd(ctx, ies.client, path)
+	data, err := ies.loadIAMConfigBytes(ctx, path)
 	if err != nil {
 		return err
 	}
-	return getIAMConfig(item, data, path)
+	return json.Unmarshal(data, item)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) {
@@ -136,11 +145,19 @@ func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([
 	if err != nil {
 		return nil, err
 	}
-	return decryptData(data, path)
+	data, err = decryptData(data, path)
+	if err == nil {
+		ies.index.observe(path, data)
+	}
+	return data, err
 }
 
 func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error {
-	return deleteKeyEtcd(ctx, ies.client, path)
+	if err := deleteKeyEtcd(ctx, ies.client, path); err != nil {
+		return err
+	}
+	ies.index.forget(path)
+	return nil
 }
 
 func (ies *IAMEtcdStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, _ int) error {
@@ -162,6 +179,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
 		return err
 	}
 
+	if p.Deleted {
+		return errNoSuchPolicy
+	}
 	m[policy] = p
 	return nil
 }
@@ -181,7 +201,11 @@ func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValu
 		return err
 	}
 
+	ies.index.observe(string(kvs.Key), data)
 	policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key)))
+	if p.Deleted {
+		return errNoSuchPolicy
+	}
 	m[policy] = p
 	return nil
 }
@@ -207,7 +231,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]Policy
 
 func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
 	var u UserIdentity
-	err := getIAMConfig(&u, userkv.Value, string(userkv.Key))
+	err := ies.decodeIAMConfig(&u, userkv.Value, string(userkv.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -219,10 +243,14 @@ func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue,
 }
 
 func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error {
+	if u.Deleted {
+		if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
+			bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
+		}
+		return errNoSuchUser
+	}
 	if u.Credentials.IsExpired() {
-		// Delete expired identity.
-		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
+		bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
 		return nil
 	}
 	if u.Credentials.AccessKey == "" {
@@ -231,16 +259,17 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
-			if u.Credentials.IsTemp() {
-				// We should delete such that the client can re-request
-				// for the expiring credentials.
-				deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
-				deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
-			}
+			// A temporarily unavailable signing key is not proof of expiration.
 			return nil
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
+	if err := checkIAMParentRevision(ctx, ies, u.Credentials); err != nil {
+		if errors.Is(err, errIAMStaleUpdate) {
+			return errNoSuchUser
+		}
+		return err
+	}
 	if u.Credentials.Description == "" {
 		u.Credentials.Description = u.Credentials.Comment
 	}
@@ -258,6 +287,9 @@ func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userTyp
 		}
 		return "", err
 	}
+	if u.Deleted {
+		return "", errNoSuchUser
+	}
 	return u.Credentials.SecretKey, nil
 }
 
@@ -274,6 +306,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 }
 
 func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
+	ctx = withIAMExpirationCleanup(ctx)
 	var basePrefix string
 	switch userType {
 	case svcUser:
@@ -312,6 +345,9 @@ func (ies *IAMEtcdStore) loadGroup(ctx context.Context, group string, m map[stri
 		}
 		return err
 	}
+	if gi.Deleted {
+		return errNoSuchGroup
+	}
 	m[group] = gi
 	return nil
 }
@@ -349,13 +385,16 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
 		}
 		return err
 	}
+	if !ies.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+		return errNoSuchPolicy
+	}
 	m.Store(name, p)
 	return nil
 }
 
-func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
+func (ies *IAMEtcdStore) getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
 	var p MappedPolicy
-	err := getIAMConfig(&p, kv.Value, string(kv.Key))
+	err := ies.decodeIAMConfig(&p, kv.Value, string(kv.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -363,6 +402,9 @@ func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy],
 		return err
 	}
 	name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json")
+	if !ies.index.mappingAllowed(string(kv.Key), p) {
+		return errNoSuchPolicy
+	}
 	m.Store(name, p)
 	return nil
 }
@@ -392,61 +434,13 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Parse all policies mapping to create the proper data model
 	for _, kv := range r.Kvs {
-		if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if err = ies.getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
 			return err
 		}
 	}
 	return nil
 }
 
-func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
-	return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
-}
-
-func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
-	return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
-}
-
-func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
-	return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
-}
-
-func (ies *IAMEtcdStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
-	return ies.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
-}
-
-func (ies *IAMEtcdStore) deletePolicyDoc(ctx context.Context, name string) error {
-	err := ies.deleteIAMConfig(ctx, getPolicyDocPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
-	err := ies.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
-	err := ies.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
-	if err == errConfigNotFound {
-		err = errNoSuchUser
-	}
-	return err
-}
-
-func (ies *IAMEtcdStore) deleteGroupInfo(ctx context.Context, name string) error {
-	err := ies.deleteIAMConfig(ctx, getGroupInfoPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchGroup
-	}
-	return err
-}
-
 func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWatchEvent {
 	ch := make(chan iamWatchEvent)
 

cmd/iam-group-revision.go

@@ -0,0 +1,164 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"maps"
+	"slices"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+type (
+	iamGroupGrantsKey   struct{}
+	iamGroupMutationKey struct{}
+	iamGroupMutation    struct {
+		Members    []string
+		Remove     bool
+		StatusOnly bool
+	}
+)
+
+// Merge the intended mutation with the record read under the distributed
+// revision lock, not the older cache used to prepare the request.
+func mergeIAMGroupMutation(ctx context.Context, previous GroupInfo, next *GroupInfo) {
+	op, ok := ctx.Value(iamGroupMutationKey{}).(iamGroupMutation)
+	if !ok || previous.Deleted || previous.Version == 0 {
+		return
+	}
+	members := set.CreateStringSet(previous.Members...)
+	grants := maps.Clone(previous.MemberGrants)
+	if grants == nil {
+		grants = make(map[string]time.Time)
+	}
+	switch {
+	case op.StatusOnly:
+		// Only the status changes.
+	case op.Remove:
+		for _, member := range op.Members {
+			members.Remove(member)
+			delete(grants, member)
+		}
+		next.Status = previous.Status
+	default:
+		requested := set.CreateStringSet(next.Members...)
+		for _, member := range op.Members {
+			if !requested.Contains(member) {
+				continue
+			}
+			at := next.MemberGrants[member]
+			if at.Before(grants[member]) {
+				continue
+			}
+			members.Add(member)
+			grants[member] = at
+		}
+		next.Status = previous.Status
+	}
+	next.Members, next.MemberGrants = members.ToSlice(), grants
+	slices.Sort(next.Members)
+}
+
+// A non-nil map is supplied by the versioned peer envelope, including for
+// snapshots. Missing times are unknown, never the snapshot's newer timestamp.
+func withIAMGroupGrants(ctx context.Context, grants map[string]time.Time) context.Context {
+	return context.WithValue(ctx, iamGroupGrantsKey{}, grants)
+}
+
+func (c *iamCache) effectiveGroupMembers(gi GroupInfo) []string {
+	var members []string
+	for _, member := range gi.Members {
+		if c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
+			members = append(members, member)
+		}
+	}
+	return members
+}
+
+func (c *iamCache) effectiveUserGroups(user string) []string {
+	var groups []string
+	for group := range c.iamUserGroupMemberships[user] {
+		gi, ok := c.iamGroupsMap[group]
+		r := c.revisions.get(getGroupInfoPath(group))
+		if r.RevokedBefore.After(gi.RevokedBefore) {
+			gi.RevokedBefore = r.RevokedBefore
+		}
+		if ok && !r.Deleted && c.groupMemberAllowed(user, gi.MemberGrants[user], gi.RevokedBefore) {
+			groups = append(groups, group)
+		}
+	}
+	return groups
+}
+
+func (c *iamCache) addGroupMembers(ctx context.Context, gi GroupInfo, members []string) (GroupInfo, error) {
+	grants, versioned := ctx.Value(iamGroupGrantsKey{}).(map[string]time.Time)
+	if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(gi.RevokedBefore) {
+		gi.RevokedBefore = boundary
+	}
+	origin, replicated := iamReplicationTime(ctx)
+	gi.Members = slices.Clone(gi.Members)
+	gi.MemberGrants = maps.Clone(gi.MemberGrants)
+	if gi.MemberGrants == nil {
+		gi.MemberGrants = make(map[string]time.Time)
+	}
+	current := set.CreateStringSet(gi.Members...)
+	gi.UpdatedAt = UTCNow()
+	if replicated {
+		gi.UpdatedAt = origin
+	}
+	for _, member := range members {
+		at := gi.UpdatedAt
+		r := c.userRevocation(member)
+		if replicated {
+			switch {
+			case versioned:
+				at = grants[member]
+				if at.After(origin) {
+					return gi, errInvalidArgument
+				}
+			case !r.RevokedBefore.IsZero() || r.Deleted || !gi.RevokedBefore.IsZero():
+				// Legacy snapshots cannot prove a post-revocation grant.
+				continue
+			case current.Contains(member):
+				continue
+			}
+			if !c.groupMemberAllowed(member, at, gi.RevokedBefore) {
+				continue
+			}
+		} else {
+			if current.Contains(member) && c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
+				continue // Editing the group is not reissuing every grant.
+			}
+			if !at.After(gi.RevokedBefore) {
+				at = gi.RevokedBefore.Add(time.Nanosecond)
+			}
+			if !at.After(r.RevokedBefore) {
+				at = r.RevokedBefore.Add(time.Nanosecond)
+			}
+			if !at.After(gi.MemberGrants[member]) {
+				at = gi.MemberGrants[member].Add(time.Nanosecond)
+			}
+			if at.After(gi.UpdatedAt) {
+				gi.UpdatedAt = at
+			}
+		}
+		u, ok := c.iamUsersMap[member]
+		if !ok {
+			return gi, errNoSuchUser
+		}
+		if u.Credentials.IsTemp() || u.Credentials.IsServiceAccount() {
+			return gi, errIAMActionNotAllowed
+		}
+		if previous := gi.MemberGrants[member]; previous.After(at) {
+			continue
+		}
+		current.Add(member)
+		gi.MemberGrants[member] = at
+	}
+	gi.Members = current.ToSlice()
+	slices.Sort(gi.Members)
+	return gi, nil
+}

cmd/iam-healing-leadership_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestIAMHealingResumesAfterLeadershipLoss(t *testing.T) {
+	previous := globalLeaderLock
+	locks := make(chan LockContext)
+	globalLeaderLock = &sharedLock{lockContext: locks}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	c := &SiteReplicationSys{}
+	go func() { c.startHealRoutine(ctx, nil); close(done) }()
+	t.Cleanup(func() {
+		cancel()
+		select {
+		case <-done:
+		case locks <- LockContext{ctx: ctx}:
+		}
+		<-done
+		globalLeaderLock = previous
+	})
+	first, loseFirst := context.WithCancel(ctx)
+	defer loseFirst()
+	select {
+	case locks <- LockContext{ctx: first}:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not acquire its first leader context")
+	}
+	loseFirst() // A transient quorum loss cancels the distributed lease.
+	select {
+	case <-done:
+		t.Fatal("healer permanently exited after temporary leadership loss")
+	case locks <- LockContext{ctx: ctx}:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not wait for reacquired leadership")
+	}
+	// Shutdown must also interrupt the wait for leadership after lease loss.
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("healer did not stop with its owning context")
+	}
+}
+
+func TestIAMHealingLeadershipWaitCancels(t *testing.T) {
+	previous := globalLeaderLock
+	locks := make(chan LockContext)
+	globalLeaderLock = &sharedLock{lockContext: locks}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	c := &SiteReplicationSys{}
+	go func() { c.startHealRoutine(ctx, nil); close(done) }()
+	cancel()
+	t.Cleanup(func() {
+		select {
+		case <-done:
+		case locks <- LockContext{ctx: ctx}:
+		}
+		<-done
+		globalLeaderLock = previous
+	})
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("healer ignored shutdown while waiting for leadership")
+	}
+}

cmd/iam-healing-performance_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+// Measures steady-state index traversal, sorting and the capability request.
+// The network peer acknowledges real batches but performs no disk I/O; this
+// benchmark deliberately does not claim durable catch-up throughput.
+func BenchmarkIAMRevisionConvergedHealing(b *testing.B) {
+	for _, n := range []int{1000, 10000} {
+		b.Run(fmt.Sprint(n), func(b *testing.B) {
+			ctx, sys, _ := prepareIAMRevisionFixture(b)
+			_, err := sys.CreateUser(ctx, "benchmark-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
+			mustIAM(b, err)
+			for i := range n {
+				at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
+				data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
+				mustIAM(b, err)
+				sys.store.revisionIndex().observe(getUserIdentityPath(fmt.Sprintf("deleted-%06d", i), regUser), data)
+			}
+			var puts atomic.Int64
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.Path == "/minio/health/live" {
+					w.WriteHeader(http.StatusOK)
+					return
+				}
+				if r.Method == http.MethodPut {
+					puts.Add(1)
+				}
+				_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: "benchmark-peer", Digest: "constant"}})
+			}))
+			defer server.Close()
+			c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "benchmark-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
+			mustIAM(b, c.healIAMDeletions(ctx))
+			before := puts.Load()
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				mustIAM(b, c.healIAMDeletions(ctx))
+			}
+			b.StopTimer()
+			b.ReportMetric(float64(puts.Load()-before)/float64(b.N), "PUT/op")
+			if puts.Load() != before {
+				b.Fatal("steady-state healing replayed acknowledged records")
+			}
+		})
+	}
+}

cmd/iam-object-store.go

@@ -45,6 +45,7 @@ type IAMObjectStore struct {
 	sync.RWMutex
 
 	*iamCache
+	index iamRevisionIndex
 
 	usersSysType UsersSysType
 
@@ -52,13 +53,17 @@ type IAMObjectStore struct {
 }
 
 func newIAMObjectStore(objAPI ObjectLayer, usersSysType UsersSysType) *IAMObjectStore {
-	return &IAMObjectStore{
+	store := &IAMObjectStore{
 		iamCache:     newIamCache(),
 		objAPI:       objAPI,
 		usersSysType: usersSysType,
 	}
+	store.revisions = &store.index
+	return store
 }
 
+func (iamOS *IAMObjectStore) revisionIndex() *iamRevisionIndex { return &iamOS.index }
+
 func (iamOS *IAMObjectStore) rlock() *iamCache {
 	iamOS.RLock()
 	return iamOS.iamCache
@@ -87,6 +92,7 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
 	if err != nil {
 		return err
 	}
+	plain := data
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, objPath),
@@ -95,7 +101,11 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
 			return err
 		}
 	}
-	return saveConfig(ctx, iamOS.objAPI, objPath, data)
+	if err := saveConfig(ctx, iamOS.objAPI, objPath, data); err != nil {
+		return err
+	}
+	iamOS.index.observe(objPath, plain)
+	return nil
 }
 
 func decryptData(data []byte, objPath string) ([]byte, error) {
@@ -133,6 +143,7 @@ func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context,
 	if err != nil {
 		return nil, meta, err
 	}
+	iamOS.index.observe(objPath, data)
 	return data, meta, nil
 }
 
@@ -146,7 +157,11 @@ func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item any, objPat
 }
 
 func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) error {
-	return deleteConfig(ctx, iamOS.objAPI, path)
+	if err := deleteConfig(ctx, iamOS.objAPI, path); err != nil {
+		return err
+	}
+	iamOS.index.forget(path)
+	return nil
 }
 
 func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error {
@@ -171,6 +186,10 @@ func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy
 			return err
 		}
 
+		if p.Deleted {
+			return errNoSuchPolicy
+		}
+
 		if p.Version == 0 {
 			// This means that policy was in the old version (without any
 			// timestamp info). We fetch the mod time of the file and save
@@ -200,6 +219,10 @@ func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (Pol
 		return p, err
 	}
 
+	if p.Deleted {
+		return PolicyDoc{}, errNoSuchPolicy
+	}
+
 	if p.Version == 0 {
 		// This means that policy was in the old version (without any
 		// timestamp info). We fetch the mod time of the file and save
@@ -245,6 +268,9 @@ func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, use
 		}
 		return "", err
 	}
+	if u.Deleted {
+		return "", errNoSuchUser
+	}
 	return u.Credentials.SecretKey, nil
 }
 
@@ -258,10 +284,15 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
 		return u, err
 	}
 
+	if u.Deleted {
+		if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
+			bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
+		}
+		return UserIdentity{}, errNoSuchUser
+	}
+
 	if u.Credentials.IsExpired() {
-		// Delete expired identity - ignoring errors here.
-		iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
-		iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
+		bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
 		return u, errNoSuchUser
 	}
 
@@ -272,16 +303,18 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
-			if u.Credentials.IsTemp() {
-				// We should delete such that the client can re-request
-				// for the expiring credentials.
-				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
-				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
-			}
-			return u, errNoSuchUser
+			// During startup the site signing key may not be available yet.
+			// Reject this load without deleting a credential that has not expired.
+			return UserIdentity{}, errNoSuchUser
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
+	if err := checkIAMParentRevision(ctx, iamOS, u.Credentials); err != nil {
+		if errors.Is(err, errIAMStaleUpdate) {
+			return UserIdentity{}, errNoSuchUser
+		}
+		return UserIdentity{}, err
+	}
 
 	if u.Credentials.Description == "" {
 		u.Credentials.Description = u.Credentials.Comment
@@ -320,6 +353,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 }
 
 func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
+	ctx = withIAMExpirationCleanup(ctx)
 	var basePrefix string
 	switch userType {
 	case svcUser:
@@ -354,6 +388,9 @@ func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[
 		}
 		return err
 	}
+	if g.Deleted {
+		return errNoSuchGroup
+	}
 	m[group] = g
 	return nil
 }
@@ -391,6 +428,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
 			goto retry
 		}
 
+		if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+			return errNoSuchPolicy
+		}
 		m.Store(name, p)
 		return nil
 	}
@@ -405,6 +445,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name
 		}
 		return p, err
 	}
+	if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
+		return MappedPolicy{}, errNoSuchPolicy
+	}
 	return p, nil
 }
 
@@ -824,54 +867,6 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	return nil
 }
 
-func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
-	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
-}
-
-func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
-	return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
-}
-
-func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
-	return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
-}
-
-func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
-	return iamOS.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
-}
-
-func (iamOS *IAMObjectStore) deletePolicyDoc(ctx context.Context, name string) error {
-	err := iamOS.deleteIAMConfig(ctx, getPolicyDocPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
-	err := iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
-	if err == errConfigNotFound {
-		err = errNoSuchPolicy
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
-	err := iamOS.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
-	if err == errConfigNotFound {
-		err = errNoSuchUser
-	}
-	return err
-}
-
-func (iamOS *IAMObjectStore) deleteGroupInfo(ctx context.Context, name string) error {
-	err := iamOS.deleteIAMConfig(ctx, getGroupInfoPath(name))
-	if err == errConfigNotFound {
-		err = errNoSuchGroup
-	}
-	return err
-}
-
 // Lists objects in the minioMetaBucket at the given path prefix. All returned
 // items have the pathPrefix removed from their names.
 func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr[string] {

cmd/iam-performance_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// Uses APIs shared with the pre-revision tree so the same benchmark can be
+// overlaid on that tree for a comparable local baseline.
+func prepareIAMPerformanceFixture(b *testing.B) (context.Context, *IAMSys) {
+	b.Helper()
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		b.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err != nil {
+		b.Fatal(err)
+	}
+	initAllSubsystems(ctx)
+	globalIAMSys.initStore(obj, nil)
+	if err := globalIAMSys.Load(ctx, true); err != nil {
+		b.Fatal(err)
+	}
+	b.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
+	return ctx, globalIAMSys
+}
+
+func BenchmarkIAMCachedCredential(b *testing.B) {
+	for _, kind := range []string{"user", "service", "sts"} {
+		b.Run(kind, func(b *testing.B) {
+			ctx, sys := prepareIAMPerformanceFixture(b)
+			const parent = "benchmark-parent"
+			_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
+			if err != nil {
+				b.Fatal(err)
+			}
+			key := parent
+			if kind == "service" {
+				c, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "benchmark-service", secretKey: "benchmark-service-password"})
+				if err != nil {
+					b.Fatal(err)
+				}
+				key = c.AccessKey
+			}
+			if kind == "sts" {
+				secret, err := getTokenSigningKey()
+				if err != nil {
+					b.Fatal(err)
+				}
+				c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+				if err != nil {
+					b.Fatal(err)
+				}
+				c.ParentUser = parent
+				if _, err := sys.SetTempUser(ctx, c.AccessKey, c, ""); err != nil {
+					b.Fatal(err)
+				}
+				key = c.AccessKey
+			}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for b.Loop() {
+				if _, ok := sys.store.GetUser(key); !ok {
+					b.Fatal("credential missing")
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkIAMSetTempUser(b *testing.B) {
+	ctx, sys := prepareIAMPerformanceFixture(b)
+	const parent = "benchmark-sts-parent"
+	_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
+	if err != nil {
+		b.Fatal(err)
+	}
+	secret, err := getTokenSigningKey()
+	if err != nil {
+		b.Fatal(err)
+	}
+	cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+	if err != nil {
+		b.Fatal(err)
+	}
+	cred.ParentUser = parent
+	b.ReportAllocs()
+	b.ResetTimer()
+	for b.Loop() {
+		if _, err := sys.SetTempUser(ctx, cred.AccessKey, cred, "readwrite"); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+// Run with -benchtime=1x. Preparation is outside the timer; each measured load
+// sees a fresh set of expired reusable service-account records.
+func BenchmarkIAMColdLoadExpiredServices(b *testing.B) {
+	for _, count := range []int{100, 1000} {
+		b.Run(fmt.Sprint(count), func(b *testing.B) {
+			ctx, sys := prepareIAMPerformanceFixture(b)
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				b.StopTimer()
+				for j := 0; j < count; j++ {
+					key := fmt.Sprintf("expired-benchmark-%d-%d", i, j)
+					u := UserIdentity{Version: 1, UpdatedAt: UTCNow().Add(-2 * time.Hour), Credentials: auth.Credentials{AccessKey: key, SecretKey: "expired-benchmark-password", ParentUser: "absent-idp-parent", Expiration: UTCNow().Add(-time.Hour), Status: auth.AccountOn}}
+					if err := sys.store.saveIAMConfig(ctx, &u, getUserIdentityPath(key, svcUser)); err != nil {
+						b.Fatal(err)
+					}
+				}
+				b.StartTimer()
+				if err := sys.store.LoadIAMCache(ctx, true); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

cmd/iam-replay_test.go

@@ -0,0 +1,168 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func TestReviewIAMRevokedUserReplay(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "review-revoked-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
+	created, err := globalIAMSys.CreateUser(ctx, user, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	policyAt, err := globalIAMSys.PolicyDBSet(ctx, user, "readwrite", regUser, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "review-bucket", ObjectName: "review-object"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("seed must allow object read")
+	}
+	if err := globalIAMSys.DeleteUser(ctx, user, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("deletion did not remove initial permission")
+	}
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Errorf("revoked user restored by an older replicated create, GetUserInfo error = %v", err)
+	}
+	if err := globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: user, UserType: int(regUser), Policy: "readwrite"}, policyAt); err != nil {
+		t.Fatal(err)
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Error("older replicated identity and policy events restored revoked S3 read permission")
+	}
+}
+
+func TestReviewIAMSourceTimestampOrder(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "review-ordered-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour)
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Fatalf("newer source deletion skipped after delayed creation, GetUserInfo error = %v", err)
+	}
+}
+
+// A user's old group grant must not return after deletion and deliberate recreation.
+func TestR3CandidateOldGroupReplayAfterRecreation(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user, group := "r3-group-member", "r3-granting-group"
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
+	peer := &globalSiteReplicationSys
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
+	add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
+	must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
+	must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("fixture must grant through group")
+	}
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("recreation must start without deleted group membership")
+	}
+	must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("old group event restored the deleted user's read grant after recreation and durable reload")
+	}
+}
+
+// The user delete is also a revocation of its earlier group memberships.
+func TestR3CandidateLateDeleteRetainsOldGroupGrant(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user, group := "r3-late-member", "r3-late-group"
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
+	peer := &globalSiteReplicationSys
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
+	must(peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin.Add(time.Minute)))
+	must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
+	if !globalIAMSys.IsAllowed(args) {
+		t.Fatal("fixture must grant through group")
+	}
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
+	must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+	must(globalIAMSys.store.LoadIAMCache(ctx, false))
+	if _, ok := globalIAMSys.GetUser(ctx, user); !ok {
+		t.Fatal("newer identity must survive")
+	}
+	if globalIAMSys.IsAllowed(args) {
+		t.Fatal("late user deletion retained the older group grant on the recreated identity")
+	}
+}

cmd/iam-replication-protocol.go

@@ -0,0 +1,321 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/policy"
+)
+
+const (
+	iamRevisionProtocol  = 1
+	iamRevisionPeerPath  = "/v3/site-replication/peer/iam-revisions"
+	iamUserBoundaryType  = "silo-user-revocation"
+	iamGroupBoundaryType = "silo-group-revocation"
+	maxIAMRevisionBatch  = 128
+)
+
+var iamRevisionInstance = mustGetUUID()
+
+type iamUserBoundary struct {
+	User   string    `json:"user"`
+	Before time.Time `json:"before"`
+}
+
+type iamGroupBoundary struct {
+	Group  string    `json:"group"`
+	Before time.Time `json:"before"`
+}
+
+// The server owns this additive protocol, without changing the client SDK or
+// overloading a policy/document field. Old servers reject the dedicated route
+// before applying any change that would lose revocation or member metadata.
+type iamReplicationItem struct {
+	madmin.SRIAMItem
+	GroupGrants     map[string]time.Time `json:"groupGrants,omitempty"`
+	GroupSnapshot   bool                 `json:"groupSnapshot,omitempty"`
+	UserRevocation  *iamUserBoundary     `json:"userRevocation,omitempty"`
+	GroupRevocation *iamGroupBoundary    `json:"groupRevocation,omitempty"`
+	RevokedBefore   time.Time            `json:"revokedBefore,omitempty"`
+}
+
+type iamRevisionBatch struct {
+	Version int                  `json:"version"`
+	Items   []iamReplicationItem `json:"items"`
+}
+
+type iamRevisionStatus struct {
+	Version  int    `json:"version"`
+	Node     string `json:"node"`
+	Instance string `json:"instance"`
+	Digest   string `json:"digest"`
+}
+
+type iamRevisionResponse struct {
+	iamRevisionStatus
+	Errors []string `json:"errors,omitempty"`
+}
+
+type iamRevisionBatchError struct{ failures []string }
+
+func (e *iamRevisionBatchError) Error() string {
+	return "IAM revision batch: " + strings.Join(e.failures, "; ")
+}
+
+type iamRevisionProgress struct {
+	Instances    map[string]string
+	Acknowledged map[string]string
+}
+
+type iamRevisionMetrics struct {
+	healFailures       atomic.Uint64
+	healLastSuccess    atomic.Int64
+	healDurationMillis atomic.Int64
+}
+
+func iamRevisionDigest(items map[string]iamRevision) string {
+	paths := make([]string, 0, len(items))
+	for path := range items {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	h := sha256.New()
+	for _, path := range paths {
+		r := items[path]
+		fmt.Fprintf(h, "%q %s %t %s\n", path, r.timestamp().UTC().Format(time.RFC3339Nano), r.Deleted, r.RevokedBefore.UTC().Format(time.RFC3339Nano))
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func (store *IAMStoreSys) iamRevisionStatus() iamRevisionStatus {
+	node := globalLocalNodeName
+	if node == "" {
+		node = "local"
+	}
+	return iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: iamRevisionInstance, Digest: store.revisionIndex().digest()}
+}
+
+func executeIAMRevisionRequest(ctx context.Context, client *madmin.AdminClient, method string, batch *iamRevisionBatch) (status iamRevisionStatus, err error) {
+	var content []byte
+	if batch != nil {
+		content, err = json.Marshal(batch)
+		if err != nil {
+			return status, err
+		}
+	}
+	resp, err := client.ExecuteMethod(ctx, method, madmin.RequestData{RelPath: iamRevisionPeerPath, QueryValues: url.Values{"api-version": {madmin.SiteReplAPIVersion}}, Content: content})
+	if resp != nil {
+		defer xhttp.DrainBody(resp.Body)
+	}
+	if err != nil {
+		return status, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		var remote madmin.ErrorResponse
+		if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&remote) == nil && remote.Code != "" {
+			return status, remote
+		}
+		return status, fmt.Errorf("IAM revision protocol requires upgraded peers: %s", resp.Status)
+	}
+	var response iamRevisionResponse
+	if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&response); err != nil {
+		return status, err
+	}
+	status = response.iamRevisionStatus
+	if status.Version != iamRevisionProtocol || status.Node == "" || status.Instance == "" || status.Digest == "" {
+		return status, errors.New("peer did not acknowledge the IAM revision protocol")
+	}
+	if len(response.Errors) != 0 {
+		return status, &iamRevisionBatchError{failures: response.Errors}
+	}
+	return status, nil
+}
+
+type (
+	iamRecordBoundaryKey struct{}
+	iamGroupSnapshotKey  struct{}
+)
+
+func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) {
+	out := iamReplicationItem{SRIAMItem: item}
+	if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil {
+		var key string
+		if item.SvcAccChange.Create != nil {
+			key = item.SvcAccChange.Create.AccessKey
+		} else if item.SvcAccChange.Update != nil {
+			key = item.SvcAccChange.Update.AccessKey
+		}
+		if key != "" {
+			r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(key, svcUser))
+			if err != nil {
+				return out, err
+			}
+			if r.Deleted || r.timestamp().After(item.UpdatedAt) {
+				return out, errIAMStaleUpdate
+			}
+			out.RevokedBefore = r.RevokedBefore
+		}
+	}
+	if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && !item.GroupInfo.UpdateReq.IsRemove {
+		out.GroupSnapshot = true
+		var gi GroupInfo
+		if err := globalIAMSys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath(item.GroupInfo.UpdateReq.Group)); err != nil {
+			return out, err
+		}
+		// The matching persisted snapshot carries member grant times. If a
+		// later write won before sending, propagate that whole newer state.
+		if gi.Deleted {
+			return out, errIAMStaleUpdate
+		}
+		out.UpdatedAt = gi.UpdatedAt
+		out.RevokedBefore = gi.RevokedBefore
+		out.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: item.GroupInfo.UpdateReq.Group, Status: madmin.GroupStatus(gi.Status)}}
+		cache := globalIAMSys.store.rlock()
+		out.GroupInfo.UpdateReq.Members = cache.effectiveGroupMembers(gi)
+		globalIAMSys.store.runlock()
+		out.GroupGrants = make(map[string]time.Time, len(out.GroupInfo.UpdateReq.Members))
+		for _, member := range out.GroupInfo.UpdateReq.Members {
+			out.GroupGrants[member] = gi.MemberGrants[member]
+		}
+	}
+	if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, getGroupInfoPath(item.GroupInfo.UpdateReq.Group))
+		if err != nil {
+			return out, err
+		}
+		if !r.Deleted && !r.RevokedBefore.IsZero() {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+			out.UpdatedAt = r.RevokedBefore
+		}
+	}
+	if item.Type == madmin.SRIAMItemIAMUser && item.IAMUser != nil {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(item.IAMUser.AccessKey, regUser))
+		if err != nil {
+			return out, err
+		}
+		if item.IAMUser.IsDeleteReq && !r.Deleted && !r.RevokedBefore.IsZero() {
+			out.Type = iamUserBoundaryType
+			out.IAMUser = nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+			out.UpdatedAt = r.RevokedBefore
+		} else if !item.IAMUser.IsDeleteReq {
+			if r.Deleted || r.timestamp().After(item.UpdatedAt) {
+				return out, errIAMStaleUpdate
+			}
+			out.RevokedBefore = r.RevokedBefore
+		}
+	}
+	return out, nil
+}
+
+func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error {
+	if item.GroupInfo != nil {
+		if item.GroupSnapshot {
+			ctx = context.WithValue(ctx, iamGroupSnapshotKey{}, true)
+		}
+		// A nil map also explicitly denotes unknown legacy grants. Do not
+		// turn an unrelated group edit into a new grant after a revocation.
+		ctx = withIAMGroupGrants(ctx, item.GroupGrants)
+	}
+	if !item.RevokedBefore.IsZero() {
+		if item.RevokedBefore.After(item.UpdatedAt) {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore)
+	}
+	switch item.Type {
+	case iamUserBoundaryType:
+		if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		return iamReplicationError(globalIAMSys.DeleteUser(withIAMReplicationTime(ctx, item.UserRevocation.Before), item.UserRevocation.User, true))
+	case iamGroupBoundaryType:
+		if item.GroupRevocation == nil || item.GroupRevocation.Group == "" || item.GroupRevocation.Before.IsZero() {
+			return errSRInvalidRequest(errInvalidArgument)
+		}
+		_, err := globalIAMSys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, item.GroupRevocation.Before), item.GroupRevocation.Group, nil)
+		return iamReplicationError(err)
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
+		}
+		p, err := policy.ParseConfig(bytes.NewReader(item.Policy))
+		if err != nil {
+			return err
+		}
+		if p.IsEmpty() {
+			p = nil
+		}
+		return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, p, item.UpdatedAt)
+	case madmin.SRIAMItemSvcAcc:
+		return globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
+	case madmin.SRIAMItemPolicyMapping:
+		return globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
+	case madmin.SRIAMItemSTSAcc:
+		return globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
+	case madmin.SRIAMItemIAMUser:
+		return globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
+	case madmin.SRIAMItemGroupInfo:
+		return globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
+	default:
+		return errSRInvalidRequest(errInvalidArgument)
+	}
+}
+
+func (a adminAPIHandlers) SRPeerIAMRevisions(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	if obj, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction); obj == nil {
+		return
+	}
+	var failures []string
+	if r.Method == http.MethodPut {
+		var batch iamRevisionBatch
+		if err := parseJSONBody(ctx, r.Body, &batch, ""); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if batch.Version != iamRevisionProtocol || len(batch.Items) == 0 || len(batch.Items) > maxIAMRevisionBatch {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
+			return
+		}
+		for i, item := range batch.Items {
+			if err := applyIAMReplicationItem(ctx, item); err != nil {
+				failures = append(failures, fmt.Sprintf("item %d (%s): %v", i, item.Type, err))
+			}
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: globalIAMSys.store.iamRevisionStatus(), Errors: failures})
+}
+
+// A site endpoint can balance requests across nodes sharing durable IAM state.
+// Switching between known node incarnations preserves ACKs; a new incarnation
+// conservatively invalidates them so restoring an old backend cannot inherit
+// acknowledgements from before the restore.
+func (p *iamRevisionProgress) observePeer(status iamRevisionStatus) {
+	if p.Instances == nil {
+		p.Instances = make(map[string]string)
+	}
+	if p.Instances[status.Node] != status.Instance || p.Acknowledged == nil {
+		p.Instances[status.Node] = status.Instance
+		p.Acknowledged = make(map[string]string)
+	}
+}

cmd/iam-replication-protocol_test.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+func TestIAMRevisionProtocolDoesNotFallBackToLegacy(t *testing.T) {
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
+			t.Errorf("unsafe fallback path: %s", r.URL.Path)
+		}
+		w.WriteHeader(http.StatusNotFound)
+		_, _ = w.Write([]byte(`{"Code":"NotImplemented","Message":"old server"}`))
+	}))
+	defer server.Close()
+	client, err := madmin.New(strings.TrimPrefix(server.URL, "http://"), "test-access", "valid-test-secret", false)
+	mustIAM(t, err)
+	_, err = executeIAMRevisionRequest(context.Background(), client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{{SRIAMItem: madmin.SRIAMItem{Type: iamUserBoundaryType}, UserRevocation: &iamUserBoundary{User: "recreated", Before: UTCNow()}}}})
+	if err == nil || requests.Load() != 1 {
+		t.Fatalf("old peer must reject without fallback, err=%v requests=%d", err, requests.Load())
+	}
+}
+
+type iamNoHealingScanStore struct{ IAMStorageAPI }
+
+func (s *iamNoHealingScanStore) listIAMConfigPaths(context.Context) ([]string, error) {
+	panic("healing must use the loaded revision index")
+}
+
+func TestIAMRevisionHealingAcknowledgements(t *testing.T) {
+	for _, balanced := range []bool{false, true} {
+		t.Run(fmt.Sprintf("load_balanced_%t", balanced), func(t *testing.T) { testIAMRevisionHealingAcknowledgements(t, balanced) })
+	}
+}
+
+func testIAMRevisionHealingAcknowledgements(t *testing.T, balanced bool) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "ack-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	for i := range maxIAMRevisionBatch*2 + 1 {
+		at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
+		mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath(fmt.Sprintf("ack-%04d", i), regUser)))
+	}
+	sys.store.IAMStorageAPI = &iamNoHealingScanStore{IAMStorageAPI: sys.store.IAMStorageAPI}
+	var mu sync.Mutex
+	var puts, gets int
+	var applied int
+	instance := "boot-1"
+	failSecondBatch := true
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		mu.Lock()
+		defer mu.Unlock()
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
+			t.Errorf("unexpected request: %s", r.URL.Path)
+			w.WriteHeader(404)
+			return
+		}
+		var failures []string
+		if r.Method == http.MethodGet {
+			gets++
+		} else {
+			puts++
+			var batch iamRevisionBatch
+			if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+				t.Error(err)
+				w.WriteHeader(400)
+				return
+			}
+			if len(batch.Items) > maxIAMRevisionBatch {
+				t.Error("batch exceeds limit")
+			}
+			if failSecondBatch && puts == 2 {
+				failures = []string{"injected item error"}
+			} else {
+				applied += len(batch.Items)
+			}
+		}
+		node := "node-1"
+		if balanced {
+			node = fmt.Sprintf("node-%d", (gets+puts)%2+1)
+		}
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: node + instance, Digest: fmt.Sprintf("%d", applied)}, Errors: failures})
+	}))
+	defer server.Close()
+	c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "ack-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
+	if err := c.healIAMDeletions(ctx); err == nil {
+		t.Fatal("item failure was hidden")
+	}
+	mu.Lock()
+	if puts != 3 || applied != maxIAMRevisionBatch+1 {
+		t.Errorf("failed middle batch blocked later revocations: puts=%d applied=%d", puts, applied)
+	}
+	failSecondBatch = false
+	applied++ // Unrelated remote mutation changes its digest.
+	mu.Unlock()
+	at := UTCNow()
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath("ack-new-local", regUser)))
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	if puts != 5 {
+		t.Errorf("did not resume at unacknowledged batch: puts=%d", puts)
+	}
+	mu.Unlock()
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	if puts != 5 || gets != 3 {
+		t.Errorf("converged records were replayed: puts=%d gets=%d", puts, gets)
+	}
+	instance = "boot-2"
+	mu.Unlock()
+	mustIAM(t, c.healIAMDeletions(ctx))
+	mu.Lock()
+	defer mu.Unlock()
+	if puts != 8 {
+		t.Fatalf("peer restart reused an old acknowledgement: puts=%d", puts)
+	}
+}
+
+func TestIAMRevisionIndexRebuildsFromStorage(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	const user = "index-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-parent-password", Status: madmin.AccountEnabled}
+	_, err := sys.CreateUser(ctx, user, req)
+	mustIAM(t, err)
+	mustIAM(t, sys.DeleteUser(ctx, user, false))
+	before := sys.store.revisionIndex().snapshot()
+	store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+	mustIAM(t, store.LoadIAMCache(ctx, true))
+	if iamRevisionDigest(before) != iamRevisionDigest(store.revisionIndex().snapshot()) {
+		t.Fatal("ordinary IAM loading did not restore the deletion index")
+	}
+	_, err = store.AddUser(ctx, user, req)
+	mustIAM(t, err)
+	r := store.revisionIndex().get(getUserIdentityPath(user, regUser))
+	if r.Deleted || r.RevokedBefore.IsZero() {
+		t.Fatal("recreation discarded the retained boundary")
+	}
+	if r.Credentials.SecretKey != "" || r.Credentials.SessionToken != "" {
+		t.Fatal("index retained credentials")
+	}
+}

cmd/iam-revision-boundary_test.go

@@ -0,0 +1,393 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+	"github.com/pgsty/silo-pkg/v3/policy"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+func prepareIAMRevisionFixture(t testing.TB, backend ...string) (context.Context, *IAMSys, ObjectLayer) {
+	t.Helper()
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	disks, err := getRandomDisks(1)
+	mustIAM(t, err)
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	mustIAM(t, err)
+	initAllSubsystems(ctx)
+	// Deliberately omit the periodic refresh goroutine. Fault injection can
+	// replace this fixture's storage interface without racing initialization.
+	var client *etcd.Client
+	if len(backend) != 0 && backend[0] == "etcd" {
+		endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+		if endpoint == "" {
+			cancel()
+			obj.Shutdown(context.Background())
+			os.RemoveAll(disks[0])
+			t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+		}
+		client, err = etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
+		mustIAM(t, err)
+		prefix := fmt.Sprintf("/silo-boundary-test/%d/", time.Now().UnixNano())
+		client.KV = namespace.NewKV(client.KV, prefix)
+		client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
+		t.Cleanup(func() { client.Delete(context.Background(), "", etcd.WithPrefix()); client.Close() })
+	}
+	globalIAMSys.initStore(obj, client)
+	mustIAM(t, globalIAMSys.Load(ctx, true))
+	t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
+	return ctx, globalIAMSys, obj
+}
+
+func mustIAM(t testing.TB, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+var errIAMInjectedWrite = errors.New("injected IAM persistence failure")
+
+type iamFailingCleanupStore struct {
+	IAMStorageAPI
+	parentPath   string
+	beforeCommit bool
+}
+
+func (s *iamFailingCleanupStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
+	if s.beforeCommit || path != s.parentPath {
+		return errIAMInjectedWrite
+	}
+	return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
+}
+
+func TestIAMRevocationCommitBoundary(t *testing.T) {
+	for _, before := range []bool{true, false} {
+		name := "after_identity_commit"
+		if before {
+			name = "before_identity_commit"
+		}
+		t.Run(name, func(t *testing.T) {
+			ctx, sys, obj := prepareIAMRevisionFixture(t)
+			const user = "commit-boundary-user"
+			origin := UTCNow().Add(-time.Hour)
+			req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+			_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
+			mustIAM(t, err)
+			_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(time.Minute)), user, "readwrite", regUser, false)
+			mustIAM(t, err)
+			_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(time.Minute)), "commit-group", []string{user})
+			mustIAM(t, err)
+			_, err = sys.PolicyDBSet(ctx, "commit-group", "readwrite", regUser, true)
+			mustIAM(t, err)
+			child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), user, nil, newServiceAccountOpts{accessKey: "commit-child", secretKey: "valid-child-password"})
+			mustIAM(t, err)
+			args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+			if !sys.IsAllowed(args) {
+				t.Fatal("fixture has no grant")
+			}
+			siblingStore := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			mustIAM(t, siblingStore.LoadIAMCache(ctx, true))
+			sibling := &IAMSys{store: siblingStore, usersSysType: MinIOUsersSysType}
+			tg, err := grid.SetupTestGrid(2)
+			mustIAM(t, err)
+			defer tg.Cleanup()
+			var notifications atomic.Int32
+			mustIAM(t, deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+				notifications.Add(1)
+				if err := sibling.LoadUserAfterDelete(ctx, r.Get(peerRESTUser)); err != nil {
+					return grid.NoPayload{}, grid.NewRemoteErr(err)
+				}
+				return grid.NoPayload{}, nil
+			}))
+			host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+			mustIAM(t, err)
+			globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{host: host, gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}}}
+			original := sys.store.IAMStorageAPI
+			sys.store.IAMStorageAPI = &iamFailingCleanupStore{IAMStorageAPI: original, parentPath: getUserIdentityPath(user, regUser), beforeCommit: before}
+			boundary := origin.Add(2 * time.Minute)
+			err = sys.DeleteUser(withIAMReplicationTime(ctx, boundary), user, true)
+			if !errors.Is(err, errIAMInjectedWrite) {
+				t.Fatalf("expected write failure, got %v", err)
+			}
+			sys.store.IAMStorageAPI = original
+			r, err := loadIAMRevision(ctx, original, getUserIdentityPath(user, regUser))
+			mustIAM(t, err)
+			if before {
+				if r.Deleted || !sys.IsAllowed(args) || !sibling.IsAllowed(args) || notifications.Load() != 0 {
+					t.Fatal("failure before commit changed the identity or grant")
+				}
+				return
+			}
+			if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
+				t.Fatal("cleanup failure lost durable revocation")
+			}
+			if sys.IsAllowed(args) || sibling.IsAllowed(args) || notifications.Load() != 1 {
+				t.Fatal("cleanup failure retained old permission")
+			}
+			// Subsequent fixture writes need no additional RPC handlers.
+			globalNotificationSys = &NotificationSys{}
+			// Recreate after the partial cleanup. The old mapping, group member
+			// and child still exist in storage; none may authorize this identity.
+			_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), user, req)
+			mustIAM(t, err)
+			reloaded := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			mustIAM(t, reloaded.LoadIAMCache(ctx, true))
+			fresh := &IAMSys{store: reloaded, usersSysType: MinIOUsersSysType}
+			if fresh.IsAllowed(args) {
+				t.Fatal("cold reload restored partially cleaned-up grants")
+			}
+			if _, ok := reloaded.GetUser(child.AccessKey); ok {
+				t.Fatal("cold reload restored the old child")
+			}
+			gd, err := reloaded.GetGroupDescription("commit-group")
+			mustIAM(t, err)
+			if len(gd.Members) != 0 {
+				t.Fatalf("listing exposed a revoked group relation: %v", gd.Members)
+			}
+			_, err = sys.AddUsersToGroup(ctx, "commit-group", []string{user})
+			mustIAM(t, err)
+			if !sys.IsAllowed(args) {
+				t.Fatal("explicit new group grant was not accepted")
+			}
+		})
+	}
+}
+
+func TestIAMGroupGrantVersionsSurviveSnapshotsAndRecreation(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	for _, user := range []string{"grant-alice", "grant-bob"} {
+		_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
+		mustIAM(t, err)
+	}
+	grant := origin.Add(time.Minute)
+	_, err := sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), "grant-group", []string{"grant-alice"})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(ctx, "grant-group", "readwrite", regUser, true)
+	mustIAM(t, err)
+	boundary := origin.Add(2 * time.Minute)
+	mustIAM(t, sys.DeleteUser(withIAMReplicationTime(ctx, boundary), "grant-alice", false))
+	_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), "grant-alice", req)
+	mustIAM(t, err)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), "grant-group", []string{"grant-bob"})
+	mustIAM(t, err)
+	_, err = sys.SetGroupStatus(withIAMReplicationTime(ctx, origin.Add(5*time.Minute)), "grant-group", true)
+	mustIAM(t, err)
+	var gi GroupInfo
+	mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
+	if !gi.MemberGrants["grant-alice"].Equal(grant) {
+		t.Fatal("unrelated group edits refreshed an old grant")
+	}
+	args := policy.Args{AccountName: "grant-alice", Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	for _, stale := range []time.Time{grant, boundary, {}} {
+		item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(6 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: "grant-group", Members: []string{"grant-alice", "grant-bob"}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{"grant-alice": stale, "grant-bob": origin.Add(4 * time.Minute)}}
+		mustIAM(t, applyIAMReplicationItem(ctx, item))
+		mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+		if sys.IsAllowed(args) {
+			t.Fatalf("snapshot restored revoked grant %s", stale)
+		}
+		gd, err := sys.GetGroupDescription("grant-group")
+		mustIAM(t, err)
+		if len(gd.Members) != 1 || gd.Members[0] != "grant-bob" {
+			t.Fatalf("inconsistent effective members: %v", gd.Members)
+		}
+	}
+	// Only an explicit post-revocation grant restores access.
+	freshAt, err := sys.AddUsersToGroup(ctx, "grant-group", []string{"grant-alice"})
+	mustIAM(t, err)
+	if !sys.IsAllowed(args) {
+		t.Fatal("explicit regrant rejected")
+	}
+	mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+	mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
+	if !gi.MemberGrants["grant-alice"].Equal(freshAt) {
+		t.Fatal("new grant version was not persisted")
+	}
+	if !gi.MemberGrants["grant-bob"].Equal(origin.Add(4 * time.Minute)) {
+		t.Fatal("regranting Alice changed Bob's grant")
+	}
+}
+
+func TestIAMGroupRevocationCommitAndRecreation(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) { testIAMGroupRevocationCommitAndRecreation(t, backend) })
+	}
+}
+
+func testIAMGroupRevocationCommitAndRecreation(t *testing.T, backend string) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
+	origin := UTCNow().Add(-time.Hour)
+	user, group := "group-boundary-user", "group-boundary"
+	_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	grant, boundary := origin.Add(time.Minute), origin.Add(2*time.Minute)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
+	mustIAM(t, err)
+	// A newer mapping must not veto the authoritative group deletion.
+	_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, boundary), group, nil)
+	mustIAM(t, err)
+	r, err := loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
+	mustIAM(t, err)
+	if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
+		t.Fatal("newer mapping swallowed group deletion")
+	}
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), group, nil)
+	mustIAM(t, err)
+	for _, at := range []time.Time{grant, boundary, {}} {
+		item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(5 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{user: at}}
+		mustIAM(t, applyIAMReplicationItem(ctx, item))
+		gd, err := sys.GetGroupDescription(group)
+		mustIAM(t, err)
+		if len(gd.Members) != 0 {
+			t.Fatalf("group recreation restored grant %s", at)
+		}
+	}
+	_, err = sys.AddUsersToGroup(ctx, group, []string{user})
+	mustIAM(t, err)
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	if !sys.IsAllowed(args) {
+		t.Fatal("explicit group regrant was rejected")
+	}
+	// The newer live snapshot may arrive before an older group deletion.
+	lateBoundary := origin.Add(6 * time.Minute)
+	_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, lateBoundary), group, nil)
+	mustIAM(t, err)
+	r, err = loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
+	mustIAM(t, err)
+	if r.Deleted || !r.RevokedBefore.Equal(lateBoundary) {
+		t.Fatal("late deletion lost the live group's revocation boundary")
+	}
+	// The old mapping is now revoked; a new explicit mapping restores access.
+	if sys.IsAllowed(args) {
+		t.Fatal("late group boundary retained an old mapping")
+	}
+	_, err = sys.PolicyDBSet(ctx, group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+	if es, ok := sys.store.IAMStorageAPI.(*IAMEtcdStore); ok {
+		store.IAMStorageAPI = newIAMEtcdStore(es.client, MinIOUsersSysType)
+	}
+	mustIAM(t, store.LoadIAMCache(ctx, true))
+	fresh := &IAMSys{store: store, usersSysType: MinIOUsersSysType}
+	if !fresh.IsAllowed(args) {
+		t.Fatal("reload lost explicit grants after a retained group boundary")
+	}
+	item, err := globalSiteReplicationSys.replicationItem(ctx, madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group}}, UpdatedAt: r.timestamp()})
+	mustIAM(t, err)
+	if !item.RevokedBefore.Equal(lateBoundary) || !item.GroupGrants[user].After(lateBoundary) {
+		t.Fatal("group snapshot lost revision metadata")
+	}
+}
+
+// A committed revision is observable before all cached dependents have been
+// cleaned up. Every authorization read must apply that boundary in this window.
+func TestIAMCachedMappingHonorsCommittedRevision(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	parent := "cached-external-parent"
+	_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), parent, "readwrite", stsUser, false)
+	mustIAM(t, err)
+	policies, err := sys.PolicyDBGet(parent)
+	mustIAM(t, err)
+	if len(policies) == 0 {
+		t.Fatal("fixture has no STS-parent mapping")
+	}
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Deleted: true, UpdatedAt: origin.Add(time.Minute)}, getMappedPolicyPath(parent, stsUser, false)))
+	policies, err = sys.PolicyDBGet(parent)
+	mustIAM(t, err)
+	if len(policies) != 0 {
+		t.Fatal("cached STS mapping ignored its own namespace tombstone")
+	}
+
+	user, group := "cached-group-user", "cached-group"
+	_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	grant := origin.Add(5 * time.Minute)
+	_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), group, "readwrite", regUser, true)
+	mustIAM(t, err)
+	args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
+	if !sys.IsAllowed(args) {
+		t.Fatal("fixture has no group grant")
+	}
+	// A late deletion preserves the newer member grant but revokes the older
+	// policy mapping. Simulate the interval before mapping cleanup completes.
+	gi := GroupInfo{Version: 1, Status: statusEnabled, Members: []string{user}, MemberGrants: map[string]time.Time{user: grant}, UpdatedAt: grant, RevokedBefore: origin.Add(2 * time.Minute)}
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &gi, getGroupInfoPath(group)))
+	if sys.IsAllowed(args) {
+		t.Fatal("cached group mapping ignored the committed group boundary")
+	}
+	gd, err := sys.GetGroupDescription(group)
+	mustIAM(t, err)
+	if gd.Policy != "" {
+		t.Fatal("group listing exposed a revoked mapping")
+	}
+}
+
+type (
+	iamExpiryLockFailure struct {
+		ObjectLayer
+		path string
+	}
+	iamFailedExpiryLock struct{ RWLocker }
+)
+
+func (o *iamExpiryLockFailure) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path+".revision-lock" {
+		return &iamFailedExpiryLock{RWLocker: lock}
+	}
+	return lock
+}
+
+func (l *iamFailedExpiryLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) {
+	return LockContext{}, errIAMInjectedWrite
+}
+
+func TestIAMExpiredCredentialCleanupDoesNotBlockLoading(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "healthy-user", madmin.AddOrUpdateUserReq{SecretKey: "healthy-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	_, err = sys.PolicyDBSet(ctx, "healthy-user", "readwrite", regUser, false)
+	mustIAM(t, err)
+	c, _, err := sys.NewServiceAccount(ctx, "healthy-user", nil, newServiceAccountOpts{accessKey: "expired-service", secretKey: "expired-service-password"})
+	mustIAM(t, err)
+	c.Expiration = UTCNow().Add(-time.Hour)
+	path := getUserIdentityPath(c.AccessKey, svcUser)
+	mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: c, UpdatedAt: UTCNow()}, path))
+	// A cold loader sees the existing version but cannot acquire the cleanup
+	// write lock. Healthy users must still load; the expired one stays denied.
+	fresh := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(&iamExpiryLockFailure{ObjectLayer: obj, path: path}, MinIOUsersSysType)}
+	mustIAM(t, fresh.LoadIAMCache(ctx, true))
+	if _, ok := fresh.GetUser("healthy-user"); !ok {
+		t.Fatal("cleanup failure prevented healthy IAM state from loading")
+	}
+	if _, ok := fresh.GetUser(c.AccessKey); ok {
+		t.Fatal("cleanup failure admitted an expired service account")
+	}
+	r, err := loadIAMRevision(ctx, fresh, path)
+	mustIAM(t, err)
+	if r.Deleted || !r.Credentials.IsExpired() {
+		t.Fatal("failed cleanup lost the existing expired revision")
+	}
+}

cmd/iam-revision-index.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+// This index is rebuilt by the existing IAM loaders and updated by successful
+// storage operations. It avoids a second full IAM walk during every heal pass.
+// It is an optimization of the durable records, never a reason to delete them.
+// The index contains no secrets or grants.
+type iamParentRevision struct {
+	deleted bool
+	before  time.Time
+}
+
+type iamRevisionIndex struct {
+	mu         sync.RWMutex
+	items      map[string]iamRevision
+	parents    map[string]iamParentRevision
+	floors     map[string]time.Time
+	generation uint64
+}
+
+func (idx *iamRevisionIndex) observe(path string, data []byte) {
+	if !strings.HasPrefix(path, iamConfigPrefix+"/") {
+		return
+	}
+	var r iamRevision
+	if json.Unmarshal(data, &r) != nil {
+		return // The caller reports malformed data using its normal decoder.
+	}
+	r.Credentials = auth.Credentials{ParentUser: r.Credentials.ParentUser, Expiration: r.Credentials.Expiration}
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+	if strings.HasPrefix(path, iamConfigUsersPrefix) {
+		// Keep a compact name-keyed view for the authentication hot path;
+		// constructing a config path on every S3 request allocates needlessly.
+		defer func() {
+			name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+			if current, ok := idx.items[path]; ok {
+				if idx.parents == nil {
+					idx.parents = make(map[string]iamParentRevision)
+				}
+				idx.parents[name] = iamParentRevision{deleted: current.Deleted, before: current.RevokedBefore}
+			} else {
+				delete(idx.parents, name)
+			}
+		}()
+	}
+	if floor, ok := idx.floors[path]; ok && r.timestamp().Before(floor) {
+		return
+	}
+	if previous, ok := idx.items[path]; ok {
+		// A concurrent read that began before a write must not roll it back.
+		if previous.timestamp().After(r.timestamp()) || (previous.Deleted && !r.Deleted && !r.timestamp().After(previous.timestamp())) {
+			return
+		}
+		if previous.RevokedBefore.After(r.RevokedBefore) {
+			r.RevokedBefore = previous.RevokedBefore
+		}
+		if previous.timestamp().Equal(r.timestamp()) && previous.Deleted == r.Deleted && previous.RevokedBefore.Equal(r.RevokedBefore) {
+			return
+		}
+	}
+	if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
+		if _, tracked := idx.items[path]; tracked {
+			delete(idx.items, path)
+			idx.generation++
+		}
+		delete(idx.floors, path)
+		return
+	}
+	if !r.Deleted && r.RevokedBefore.IsZero() {
+		_, tracked := idx.items[path]
+		_, hasFloor := idx.floors[path]
+		if tracked || hasFloor {
+			if idx.floors == nil {
+				idx.floors = make(map[string]time.Time)
+			}
+			idx.floors[path] = r.timestamp()
+		}
+		if tracked {
+			delete(idx.items, path)
+			idx.generation++
+		}
+		return
+	}
+	if idx.items == nil {
+		idx.items = make(map[string]iamRevision)
+	}
+	idx.items[path] = r
+	delete(idx.floors, path)
+	idx.generation++
+}
+
+func (idx *iamRevisionIndex) get(path string) iamRevision {
+	if idx == nil {
+		return iamRevision{}
+	}
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return idx.items[path]
+}
+
+func (idx *iamRevisionIndex) snapshot() map[string]iamRevision {
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+	for path, r := range idx.items {
+		if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
+			delete(idx.items, path)
+			delete(idx.floors, path)
+			idx.generation++
+		}
+	}
+	return maps.Clone(idx.items)
+}
+
+func (idx *iamRevisionIndex) count() int {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return len(idx.items)
+}
+
+// A process-local generation plus the protocol's instance ID is sufficient
+// for acknowledgements. Avoid hashing the entire index on every IAM write.
+func (idx *iamRevisionIndex) digest() string {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return fmt.Sprintf("%x:%x", idx.generation, len(idx.items))
+}
+
+func (idx *iamRevisionIndex) forget(path string) {
+	idx.mu.Lock()
+	if _, ok := idx.items[path]; ok {
+		delete(idx.items, path)
+		idx.generation++
+	}
+	delete(idx.floors, path)
+	if strings.HasPrefix(path, iamConfigUsersPrefix) {
+		delete(idx.parents, strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile))
+	}
+	idx.mu.Unlock()
+}
+
+func (c *iamCache) userRevocation(user string) iamRevision {
+	r := c.revisions.parentRevision(user)
+	if u, ok := c.iamUsersMap[user]; ok && u.RevokedBefore.After(r.RevokedBefore) {
+		r.RevokedBefore = u.RevokedBefore
+	}
+	return r
+}
+
+func (c *iamCache) groupMemberAllowed(member string, grantedAt, groupBoundary time.Time) bool {
+	r := c.userRevocation(member)
+	return !r.Deleted && (r.RevokedBefore.IsZero() || grantedAt.After(r.RevokedBefore)) && (groupBoundary.IsZero() || grantedAt.After(groupBoundary))
+}
+
+func iamMappingParentPath(path string) string {
+	kind, name, ok := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+	if !ok {
+		return ""
+	}
+	name = strings.TrimSuffix(name, ".json")
+	switch kind {
+	case "users", "sts-users":
+		return getUserIdentityPath(name, regUser)
+	case "service-accounts":
+		return getUserIdentityPath(name, svcUser)
+	case "groups":
+		return getGroupInfoPath(name)
+	}
+	return ""
+}
+
+func (idx *iamRevisionIndex) mappingAllowed(path string, mp MappedPolicy) bool {
+	if mp.Deleted || idx.get(path).Deleted {
+		return false
+	}
+	r := idx.get(iamMappingParentPath(path))
+	return !r.Deleted && (r.RevokedBefore.IsZero() || mp.UpdatedAt.After(r.RevokedBefore))
+}
+
+// Apply the persisted commit boundary even before dependent cache cleanup has
+// completed. The map namespace is part of the authorization record's identity.
+func (c *iamCache) cachedMappedPolicy(name string, userType IAMUserType, isGroup bool) (MappedPolicy, bool) {
+	var mp MappedPolicy
+	var ok bool
+	switch {
+	case isGroup:
+		mp, ok = c.iamGroupPolicyMap.Load(name)
+	case userType == stsUser:
+		mp, ok = c.iamSTSPolicyMap.Load(name)
+	default:
+		mp, ok = c.iamUserPolicyMap.Load(name)
+	}
+	if !ok || !c.revisions.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), mp) {
+		return MappedPolicy{}, false
+	}
+	return mp, true
+}
+
+func (idx *iamRevisionIndex) parentRevision(user string) iamRevision {
+	if idx == nil {
+		return iamRevision{}
+	}
+	idx.mu.RLock()
+	p := idx.parents[user]
+	idx.mu.RUnlock()
+	return iamRevision{Deleted: p.deleted, RevokedBefore: p.before}
+}

cmd/iam-revision-lock_test.go

@@ -0,0 +1,305 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+type iamRevisionLockObserver struct {
+	ObjectLayer
+	path    string
+	waiting chan struct{}
+	once    sync.Once
+}
+
+func (o *iamRevisionLockObserver) NewNSLock(bucket string, objects ...string) RWLocker {
+	lock := o.ObjectLayer.NewNSLock(bucket, objects...)
+	if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path {
+		return &iamRevisionObservedLock{RWLocker: lock, observe: func() { o.once.Do(func() { close(o.waiting) }) }}
+	}
+	return lock
+}
+
+type iamRevisionObservedLock struct {
+	RWLocker
+	observe func()
+}
+
+func (l *iamRevisionObservedLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	l.observe()
+	return l.RWLocker.GetLock(ctx, timeout)
+}
+
+type iamRevisionWatchObserver struct {
+	etcd.Watcher
+	waiting chan struct{}
+	once    sync.Once
+}
+
+func (w *iamRevisionWatchObserver) Watch(ctx context.Context, key string, opts ...etcd.OpOption) etcd.WatchChan {
+	w.once.Do(func() { close(w.waiting) })
+	return w.Watcher.Watch(ctx, key, opts...)
+}
+
+// Simulate an unavailable cleanup RPC. Mutex.Lock calls Delete after its wait
+// is canceled; that RPC must inherit a deadline too, not Client.Ctx() forever.
+type iamRevisionCleanupBlocker struct {
+	etcd.KV
+	release chan struct{}
+}
+
+func (b *iamRevisionCleanupBlocker) Delete(ctx context.Context, key string, opts ...etcd.OpOption) (*etcd.DeleteResponse, error) {
+	if strings.Contains(key, "/iam-revision-locks/") {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-b.release:
+		}
+	}
+	return b.KV.Delete(ctx, key, opts...)
+}
+
+type iamRevisionReadBlocker struct {
+	IAMStorageAPI
+	path    string
+	after   int
+	waiting chan struct{}
+}
+
+func (b *iamRevisionReadBlocker) loadIAMConfig(ctx context.Context, item any, path string) error {
+	if path == b.path {
+		b.after--
+		if b.after == 0 {
+			close(b.waiting)
+			<-ctx.Done()
+			return ctx.Err()
+		}
+	}
+	return b.IAMStorageAPI.loadIAMConfig(ctx, item, path)
+}
+
+func TestIAMRevisionReadDoesNotBlockAuthentication(t *testing.T) {
+	for _, stage := range []struct {
+		name   string
+		offset time.Duration
+	}{{"deletion", time.Minute}, {"retained_revocation", -time.Minute}} {
+		t.Run(stage.name, func(t *testing.T) {
+			resetTestGlobals()
+			t.Cleanup(resetTestGlobals)
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			disks, err := getRandomDisks(1)
+			if err != nil {
+				t.Fatal(err)
+			}
+			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() {
+				obj.Shutdown(context.Background())
+				os.RemoveAll(disks[0])
+			})
+			store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+			const user = "read-blocked-parent"
+			created, err := store.AddUser(ctx, user, madmin.AddOrUpdateUserReq{SecretKey: "original-password", Status: madmin.AccountEnabled})
+			if err != nil {
+				t.Fatal(err)
+			}
+			blocked := &iamRevisionReadBlocker{IAMStorageAPI: store.IAMStorageAPI, path: getUserIdentityPath(user, regUser), after: 1, waiting: make(chan struct{})}
+			store.IAMStorageAPI = blocked
+			done := make(chan error, 1)
+			go func() {
+				done <- store.DeleteUser(withIAMReplicationTime(ctx, created.Add(stage.offset)), user, regUser)
+			}()
+			defer func() { cancel(); <-done }()
+			select {
+			case <-blocked.waiting:
+			case <-time.After(5 * time.Second):
+				t.Fatal("revision read was not attempted")
+			}
+			read := make(chan bool, 1)
+			go func() {
+				u, ok := store.GetUser(user)
+				read <- ok && u.Credentials.SecretKey == "original-password"
+			}()
+			select {
+			case ok := <-read:
+				if !ok {
+					t.Fatal("pending revision read changed the cached identity")
+				}
+			case <-time.After(time.Second):
+				t.Fatal("revision read blocked cached authentication")
+			}
+		})
+	}
+}
+
+func TestIAMRevisionLockContention(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+			if backend == "etcd" && endpoint == "" {
+				t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+			}
+			for _, outcome := range []string{"release", "cancel", "default_timeout"} {
+				t.Run(outcome, func(t *testing.T) {
+					resetTestGlobals()
+					t.Cleanup(resetTestGlobals)
+					ctx, cancel := context.WithCancel(context.Background())
+					defer cancel()
+					oldTimeout := defaultContextTimeout
+					defaultContextTimeout = 2 * time.Second
+					t.Cleanup(func() { defaultContextTimeout = oldTimeout })
+					must := func(err error) {
+						t.Helper()
+						if err != nil {
+							t.Fatal(err)
+						}
+					}
+					const user = "contended-user"
+					path := getUserIdentityPath(user, regUser)
+					waiting := make(chan struct{})
+					var store *IAMStoreSys
+					var hold func() func()
+					unblockCleanup := func() {}
+					if backend == "object" {
+						disks, err := getRandomDisks(1)
+						must(err)
+						obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+						must(err)
+						t.Cleanup(func() {
+							obj.Shutdown(context.Background())
+							os.RemoveAll(disks[0])
+						})
+						observed := &iamRevisionLockObserver{ObjectLayer: obj, path: path + ".revision-lock", waiting: waiting}
+						store = &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
+						hold = func() func() {
+							lock := obj.NewNSLock(minioMetaBucket, observed.path)
+							lc, err := lock.GetLock(ctx, newDynamicTimeout(time.Second, time.Second))
+							must(err)
+							store.IAMStorageAPI.(*IAMObjectStore).objAPI = observed
+							return func() { lock.Unlock(lc) }
+						}
+					} else {
+						client, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: time.Second})
+						must(err)
+						t.Cleanup(func() { client.Close() })
+						prefix := fmt.Sprintf("/silo-lock-test/%d/", time.Now().UnixNano())
+						client.KV = namespace.NewKV(client.KV, prefix)
+						client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
+						store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(client, MinIOUsersSysType)}
+						hold = func() func() {
+							session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+							must(err)
+							lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+							must(lock.Lock(ctx))
+							client.Watcher = &iamRevisionWatchObserver{Watcher: client.Watcher, waiting: waiting}
+							blocker := &iamRevisionCleanupBlocker{KV: client.KV, release: make(chan struct{})}
+							client.KV = blocker
+							unblockCleanup = sync.OnceFunc(func() { close(blocker.release) })
+							t.Cleanup(unblockCleanup)
+							return func() { session.Close() }
+						}
+					}
+					request := func(secret string) madmin.AddOrUpdateUserReq {
+						return madmin.AddOrUpdateUserReq{SecretKey: secret, Status: madmin.AccountEnabled}
+					}
+					_, err := store.AddUser(ctx, user, request("original-password"))
+					must(err)
+					release := sync.OnceFunc(hold())
+					t.Cleanup(release)
+					writeCtx, cancelWrite := context.WithCancel(ctx)
+					defer cancelWrite()
+					first, second := make(chan error, 1), make(chan error, 1)
+					var writers sync.WaitGroup
+					t.Cleanup(func() {
+						cancelWrite()
+						unblockCleanup()
+						release()
+						writers.Wait()
+					})
+					writers.Go(func() {
+						_, err := store.AddUser(writeCtx, user, request("first-password"))
+						first <- err
+					})
+					select {
+					case <-waiting:
+					case <-time.After(5 * time.Second):
+						t.Fatal("writer did not attempt the held revision lock")
+					}
+					// A second writer must queue without taking the cache's RWMutex:
+					// Go's writer preference would otherwise block every new reader.
+					writers.Go(func() {
+						_, err := store.AddUser(ctx, user, request("second-password"))
+						second <- err
+					})
+					select {
+					case err := <-second:
+						t.Fatalf("second writer bypassed the first: %v", err)
+					case <-time.After(50 * time.Millisecond):
+					}
+					read := make(chan UserIdentity, 1)
+					go func() {
+						u, _ := store.GetUser(user)
+						read <- u
+					}()
+					select {
+					case u := <-read:
+						if u.Credentials.SecretKey != "original-password" {
+							t.Fatal("pending write changed the cached credential")
+						}
+					case <-time.After(time.Second):
+						t.Fatal("distributed lock contention blocked cached authentication")
+					}
+					switch outcome {
+					case "release":
+						release()
+					case "cancel":
+						cancelWrite()
+					}
+					select {
+					case err := <-first:
+						if outcome == "release" {
+							must(err)
+						} else if err == nil {
+							t.Fatal("canceled or timed-out write succeeded")
+						}
+					case <-time.After(5 * time.Second):
+						t.Fatal("lock wait or cancellation cleanup exceeded its deadline")
+					}
+					release()
+					select {
+					case err := <-second:
+						must(err)
+					case <-time.After(5 * time.Second):
+						t.Fatal("queued writer did not recover after the first completed")
+					}
+					cached, ok := store.GetUser(user)
+					if !ok || cached.Credentials.SecretKey != "second-password" {
+						t.Fatal("cached write order was lost")
+					}
+					var persisted UserIdentity
+					must(store.loadIAMConfig(ctx, &persisted, path))
+					if persisted.Credentials.SecretKey != cached.Credentials.SecretKey || !persisted.UpdatedAt.Equal(cached.UpdatedAt) {
+						t.Fatal("persistent and cached revisions differ")
+					}
+				})
+			}
+		})
+	}
+}

cmd/iam-revision.go

@@ -0,0 +1,749 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+var errIAMStaleUpdate = errors.New("IAM update predates a stored revision or revocation")
+
+// The parent is still live; callers must not broadcast a user deletion when
+// only its revocation boundary was retained.
+var errIAMRevocationRetained = errors.New("IAM revocation recorded without deleting the record")
+
+// A revocation advances the boundary even when a newer identity already
+// exists. Keep this operation distinct from replacing/deleting that identity.
+type iamUserRevocation struct {
+	UserIdentity
+	retained bool
+}
+
+type iamGroupRevocation struct {
+	GroupInfo
+	retained     bool
+	requireEmpty bool
+}
+
+// Natural expiration is distinct from revoking a live credential. An expired
+// immutable STS token can be removed; a reusable service-account key retains
+// its revision so an older non-expiring credential cannot return.
+type iamExpireIdentity struct{}
+
+// The authoritative revocation is durable even if dependent cleanup fails.
+// Callers must publish it to sibling caches before returning the error.
+type iamCommittedCleanupError struct {
+	err      error
+	retained bool
+}
+
+func (e *iamCommittedCleanupError) Error() string {
+	return "IAM revocation committed; cleanup failed: " + e.err.Error()
+}
+func (e *iamCommittedCleanupError) Unwrap() error { return e.err }
+
+type iamReplicationTimeKey struct{}
+
+func withIAMReplicationTime(ctx context.Context, at time.Time) context.Context {
+	return context.WithValue(ctx, iamReplicationTimeKey{}, at)
+}
+
+func iamReplicationTime(ctx context.Context) (time.Time, bool) {
+	at, ok := ctx.Value(iamReplicationTimeKey{}).(time.Time)
+	return at, ok
+}
+
+func iamReplicationError(err error) error {
+	if errors.Is(err, errIAMStaleUpdate) {
+		// Retrying an obsolete event cannot change the result.
+		return nil
+	}
+	return wrapSRErr(err)
+}
+
+// Deletions occupy the original IAM config path. They contain no secret or
+// grant and are hidden by the normal loaders, but remain available to heal
+// and to timestamp comparisons after a restart. Do not age them out: a peer
+// can be offline indefinitely.
+type iamRevision struct {
+	UpdatedAt     time.Time        `json:"updatedAt"`
+	UpdateDate    time.Time        `json:"UpdateDate"`
+	Deleted       bool             `json:"deleted"`
+	RevokedBefore time.Time        `json:"revokedBefore"`
+	ExpiresAt     time.Time        `json:"expiresAt,omitempty"`
+	Credentials   auth.Credentials `json:"credentials"`
+}
+
+func (r iamRevision) timestamp() time.Time {
+	if r.UpdateDate.After(r.UpdatedAt) {
+		return r.UpdateDate
+	}
+	return r.UpdatedAt
+}
+
+func loadIAMRevision(ctx context.Context, store IAMStorageAPI, path string) (iamRevision, error) {
+	var r iamRevision
+	err := store.loadIAMConfig(ctx, &r, path)
+	if errors.Is(err, errConfigNotFound) {
+		err = nil
+	}
+	return r, err
+}
+
+func (store *IAMStoreSys) checkIAMRevision(ctx context.Context, path string, deleting bool) error {
+	at, replicated := iamReplicationTime(ctx)
+	if !replicated {
+		return nil
+	}
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		r, err := loadIAMRevision(ctx, store.IAMStorageAPI, path)
+		if err != nil {
+			return err
+		}
+		if r.timestamp().After(at) || (r.Deleted && !deleting && !at.After(r.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		return nil
+	})
+}
+
+// This signed claim records the parent's revocation boundary at issuance.
+// Unlike UpdatedAt, it cannot advance when an offline site edits an old child.
+// It travels in the existing service-account Claims and STS SessionToken fields.
+const iamParentRevocationClaim = "siloParentRevocation"
+
+func setIAMParentRevocationClaim(ctx context.Context, store IAMStorageAPI, parent string, claims map[string]any) error {
+	delete(claims, iamParentRevocationClaim)
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted {
+		return errIAMStaleUpdate
+	}
+	if !r.RevokedBefore.IsZero() {
+		claims[iamParentRevocationClaim] = r.RevokedBefore.Format(time.RFC3339Nano)
+	}
+	return nil
+}
+
+func iamCredentialSurvivesRevocation(cred auth.Credentials, at time.Time) bool {
+	if at.IsZero() {
+		return true
+	}
+	s, _ := cred.Claims[iamParentRevocationClaim].(string)
+	issuedAfter, err := time.Parse(time.RFC3339Nano, s)
+	return err == nil && !issuedAfter.Before(at)
+}
+
+// Parent revocations delete old children even if an offline peer has edited
+// them later. Preserve children that prove issuance after this revocation.
+func iamChildDeletionContext(ctx context.Context, child UserIdentity) (context.Context, bool) {
+	if at, replicated := iamReplicationTime(ctx); replicated {
+		if !at.IsZero() && iamCredentialSurvivesRevocation(child.Credentials, at) {
+			return ctx, false
+		}
+		if child.UpdatedAt.After(at) {
+			ctx = withIAMReplicationTime(ctx, child.UpdatedAt)
+		}
+	}
+	return ctx, true
+}
+
+// A delayed service account or STS event must not outlive deletion of its
+// built-in parent. The caller must populate Claims from the verified token.
+func checkIAMParentRevision(ctx context.Context, store IAMStorageAPI, cred auth.Credentials) error {
+	parent := cred.ParentUser
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted || !iamCredentialSurvivesRevocation(cred, r.RevokedBefore) {
+		return errIAMStaleUpdate
+	}
+	return nil
+}
+
+// Called with the IAM writer mutex and cache lock held. Persistence only
+// touches the caller's record, not the cache. Keep writers serialized while
+// allowing cached authentication reads throughout storage and lock waits.
+func (store *IAMStoreSys) withIAMStorage(ctx context.Context, fn func(context.Context) error) error {
+	store.IAMStorageAPI.unlock()
+	defer store.IAMStorageAPI.lock()
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	return fn(ctx)
+}
+
+func (store *IAMStoreSys) saveIAMRevision(ctx context.Context, path string, item any, opts ...options) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return saveIAMRevision(ctx, store.IAMStorageAPI, path, item, opts...)
+	})
+}
+
+func (store *IAMStoreSys) checkIAMParentRevision(ctx context.Context, cred auth.Credentials) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return checkIAMParentRevision(ctx, store.IAMStorageAPI, cred)
+	})
+}
+
+// Update the caller's record with the persisted revision before it is cached.
+func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item any, opts ...options) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	// Serialize compare-and-write across nodes, as well as goroutines. Use a
+	// separate lock name so saving the config does not reacquire this lock.
+	switch s := store.(type) {
+	case *IAMObjectStore:
+		lock := s.objAPI.NewNSLock(minioMetaBucket, path+".revision-lock")
+		lc, err := lock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		defer lock.Unlock(lc)
+		ctx = lc.Context()
+	case *IAMEtcdStore:
+		// Mutex.Lock also uses Client.Ctx() for cleanup after cancellation.
+		// Borrow the existing services with the operation's bounded context;
+		// never close this facade, which does not own those services.
+		client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(s.client.GetLogger()))
+		client.KV, client.Lease, client.Watcher = s.client.KV, s.client.Lease, s.client.Watcher
+		session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+		if err != nil {
+			return err
+		}
+		defer func() {
+			session.Orphan()
+			// A canceled operation must still release its lease when etcd is
+			// reachable. If it is unavailable, stop waiting and let it expire.
+			cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultContextTimeout)
+			defer cancel()
+			_, _ = s.client.Revoke(cleanupCtx, session.Lease())
+		}()
+		lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+		if err = lock.Lock(ctx); err != nil {
+			return err
+		}
+		// Revoking the session lease releases the lock, including on cancellation.
+	}
+	previous, err := loadIAMRevision(ctx, store, path)
+	if err != nil {
+		return err
+	}
+	if _, expiring := item.(*iamExpireIdentity); expiring {
+		sts := strings.HasPrefix(path, iamConfigSTSPrefix)
+		if previous.Deleted {
+			if sts && !previous.ExpiresAt.IsZero() && UTCNow().After(previous.ExpiresAt) {
+				return expireIAMSTSConfig(ctx, store, path)
+			}
+			return nil
+		}
+		if previous.timestamp().IsZero() || !previous.Credentials.IsExpired() {
+			return nil
+		}
+		if sts {
+			return expireIAMSTSConfig(ctx, store, path)
+		}
+		item = &UserIdentity{Version: 1, Deleted: true}
+		ctx = withIAMReplicationTime(ctx, previous.timestamp())
+	}
+	var revocation *iamUserRevocation
+	if op, ok := item.(*iamUserRevocation); ok {
+		revocation = op
+		op.UserIdentity = UserIdentity{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.UserIdentity = UserIdentity{Version: 1, Credentials: previous.Credentials, UpdatedAt: previous.timestamp(), RevokedBefore: origin}
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.UserIdentity
+	}
+	var groupRevocation *iamGroupRevocation
+	if op, ok := item.(*iamGroupRevocation); ok {
+		groupRevocation = op
+		var group GroupInfo
+		if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+			return err
+		}
+		if op.requireEmpty && !group.Deleted {
+			for _, member := range group.Members {
+				r := store.revisionIndex().get(getUserIdentityPath(member, regUser))
+				at := group.MemberGrants[member]
+				if !r.Deleted && (r.RevokedBefore.IsZero() || at.After(r.RevokedBefore)) && (group.RevokedBefore.IsZero() || at.After(group.RevokedBefore)) {
+					return errGroupNotEmpty
+				}
+			}
+		}
+		op.GroupInfo = GroupInfo{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.GroupInfo = group
+			op.RevokedBefore = origin
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.GroupInfo
+	}
+	var at *time.Time
+	var deleted bool
+	switch v := item.(type) {
+	case *UserIdentity:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *GroupInfo:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if !deleted {
+			var group GroupInfo
+			if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+				return err
+			}
+			mergeIAMGroupMutation(ctx, group, v)
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *MappedPolicy:
+		at, deleted = &v.UpdatedAt, v.Deleted
+	case *PolicyDoc:
+		at, deleted = &v.UpdateDate, v.Deleted
+	default:
+		return errInvalidArgument
+	}
+	if strings.HasPrefix(path, iamConfigSTSPrefix) && previous.Deleted && !deleted {
+		// STS access keys identify immutable tokens, not reusable user names.
+		return errIAMStaleUpdate
+	}
+	if origin, replicated := iamReplicationTime(ctx); replicated {
+		*at = origin
+		if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		if strings.HasPrefix(path, iamConfigServiceAccountsPrefix) && !deleted && previous.Credentials.AccessKey != "" && previous.timestamp().Equal(origin) {
+			// Duplicate service snapshots are acknowledgements, not new creates
+			// or edits. Reload the winner without writing, so even a stale
+			// sibling cache is refreshed by the retry before acknowledging it.
+			return store.loadIAMConfig(ctx, item, path)
+		}
+		if previous.Deleted && deleted && !origin.After(previous.timestamp()) {
+			// An already-applied tombstone needs no further persistent write.
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+	} else {
+		if previous.Deleted && deleted {
+			// A peer notification without an originating revision must not
+			// advance a tombstone past a subsequent deliberate recreation.
+			*at = previous.timestamp()
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+		if at.IsZero() {
+			*at = UTCNow()
+		}
+		if !at.After(previous.timestamp()) {
+			*at = previous.timestamp().Add(time.Nanosecond)
+		}
+	}
+	if v, ok := item.(*UserIdentity); ok {
+		if deleted {
+			// Retain only the parent name for root-account exclusion during heal.
+			v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
+			v.RevokedBefore = *at
+			if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) {
+				// The signed STS token cannot authorize beyond this time, even
+				// if an offline site replays it with a newer event timestamp.
+				v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime)
+				opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
+			}
+		} else {
+			if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
+				claims, err := extractJWTClaims(*v)
+				if err != nil {
+					return err
+				}
+				v.Credentials.Claims = claims.Map()
+			}
+			if err = checkIAMParentRevision(ctx, store, v.Credentials); err != nil {
+				return err
+			}
+		}
+	}
+	if v, ok := item.(*GroupInfo); ok && deleted {
+		v.RevokedBefore = *at
+		v.Members, v.MemberGrants = nil, nil
+	}
+	if _, ok := item.(*MappedPolicy); ok && !deleted {
+		if parentPath := iamMappingParentPath(path); parentPath != "" {
+			parent, err := loadIAMRevision(ctx, store, parentPath)
+			if err != nil {
+				return err
+			}
+			if parent.Deleted {
+				return errIAMStaleUpdate
+			}
+			if !parent.RevokedBefore.IsZero() && !at.After(parent.RevokedBefore) {
+				if _, replicated := iamReplicationTime(ctx); replicated {
+					return errIAMStaleUpdate
+				}
+				*at = parent.RevokedBefore.Add(time.Nanosecond)
+			}
+		}
+	}
+	if err := store.saveIAMConfig(ctx, item, path, opts...); err != nil {
+		return err
+	}
+	if revocation != nil && revocation.retained {
+		return errIAMRevocationRetained
+	}
+	if groupRevocation != nil && groupRevocation.retained {
+		return errIAMRevocationRetained
+	}
+	return nil
+}
+
+func (iamOS *IAMObjectStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var paths []string
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+"/") {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+		paths = append(paths, iamConfigPrefix+"/"+item.Item)
+	}
+	return paths, nil
+}
+
+func (ies *IAMEtcdStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	r, err := ies.client.Get(ctx, iamConfigPrefix+"/", etcd.WithPrefix(), etcd.WithKeysOnly())
+	if err != nil {
+		return nil, err
+	}
+	paths := make([]string, 0, len(r.Kvs))
+	for _, kv := range r.Kvs {
+		paths = append(paths, string(kv.Key))
+	}
+	return paths, nil
+}
+
+func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool) {
+	if (strings.HasPrefix(path, iamConfigUsersPrefix) || strings.HasPrefix(path, iamConfigGroupsPrefix)) && !r.RevokedBefore.IsZero() {
+		// Recreating a parent does not cancel its older revocation of derived
+		// credentials. Replay this boundary even after the parent is live again.
+		r.Deleted = true
+		r.UpdatedAt, r.UpdateDate = r.RevokedBefore, time.Time{}
+	}
+	if !r.Deleted {
+		return item, false
+	}
+	item.UpdatedAt = r.timestamp()
+	switch {
+	case strings.HasPrefix(path, iamConfigUsersPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+		item.Type = madmin.SRIAMItemIAMUser
+		item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
+	case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
+		if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
+			return item, false
+		}
+		item.Type = madmin.SRIAMItemSvcAcc
+		item.SvcAccChange = &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: name}}
+	case strings.HasPrefix(path, iamConfigGroupsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigGroupsPrefix), "/"+iamGroupMembersFile)
+		item.Type = madmin.SRIAMItemGroupInfo
+		item.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: name, IsRemove: true}}
+	case strings.HasPrefix(path, iamConfigPoliciesPrefix):
+		item.Type = madmin.SRIAMItemPolicy
+		item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigPoliciesPrefix), "/"+iamPolicyFile)
+	case strings.HasPrefix(path, iamConfigPolicyDBPrefix):
+		prefix, name, found := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+		if !found {
+			return item, false
+		}
+		typ := regUser
+		switch prefix {
+		case "sts-users":
+			typ = stsUser
+		case "service-accounts":
+			typ = svcUser
+		}
+		item.Type = madmin.SRIAMItemPolicyMapping
+		item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
+	default:
+		// Expired STS credentials are not replayed. Parent revocations and
+		// their retained timestamp reject delayed copies of derived tokens.
+		return item, false
+	}
+	return item, true
+}
+
+func iamDeletionPath(item madmin.SRIAMItem) string {
+	switch item.Type {
+	case madmin.SRIAMItemIAMUser:
+		if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
+			return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
+		}
+	case madmin.SRIAMItemSvcAcc:
+		if item.SvcAccChange != nil && item.SvcAccChange.Delete != nil {
+			return getUserIdentityPath(item.SvcAccChange.Delete.AccessKey, svcUser)
+		}
+	case madmin.SRIAMItemGroupInfo:
+		if item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+			return getGroupInfoPath(item.GroupInfo.UpdateReq.Group)
+		}
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return getPolicyDocPath(item.Name)
+		}
+	case madmin.SRIAMItemPolicyMapping:
+		if p := item.PolicyMapping; p != nil && p.Policy == "" {
+			return getMappedPolicyPath(p.UserOrGroup, IAMUserType(p.UserType), p.IsGroup)
+		}
+	}
+	return ""
+}
+
+func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
+	started := time.Now()
+	defer func() {
+		c.iamRevisionMetrics.healDurationMillis.Store(time.Since(started).Milliseconds())
+		if err != nil {
+			c.iamRevisionMetrics.healFailures.Add(1)
+		} else {
+			c.iamRevisionMetrics.healLastSuccess.Store(time.Now().Unix())
+		}
+	}()
+	c.iamHealMu.Lock()
+	defer c.iamHealMu.Unlock()
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	snapshot := globalIAMSys.store.revisionIndex().snapshot()
+	paths := make([]string, 0, len(snapshot))
+	for path := range snapshot {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	byType := make(map[string][]iamReplicationItem)
+	for _, path := range paths {
+		r := snapshot[path]
+		item, ok := iamDeletionItem(path, r)
+		if !ok {
+			continue
+		}
+		out := iamReplicationItem{SRIAMItem: item}
+		if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
+			out.Type, out.IAMUser = iamUserBoundaryType, nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+		}
+		if item.Type == madmin.SRIAMItemGroupInfo && !r.Deleted {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+		}
+		byType[item.Type] = append(byType[item.Type], out)
+	}
+	var items []iamReplicationItem
+	for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
+		items = append(items, byType[typ]...)
+	}
+	if len(items) == 0 {
+		return nil
+	}
+	if c.iamRevisionProgress == nil {
+		c.iamRevisionProgress = make(map[string]iamRevisionProgress)
+	}
+	for id := range c.iamRevisionProgress {
+		if _, present := c.state.Peers[id]; !present {
+			delete(c.iamRevisionProgress, id)
+		}
+	}
+	var progressMu sync.Mutex
+	cerr := c.concDo(nil, func(id string, p madmin.PeerInfo) error {
+		// Bound each pass, but retain acknowledgements independently of the
+		// pass deadline or unrelated changes at either site.
+		peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+		defer cancel()
+		client, err := c.getAdminClient(peerCtx, id)
+		if err != nil {
+			return err
+		}
+		remote, err := executeIAMRevisionRequest(peerCtx, client, http.MethodGet, nil)
+		if err != nil {
+			return err
+		}
+		progressMu.Lock()
+		progress := c.iamRevisionProgress[id]
+		progressMu.Unlock()
+		progress.observePeer(remote)
+		defer func() {
+			progressMu.Lock()
+			c.iamRevisionProgress[id] = progress
+			progressMu.Unlock()
+		}()
+		var pending []iamReplicationItem
+		for _, item := range items {
+			path, version := iamReplicationMarker(item)
+			if progress.Acknowledged[path] != version {
+				pending = append(pending, item)
+			}
+		}
+		// Acknowledgements are only a replay optimization, never GC proof.
+		for path := range progress.Acknowledged {
+			if _, retained := snapshot[path]; !retained {
+				delete(progress.Acknowledged, path)
+			}
+		}
+		var failures []error
+		for next := 0; next < len(pending); {
+			end := min(next+maxIAMRevisionBatch, len(pending))
+			batch := pending[next:end]
+			remote, err = executeIAMRevisionRequest(peerCtx, client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: batch})
+			if err != nil {
+				var batchErr *iamRevisionBatchError
+				if !errors.As(err, &batchErr) {
+					return errors.Join(append(failures, err)...)
+				}
+				failures = append(failures, err)
+			} else {
+				progress.observePeer(remote)
+				for _, item := range batch {
+					path, version := iamReplicationMarker(item)
+					progress.Acknowledged[path] = version
+				}
+			}
+			next = end
+		}
+		return errors.Join(failures...)
+	}, "IAM revision convergence")
+	return errors.Unwrap(cerr)
+}
+
+func iamReplicationMarker(item iamReplicationItem) (path, version string) {
+	path = iamDeletionPath(item.SRIAMItem)
+	if item.UserRevocation != nil {
+		path = getUserIdentityPath(item.UserRevocation.User, regUser)
+	}
+	if item.GroupRevocation != nil {
+		path = getGroupInfoPath(item.GroupRevocation.Group)
+	}
+	return path, item.Type + ":" + item.UpdatedAt.UTC().Format(time.RFC3339Nano)
+}
+
+func (store *IAMStoreSys) savePolicyDoc(ctx context.Context, policyName string, p *PolicyDoc) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(policyName), p)
+}
+
+func (store *IAMStoreSys) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp *MappedPolicy, opts ...options) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), mp, opts...)
+}
+
+func (store *IAMStoreSys) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u *UserIdentity, opts ...options) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), u, opts...)
+}
+
+func (store *IAMStoreSys) saveGroupInfo(ctx context.Context, name string, gi *GroupInfo) error {
+	return store.saveIAMRevision(ctx, getGroupInfoPath(name), gi)
+}
+
+func (store *IAMStoreSys) deletePolicyDoc(ctx context.Context, name string) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(name), &PolicyDoc{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), &MappedPolicy{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), &UserIdentity{Version: 1, Deleted: true})
+}
+
+// Called under the identity's distributed revision lock, after verifying that
+// its immutable STS token (or early-revocation retention) has expired. Only the
+// old token-key mapping is removed; the reusable parent mapping is unaffected.
+func expireIAMSTSConfig(ctx context.Context, store IAMStorageAPI, path string) error {
+	key := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
+	if err := store.deleteIAMConfig(ctx, getMappedPolicyPath(key, stsUser, false)); err != nil && !errors.Is(err, errConfigNotFound) {
+		return err
+	}
+	return store.deleteIAMConfig(ctx, path)
+}
+
+type (
+	iamExpirationCleanupKey   struct{}
+	iamExpirationCleanupState struct{ failed atomic.Bool }
+)
+
+func withIAMExpirationCleanup(ctx context.Context) context.Context {
+	if _, ok := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState); ok {
+		return ctx
+	}
+	return context.WithValue(ctx, iamExpirationCleanupKey{}, &iamExpirationCleanupState{})
+}
+
+func bestEffortIAMExpiration(ctx context.Context, store IAMStorageAPI, path string) {
+	state, _ := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState)
+	if ctx.Err() != nil || (state != nil && state.failed.Load()) {
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	// Failure leaves the expired record and its existing version intact.
+	// Stop optional reclamation for this load, while still loading healthy
+	// users. Healthy cleanup has no per-scan quota that could build a backlog.
+	if err := saveIAMRevision(ctx, store, path, &iamExpireIdentity{}); err != nil {
+		if state != nil {
+			state.failed.Store(true)
+		}
+		iamLogIf(ctx, err)
+	}
+}

cmd/iam-revision_test.go

@@ -0,0 +1,330 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
+	xnet "github.com/pgsty/silo-pkg/v3/net"
+)
+
+// Count physical saves: comparing timestamps alone would miss identical
+// tombstones being rewritten on every heal pass.
+type iamRevisionWriteCounter struct {
+	IAMStorageAPI
+	data   []byte
+	writes int
+}
+
+func (s *iamRevisionWriteCounter) loadIAMConfig(_ context.Context, item any, _ string) error {
+	return json.Unmarshal(s.data, item)
+}
+
+func (s *iamRevisionWriteCounter) saveIAMConfig(_ context.Context, item any, _ string, _ ...options) error {
+	data, err := json.Marshal(item)
+	if err == nil {
+		s.data = data
+		s.writes++
+	}
+	return err
+}
+
+func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
+	at := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC)
+	for _, record := range []struct {
+		name string
+		new  func(bool) any
+	}{
+		{"user", func(deleted bool) any { return &UserIdentity{Version: 1, Deleted: deleted} }},
+		{"group", func(deleted bool) any { return &GroupInfo{Version: 1, Deleted: deleted} }},
+		{"policy", func(deleted bool) any { return &PolicyDoc{Version: 1, Deleted: deleted} }},
+		{"mapping", func(deleted bool) any { return &MappedPolicy{Version: 1, Deleted: deleted} }},
+	} {
+		t.Run(record.name, func(t *testing.T) {
+			data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
+			if err != nil {
+				t.Fatal(err)
+			}
+			store := &iamRevisionWriteCounter{data: data}
+			ctx := context.Background()
+			for range 3 {
+				// Site heal carries the original timestamp. Sibling notifications
+				// have no timestamp; both must leave an applied deletion untouched.
+				for _, replay := range []context.Context{withIAMReplicationTime(ctx, at), ctx} {
+					if err := saveIAMRevision(replay, store, record.name, record.new(true)); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			if store.writes != 0 || string(store.data) != string(data) {
+				t.Fatalf("replayed tombstone changed storage: writes=%d, record=%s", store.writes, store.data)
+			}
+			for _, deleted := range []bool{false, true} {
+				err := saveIAMRevision(withIAMReplicationTime(ctx, at.Add(-time.Second)), store, record.name, record.new(deleted))
+				if !errors.Is(err, errIAMStaleUpdate) {
+					t.Fatalf("older event accepted, deleted=%t: %v", deleted, err)
+				}
+			}
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, at), store, record.name, record.new(false)); !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("equal-time recreation accepted: %v", err)
+			}
+			newer := at.Add(time.Minute)
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer), store, record.name, record.new(true)); err != nil {
+				t.Fatal(err)
+			}
+			r, err := loadIAMRevision(ctx, store, record.name)
+			if err != nil || store.writes != 1 || !r.timestamp().Equal(newer) || !r.Deleted {
+				t.Fatalf("newer deletion did not advance storage: writes=%d, revision=%+v, error=%v", store.writes, r, err)
+			}
+			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer.Add(time.Minute)), store, record.name, record.new(false)); err != nil {
+				t.Fatalf("newer recreation rejected: %v", err)
+			}
+		})
+	}
+}
+
+// Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
+// The object-store case uses the real peer RPC and deletion handler, so a
+// spurious notification actually destroys the parent instead of only counting it.
+func testIAMRevocationReplayAfterRecreation(ctx context.Context, t *testing.T, sys *IAMSys) {
+	t.Helper()
+	peer := &globalSiteReplicationSys
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user := "heal-recreated-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
+	create := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
+	}
+	revoke := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
+	}
+	create(origin)
+	revoke(deleted)
+	create(recreated)
+	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "heal-recreated-child", secretKey: "valid-service-password",
+	})
+	must(err)
+
+	tg, err := grid.SetupTestGrid(2)
+	must(err)
+	t.Cleanup(tg.Cleanup)
+	var deletes atomic.Int32
+	server := &peerRESTServer{}
+	must(deleteUserRPC.Register(tg.Managers[1], func(req *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		deletes.Add(1)
+		return server.DeleteUserHandler(req)
+	}))
+	// Future user updates still use the normal peer reload notification.
+	must(loadUserRPC.Register(tg.Managers[1], server.LoadUserHandler))
+	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+	must(err)
+	previousNotifications := globalNotificationSys
+	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
+		host: host,
+		gridConn: func() *grid.Connection {
+			return tg.Managers[0].Connection(tg.Hosts[1])
+		},
+	}}}
+	t.Cleanup(func() { globalNotificationSys = previousNotifications })
+	assertLive := func(key string) {
+		t.Helper()
+		if _, ok := sys.GetUser(ctx, key); !ok {
+			t.Fatalf("live credential %s lost during deletion replay", key)
+		}
+	}
+	assertNoDelete := func() {
+		t.Helper()
+		if n := deletes.Load(); n != 0 {
+			t.Fatalf("retained revocation sent %d destructive sibling notifications", n)
+		}
+	}
+	for range 3 {
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+		must(err)
+		item, ok := iamDeletionItem(getUserIdentityPath(user, regUser), r)
+		if !ok || item.IAMUser == nil || !item.UpdatedAt.Equal(deleted) {
+			t.Fatal("recreated user lost its durable revocation replay")
+		}
+		must(peer.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt))
+		must(sys.store.LoadIAMCache(ctx, false))
+		assertLive(user)
+		assertLive(child.AccessKey)
+		assertNoDelete()
+	}
+
+	// A divergent site sends a previously unseen revocation between our old
+	// boundary and recreation. Retain it and revoke old children, but never
+	// turn it into an unversioned delete of the recreated parent.
+	delayed := deleted.Add(time.Minute)
+	revoke(delayed)
+	assertNoDelete()
+	must(sys.store.LoadIAMCache(ctx, false))
+	assertLive(user)
+	if _, ok := sys.GetUser(ctx, child.AccessKey); ok {
+		t.Fatal("child from before the delayed revocation remains usable")
+	}
+	r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+	must(err)
+	if r.Deleted || !r.RevokedBefore.Equal(delayed) || !r.timestamp().Equal(recreated) {
+		t.Fatalf("retained revocation damaged the recreated identity: %+v", r)
+	}
+
+	// A genuinely newer deletion must still reach siblings and remove the
+	// parent plus credentials issued under its latest revocation boundary.
+	fresh, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "heal-fresh-child", secretKey: "valid-service-password",
+	})
+	must(err)
+	latest := recreated.Add(time.Minute)
+	revoke(latest)
+	wantDeletes := int32(1)
+	if sys.HasWatcher() {
+		wantDeletes = 0
+	}
+	if n := deletes.Load(); n != wantDeletes {
+		t.Fatalf("new deletion notifications=%d, want %d", n, wantDeletes)
+	}
+	for _, key := range []string{user, fresh.AccessKey} {
+		if _, ok := sys.GetUser(ctx, key); ok {
+			t.Fatalf("newer deletion left credential %s usable", key)
+		}
+	}
+	// Exercise the actual sibling handler again against the already persisted
+	// tombstone. Its context has no revision; it must not re-stamp the record.
+	_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+	if remoteErr != nil {
+		t.Fatal(remoteErr)
+	}
+	r, err = loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
+	must(err)
+	if !r.Deleted || !r.timestamp().Equal(latest) {
+		t.Fatalf("sibling re-stamped the tombstone: got %s, want %s", r.timestamp(), latest)
+	}
+	create(latest.Add(time.Minute))
+	_, remoteErr = server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
+	if remoteErr != nil {
+		t.Fatal(remoteErr)
+	}
+	assertLive(user)
+	revoke(latest)
+	must(sys.store.LoadIAMCache(ctx, false))
+	assertLive(user)
+}
+
+// Counts what a retained revocation actually sends to sibling nodes.
+func TestIAMRevocationRetainedReloadsSibling(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	initAllSubsystems(ctx)
+	globalIAMSys.Init(ctx, obj, nil, 2*time.Second)
+	defer os.RemoveAll(disks[0])
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+
+	sys, peer := globalIAMSys, &globalSiteReplicationSys
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	user := "retained-parent"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
+	create := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
+	}
+	revoke := func(at time.Time) {
+		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
+	}
+	create(origin)
+	revoke(deleted)
+	create(recreated)
+	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
+		accessKey: "retained-child", secretKey: "valid-service-password",
+	})
+	must(err)
+
+	// A sibling shares persistent state but has an independent IAM cache.
+	sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, sys.usersSysType)}
+	must(sibling.LoadIAMCache(ctx, false))
+	if _, ok := sibling.GetUser(child.AccessKey); !ok {
+		t.Fatal("sibling fixture did not load child")
+	}
+	tg, err := grid.SetupTestGrid(2)
+	must(err)
+	t.Cleanup(tg.Cleanup)
+	var deletes, loads atomic.Int32
+	server := &peerRESTServer{}
+	must(deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		deletes.Add(1)
+		return server.DeleteUserHandler(r)
+	}))
+	must(loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
+		loads.Add(1)
+		// LoadUserHandler delegates to this same cache reload method.
+		if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), regUser); err != nil {
+			return grid.NoPayload{}, grid.NewRemoteErr(err)
+		}
+		return grid.NoPayload{}, nil
+	}))
+	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
+	must(err)
+	prev := globalNotificationSys
+	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
+		host:     host,
+		gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
+	}}}
+	t.Cleanup(func() { globalNotificationSys = prev })
+
+	delayed := deleted.Add(time.Minute)
+	revoke(delayed)
+
+	t.Logf("sibling notifications after a retained revocation: destructive=%d reload=%d", deletes.Load(), loads.Load())
+	if deletes.Load() != 0 {
+		t.Errorf("destructive sibling delete sent: %d", deletes.Load())
+	}
+	if loads.Load() == 0 {
+		t.Errorf("retained revocation did not notify the sibling")
+	}
+	if _, ok := sibling.GetUser(child.AccessKey); ok {
+		t.Error("sibling still resolves revoked child")
+	}
+	if _, ok := sibling.GetUser(user); !ok {
+		t.Error("sibling lost live parent")
+	}
+	if _, ok := sys.store.GetUser(user); !ok {
+		t.Error("live parent lost")
+	}
+	if _, ok := sys.store.GetUser(child.AccessKey); ok {
+		t.Error("revoked child still resolves on the receiving node")
+	}
+}

cmd/iam-revocation_test.go

@@ -0,0 +1,535 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+)
+
+// Exercise the persisted IAM store and the same peer handler used by site heal.
+// A delete must survive a cache reload and an older create arriving afterwards.
+func TestIAMRevocationRejectsOfflineUser(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	user := "offline-revoked-user"
+	req := madmin.AddOrUpdateUserReq{SecretKey: "test-password-valid", Status: madmin.AccountEnabled}
+	created, err := globalIAMSys.CreateUser(ctx, user, req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = globalIAMSys.DeleteUser(ctx, user, false); err != nil {
+		t.Fatal(err)
+	}
+	if err = globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
+		t.Fatalf("revoked user restored by old peer event: %v", err)
+	}
+}
+
+func TestIAMRevocationHealingContinuesAfterPeerRejectsDelete(t *testing.T) {
+	resetTestGlobals()
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	obj, disk, err := prepareFS(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	if _, err := globalIAMSys.CreateUser(ctx, "heal-sync", req); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.CreateUser(ctx, "heal-deleted", req); err != nil {
+		t.Fatal(err)
+	}
+	if err := globalIAMSys.DeleteUser(ctx, "heal-deleted", false); err != nil {
+		t.Fatal(err)
+	}
+	p, err := globalIAMSys.store.GetPolicy("readwrite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := globalIAMSys.SetPolicy(ctx, "heal-new-policy", p); err != nil {
+		t.Fatal(err)
+	}
+	var liveUpdates atomic.Int32
+	peer := func(id string, rejectDelete bool) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			switch {
+			case strings.HasSuffix(r.URL.Path, "/metainfo"):
+				_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: id})
+			case r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions":
+				if r.Method == http.MethodGet {
+					_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
+					return
+				}
+				var batch iamRevisionBatch
+				if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+					t.Error(err)
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				for _, item := range batch.Items {
+					if rejectDelete && iamDeletionPath(item.SRIAMItem) != "" {
+						w.WriteHeader(http.StatusForbidden)
+						_, _ = w.Write([]byte(`{"Code":"AccessDenied","Message":"delete rejected"}`))
+						return
+					}
+					if id == "healthy" && item.Type == madmin.SRIAMItemPolicy && item.Name == "heal-new-policy" && len(item.Policy) > 0 {
+						liveUpdates.Add(1)
+					}
+				}
+				_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
+			default:
+				t.Errorf("unexpected peer request %s", r.URL.Path)
+				w.WriteHeader(http.StatusNotFound)
+			}
+		}))
+	}
+	healthy, rejected := peer("healthy", false), peer("rejected", true)
+	defer healthy.Close()
+	defer rejected.Close()
+	c := &SiteReplicationSys{enabled: true, state: srState{
+		ServiceAccountAccessKey: "heal-sync",
+		Peers: map[string]madmin.PeerInfo{
+			globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
+			"healthy":            {Name: "healthy", DeploymentID: "healthy", Endpoint: healthy.URL},
+			"rejected":           {Name: "rejected", DeploymentID: "rejected", Endpoint: rejected.URL},
+		},
+	}}
+	if err := c.healIAMSystem(ctx, obj); err == nil {
+		t.Fatal("deletion failure was not reported")
+	}
+	if liveUpdates.Load() == 0 {
+		t.Fatal("one peer rejecting a deletion blocked unrelated live IAM healing to a healthy peer")
+	}
+}
+
+func TestIAMRevocationLifecycle(t *testing.T) {
+	testIAMRevocationLifecycle(t, nil)
+}
+
+func TestIAMRevocationEtcdLifecycle(t *testing.T) {
+	endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
+	if endpoint == "" {
+		t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
+	}
+	connection, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer connection.Close()
+	// The facade borrows the connection's services. Close the owning client,
+	// not namespace.Watcher while IAM's canceled watch loop is winding down.
+	ctx, cancel := context.WithCancel(connection.Ctx())
+	defer cancel()
+	client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(connection.GetLogger()))
+	prefix := fmt.Sprintf("/silo-revocation-test/%d/", time.Now().UnixNano())
+	client.KV = namespace.NewKV(connection.KV, prefix)
+	client.Watcher = namespace.NewWatcher(connection.Watcher, prefix)
+	client.Lease = connection.Lease
+	testIAMRevocationLifecycle(t, client)
+}
+
+func testIAMRevocationLifecycle(t *testing.T, client *etcd.Client) {
+	resetTestGlobals()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	disk := disks[0]
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
+	if err == nil {
+		initAllSubsystems(ctx)
+		globalIAMSys.Init(ctx, obj, client, 2*time.Second)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+	defer obj.Shutdown(ctx)
+	defer resetTestGlobals()
+	sys, peer := globalIAMSys, &globalSiteReplicationSys
+	must := func(t *testing.T, err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	reload := func(t *testing.T) { t.Helper(); must(t, sys.store.LoadIAMCache(ctx, false)) }
+	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
+	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
+	createUser := func(t *testing.T, name string) {
+		t.Helper()
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, origin))
+	}
+	assertAbsent := func(t *testing.T, name string) {
+		t.Helper()
+		if _, ok := sys.GetUser(ctx, name); ok {
+			t.Fatalf("revoked credential %s is usable", name)
+		}
+	}
+
+	t.Run("replay after recreation", func(t *testing.T) {
+		testIAMRevocationReplayAfterRecreation(ctx, t, sys)
+	})
+
+	t.Run("origin timestamp and recreation", func(t *testing.T) {
+		name := "revocation-recreate"
+		createUser(t, name)
+		ui, ok := sys.store.GetUser(name)
+		if !ok || !ui.UpdatedAt.Equal(origin) {
+			t.Fatalf("origin time changed: %v", ui.UpdatedAt)
+		}
+		must(t, sys.DeleteUser(ctx, name, false))
+		reload(t)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, time.Time{}))
+		assertAbsent(t, name)
+		newTime := UTCNow().Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, newTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, origin.Add(time.Second)))
+		reload(t)
+		ui, ok = sys.store.GetUser(name)
+		if !ok || !ui.UpdatedAt.Equal(newTime) || ui.RevokedBefore.IsZero() {
+			t.Fatalf("newer recreation lost, or deletion boundary missing: present=%v", ok)
+		}
+	})
+
+	t.Run("groups policies and mappings", func(t *testing.T) {
+		user, group, name := "revocation-member", "revocation-group", "revocation-policy"
+		createUser(t, user)
+		p, err := sys.store.GetPolicy("readwrite")
+		must(t, err)
+		must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
+		add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
+		for _, isGroup := range []bool{false, true} {
+			entity := user
+			if isGroup {
+				entity = group
+			}
+			mp := &madmin.SRPolicyMapping{UserOrGroup: entity, Policy: name, UserType: int(regUser), IsGroup: isGroup}
+			must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
+			_, err = sys.PolicyDBSet(ctx, entity, "", regUser, isGroup)
+			must(t, err)
+			must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
+			if _, ok := sys.store.GetMappedPolicy(entity, isGroup); ok {
+				t.Fatal("old grant restored")
+			}
+		}
+		// This receiver never saw the member-removal event preceding deletion.
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, UTCNow()))
+		must(t, sys.DeletePolicy(ctx, name, true))
+		reload(t)
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
+		must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
+		if _, err = sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
+			t.Fatalf("group restored: %v", err)
+		}
+		if _, err = sys.store.GetPolicyDoc(name); !errors.Is(err, errNoSuchPolicy) {
+			t.Fatalf("policy restored: %v", err)
+		}
+		paths, err := sys.store.listIAMConfigPaths(ctx)
+		must(t, err)
+		found := make(map[string]bool)
+		for _, path := range paths {
+			r, err := loadIAMRevision(ctx, sys.store, path)
+			must(t, err)
+			if item, ok := iamDeletionItem(path, r); ok {
+				found[iamDeletionPath(item)] = true
+				if item.UpdatedAt.IsZero() {
+					t.Fatal("undated delete replay")
+				}
+			}
+		}
+		for _, path := range []string{getGroupInfoPath(group), getPolicyDocPath(name), getMappedPolicyPath(user, regUser, false), getMappedPolicyPath(group, regUser, true)} {
+			if !found[path] {
+				t.Errorf("deletion missing from heal: %s", path)
+			}
+		}
+	})
+
+	t.Run("parent revokes service accounts and STS", func(t *testing.T) {
+		parent := "revocation-parent"
+		createUser(t, parent)
+		svc, svcAt, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "revocation-service", secretKey: "valid-service-password"})
+		must(t, err)
+		secret, err := getTokenSigningKey()
+		must(t, err)
+		sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		sts.ParentUser = parent
+		_, err = sys.SetTempUser(withIAMReplicationTime(ctx, origin), sts.AccessKey, sts, "readwrite")
+		must(t, err)
+		must(t, sys.DeleteUser(ctx, parent, false))
+		reload(t)
+		assertAbsent(t, parent)
+		assertAbsent(t, svc.AccessKey)
+		assertAbsent(t, sts.AccessKey)
+		// Recreate the parent, then deliver old child events from the offline site.
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, UTCNow()))
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: svc.AccessKey, SecretKey: svc.SecretKey}}, svcAt))
+		must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken, ParentPolicyMapping: "readwrite"}, origin))
+		reload(t)
+		assertAbsent(t, svc.AccessKey)
+		assertAbsent(t, sts.AccessKey)
+		// A freshly issued credential is still supported after deliberate recreation.
+		_, _, err = sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-service", secretKey: "valid-service-password"})
+		must(t, err)
+		if _, ok := sys.GetUser(ctx, "new-service"); !ok {
+			t.Fatal("fresh service account rejected")
+		}
+	})
+
+	t.Run("delete before first create", func(t *testing.T) {
+		name := "revocation-unseen"
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, UTCNow()))
+		createUser(t, name)
+		assertAbsent(t, name)
+		svc := "unseen-service"
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: svc}}, UTCNow()))
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "revocation-recreate", AccessKey: svc, SecretKey: "valid-service-password"}}, origin))
+		assertAbsent(t, svc)
+	})
+
+	t.Run("recreation arrives before revocation", func(t *testing.T) {
+		parent := "reordered-parent"
+		createUser(t, parent)
+		child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "reordered-child", secretKey: "valid-service-password"})
+		must(t, err)
+		newTime, deleteTime := UTCNow(), origin.Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, newTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+		assertAbsent(t, child.AccessKey)
+		reload(t)
+		assertAbsent(t, child.AccessKey)
+		u, ok := sys.GetUser(ctx, parent)
+		if !ok || !u.UpdatedAt.Equal(newTime) || !u.RevokedBefore.Equal(deleteTime) {
+			t.Fatal("reordered revocation damaged the new parent or lost its boundary")
+		}
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(parent, regUser))
+		must(t, err)
+		item, ok := iamDeletionItem(getUserIdentityPath(parent, regUser), r)
+		if !ok || !item.UpdatedAt.Equal(deleteTime) {
+			t.Fatal("recreation erased deletion replay")
+		}
+	})
+
+	t.Run("user cleanup does not supersede group deletion", func(t *testing.T) {
+		user, group := "cascade-user", "cascade-group"
+		createUser(t, user)
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin))
+		// On the origin site the group was removed before the user, but the
+		// recovering receiver processes those independent events in reverse.
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
+		must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, origin.Add(time.Minute)))
+		reload(t)
+		if _, err := sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
+			t.Fatalf("deleted group survived reordered cleanup: %v", err)
+		}
+		groups, err := sys.ListGroups(ctx)
+		must(t, err)
+		for _, name := range groups {
+			if name == group {
+				t.Fatal("deleted group listed")
+			}
+		}
+	})
+
+	t.Run("parent revocation covers later updates to existing children", func(t *testing.T) {
+		parent, key := "late-update-parent", "late-update-child"
+		createUser(t, parent)
+		_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
+		must(t, err)
+		// This site missed the deletion and subsequently edited an old child.
+		_, err = sys.UpdateServiceAccount(withIAMReplicationTime(ctx, origin.Add(2*time.Minute)), key, updateServiceAccountOpts{description: "edited while the peer was offline"})
+		must(t, err)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, origin.Add(time.Minute)))
+		assertAbsent(t, key)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(3*time.Minute)))
+		reload(t)
+		assertAbsent(t, key)
+	})
+
+	t.Run("old generation cannot return with a newer event timestamp", func(t *testing.T) {
+		parent := "generation-parent"
+		createUser(t, parent)
+		deleteTime := origin.Add(time.Minute)
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+		must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(2*time.Minute)))
+		// Another offline site issued this child under the original parent,
+		// after this site's delete/recreate. Wall-clock ordering cannot identify it.
+		late := origin.Add(3 * time.Minute)
+		must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: "old-gen-service", SecretKey: "valid-service-password"}}, late))
+		secret, err := getTokenSigningKey()
+		must(t, err)
+		sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken}, late))
+		reload(t)
+		assertAbsent(t, "old-gen-service")
+		assertAbsent(t, sts.AccessKey)
+		// A local issuer knows the new boundary and signs it into both kinds
+		// of child. Untrusted inherited claims cannot select that boundary.
+		child, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-gen-service", secretKey: "valid-service-password", claims: map[string]any{iamParentRevocationClaim: "forged"}})
+		must(t, err)
+		newClaims := map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}
+		must(t, setIAMParentRevocationClaim(ctx, sys.store, parent, newClaims))
+		fresh, err := auth.GetNewCredentialsWithMetadata(newClaims, secret)
+		must(t, err)
+		fresh.ParentUser = parent
+		_, err = sys.SetTempUser(ctx, fresh.AccessKey, fresh, "")
+		must(t, err)
+		reload(t)
+		// A periodic reload retains the STS cache. Explicitly clear it to
+		// exercise the cold credential load performed after process restart.
+		cache := sys.store.lock()
+		cache.iamSTSAccountsMap = make(map[string]UserIdentity)
+		sys.store.unlock()
+		for _, key := range []string{child.AccessKey, fresh.AccessKey} {
+			u, ok := sys.GetUser(ctx, key)
+			if !ok || !iamCredentialSurvivesRevocation(u.Credentials, deleteTime) {
+				t.Fatalf("new-generation credential %s rejected", key)
+			}
+		}
+	})
+
+	t.Run("late revocation preserves proven new-generation children", func(t *testing.T) {
+		for _, recreateFirst := range []bool{false, true} {
+			parent := fmt.Sprintf("gen-parent-%t", recreateFirst)
+			createUser(t, parent)
+			deleteTime, createTime := origin.Add(time.Minute), origin.Add(2*time.Minute)
+			if recreateFirst {
+				must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
+			}
+			key := fmt.Sprintf("gen-child-%t", recreateFirst)
+			must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: key, SecretKey: "valid-service-password", Claims: map[string]any{iamParentRevocationClaim: deleteTime.Format(time.RFC3339Nano)}}}, createTime.Add(time.Second)))
+			must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
+			if !recreateFirst {
+				assertAbsent(t, key)
+				must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
+			}
+			reload(t)
+			if _, ok := sys.GetUser(ctx, key); !ok {
+				t.Fatal("late revocation deleted a child issued by the recreated parent")
+			}
+		}
+	})
+
+	t.Run("cold loading preserves site-signed STS", func(t *testing.T) {
+		parent := "cold-sts-parent"
+		createUser(t, parent)
+		secret := "site-signing-key-valid"
+		_, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{
+			accessKey: siteReplicatorSvcAcc, secretKey: secret, allowSiteReplicatorAccount: true,
+		})
+		must(t, err)
+		setReplication := func(enabled bool) {
+			globalSiteReplicationSys.Lock()
+			globalSiteReplicationSys.enabled = enabled
+			globalSiteReplicationSys.Unlock()
+			globalSiteReplicatorCred.Set("")
+		}
+		setReplication(true)
+		defer setReplication(false)
+		cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
+		must(t, err)
+		cred.ParentUser = parent
+		_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
+		must(t, err)
+		// IAM can load before the site replication manager during startup.
+		// A signing key that is not available yet must not delete live tokens.
+		setReplication(false)
+		for range 3 {
+			unverified := make(map[string]UserIdentity)
+			_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, unverified)
+			if _, ok := unverified[cred.AccessKey]; ok {
+				t.Fatal("accepted STS before the signing key became available")
+			}
+		}
+		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
+		must(t, err)
+		if r.Credentials.SessionToken == "" {
+			t.Fatal("cold IAM load physically deleted a non-expired site-signed STS credential")
+		}
+		setReplication(true)
+		loaded := make(map[string]UserIdentity)
+		must(t, sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded))
+		if _, ok := loaded[cred.AccessKey]; !ok {
+			t.Fatal("STS credential did not recover when the signing key became available")
+		}
+	})
+
+	t.Run("unverifiable STS stay denied and expired STS are removed", func(t *testing.T) {
+		parent := "invalid-sts-parent"
+		createUser(t, parent)
+		// Keep the etcd watcher from cleaning half of the fixture before the
+		// second record is seeded; this subtest exercises the loader directly.
+		sys.store.lock()
+		defer sys.store.unlock()
+		for _, expired := range []bool{false, true} {
+			cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, "unavailable-test-signing-key")
+			must(t, err)
+			cred.ParentUser = parent
+			if expired {
+				cred.Expiration = UTCNow().Add(-time.Minute)
+			}
+			identityPath := getUserIdentityPath(cred.AccessKey, stsUser)
+			mappingPath := getMappedPolicyPath(cred.AccessKey, stsUser, false)
+			// Seed disk directly to exercise loading, including existing records
+			// whose key is unknown. The write API should not accept such tokens.
+			must(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()}, identityPath))
+			must(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Policies: "readwrite"}, mappingPath))
+			loaded := make(map[string]UserIdentity)
+			_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded)
+			if _, ok := loaded[cred.AccessKey]; ok {
+				t.Fatalf("invalid STS accepted, expired=%t", expired)
+			}
+			for _, path := range []string{identityPath, mappingPath} {
+				var record map[string]any
+				err := sys.store.loadIAMConfig(ctx, &record, path)
+				if expired {
+					if !errors.Is(err, errConfigNotFound) {
+						t.Fatalf("expired STS data not cleaned up at %s: %v", path, err)
+					}
+				} else {
+					must(t, err)
+				}
+			}
+		}
+	})
+}

cmd/iam-service-recreation_test.go

@@ -0,0 +1,239 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+)
+
+// The receiver missed a deletion and still has the previous service key.
+// A later full snapshot must replace it, including when the owner changed.
+func TestIAMServiceAccountRecreation(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		t.Run(backend, func(t *testing.T) {
+			ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+			origin := UTCNow().Add(-time.Hour)
+			for _, parent := range []string{"old-owner", "new-owner"} {
+				_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), parent, madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+				mustIAM(t, err)
+			}
+			const key = "reusable-service"
+			old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "old-owner", AccessKey: key, SecretKey: "old-service-password"}}
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+			oldIdentity, _ := sys.store.GetUser(key)
+			_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
+			mustIAM(t, err)
+			boundary, newer := origin.Add(time.Minute), origin.Add(2*time.Minute)
+			fresh := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemSvcAcc, UpdatedAt: newer, SvcAccChange: &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "new-owner", AccessKey: key, SecretKey: "new-service-password", Status: auth.AccountOff}}}, RevokedBefore: boundary}
+			mustIAM(t, applyIAMReplicationItem(ctx, fresh))
+			// A sibling may have missed the notification of the committed
+			// replacement. An equal-version retry must refresh that cache too.
+			staleCache := sys.store.lock()
+			staleCache.iamUsersMap[key] = oldIdentity
+			sys.store.unlock()
+			mustIAM(t, applyIAMReplicationItem(ctx, fresh)) // duplicate delivery is acknowledged
+			if current, _ := sys.store.GetUser(key); current.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey {
+				t.Fatal("duplicate snapshot acknowledged without refreshing the stale cache")
+			}
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+			mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: key}}, boundary))
+			mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+			u, ok := sys.store.GetUser(key)
+			if !ok || u.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey || u.Credentials.ParentUser != "new-owner" || u.Credentials.Status != auth.AccountOff || !u.UpdatedAt.Equal(newer) || !u.RevokedBefore.Equal(boundary) {
+				t.Fatal("recreation did not retain the new identity, disabled status, source version and revocation")
+			}
+			if _, ok := sys.GetUser(ctx, key); ok {
+				t.Fatal("replicated disabled service can authenticate")
+			}
+			cache := sys.store.rlock()
+			_, mapped := cache.cachedMappedPolicy(key, svcUser, false)
+			sys.store.runlock()
+			if mapped {
+				t.Fatal("recreated service inherited an older mapping")
+			}
+			_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
+			if !errors.Is(err, errIAMStaleUpdate) {
+				t.Fatalf("old service mapping replay was accepted: %v", err)
+			}
+			_, _, err = sys.NewServiceAccount(ctx, "new-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "local-service-password"})
+			if !errors.Is(err, errIAMServiceAccountNotAllowed) {
+				t.Fatalf("local duplicate creation must remain rejected: %v", err)
+			}
+			// Outbound snapshots must carry the retained service boundary too.
+			out, err := globalSiteReplicationSys.replicationItem(ctx, fresh.SRIAMItem)
+			mustIAM(t, err)
+			if !out.RevokedBefore.Equal(boundary) {
+				t.Fatal("outbound service snapshot lost its revocation")
+			}
+		})
+	}
+}
+
+func TestIAMServiceAccountReplicationRejectsOtherCredentialKinds(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	_, err := sys.CreateUser(ctx, "builtin-collision", madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	secret, err := getTokenSigningKey()
+	mustIAM(t, err)
+	token, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "builtin-collision"}, secret)
+	mustIAM(t, err)
+	token.ParentUser = "builtin-collision"
+	_, err = sys.SetTempUser(ctx, token.AccessKey, token, "")
+	mustIAM(t, err)
+	for _, key := range []string{"builtin-collision", token.AccessKey} {
+		_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), "another-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
+		if !errors.Is(err, errIAMServiceAccountNotAllowed) {
+			t.Fatalf("service replication replaced another credential kind: %v", err)
+		}
+	}
+}
+
+// SR configuration can be temporarily unreadable even though a service token
+// is signed with its own valid secret. Do not acknowledge a failed cache load.
+func TestIAMServiceAccountRetryReportsClaimLoadFailure(t *testing.T) {
+	ctx, sys, _ := prepareIAMRevisionFixture(t)
+	globalSiteReplicatorCred.RLock()
+	previousSigningKey := globalSiteReplicatorCred.secretKey
+	globalSiteReplicatorCred.RUnlock()
+	globalSiteReplicatorCred.Set("")
+	t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSigningKey) })
+	_, err := sys.CreateUser(ctx, "retry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	opts := newServiceAccountOpts{accessKey: "retry-service", secretKey: "valid-service-password"}
+	_, _, err = sys.NewServiceAccount(ctx, "retry-owner", nil, opts)
+	mustIAM(t, err)
+	old, _ := sys.store.GetUser(opts.accessKey)
+	opts.secretKey = "replacement-service-password"
+	at, err := sys.UpdateServiceAccount(ctx, opts.accessKey, updateServiceAccountOpts{secretKey: opts.secretKey})
+	mustIAM(t, err)
+	cache := sys.store.lock()
+	cache.iamUsersMap[opts.accessKey] = old // Missed sibling notification.
+	sys.store.unlock()
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = true // No site-replicator credential is installed.
+	globalSiteReplicationSys.Unlock()
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
+	if err == nil {
+		t.Fatal("acknowledged service retry despite failed claims loading")
+	}
+	if _, ok := sys.store.GetUser(opts.accessKey); ok {
+		t.Fatal("failed cache refresh retained the superseded service secret")
+	}
+	globalSiteReplicationSys.Lock()
+	globalSiteReplicationSys.enabled = false
+	globalSiteReplicationSys.Unlock()
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
+	mustIAM(t, err)
+}
+
+// A delayed snapshot still has its original absolute expiration. Reapplying
+// the local minimum issuance lifetime would leave the old unexpired key alive.
+func TestIAMServiceAccountReplicationPreservesExpiration(t *testing.T) {
+	for _, backend := range []string{"object", "etcd"} {
+		for _, action := range []string{"create", "update"} {
+			for _, expired := range []bool{false, true} {
+				name := backend + "/" + action + "/near_expiry"
+				if expired {
+					name = backend + "/" + action + "/expired"
+				}
+				t.Run(name, func(t *testing.T) {
+					ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
+					origin := UTCNow().Add(-time.Hour)
+					_, err := sys.CreateUser(ctx, "expiry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+					mustIAM(t, err)
+					old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "old-service-password"}}
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+					expires := UTCNow().Add(time.Minute)
+					if expired {
+						expires = UTCNow().Add(-time.Minute)
+					}
+					change := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
+					if action == "update" {
+						change = &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
+					}
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, change, origin.Add(2*time.Minute)))
+					u, ok := sys.store.GetUser("expiry-service")
+					if !ok || u.Credentials.SecretKey != "new-service-password" || !u.Credentials.Expiration.Equal(expires) {
+						t.Fatal("delayed snapshot lost its new secret or absolute expiration")
+					}
+					_, allowed := sys.GetUser(ctx, "expiry-service")
+					if allowed == expired {
+						t.Fatal("credential validity disagrees with its absolute expiration")
+					}
+					mustIAM(t, sys.store.LoadIAMCache(ctx, false))
+					mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
+					r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath("expiry-service", svcUser))
+					mustIAM(t, err)
+					if r.Credentials.SecretKey == old.Create.SecretKey || (expired && !r.Deleted) {
+						t.Fatal("old non-expiring credential returned after reload")
+					}
+					_, _, err = sys.NewServiceAccount(ctx, "expiry-owner", nil, newServiceAccountOpts{accessKey: "local-expiry", secretKey: "valid-service-password", expiration: &expires})
+					if !errors.Is(err, errInvalidSvcAcctExpiration) {
+						t.Fatalf("local issuance lifetime check changed: %v", err)
+					}
+				})
+			}
+		}
+	}
+}
+
+// Status-only summaries intentionally omit secrets. Different revisions must
+// still trigger live healing, and disabled identities must be eligible sources.
+func TestIAMServiceAccountHealingNewerSnapshot(t *testing.T) {
+	ctx, sys, obj := prepareIAMRevisionFixture(t)
+	origin := UTCNow().Add(-time.Hour)
+	_, err := sys.CreateUser(ctx, "heal-svc-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
+	mustIAM(t, err)
+	_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), "heal-svc-owner", nil, newServiceAccountOpts{accessKey: "heal-service", secretKey: "valid-service-password"})
+	mustIAM(t, err)
+	at, err := sys.UpdateServiceAccount(ctx, "heal-service", updateServiceAccountOpts{status: auth.AccountOff})
+	mustIAM(t, err)
+	var sent atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/minio/health/live" {
+			return
+		}
+		if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" || r.Method != http.MethodPut {
+			t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		var batch iamRevisionBatch
+		if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
+			t.Error(err)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		for _, item := range batch.Items {
+			if item.SvcAccChange != nil && item.SvcAccChange.Create != nil && item.SvcAccChange.Create.AccessKey == "heal-service" && item.SvcAccChange.Create.Status == auth.AccountOff && item.UpdatedAt.Equal(at) {
+				sent.Add(1)
+			}
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node", Instance: "boot", Digest: "fixture"}})
+	}))
+	defer server.Close()
+	peers := map[string]madmin.PeerInfo{globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()}, "remote": {Name: "remote", DeploymentID: "remote", Endpoint: server.URL}}
+	c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "heal-svc-owner", Peers: peers}}
+	local := madmin.UserInfo{Status: madmin.AccountStatus(auth.AccountOff), UpdatedAt: at}
+	remote := local
+	remote.UpdatedAt = origin
+	if isUserInfoReplicated(2, 2, []madmin.UserInfo{local, remote}) {
+		t.Fatal("status-only summaries concealed different service revisions")
+	}
+	info := srStatusInfo{Sites: peers, UserStats: map[string]map[string]srUserStatsSummary{"heal-service": {globalDeploymentID(): {userInfo: srUserInfo{UserInfo: local}}, "remote": {SRUserStatsSummary: madmin.SRUserStatsSummary{UserInfoMismatch: true}, userInfo: srUserInfo{UserInfo: remote}}}}}
+	mustIAM(t, c.healUsers(ctx, obj, "heal-service", info))
+	if sent.Load() == 0 {
+		t.Fatal("disabled newer service snapshot was not healed")
+	}
+}

cmd/iam.go

@@ -605,10 +605,22 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 		return errServerNotInitialized
 	}
 
-	for _, v := range policy.DefaultPolicies {
-		if v.Name == policyName {
-			if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound {
-				return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+	if _, replicated := iamReplicationTime(ctx); !replicated && notifyPeers {
+		for _, v := range policy.DefaultPolicies {
+			if v.Name == policyName {
+				var err error
+				if objectStore, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
+					err = checkConfig(ctx, objectStore.objAPI, getPolicyDocPath(policyName))
+				} else {
+					var r iamRevision
+					err = sys.store.loadIAMConfig(ctx, &r, getPolicyDocPath(policyName))
+				}
+				if errors.Is(err, errConfigNotFound) {
+					return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+				}
+				if err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -714,20 +726,30 @@ func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers
 		return errServerNotInitialized
 	}
 
-	if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil {
+	err := sys.store.DeleteUser(ctx, accessKey, regUser)
+	var cleanupErr *iamCommittedCleanupError
+	retained := errors.Is(err, errIAMRevocationRetained)
+	if errors.As(err, &cleanupErr) {
+		retained = cleanupErr.retained
+	} else if err != nil && !retained {
 		return err
 	}
-
-	// Notify all other MinIO peers to delete user.
+	// Publish the committed state even when dependent cleanup must be retried.
 	if notifyPeers && !sys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				iamLogIf(ctx, nerr.Err)
+		if retained {
+			sys.notifyForUser(ctx, accessKey, false)
+		} else {
+			for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
+				if nerr.Err != nil {
+					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+					iamLogIf(ctx, nerr.Err)
+				}
 			}
 		}
 	}
-
+	if cleanupErr != nil {
+		return cleanupErr
+	}
 	return nil
 }
 
@@ -1061,6 +1083,7 @@ type newServiceAccountOpts struct {
 	sessionPolicy              *policy.Policy
 	accessKey                  string
 	secretKey                  string
+	status                     string // Used by replication snapshots; local creates default to enabled.
 	name, description          string
 	expiration                 *time.Time
 	allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
@@ -1125,6 +1148,11 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 			m[k] = v
 		}
 	}
+	if _, replicated := iamReplicationTime(ctx); !replicated {
+		if err := setIAMParentRevocationClaim(ctx, sys.store, parentUser, m); err != nil {
+			return auth.Credentials{}, time.Time{}, err
+		}
+	}
 
 	var accessKey, secretKey string
 	var err error
@@ -1143,12 +1171,19 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	cred.ParentUser = parentUser
 	cred.Groups = groups
 	cred.Status = string(auth.AccountOn)
+	switch opts.status {
+	case "", auth.AccountOn, string(madmin.AccountEnabled):
+	case auth.AccountOff, string(madmin.AccountDisabled):
+		cred.Status = auth.AccountOff
+	default:
+		return auth.Credentials{}, time.Time{}, errInvalidArgument
+	}
 	cred.Name = opts.name
 	cred.Description = opts.description
 
 	if opts.expiration != nil {
 		expirationInUTC := opts.expiration.UTC()
-		if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
+		if err := validateSvcExpirationInUTC(ctx, expirationInUTC); err != nil {
 			return auth.Credentials{}, time.Time{}, err
 		}
 		cred.Expiration = expirationInUTC
@@ -1379,7 +1414,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.Credentials.IsServiceAccount() {
+	if _, replicated := iamReplicationTime(ctx); (!ok || !sa.Credentials.IsServiceAccount()) && !replicated {
 		return nil
 	}
 
@@ -1483,8 +1518,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 		}
 	}
 
-	// We ignore any errors
-	_ = sys.store.DeleteUsers(ctx, expiredUsers)
+	// Keep failed revocations visible so the next purge can retry.
+	iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
 }
 
 // purgeExpiredCredentialsForLDAP - validates if local credentials are still
@@ -1512,8 +1547,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 		return
 	}
 
-	// We ignore any errors
-	_ = sys.store.DeleteUsers(ctx, expiredUsers)
+	// Keep failed revocations visible so the next purge can retry.
+	iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
 }
 
 // updateGroupMembershipsForLDAP - updates the list of groups associated with the credential.
@@ -1934,12 +1969,12 @@ func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, membe
 	}
 
 	updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members)
-	if err != nil {
+	var cleanupErr *iamCommittedCleanupError
+	if err != nil && !errors.As(err, &cleanupErr) {
 		return updatedAt, err
 	}
-
 	sys.notifyForGroup(ctx, group)
-	return updatedAt, nil
+	return updatedAt, err
 }
 
 // SetGroupStatus - enable/disabled a group

cmd/metrics-v3-cluster-iam.go

@@ -34,6 +34,10 @@ const (
 	sinceLastSyncMillis                    = "since_last_sync_millis"
 	syncFailures                           = "sync_failures"
 	syncSuccesses                          = "sync_successes"
+	revocationRecords                      = "revocation_records"
+	revocationHealFailures                 = "revocation_heal_failures"
+	revocationHealDurationMillis           = "revocation_heal_duration_millis"
+	revocationHealLastSuccess              = "revocation_heal_last_success_timestamp_seconds"
 )
 
 var (
@@ -47,10 +51,20 @@ var (
 	sinceLastSyncMillisMD                    = NewCounterMD(sinceLastSyncMillis, "Time (in milliseconds) since last successful IAM data sync.")
 	syncFailuresMD                           = NewCounterMD(syncFailures, "Number of failed IAM data syncs since server start.")
 	syncSuccessesMD                          = NewCounterMD(syncSuccesses, "Number of successful IAM data syncs since server start.")
+	revocationRecordsMD                      = NewGaugeMD(revocationRecords, "Retained IAM deletion records and revocation boundaries in this node's index.")
+	revocationHealFailuresMD                 = NewCounterMD(revocationHealFailures, "Failed IAM revocation convergence passes since server start.")
+	revocationHealDurationMillisMD           = NewGaugeMD(revocationHealDurationMillis, "Duration of the last IAM revocation convergence pass in milliseconds.")
+	revocationHealLastSuccessMD              = NewGaugeMD(revocationHealLastSuccess, "Unix timestamp of the last successful IAM revocation convergence pass.")
 )
 
 // loadClusterIAMMetrics - `MetricsLoaderFn` for cluster IAM metrics.
 func loadClusterIAMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
+	if globalIAMSys.Initialized() {
+		m.Set(revocationRecords, float64(globalIAMSys.store.revisionIndex().count()))
+	}
+	m.Set(revocationHealFailures, float64(globalSiteReplicationSys.iamRevisionMetrics.healFailures.Load()))
+	m.Set(revocationHealDurationMillis, float64(globalSiteReplicationSys.iamRevisionMetrics.healDurationMillis.Load()))
+	m.Set(revocationHealLastSuccess, float64(globalSiteReplicationSys.iamRevisionMetrics.healLastSuccess.Load()))
 	m.Set(lastSyncDurationMillis, float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)))
 	pluginAuthNMetrics := globalAuthNPlugin.Metrics()
 	m.Set(pluginAuthnServiceFailedRequestsMinute, float64(pluginAuthNMetrics.FailedRequests))

cmd/metrics-v3.go

@@ -323,6 +323,10 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			sinceLastSyncMillisMD,
 			syncFailuresMD,
 			syncSuccessesMD,
+			revocationRecordsMD,
+			revocationHealFailuresMD,
+			revocationHealDurationMillisMD,
+			revocationHealLastSuccessMD,
 		},
 		loadClusterIAMMetrics,
 	)

cmd/object-api-options-replication_test.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/base64"
+	"net/http"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func TestPutOptsFromHeadersReplicationTimestamps(t *testing.T) {
+	stamp := time.Date(2026, 9, 15, 1, 2, 3, 123456789, time.UTC)
+	context := base64.StdEncoding.EncodeToString([]byte(`{"purpose":"tag-replication"}`))
+	for _, encryption := range []struct {
+		name    string
+		headers map[string]string
+	}{
+		{name: "none"},
+		{name: "SSE-S3", headers: map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}},
+		{name: "SSE-KMS", headers: map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS}},
+		{name: "SSE-KMS-context", headers: map[string]string{
+			xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS, xhttp.AmzServerSideEncryptionKmsID: "tag-replication-key",
+			xhttp.AmzServerSideEncryptionKmsContext: context,
+		}},
+		{name: "SSE-C", headers: ssecKeyHeaders([]byte("01234567890123456789012345678901"), false)},
+	} {
+		t.Run(encryption.name, func(t *testing.T) {
+			for _, trusted := range []bool{false, true} {
+				t.Run("trusted="+strconv.FormatBool(trusted), func(t *testing.T) {
+					for _, tagging := range []struct {
+						name, header string
+						want         time.Time
+						invalid      bool
+					}{
+						{name: "absent"},
+						{name: "nanoseconds", header: stamp.Format(time.RFC3339Nano), want: stamp},
+						{name: "offset-whitespace", header: " " + stamp.In(time.FixedZone("UTC+8", 8*60*60)).Format(time.RFC3339Nano) + " ", want: stamp},
+						{name: "invalid", header: "not-a-timestamp", invalid: true},
+					} {
+						t.Run(tagging.name, func(t *testing.T) {
+							for _, metadata := range []map[string]string{nil, {"x-amz-meta-test": "kept"}} {
+								hdr := make(http.Header)
+								wantEncryption := make(http.Header)
+								for key, value := range encryption.headers {
+									hdr.Set(key, value)
+									wantEncryption.Set(key, value)
+								}
+								hdr.Set(xhttp.MinIOSourceTaggingTimestamp, tagging.header)
+								hdr.Set(xhttp.MinIOSourceMTime, stamp.Add(-time.Hour).Format(time.RFC3339Nano))
+								hdr.Set(xhttp.MinIOSourceObjectRetentionTimestamp, stamp.Add(-time.Minute).Format(time.RFC3339Nano))
+								hdr.Set(xhttp.MinIOSourceObjectLegalHoldTimestamp, stamp.Add(-time.Second).Format(time.RFC3339Nano))
+								hdr.Set(xhttp.MinIOSourceETag, "source-etag")
+								opts, err := putOptsFromHeaders(t.Context(), hdr, metadata, trusted)
+								if trusted && tagging.invalid {
+									if err == nil || !strings.Contains(err.Error(), xhttp.MinIOSourceTaggingTimestamp) {
+										t.Fatalf("malformed trusted timestamp: got %v", err)
+									}
+									continue
+								}
+								if err != nil {
+									t.Fatal(err)
+								}
+								wantTag, wantMTime, wantRetention, wantLegalhold, wantETag := time.Time{}, time.Time{}, time.Time{}, time.Time{}, ""
+								if trusted {
+									wantTag, wantMTime = tagging.want, stamp.Add(-time.Hour)
+									wantRetention, wantLegalhold, wantETag = stamp.Add(-time.Minute), stamp.Add(-time.Second), "source-etag"
+								}
+								if !opts.ReplicationSourceTaggingTimestamp.Equal(wantTag) {
+									t.Errorf("tag timestamp=%s, want %s", opts.ReplicationSourceTaggingTimestamp, wantTag)
+								}
+								if !opts.MTime.Equal(wantMTime) || !opts.ReplicationSourceRetentionTimestamp.Equal(wantRetention) ||
+									!opts.ReplicationSourceLegalholdTimestamp.Equal(wantLegalhold) || opts.PreserveETag != wantETag || opts.ReplicationRequest != trusted {
+									t.Error("other source fields did not preserve the replication trust boundary")
+								}
+								if opts.UserDefined == nil || (metadata != nil && !reflect.DeepEqual(opts.UserDefined, metadata)) {
+									t.Errorf("metadata=%v, want nonnil map preserving %v", opts.UserDefined, metadata)
+								}
+								gotEncryption := make(http.Header)
+								if opts.ServerSideEncryption != nil {
+									opts.ServerSideEncryption.Marshal(gotEncryption)
+								}
+								if !reflect.DeepEqual(gotEncryption, wantEncryption) {
+									t.Errorf("SSE headers=%v, want %v", gotEncryption, wantEncryption)
+								}
+							}
+						})
+					}
+				})
+			}
+		})
+	}
+}

cmd/object-api-options.go

@@ -452,11 +452,11 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
 			MTime:                mtime,
 			PreserveETag:         etag,
 			ReplicationRequest:   trustedReplication,
-			// The Object Lock timestamps order replicated retention and legal
-			// hold updates. Dropping them here would leave every update on an
-			// SSE-KMS destination unordered.
+			// These timestamps order replicated retention, legal hold and tagging
+			// updates on an SSE-KMS destination.
 			ReplicationSourceLegalholdTimestamp: lholdtimestmp,
 			ReplicationSourceRetentionTimestamp: retaintimestmp,
+			ReplicationSourceTaggingTimestamp:   taggingtimestmp,
 		}
 		return op, nil
 	}

cmd/object-copy-replication-tagging_test.go

@@ -0,0 +1,143 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+)
+
+// TestAPICopyObjectReplicaTaggingTimestampUnderKMS covers signed replica COPY
+// requests through encryption, metadata replacement and disk persistence. Both
+// the single-disk and 16-disk fixtures are single-pool backends.
+func TestAPICopyObjectReplicaTaggingTimestampUnderKMS(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS})
+}
+
+func testAPICopyObjectReplicaTaggingTimestampUnderKMS(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
+	// Ignore the host free-space percentage while retaining real disk I/O.
+	for _, pool := range obj.(*erasureServerPools).serverPools {
+		for _, set := range pool.sets {
+			original := set.getDisks
+			disks := append([]StorageAPI(nil), original()...)
+			for i := range disks {
+				disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
+			}
+			set.getDisks = func() []StorageAPI { return disks }
+			defer func() { set.getDisks = original }()
+		}
+	}
+	oldKMS, oldAuto := GlobalKMS, globalAutoEncryption
+	GlobalKMS = kms.NewStub("replica-tags-key")
+	globalAutoEncryption = false
+	defer func() { GlobalKMS, globalAutoEncryption = oldKMS, oldAuto }()
+	if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
+		t.Fatal(err)
+	}
+	const tsKey = ReservedMetadataPrefixLower + TaggingTimestamp
+	stamp := time.Date(2026, 9, 15, 1, 0, 0, 123456789, time.UTC)
+	for _, mode := range []string{"none", "explicit-sse-s3", "explicit-kms", "auto-kms", "bucket-kms"} {
+		t.Run(instance+"/"+mode, func(t *testing.T) {
+			globalAutoEncryption = mode == "auto-kms"
+			if mode == "bucket-kms" {
+				sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>replica-tags-key</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
+				if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketSSEConfig, sseXML); err != nil {
+					t.Fatal(err)
+				}
+			}
+			const data = "encrypted replica copy remains readable"
+			oi, err := obj.PutObject(t.Context(), bucket, mode, mustGetPutObjReader(t, bytes.NewReader([]byte(data)), int64(len(data)), "", ""), ObjectOptions{
+				Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "key=old", tsKey: stamp.Format(time.RFC3339Nano)},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, event := range []struct {
+				name, tags, wantTags string
+				delta, wantDelta     time.Duration
+				missingTimestamp     bool
+			}{
+				{"newer", "key=new", "key=new", 2, 2, false},
+				{"stale", "key=stale", "key=new", 1, 2, false},
+				{"duplicate", "key=new", "key=new", 2, 2, false},
+				{"newer-again", "key=latest", "key=latest", 3, 3, false},
+				{"missing-timestamp", "key=unordered", "key=latest", 0, 3, true},
+			} {
+				headers := map[string]string{
+					xhttp.AmzCopySource:        "/" + bucket + "/" + mode + "?versionId=" + oi.VersionID,
+					xhttp.AmzMetadataDirective: "REPLACE", xhttp.AmzTagDirective: "REPLACE",
+					xhttp.AmzObjectTagging: event.tags, xhttp.MinIOSourceReplicationRequest: "true",
+					xhttp.AmzBucketReplicationStatus: "REPLICA", xhttp.MinIOSourceTaggingTimestamp: stamp.Add(event.delta).Format(time.RFC3339Nano),
+					xhttp.MinIOSourceMTime: oi.ModTime.Format(time.RFC3339Nano), xhttp.MinIOSourceETag: oi.ETag,
+				}
+				if event.missingTimestamp {
+					delete(headers, xhttp.MinIOSourceTaggingTimestamp)
+				}
+				if mode == "explicit-sse-s3" {
+					headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+				}
+				if mode == "explicit-kms" {
+					headers[xhttp.AmzServerSideEncryption] = "aws:kms"
+					headers[xhttp.AmzServerSideEncryptionKmsID] = "replica-tags-key"
+				}
+				req, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/"+mode+"?versionId="+oi.VersionID, 0, nil, creds.AccessKey, creds.SecretKey, headers)
+				if err != nil {
+					t.Fatal(err)
+				}
+				w := httptest.NewRecorder()
+				router.ServeHTTP(w, req)
+				if w.Code != http.StatusOK {
+					t.Fatalf("%s: COPY %d %s", event.name, w.Code, w.Body.String())
+				}
+				got, err := obj.GetObjectInfo(t.Context(), bucket, mode, ObjectOptions{VersionID: oi.VersionID})
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Logf("%s: tags=%q timestamp=%q kms=%v", event.name, got.UserTags, got.UserDefined[tsKey], crypto.S3KMS.IsEncrypted(got.UserDefined))
+				if got.UserTags != event.wantTags || got.UserDefined[tsKey] != stamp.Add(event.wantDelta).Format(time.RFC3339Nano) {
+					t.Errorf("%s: incorrect persisted tags/timestamp", event.name)
+				}
+				wantKMS := mode == "explicit-kms" || mode == "auto-kms" || mode == "bucket-kms"
+				if crypto.S3KMS.IsEncrypted(got.UserDefined) != wantKMS || crypto.S3.IsEncrypted(got.UserDefined) != (mode == "explicit-sse-s3") {
+					t.Errorf("%s: unexpected destination encryption", event.name)
+				}
+				if got.VersionID != oi.VersionID {
+					t.Errorf("%s: version=%q, want %q", event.name, got.VersionID, oi.VersionID)
+				}
+				req, err = newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/"+mode+"?versionId="+oi.VersionID, 0, nil, creds.AccessKey, creds.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				w = httptest.NewRecorder()
+				router.ServeHTTP(w, req)
+				if w.Code != http.StatusOK || w.Body.String() != data {
+					t.Fatalf("%s: GET %d %q", event.name, w.Code, w.Body.String())
+				}
+			}
+		})
+	}
+}

cmd/peer-rest-server.go

@@ -204,7 +204,9 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(mss *grid.MSS) (np grid.NoP
 		return np, grid.NewRemoteErr(errors.New("service account name is missing"))
 	}
 
-	if err := globalIAMSys.DeleteServiceAccount(context.Background(), accessKey, false); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err := globalIAMSys.LoadServiceAccount(ctx, accessKey); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 
@@ -274,7 +276,9 @@ func (s *peerRESTServer) LoadUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr
 		userType = stsUser
 	}
 
-	if err = globalIAMSys.LoadUser(context.Background(), objAPI, accessKey, userType); err != nil {
+	ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
+	defer cancel()
+	if err = globalIAMSys.LoadUser(ctx, objAPI, accessKey, userType); err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 

cmd/replication-content-encoding_test.go

@@ -0,0 +1,276 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"encoding/xml"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio/internal/auth"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// Exercise authenticated handlers and actual disk metadata, including the
+// response headers consumers see after replication has completed.
+func TestAPIReplicaContentEncoding(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIReplicaContentEncoding})
+}
+
+func testAPIReplicaContentEncoding(obj ObjectLayer, instance, bucket string, router http.Handler, owner auth.Credentials, t *testing.T) {
+	ordinary := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject"`)
+	replicator := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
+	for _, mode := range []string{"ordinary", "untrusted-marker", "replica"} {
+		for _, tc := range []struct{ name, wire, want string }{
+			{"bare", "aws-chunked", ""}, {"mixed", "aws-chunked,gzip", "gzip"}, {"gzip", "gzip", "gzip"},
+		} {
+			for _, operation := range []string{"put", "copy-replace", "multipart"} {
+				t.Run(instance+"/"+mode+"/"+tc.name+"/"+operation, func(t *testing.T) {
+					object := mode + "/" + tc.name + "/" + operation
+					payload := replicaEncodingPayload(t, tc.want)
+					creds := ordinary
+					headers := map[string]string{xhttp.ContentEncoding: tc.wire, xhttp.ContentType: "application/octet-stream", "X-Amz-Meta-Source": "encoding-test"}
+					if mode != "ordinary" {
+						headers[xhttp.MinIOSourceReplicationRequest] = "true"
+					}
+					if mode == "replica" {
+						creds = replicator
+						headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
+					}
+					send := func(method, target string, data []byte, hdrs map[string]string) *httptest.ResponseRecorder {
+						t.Helper()
+						req, err := newTestSignedRequestV4(method, target, int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, hdrs)
+						if err != nil {
+							t.Fatal(err)
+						}
+						return replicaEncodingServe(t, router, req, http.StatusOK)
+					}
+					switch operation {
+					case "put":
+						if strings.Contains(tc.wire, "aws-chunked") {
+							req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), payload, creds, headers)
+							replicaEncodingServe(t, router, req, http.StatusOK)
+						} else {
+							send(http.MethodPut, getPutObjectURL("", bucket, object), payload, headers)
+						}
+					case "copy-replace":
+						source := object + "-source"
+						if _, err := obj.PutObject(t.Context(), bucket, source, mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{}); err != nil {
+							t.Fatal(err)
+						}
+						headers[xhttp.AmzCopySource] = url.QueryEscape("/" + bucket + "/" + source)
+						headers[xhttp.AmzMetadataDirective] = replaceDirective
+						send(http.MethodPut, getCopyObjectURL("", bucket, object), nil, headers)
+					case "multipart":
+						rec := send(http.MethodPost, getNewMultipartURL("", bucket, object), nil, headers)
+						var init InitiateMultipartUploadResponse
+						if err := xml.Unmarshal(rec.Body.Bytes(), &init); err != nil {
+							t.Fatal(err)
+						}
+						// Part/completion metadata must not replace the encoding saved at initiation.
+						partHeaders := map[string]string{xhttp.ContentEncoding: "br"}
+						if mode == "replica" {
+							partHeaders[xhttp.MinIOSourceReplicationRequest] = "true"
+							partHeaders[xhttp.AmzBucketReplicationStatus] = "REPLICA"
+						}
+						part := send(http.MethodPut, getPutObjectPartURL("", bucket, object, init.UploadID, "1"), payload, partHeaders)
+						partETags := part.Header()[xhttp.ETag]
+						if len(partETags) != 1 {
+							t.Fatalf("missing part ETag: %#v", part.Header())
+						}
+						complete, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(partETags[0])}}})
+						if err != nil {
+							t.Fatal(err)
+						}
+						send(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, init.UploadID), complete, partHeaders)
+					}
+					assertReplicaEncodingObject(t, obj, router, owner, bucket, object, tc.want, payload)
+					info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if got := info.UserDefined[xhttp.AmzBucketReplicationStatus]; (got == "REPLICA") != (mode == "replica") {
+						t.Errorf("replica status %q for mode %s", got, mode)
+					}
+					if info.ContentType != "application/octet-stream" {
+						t.Errorf("content-type=%q", info.ContentType)
+					}
+					if value, ok := caseInsensitiveMap(info.UserDefined).Lookup("x-amz-meta-source"); !ok || value != "encoding-test" {
+						t.Errorf("user metadata lost: %#v", info.UserDefined)
+					}
+				})
+			}
+		}
+	}
+	t.Run(instance+"/unauthorized-replica", func(t *testing.T) {
+		object := "denied-replica"
+		req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), []byte("denied"), ordinary, map[string]string{xhttp.ContentEncoding: "aws-chunked", xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA"})
+		rec := replicaEncodingServe(t, router, req, http.StatusForbidden)
+		var response APIErrorResponse
+		if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+			t.Fatal(err)
+		}
+		if response.Code != "AccessDenied" {
+			t.Fatalf("expected permission denial, got %s", response.Code)
+		}
+		if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil {
+			t.Error("denied replica created an object")
+		}
+	})
+}
+
+func replicaEncodingPayload(t *testing.T, encoding string) []byte {
+	t.Helper()
+	data := bytes.Repeat([]byte("replica encoding payload\n"), 128)
+	if encoding != "gzip" {
+		return data
+	}
+	var b bytes.Buffer
+	w := gzip.NewWriter(&b)
+	if _, err := w.Write(data); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return b.Bytes()
+}
+
+func replicaEncodingStream(t *testing.T, target string, data []byte, creds auth.Credentials, headers map[string]string) *http.Request {
+	t.Helper()
+	const chunkSize = 64
+	body := bytes.NewReader(data)
+	req, err := newTestStreamingRequest(http.MethodPut, target, int64(len(data)), chunkSize, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	now := UTCNow()
+	signature, err := signStreamingRequest(req, creds.AccessKey, creds.SecretKey, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req, err = assembleStreamingChunks(req, body, chunkSize, creds.SecretKey, signature, now)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return req
+}
+
+func replicaEncodingServe(t *testing.T, router http.Handler, req *http.Request, want int) *httptest.ResponseRecorder {
+	t.Helper()
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	if rec.Code != want {
+		t.Fatalf("%s %s: status=%d want=%d body=%s", req.Method, req.URL, rec.Code, want, rec.Body.String())
+	}
+	return rec
+}
+
+func assertReplicaEncodingObject(t *testing.T, obj ObjectLayer, router http.Handler, creds auth.Credentials, bucket, object, encoding string, data []byte) {
+	t.Helper()
+	info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.ContentEncoding != encoding {
+		t.Errorf("persisted content-encoding=%q want=%q", info.ContentEncoding, encoding)
+	}
+	if encoding == "" {
+		if _, present := info.UserDefined["content-encoding"]; present {
+			t.Error("transport-only content-encoding key persisted")
+		}
+	}
+	for _, method := range []string{http.MethodGet, http.MethodHead} {
+		req, err := newTestSignedRequestV4(method, getPutObjectURL("", bucket, object), 0, nil, creds.AccessKey, creds.SecretKey, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		rec := replicaEncodingServe(t, router, req, http.StatusOK)
+		if got := rec.Header().Get(xhttp.ContentEncoding); got != encoding {
+			t.Errorf("%s content-encoding=%q want=%q", method, got, encoding)
+		}
+		if encoding == "" {
+			if _, present := rec.Header()[xhttp.ContentEncoding]; present {
+				t.Errorf("%s sent an empty/transport encoding header", method)
+			}
+		}
+		if method == http.MethodGet && !bytes.Equal(rec.Body.Bytes(), data) {
+			t.Errorf("GET body differs: got %d bytes want %d", rec.Body.Len(), len(data))
+		}
+	}
+}
+
+func TestAPISnowballReplicaContentEncoding(t *testing.T) {
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
+		for _, tc := range []struct {
+			name string
+			pax  map[string]string
+			want string
+		}{
+			{name: "no-pax"},
+			{name: "pax-without-encoding", pax: map[string]string{"minio.metadata.Content-Type": "application/octet-stream"}},
+			{name: "pax-bare", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked"}},
+			{name: "pax-mixed", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked,gzip"}, want: "gzip"},
+		} {
+			t.Run(instance+"/"+tc.name, func(t *testing.T) {
+				object := "snowball/" + tc.name
+				data := replicaEncodingPayload(t, tc.want)
+				var archive bytes.Buffer
+				tw := tar.NewWriter(&archive)
+				if err := tw.WriteHeader(&tar.Header{Name: object, Mode: 0o600, Size: int64(len(data)), PAXRecords: tc.pax}); err != nil {
+					t.Fatal(err)
+				}
+				if _, err := tw.Write(data); err != nil {
+					t.Fatal(err)
+				}
+				if err := tw.Close(); err != nil {
+					t.Fatal(err)
+				}
+				var ordinaryMetadata map[string]string
+				// An unauthorized entry in a REPLICA request is rejected. Compare the
+				// same archive across ordinary and authorized replica requests instead.
+				for _, replica := range []bool{false, true} {
+					headers := map[string]string{
+						xhttp.ContentEncoding: "aws-chunked", xhttp.AmzSnowballExtract: "true",
+						xhttp.ContentType: "application/x-tar", xhttp.CacheControl: "max-age=123",
+						"X-Amz-Meta-Archive": "outer-request",
+					}
+					if replica {
+						headers[xhttp.MinIOSourceReplicationRequest] = "true"
+						headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
+					}
+					req := replicaEncodingStream(t, getPutObjectURL("", bucket, "archive.tar"), archive.Bytes(), creds, headers)
+					replicaEncodingServe(t, router, req, http.StatusOK)
+					assertReplicaEncodingObject(t, obj, router, creds, bucket, object, tc.want, data)
+					info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					metadata := maps.Clone(info.UserDefined)
+					for _, key := range []string{xhttp.AmzBucketReplicationStatus, ReservedMetadataPrefixLower + ReplicaStatus, ReservedMetadataPrefixLower + ReplicaTimestamp, "etag"} {
+						delete(metadata, key)
+					}
+					if !replica {
+						ordinaryMetadata = metadata
+					} else if !reflect.DeepEqual(metadata, ordinaryMetadata) {
+						t.Errorf("replica inherited ordinary archive metadata: got %#v want %#v", metadata, ordinaryMetadata)
+					}
+				}
+			})
+		}
+	}})
+}

cmd/replication-metadata_test.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+	"maps"
+	"net/http"
+	"net/textproto"
+	"reflect"
+	"strings"
+	"testing"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func TestExtractReplicationMetadataPreservesNormalizedMetadata(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		wire []string
+		want string
+	}{
+		{name: "absent"},
+		{name: "transport-only", wire: []string{"aws-chunked"}},
+		{name: "mixed", wire: []string{"aws-chunked,gzip"}, want: "gzip"},
+		{name: "gzip", wire: []string{"gzip"}, want: "gzip"},
+		{name: "transport-last", wire: []string{"gzip,aws-chunked"}, want: "gzip"},
+		{name: "multiple-values", wire: []string{"aws-chunked", "gzip"}, want: "gzip"},
+		// Preserve the existing exact-token grammar; whitespace is not normalized here.
+		{name: "space-before-gzip", wire: []string{"aws-chunked, gzip"}, want: " gzip"},
+		{name: "space-before-transport", wire: []string{"gzip, aws-chunked"}, want: "gzip, aws-chunked"},
+	} {
+		for _, lowercase := range []bool{false, true} {
+			name := tc.name + "/canonical"
+			if lowercase {
+				name = tc.name + "/lowercase"
+			}
+			t.Run(name, func(t *testing.T) {
+				header := http.Header{
+					"Content-Type":      []string{"application/octet-stream"},
+					"X-Amz-Meta-Source": []string{"raw"},
+					"X-Minio-Replication-Server-Side-Encryption-Sealed-Key":     []string{"sealed-key"},
+					"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
+					"X-Minio-Replication-Server-Side-Encryption-Iv":             []string{"iv"},
+					"X-Minio-Replication-Encrypted-Multipart":                   []string{""},
+					"X-Minio-Replication-Actual-Object-Size":                    []string{"1"},
+					ReplicationSsecChecksumHeader:                               []string{"checksum"},
+					xhttp.AmzMetaUnencryptedContentLength:                       []string{"injected-length"},
+					xhttp.AmzMetaUnencryptedContentMD5:                          []string{"injected-md5"},
+				}
+				if tc.wire != nil {
+					header[xhttp.ContentEncoding] = tc.wire
+				}
+				if lowercase {
+					h := make(http.Header, len(header))
+					for k, v := range header {
+						h[strings.ToLower(k)] = v
+					}
+					header = h
+				}
+				metadata, err := extractMetadata(t.Context(), textproto.MIMEHeader(header))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if metadata["content-encoding"] != tc.want {
+					t.Fatalf("ordinary encoding=%q want=%q", metadata["content-encoding"], tc.want)
+				}
+				for _, internal := range replicationToInternalHeaders {
+					if _, ok := metadata[internal]; ok {
+						t.Fatalf("ordinary request accepted internal field %s", internal)
+					}
+				}
+				// Callers own ordinary metadata and may transform it after extraction.
+				metadata["content-type"] = "application/wasm"
+				for k := range metadata {
+					if strings.EqualFold(k, "x-amz-meta-source") {
+						metadata[k] = "caller"
+					}
+				}
+				want := maps.Clone(metadata)
+				maps.Copy(want, map[string]string{
+					"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "sealed-key",
+					"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+					"X-Minio-Internal-Server-Side-Encryption-Iv":             "iv",
+					"X-Minio-Internal-Encrypted-Multipart":                   "",
+					"X-Minio-Internal-Actual-Object-Size":                    "1",
+					ReplicationSsecChecksumHeader:                            "checksum",
+				})
+				for range 2 {
+					if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
+						t.Fatal(err)
+					}
+					if !reflect.DeepEqual(metadata, want) {
+						t.Errorf("restoration changed normalized metadata: got %#v want %#v", metadata, want)
+					}
+				}
+				if tc.want == "" {
+					if _, present := metadata["content-encoding"]; present {
+						t.Error("transport-only content-encoding key restored")
+					}
+				}
+				for _, key := range []string{xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5} {
+					if _, present := caseInsensitiveMap(metadata).Lookup(key); present {
+						t.Errorf("redacted metadata restored: %s", key)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestExtractReplicationMetadataNilHeader(t *testing.T) {
+	metadata := map[string]string{"content-type": "application/wasm"}
+	want := maps.Clone(metadata)
+	if err := extractReplicationMetadataFromMime(t.Context(), nil, metadata); err != errInvalidArgument {
+		t.Fatalf("nil header: got %v want %v", err, errInvalidArgument)
+	}
+	if !reflect.DeepEqual(metadata, want) {
+		t.Fatalf("nil input changed metadata: %#v", metadata)
+	}
+}

cmd/site-replication-metadata-gate_test.go

@@ -274,6 +274,10 @@ func TestBucketMetadataInitialSyncPhysicalCreated(t *testing.T) {
 					}
 					events = append(events, event)
 				}
+				if r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions" {
+					_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "initial-peer", Instance: "initial-boot", Digest: "ack"}})
+					return
+				}
 				w.WriteHeader(http.StatusOK)
 			}))
 			defer peer.Close()

cmd/site-replication.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"maps"
 	"math/rand"
+	"net/http"
 	"net/url"
 	"reflect"
 	"runtime"
@@ -209,7 +210,11 @@ type SiteReplicationSys struct {
 	// In-memory and persisted multi-site replication state.
 	state srState
 
-	iamMetaCache srIAMCache
+	iamMetaCache        srIAMCache
+	healOnce            sync.Once // Configuration reloads must not spawn more healing loops.
+	iamHealMu           sync.Mutex
+	iamRevisionProgress map[string]iamRevisionProgress
+	iamRevisionMetrics  iamRevisionMetrics
 }
 
 type srState srStateV1
@@ -234,7 +239,7 @@ type srStateData struct {
 
 // Init - initialize the site replication manager.
 func (c *SiteReplicationSys) Init(ctx context.Context, objAPI ObjectLayer) error {
-	go c.startHealRoutine(ctx, objAPI)
+	c.healOnce.Do(func() { go c.startHealRoutine(ctx, objAPI) })
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
 		err := c.loadFromDisk(ctx, objAPI)
@@ -1257,13 +1262,36 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 		return nil
 	}
 
+	if path := iamDeletionPath(item); path != "" {
+		r, err := loadIAMRevision(ctx, globalIAMSys.store, path)
+		if err != nil {
+			return err
+		}
+		if !r.Deleted && ((item.Type != madmin.SRIAMItemIAMUser && item.Type != madmin.SRIAMItemGroupInfo) || r.RevokedBefore.IsZero()) {
+			// A concurrent recreation has already superseded this delete.
+			return nil
+		}
+		item.UpdatedAt = r.timestamp()
+		if !r.Deleted {
+			item.UpdatedAt = r.RevokedBefore
+		}
+	}
+
+	versioned, err := c.replicationItem(ctx, item)
+	if errors.Is(err, errIAMStaleUpdate) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
 	cerr := c.concDo(nil, func(d string, p madmin.PeerInfo) error {
 		admClient, err := c.getAdminClient(ctx, d)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		return c.annotatePeerErr(p.Name, replicateIAMItem, admClient.SRPeerReplicateIAMItem(ctx, item))
+		_, err = executeIAMRevisionRequest(ctx, admClient, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{versioned}})
+		return c.annotatePeerErr(p.Name, replicateIAMItem, err)
 	},
 		replicateIAMItem,
 	)
@@ -1273,6 +1301,7 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 // PeerAddPolicyHandler - copies IAM policy to local. A nil policy argument,
 // causes the named policy to be deleted.
 func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyName string, p *policy.Policy, updatedAt time.Time) error {
+	ctx = withIAMReplicationTime(ctx, updatedAt)
 	var err error
 	// skip overwrite of local update if peer sent stale info
 	if !updatedAt.IsZero() {
@@ -1286,18 +1315,19 @@ func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyNam
 		_, err = globalIAMSys.SetPolicy(ctx, policyName, *p)
 	}
 	if err != nil {
-		return wrapSRErr(err)
+		return iamReplicationError(err)
 	}
 	return nil
 }
 
 // PeerIAMUserChangeHandler - copies IAM user to local.
 func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, change *madmin.SRIAMUser, updatedAt time.Time) error {
+	ctx = withIAMReplicationTime(ctx, updatedAt)
 	if change == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
 	// skip overwrite of local update if peer sent stale info
-	if !updatedAt.IsZero() {
+	if !change.IsDeleteReq && !updatedAt.IsZero() {
 		if ui, err := globalIAMSys.GetUserInfo(ctx, change.AccessKey); err == nil && ui.UpdatedAt.After(updatedAt) {
 			return nil
 		}
@@ -1326,13 +1356,14 @@ func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, chang
 		}
 	}
 	if err != nil {
-		return wrapSRErr(err)
+		return iamReplicationError(err)
 	}
 	return nil
 }
 
 // PeerGroupInfoChangeHandler - copies group changes to local.
 func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, change *madmin.SRGroupInfo, updatedAt time.Time) error {
+	ctx = withIAMReplicationTime(ctx, updatedAt)
 	if change == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
@@ -1340,7 +1371,7 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
 	var err error
 
 	// skip overwrite of local update if peer sent stale info
-	if !updatedAt.IsZero() {
+	if !updatedAt.IsZero() && (!updReq.IsRemove || len(updReq.Members) != 0) {
 		if gd, err := globalIAMSys.GetGroupDescription(updReq.Group); err == nil && gd.UpdatedAt.After(updatedAt) {
 			return nil
 		}
@@ -1349,7 +1380,8 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
 	if updReq.IsRemove {
 		_, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
 	} else {
-		if updReq.Status != "" && len(updReq.Members) == 0 {
+		snapshot, _ := ctx.Value(iamGroupSnapshotKey{}).(bool)
+		if !snapshot && updReq.Status != "" && len(updReq.Members) == 0 {
 			_, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
 		} else {
 			if globalIAMSys.LDAPConfig.Enabled() {
@@ -1365,13 +1397,14 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
 		}
 	}
 	if err != nil && !errors.Is(err, errNoSuchGroup) {
-		return wrapSRErr(err)
+		return iamReplicationError(err)
 	}
 	return nil
 }
 
 // PeerSvcAccChangeHandler - copies service-account change to local.
 func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange, updatedAt time.Time) error {
+	ctx = withIAMReplicationTime(ctx, updatedAt)
 	if change == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
@@ -1382,7 +1415,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 		if len(change.Create.SessionPolicy) > 0 {
 			sp, err = policy.ParseConfig(bytes.NewReader(change.Create.SessionPolicy))
 			if err != nil {
-				return wrapSRErr(err)
+				return iamReplicationError(err)
 			}
 		}
 		// skip overwrite of local update if peer sent stale info
@@ -1394,6 +1427,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 		opts := newServiceAccountOpts{
 			accessKey:     change.Create.AccessKey,
 			secretKey:     change.Create.SecretKey,
+			status:        change.Create.Status,
 			sessionPolicy: sp,
 			claims:        change.Create.Claims,
 			name:          change.Create.Name,
@@ -1402,7 +1436,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 		}
 		_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
 		if err != nil {
-			return wrapSRErr(err)
+			return iamReplicationError(err)
 		}
 
 	case change.Update != nil:
@@ -1411,7 +1445,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 		if len(change.Update.SessionPolicy) > 0 {
 			sp, err = policy.ParseConfig(bytes.NewReader(change.Update.SessionPolicy))
 			if err != nil {
-				return wrapSRErr(err)
+				return iamReplicationError(err)
 			}
 		}
 		// skip overwrite of local update if peer sent stale info
@@ -1431,7 +1465,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 
 		_, err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts)
 		if err != nil {
-			return wrapSRErr(err)
+			return iamReplicationError(err)
 		}
 
 	case change.Delete != nil:
@@ -1442,7 +1476,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 			}
 		}
 		if err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true); err != nil {
-			return wrapSRErr(err)
+			return iamReplicationError(err)
 		}
 	}
 
@@ -1451,24 +1485,17 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 
 // PeerPolicyMappingHandler - copies policy mapping to local.
 func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping, updatedAt time.Time) error {
+	ctx = withIAMReplicationTime(ctx, updatedAt)
 	if mapping == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
-	// skip overwrite of local update if peer sent stale info
-	if !updatedAt.IsZero() {
-		mp, ok := globalIAMSys.store.GetMappedPolicy(mapping.Policy, mapping.IsGroup)
-		if ok && mp.UpdatedAt.After(updatedAt) {
-			return nil
-		}
-	}
-
 	// When LDAP is enabled, we verify that the user or group exists in LDAP and
 	// use the normalized form of the entityName (which will be an LDAP DN).
 	userType := IAMUserType(mapping.UserType)
 	isGroup := mapping.IsGroup
 	entityName := mapping.UserOrGroup
 
-	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser {
+	if mapping.Policy != "" && globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser {
 		// Validate that the user or group exists in LDAP and use the normalized
 		// form of the entityName (which will be an LDAP DN).
 		var err error
@@ -1491,19 +1518,20 @@ func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mappi
 			entityName = foundUserDN.NormDN
 		}
 		if err != nil {
-			return wrapSRErr(err)
+			return iamReplicationError(err)
 		}
 	}
 
 	_, err := globalIAMSys.PolicyDBSet(ctx, entityName, mapping.Policy, userType, isGroup)
 	if err != nil {
-		return wrapSRErr(err)
+		return iamReplicationError(err)
 	}
 	return nil
 }
 
 // PeerSTSAccHandler - replicates STS credential locally.
 func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential, updatedAt time.Time) error {
+	ctx = withIAMReplicationTime(ctx, updatedAt)
 	if stsCred == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
@@ -1555,7 +1583,7 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
 
 	// Set these credentials to IAM.
 	if _, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, stsCred.ParentPolicyMapping); err != nil {
-		return fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err)
+		return iamReplicationError(fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err))
 	}
 
 	return nil
@@ -4193,7 +4221,8 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 				}
 
 				info.UserInfoMap[k] = madmin.UserInfo{
-					Status: madmin.AccountStatus(v.Credentials.Status),
+					Status:    madmin.AccountStatus(v.Credentials.Status),
+					UpdatedAt: v.UpdatedAt,
 				}
 			}
 		}
@@ -4549,9 +4578,25 @@ func (c *SiteReplicationSys) PeerStateEditReq(ctx context.Context, arg madmin.SR
 const siteHealTimeInterval = 30 * time.Second
 
 func (c *SiteReplicationSys) startHealRoutine(ctx context.Context, objAPI ObjectLayer) {
-	ctx, cancel := globalLeaderLock.GetLock(ctx)
-	defer cancel()
+	for ctx.Err() == nil {
+		var leadership LockContext
+		select {
+		case <-ctx.Done():
+			return
+		case leadership = <-globalLeaderLock.lockContext:
+		}
+		if leadership.Context().Err() != nil {
+			continue
+		}
+		leaderCtx, cancel := mergeContext(leadership.Context(), ctx)
+		c.healWithLeadership(leaderCtx, objAPI)
+		cancel()
+		// Quorum loss cancels a leadership lease, not the subsystem. Wait
+		// for a new lease so revocations can still reach offline sites.
+	}
+}
 
+func (c *SiteReplicationSys) healWithLeadership(ctx context.Context, objAPI ObjectLayer) {
 	healTimer := time.NewTimer(siteHealTimeInterval)
 	defer healTimer.Stop()
 
@@ -5170,13 +5215,16 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
 }
 
 func (c *SiteReplicationSys) healIAMSystem(ctx context.Context, objAPI ObjectLayer) error {
+	// A peer rejecting a deletion must not stop unrelated live updates from
+	// reaching healthy peers. Retain and report the error for the next retry.
+	deletionErr := c.healIAMDeletions(ctx)
 	info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
 		Users:    true,
 		Policies: true,
 		Groups:   true,
 	})
 	if err != nil {
-		return err
+		return errors.Join(deletionErr, err)
 	}
 	for policy := range info.PolicyStats {
 		c.healPolicies(ctx, objAPI, policy, info)
@@ -5194,7 +5242,7 @@ func (c *SiteReplicationSys) healIAMSystem(ctx context.Context, objAPI ObjectLay
 		c.healGroupPolicies(ctx, objAPI, group, info)
 	}
 
-	return nil
+	return deletionErr
 }
 
 // heal iam policies present on this site to peers, provided current cluster has the most recent update.
@@ -5429,8 +5477,10 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 
 		peerName := info.Sites[dID].Name
 
-		u, ok := globalIAMSys.GetUser(ctx, user)
-		if !ok {
+		// Disabled identities are valid replication sources. CheckKey returns
+		// their stored record even though authentication is denied.
+		u, _, err := globalIAMSys.CheckKey(ctx, user)
+		if err != nil || u.Credentials.AccessKey == "" || u.Credentials.IsExpired() {
 			continue
 		}
 		creds := u.Credentials
@@ -5629,7 +5679,10 @@ func isGroupDescEqual(g1, g2 madmin.GroupDesc) bool {
 }
 
 func isUserInfoEqual(u1, u2 madmin.UserInfo) bool {
-	if u1.PolicyName != u2.PolicyName ||
+	// Full-site summaries omit secrets and claims. Equal status alone cannot
+	// distinguish a recreated identity or an edited service-account policy.
+	if !u1.UpdatedAt.Equal(u2.UpdatedAt) ||
+		u1.PolicyName != u2.PolicyName ||
 		u1.Status != u2.Status ||
 		u1.SecretKey != u2.SecretKey {
 		return false

cmd/sts-handlers.go

@@ -624,6 +624,10 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 
 	claims[expClaim] = UTCNow().Add(duration).Unix()
 	claims[parentClaim] = user.AccessKey
+	if err := setIAMParentRevocationClaim(ctx, globalIAMSys.store, user.AccessKey, claims); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
+		return
+	}
 
 	tokenRevokeType := r.Form.Get(stsRevokeTokenType)
 	if tokenRevokeType != "" {

docs/investigations/r4/consensus.md

@@ -0,0 +1,33 @@
+# R4 plan consensus and review disposition
+
+## Agreed version
+
+- Plan: [plan v1](plan-v1.md), SHA-256 `ad539f2071155de6955b583991684ed33c4bfe2e29660005840cdc97d7e1a754`. The frozen file remains unchanged.
+- Baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`.
+- Actual reviewer: Claude Code 2.1.270, every assistant model in the review stream is `claude-opus-5`; explicit `--effort max`.
+- Opus: **GO_WITH_NONBLOCKING_NOTES**, zero blockers; explicitly agrees that this exact plan can enter local implementation. [Unedited returned review](opus-v1-review.md), [machine-readable provenance](opus-v1.metadata.json).
+- Codex: agrees that adding the already-parsed timestamp to the KMS literal fixes R4, and accepts the nonblocking dispositions below. **No blocking disagreement remains on plan v1.** No production source edits were made before this record was saved.
+- The agreement permits the planned local implementation and tests; it is not implementation acceptance, a merge decision or production release approval.
+
+## Item-by-item disposition
+
+| Opus ID | Disposition |
+|---|---|
+| R4-01 | Accepted citation correction here, leaving the agreed hash frozen: `ReplicaLockReconcile` is at baseline `object-handlers.go:1847`; encryption merge is at `:1903`. |
+| R4-02 | Accepted scope clarification: ErasureSD and Erasure16 are both single-pool local backends. KMS rewrites use PutObject under-lock reconciliation. Multi-pool and multi-site validation are optional and deferred to the wider integration gate. Test comments and the final report will identify this boundary. |
+| R4-03 | Accepted wording clarification: source encryption alone does not request destination encryption. Source-only SSE-C copy headers do not prevent destination bucket/default auto-KMS from selecting KMS. The three destination trigger categories stay unchanged. |
+| R4-04 | Accepted intent. The regression matrix uses identical expected mtime, ETag, trust and all three source timestamps across all encryption modes, giving field-by-field equivalence without constructing expected values through the production function. The temporary expanded baseline matrix fails only trusted valid KMS tag timestamps. |
+| R4-05 | Accepted optional test within the existing scope: a signed KMS COPY with nonempty tags and no source tag timestamp must preserve the stored value/time. This adds evidence, not production behavior. |
+| R4-06 | Registered as a separate unverified-impact finding: KMS construction also omits `ProxyHeaderSet`, `ProxyRequest`, `Speedtest` relative to `getDefaultOpts`. No R4 fix or correctness claim for those flags. Send the observation to the parent for separate triage; do not assign it to R5. |
+| R4-07 | Accepted. Assertions target final disk state; the REPLACE handler rebuilds metadata, while final stored-tag rejection occurs under the storage write lock. HTTP 200 alone is not acceptance. |
+| R4-08 | Resolved provenance uncertainty by Codex: SHA-256 recomputed before/after review, baseline identity and current GitHub main/PR query captured in `baseline-identity.txt`. History was inspected locally with `git blame` / `git show`. Opus's read-only tools did not independently recompute the hash or check GitHub; those facts remain attributed to the local commands. |
+
+## Raw evidence
+
+Directory: `/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/`.
+
+- `review-prompt-v1.md`, `opus-review-v1.jsonl`, `opus-review-v1.stderr.log`, `opus-review-v1.exit`.
+- `baseline-identity.txt`, `r4_repro_test.go`, `overlay.json`, `baseline-repro.log`.
+- `options_repro_test.go`, `options-overlay.json`, `baseline-options.log`.
+
+The stream includes an attempted Write to Claude's own plan file. Its tool was disabled; the reviewer returned the full result in text and did not edit production source. The successful result and actual assistant models are checked separately from rate-limit status and auxiliary-model usage.

docs/investigations/r4/implementation-review.md

@@ -0,0 +1,39 @@
+已完成独立复核：读了三份改动文件在 HEAD 的实际内容、`putOptsFromHeaders` 全函数与两条返回路径、`CopyObjectHandler` 的默认加密与标签排序段、`reconcileStoredObjectTags` 的三处生产调用点，以及全部指定日志与证据文件。
+
+# R4 实现复核结论
+
+**Verdict: GO_WITH_NONBLOCKING_NOTES（0 阻断项）**
+
+- 复核 HEAD：`dbcf8dec589deb5d91e17d295cb70997635f5b55`
+- 代码/测试 diff SHA-256（按提供值记录）：`c8cd6648f8ecea835ec74a038cdeaa82acaa3f36250395f97ead3260dc2fc0a5`
+  - 本会话无 shell，未重算该哈希；改为逐行比对 diff 与工作树三份文件，内容完全一致（`cmd/object-api-options.go`、`cmd/object-api-options-replication_test.go`、`cmd/object-copy-replication-tagging_test.go`）。
+
+## 核验到的事实
+
+- 生产改动确为一个字段 + 相邻注释：`cmd/object-api-options.go:459` 的 `ReplicationSourceTaggingTimestamp: taggingtimestmp`，变量来自 `:419-425` 已解析值，与非 KMS 路径 `:473` 对齐。未动解析、信任判定、KMS key/context、返回结构。
+- 影响面封闭：全仓该字段唯一消费点是 `cmd/object-handlers.go:1820`（COPY 标签排序）。PUT/POST/multipart 虽同经 `putOptsFromReq`，但无消费者，故不可能回归——与 R4/R5 切分一致。
+- 三条 KMS 触发路径真实可达：`object-handlers.go:1428-1433` 在 `copyDstOpts`（`:1454`）之前套用目的端默认；`bucket-sse-config.go:139-151` 在 `nil 配置 + AutoEncrypt` 与桶默认 KMS 两种情况下都写入 `aws:kms`，因此 explicit / auto / bucket 三种模式均进入 KMS 分支。
+- 回归证明成立：`baseline-final.log` 用 `-overlay` 换回未修复 constructor，失败面精确为「trusted × 有效标签时间戳 × SSE-KMS / SSE-KMS-context」和 6 个 KMS COPY 子测试（`tags="key=old"`、`kms=true`、HTTP 200），`none`/`SSE-S3`/`SSE-C`/非 trusted 全通过。修复后 `focused.log:194-204` 全 PASS。
+- 测试确实覆盖被要求的维度：信任边界（trusted=false 时 mtime/ETag/三时间戳全归零）、错误路径（trusted + 畸形值必须报错且错误串含头名）、SSE 序列化回环（KMS keyID/context 原样还原）、磁盘终态（每事件 `obj.GetObjectInfo` 读真实盘）、版本一致性、签名 GET 明文可读。全局 `GlobalKMS`/`globalAutoEncryption`/`set.getDisks` 均 defer 还原。
+- `race` exit 0、`vet` 空输出、`golangci-lint` 0 issues，均记录了与 HEAD 一致的三文件哈希。
+- 未发现 `verification.md` / `verification.json` / `consensus.md` 中与日志矛盾的陈述。（评审者版本/模型/effort 这类 provenance 声明不在我可验证范围，未作背书。）
+
+## 发现清单
+
+| ID | 内容 | 阻断 |
+|---|---|---|
+| IMPL-01 | 单字段修复正确且充分，位置、变量、注释与 `:473` 语义一致 | 否（确认项） |
+| IMPL-02 | 基线失败/修复通过的判别力成立，对照组不误报 | 否（确认项） |
+| IMPL-03 | KMS 字面量相对 `getDefaultOpts` 仍缺 `ProxyHeaderSet`/`ProxyRequest`/`Speedtest`（`object-api-options.go:40-44` vs `:449-460`）。R4 范围外，已登记为 R4-06 | 否，不设为新合并门槛 |
+| IMPL-04 | `metadata-directive: REPLACE` 下 `getCpObjMetadataFromHeader`（`:1143-1156`）返回全新 map，故 `:1818` 的 `lastTaggingTimestamp` 为空、`:1822` 解析失败使 handler 侧比较恒「incoming 胜」；真正的 stale 拒绝发生在写锁内的 `reconcileStoredObjectTags`（`erasure-object.go:1312-1315`）。测试终态断言仍正确，文档 R4-07 已明示此分工 | 否（R5 上下文） |
+| IMPL-05 | 测试卫生：`bucket-kms` 模式写入的 `bucketSSEConfig` 未还原，仅因它是最后一个 mode、且 `ExecObjectLayerAPITest` 每后端重建对象层并 `resetTestGlobals()` 才安全；后续若在其后追加 mode 会继承默认 KMS | 否 |
+| IMPL-06 | `object-api-options-replication_test.go:35` 局部变量名 `context` 遮蔽标准包名（本文件未导入该包），纯观感 | 否 |
+| IMPL-07 | `focused.log` exit 1 的唯一失败是既有 `TestAPICopyObjectReplicaRetentionRemovalUnderBucketKMS`（`replication-trust_test.go:1284`，"Storage reached its minimum free drive threshold"），属本机磁盘余量环境问题，非本次引入；容量 overlay 是测试专用、未提交。新增 COPY 测试自带 `tagTestCapacityDisk` 包装，不受该阈值影响 | 否 |
+
+**没有发现阻断性正确性问题。** 生产语义、存储格式、API 与既有排序规则均未改变，无任何既有测试断言旧（缺陷）行为。
+
+## 合并适配性
+
+`dbcf8dec5` 直接位于实时 main `9ebe81c1b` 之上，可快进合并。按仓库 CI 通过为前提，本实现适合合入 main。
+
+*（我未运行任何测试，也未查询 GitHub；以上仅基于源码阅读与所提供日志。）*

docs/investigations/r4/implementation-review.metadata.json

@@ -0,0 +1,33 @@
+{
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "reviewed_head": "dbcf8dec589deb5d91e17d295cb70997635f5b55",
+  "requested_model": "claude-opus-5",
+  "requested_effort": "max",
+  "cli_version": "2.1.270",
+  "diff_sha256": "c8cd6648f8ecea835ec74a038cdeaa82acaa3f36250395f97ead3260dc2fc0a5",
+  "started_at": "2026-09-15T15:59:41.004963+00:00",
+  "status": "completed",
+  "command": "/opt/homebrew/bin/claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --add-dir /Users/vonng/tmp/silo-r4-evidence-20260915-a9cb --output-format stream-json --verbose",
+  "completed_at": "2026-09-15T16:03:52.331640+00:00",
+  "assistant_models": [
+    "claude-opus-5"
+  ],
+  "observed_model": "claude-opus-5",
+  "verdict": "GO_WITH_NONBLOCKING_NOTES",
+  "blocking_findings": 0,
+  "session_id": "c09bc4fb-85f1-4af8-a1de-c453398e4a20",
+  "duration_ms": 161469,
+  "subtype": "success",
+  "is_error": false,
+  "used_tools": {
+    "Read": 20,
+    "Glob": 3,
+    "Grep": 14,
+    "ExitPlanMode": 1
+  },
+  "raw_stream": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/merge-review-1/opus.jsonl",
+  "stream_sha256": "875f617454b27e15ab44d9b777de89643ee352e0ccb948faad570fc546261acc",
+  "review_sha256": "962ff88d2ffcb75cd692ec17017411d624de80dc120f8dedc675e0fe25009335",
+  "prompt_sha256": "e1341c721161229431b943ba18d89b740e94470803c099b9ae3d597fd50544a4",
+  "review_extraction": "The substantive review is an earlier assistant text block; result.result only repeats CLI plan-mode merge limitations. Full raw stream and all assistant text are retained."
+}

docs/investigations/r4/merge-verification.json

@@ -0,0 +1,84 @@
+{
+  "original_reviewed_head": "dbcf8dec589deb5d91e17d295cb70997635f5b55",
+  "dco_signed_equivalent_head": "03027727d1d1b97d8beb83ac55569ea9a83dab23",
+  "notice_equivalence": {
+    "cmd/object-api-options-replication_test.go": {
+      "before_sha256": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+      "after_sha256": "c21fc8889a079085d9a882499a1cbe868278a3517580651f3bed1102e2a6aef8",
+      "package_body_sha256": "096f143c0b0a068581f9bb892f35ded0d65b6b60ab711f043236d27fbf51ca33",
+      "body_unchanged": true
+    },
+    "cmd/object-copy-replication-tagging_test.go": {
+      "before_sha256": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3",
+      "after_sha256": "73f066ed7258d430f078ecc90e551ece878bd3d4672bc762094d434ff8fec23d",
+      "package_body_sha256": "6b5173db2ded2d54055073c3259be208a4d7c8eac0367687082877f1fd3bef15",
+      "body_unchanged": true
+    }
+  },
+  "checks": {
+    "verifiers": {
+      "command": [
+        "make",
+        "verifiers",
+        "GOLANGCI=/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/merge-review-1/golangci-serial"
+      ],
+      "exit_code": 0,
+      "started_at": "2026-09-15T16:04:55.536605+00:00",
+      "finished_at": "2026-09-15T16:07:03.252820+00:00",
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2",
+        "GOFLAGS": "-p=2"
+      },
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "c21fc8889a079085d9a882499a1cbe868278a3517580651f3bed1102e2a6aef8",
+        "cmd/object-copy-replication-tagging_test.go": "73f066ed7258d430f078ecc90e551ece878bd3d4672bc762094d434ff8fec23d"
+      },
+      "log_sha256": "e42a5bb55f5c1ebfcf02cebebf6d82cf1ec5a2d74590cdf838deba16dd80bfdf"
+    },
+    "build": {
+      "command": [
+        "make",
+        "build"
+      ],
+      "exit_code": 0,
+      "started_at": "2026-09-15T16:07:03.253715+00:00",
+      "finished_at": "2026-09-15T16:07:35.594062+00:00",
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2",
+        "GOFLAGS": "-p=2"
+      },
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "c21fc8889a079085d9a882499a1cbe868278a3517580651f3bed1102e2a6aef8",
+        "cmd/object-copy-replication-tagging_test.go": "73f066ed7258d430f078ecc90e551ece878bd3d4672bc762094d434ff8fec23d"
+      },
+      "log_sha256": "6ba9b545236be964861749c72e7609edf12b8f470df30d1ede8fd62f497e629b"
+    },
+    "binary-version": {
+      "command": [
+        "./silo",
+        "--version"
+      ],
+      "exit_code": 0,
+      "started_at": "2026-09-15T16:07:35.594918+00:00",
+      "finished_at": "2026-09-15T16:07:37.616706+00:00",
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2",
+        "GOFLAGS": "-p=2"
+      },
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "c21fc8889a079085d9a882499a1cbe868278a3517580651f3bed1102e2a6aef8",
+        "cmd/object-copy-replication-tagging_test.go": "73f066ed7258d430f078ecc90e551ece878bd3d4672bc762094d434ff8fec23d"
+      },
+      "log_sha256": "36317d06b691593fe0d74f88d053a24485500c15fc2001e857f2fc6fa5ba752a"
+    }
+  },
+  "binary_version": "silo version DEVELOPMENT.2026-09-15T16-03-52Z (commit-id=03027727d1d1b97d8beb83ac55569ea9a83dab23)\nRuntime: go1.27.1 darwin/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-2025 MinIO, Inc.\nModifications: Copyright 2025-2026 PGSTY\nSource compatibility: based on MinIO technology\n",
+  "raw_evidence_directory": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/merge-review-1",
+  "all_function_and_test_bodies_identical_to_opus_reviewed_version": true
+}

docs/investigations/r4/merge-verification.md

@@ -0,0 +1,36 @@
+# R4 合并前复核
+
+用户已明确追加授权：使用 Opus 5 max 核实最终实现，确认无误后合并 main。本轮授权取代此前只交付本地补丁的范围限制。
+
+## 真实实现评审
+
+- 独立新调用：Claude Code 2.1.270，`--model claude-opus-5 --effort max`。
+- 复核代码提交：`dbcf8dec589deb5d91e17d295cb70997635f5b55`；当时实时 main 与 fetch 结果均为 `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`。
+- 实际 assistant 模型只有 `claude-opus-5`。结论 **GO_WITH_NONBLOCKING_NOTES，0 阻断项**，明确表示仓库 CI 通过后适合合入 main。
+- [原始实现评审正文](implementation-review.md)、[实际模型与输出哈希](implementation-review.metadata.json) 已保存。
+- 原始流、全部 assistant 正文与最终 result 位于 `/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/merge-review-1/`。实质评审出现在较早的 assistant 消息；最终 result 只重复 Claude 只读会话不能自行合并的工具限制，不是对修复结论的撤回。本任务由 Codex 按用户明确授权完成合并。
+
+## 意见处置
+
+| 条目 | 处置 |
+|---|---|
+| IMPL-01 / IMPL-02 | 确认单字段修复和基线失败/修复通过的测试判别力，无需追加修改。 |
+| IMPL-03 | Proxy/Speedtest 选项遗漏已交父任务单独核验，维持范围外，不纳入 R4 合并。 |
+| IMPL-04 | REPLACE 请求的旧标签拒绝由写锁内对账完成，测试断言真实落盘状态，已有文档准确说明。 |
+| IMPL-05 | 当前测试固定以 bucket-kms 为最后一种模式，且每后端重新初始化；现有执行顺序安全。后续增添模式需同步隔离桶默认配置，本次保持已评审测试逻辑。 |
+| IMPL-06 | 局部变量 context 命名建议为可选观感项，不改动已评审逻辑。 |
+| IMPL-07 | 既有锁测试的磁盘余量限制及仅测试容量 overlay 已如实记录；新测试和 race 不使用生产代码 overlay。 |
+
+## 提交规范调整
+
+按 `CONTRIBUTING.md` 补齐提交作者对应的 DCO sign-off，并将两个新原创测试文件的文件头改为 `Copyright (c) 2026 Feng Ruohang`，保留 AGPL-3.0-or-later。原有生产文件的继承声明保持原样。
+
+生产函数和测试的 `package cmd` 之后内容与 Opus 审查版本逐字节相同。`merge-review-1/notice-equivalence.json` 记录了旧/新文件哈希及不变的代码正文哈希。原 `verification.json` 保留当时原始验证记录，不覆盖历史哈希；本轮 PR 的 CI 对最终提交重新验证。
+
+## 合并门槛
+
+`make verifiers` 已通过：全仓 lint 为 0 issues，生成文件检查通过，rebrand 兼容性清单未变化，交付/运行时标识检查和 entrypoint 参数兼容性测试通过。首次执行曾遇到其他任务持有 golangci-lint 进程锁；使用工具自带 `--allow-serial-runners` 串行等待后完成全部检查。可选 typos 工具未安装，由仓库 Makefile 按既有规则跳过。
+
+`make build` 通过，已生成本地 `silo` 并成功执行 `./silo --version`。最终三个源文件哈希与本轮校验记录一致，详情见 [本轮验证清单](merge-verification.json)。
+
+接下来由 PR CI 验证最终候选，并在合并前再次核对 main 和精确 PR head。CI 与合并事实以 GitHub PR 状态和本机原始合并证据为准，评审意见不等同于合并或发布。

docs/investigations/r4/opus-v1-review.md

@@ -0,0 +1,46 @@
+## R4 独立评审（Opus 5.0，只读复核）
+
+**计划**：plan v1 — `docs/investigations/r4/plan-v1.md`
+**哈希（按任务给定）**：`ad539f2071155de6955b583991684ed33c4bfe2e29660005840cdc97d7e1a754`
+**Baseline**：`9ebe81c1b3611f9cc73e676b5b741c2be62c467a`
+
+### 裁定：GO_WITH_NONBLOCKING_NOTES
+
+无阻断项。**我同意这份 exact plan（上述哈希）可以进入本地实现。** 下列 R4-01…R4-08 均为非阻断；其中 R4-02/04/05 的测试建议是**可选**的，不自动并入必做范围。
+
+（说明：本会话 Write 工具被禁用，评审仅以正文返回，未写入任何文件，也未改动任何源码。）
+
+### 我实际核验到的关键事实（支撑"单字段补丁正确且充分"）
+
+1. **缺陷确认**：`cmd/object-api-options.go:449-460` 的 KMS 字面量带了 MTime/PreserveETag/ReplicationRequest + 两个 Object Lock 时间戳，独缺 tagging；默认路径 `:473` 有。补丁片段中的变量名 `taggingtimestmp` 与 `:419` 完全一致，可直接编译；gofmt 对齐由更长的两个 Lock 键决定，不会扰动他行。
+2. **影响面封闭**：全仓 `ReplicationSourceTaggingTimestamp` 只在 `cmd/object-handlers.go:1820` 被读取（定义于 `object-api-interface.go:99`）。因此该字段对 PUT/分段路径天然无效果——既印证 R4/R5 的切分合理，也说明补丁不可能回归其他路径。
+3. **充分性的关键点（我重点查证的风险）**：`encMetadata` 只有在 SSE-C 轮换分支 `object-handlers.go:1648-1659` 才批量快照全部保留键，而该分支与 KMS options 分支互斥（目的端是 SSE-C 时 `crypto.S3KMS.IsRequested` 为假）。故 `:1903` 的 `maps.Copy(srcInfo.UserDefined, encMetadata)` **不会**覆盖 KMS COPY 新写入的 tags/时间戳 —— 单字段补丁在 R4 边界内充分。
+4. **三个触发点准确**：`bucket-sse-config.go:135-153`（显式请求优先 → nil 配置 + AutoEncrypt → KMS → bucket 默认 KMS 写 header+keyID；默认 AES 走 AES 分支），配合 `object-handlers.go:1428-1433` 仅在非联邦时套用目的端默认。
+5. **REPLACE 副本路径准确**：`reconcileStoredObjectTags`（`erasure-server-pool-consistency.go:232-243`）语义即"存量有效时间戳胜过缺失/更旧/相等的 incoming，并连同 tag 值一起还原"。KMS 目的端因 `isTargetEncrypted` 使 `metadataOnly=false`，实际落到 `erasure-server-pool.go:1499-1513`（`ReplicaLockReconcile` 经 `:1509` 透传）→ `erasure-object.go:1276-1316`，在 `cloneMSS`(:1324) 之前于写锁内完成对账；纯元数据路径走 `erasure-object.go:136-139`。计划同时引用 `:136` 与 `:1509`，判断正确。
+6. **证据可信**：`baseline-repro.log` 中 options 用例非 KMS 保留 `...123456789Z`、KMS 返回零值；COPY 用例 6/6（ErasureSD + Erasure16 × explicit/auto/bucket KMS）失败，且均为 200、`kms=true`、明文 GET 通过、tags 停在 `key=old`。即"请求成功、加密正常，但复制标签被静默丢弃"，与计划表述一致，未夸大。
+7. **修复后推演**：newer/stale/duplicate/newer-again 在 handler(:1817-1833) 与写锁对账的双重排序下分别得到 new/new/new/latest，与测试期望吻合；旧发送端不带 `X-Minio-Source-Tagging-Timestamp` 时仍为零值 → 行为不变，兼容性主张成立。
+
+### 问题清单
+
+| ID | 阻断 | 内容与建议 |
+|---|---|---|
+| **R4-01** | 否 | 行号漂移：计划写的 `1851/1910`，实际是 `object-handlers.go:1847`（`ReplicaLockReconcile`）与 `:1903`（encMetadata merge）。建议更正引用。 |
+| **R4-02** | 否（建议可选） | `ExecObjectLayerAPITest` 两种后端均为**单 pool**（`test-utils_test.go:216` `mustGetPoolEndpoints(0, ...)`），故 `erasure-server-pool.go:1443` 多池分支未被覆盖；且 KMS 目的端命中的是 PutObject 重写对账而非 `CopyObject:136`。建议在计划或测试注释中点明"单盘/16 盘均为单池"；补多池覆盖**可选**，不必进必做范围。 |
+| **R4-03** | 否 | 措辞：`crypto.Requested`（`internal/crypto/sse.go:74`）只检查**目的端** SSE 头，因此仅带 SSE-C *copy-source* 头的请求在 KMS 默认桶/自动加密下仍会进入 KMS 分支（归入触发点 2/3，枚举仍完整）。建议澄清 "source encryption alone…" 一句。 |
+| **R4-04** | 否（**可选**） | 建议在 options 矩阵里加一条 KMS 分支 vs 默认分支的**逐字段等价断言**（MTime/PreserveETag/ReplicationRequest/三个复制时间戳）。这是阻止第三次复发最廉价的护栏（2021 漏、2026 补了两个 Lock 时间戳仍漏此项）。计划第 1 条已基本覆盖，此为结构化建议。 |
+| **R4-05** | 否（**可选**） | 建议加一例"KMS 目的端 + 有 tags 但无 tagging 时间戳头 → 存量不变"，把兼容性主张钉在 handler 层而不仅在 options 层。 |
+| **R4-06** | 否（范围外，仅登记） | 同一 KMS 字面量相对 `getDefaultOpts`(`object-api-options.go:40-44`) 还遗漏 `ProxyHeaderSet/ProxyRequest/Speedtest`；`opts.Speedtest` 在 `erasure-object.go:1625` 被读取，全局自动加密下 speedtest PUT 会丢该标志。**不要在 R4 修**，且当前也不在 R5 声明范围内，建议单列条目登记。 |
+| **R4-07** | 否 | REPLACE 时 `getCpObjMetadataFromHeader:1143-1156` 会重建 map，`lastTaggingTimestamp` 为空 → handler 对 stale 事件**恒接受**，真正的拒绝来自写锁内对账。因此回归测试必须断言**最终落盘状态**（现有复现已如此），不要改为断言 handler 层行为。 |
+| **R4-08** | 否（不确定性） | 本会话无 shell，无法独立复算计划 SHA-256、验证 `c4373ef290 / b2dca43fda / cfefc049c` 历史归属与 PR #184/#187。可由 `shasum -a 256 docs/investigations/r4/plan-v1.md` 与 `git log -L` 输出消解；均不影响补丁正确性。`cfefc049c` 的 KMS context 编码修复实体（`:436-444` 的 `sdkContext`）仍在，补丁不触碰。 |
+
+### 对计划各主张的逐项裁定
+
+- 单字段补丁**正确且对本 bounded issue 充分**：同意（依据 2/3/7）。
+- 三个目的端 KMS 触发点**描述准确**：同意（R4-03 仅措辞澄清）。
+- 当前 REPLACE 副本路径**描述准确**：同意（R4-01/02 属引用精度）。
+- 回归矩阵与存量状态说明**充分**：同意；存量部分"不自动回填、丢失源时间不可重建、并列/更旧事件不保证修复"的表述与 `reconcileStoredObjectTags` 实际语义一致。
+- 信任边界、错误行为、加密 key/context、tie 语义、兼容性：补丁均未触碰，维持不变。
+
+### 交付
+
+本轮为**计划共识**，非实现验收。我未作任何源码或文件修改；R4 可按 plan v1 在本地实施，实施后的差异与测试证据需另行验收。

docs/investigations/r4/opus-v1.metadata.json

@@ -0,0 +1,36 @@
+{
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "plan": "docs/investigations/r4/plan-v1.md",
+  "plan_sha256": "ad539f2071155de6955b583991684ed33c4bfe2e29660005840cdc97d7e1a754",
+  "requested_model": "claude-opus-5",
+  "requested_effort": "max",
+  "cli_version": "2.1.270",
+  "started_at": "2026-09-15T15:45:46.438630+00:00",
+  "status": "completed",
+  "raw_output": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/opus-review-v1.jsonl",
+  "raw_stderr": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/opus-review-v1.stderr.log",
+  "command": "/opt/homebrew/bin/claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --add-dir /Users/vonng/tmp/silo-r4-evidence-20260915-a9cb --output-format stream-json --verbose",
+  "completed_at": "2026-09-15T15:49:48.150062+00:00",
+  "assistant_models": [
+    "claude-opus-5"
+  ],
+  "observed_model": "claude-opus-5",
+  "verdict": "GO_WITH_NONBLOCKING_NOTES",
+  "blocking_findings": 0,
+  "result_subtype": "success",
+  "is_error": false,
+  "session_id": "63e14a68-8565-41fa-9746-3e405fb63e9f",
+  "duration_ms": 161784,
+  "num_turns": 35,
+  "used_tools": {
+    "Read": 18,
+    "Glob": 4,
+    "Grep": 11,
+    "Write": 1
+  },
+  "stream_sha256": "eb0918d8a6185b180dddcfc664a96682f05502ecf3b686b08a0547f09879d57d",
+  "review_sha256": "e1dc12dd99326ae432623ff8de201813e6e84e7ed16a5556c21f9c514d663676",
+  "prompt_sha256": "07c225beff1523e056c154b3a387cf1ae345def4b4d0173065b882140ac5abdf",
+  "tool_scope_note": "Read/Grep/Glob allowed. Claude attempted Write to its own plan; the tool was disabled and no file was written. git diff before consensus showed no production source changes.",
+  "auxiliary_model_note": "assistant_models records actual reviewing assistant messages. Auxiliary usage is distinct. --effort max is explicit in the command, not inferred from model usage."
+}

docs/investigations/r4/plan-v1.md

@@ -0,0 +1,65 @@
+# R4 plan v1: preserve the replicated tag timestamp for SSE-KMS
+
+## Baseline and ownership
+
+- Baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`, verified against GitHub main on 2026-09-15.
+- Branch: `codex/r4-kms-tag-timestamp`; worktree: `/Users/vonng/.codex/worktrees/a9cb/silo`.
+- Live open PRs at inspection: #184 and #187, neither owns this options change.
+- The worktree lacks the ignored `AGENTS.md`; the parent explicitly confirms `/Users/vonng/pgsty/silo/AGENTS.md` applies. Maintain the PGSTY product graph and inexpensive compatibility.
+- R4 owns only the missing field in `cmd/object-api-options.go` and its regression tests. R5 owns DELETE/empty tags, PUT/multipart receiving, sender propagation and full receiver ordering. R4 will supply a standalone source patch to R5; neither task edits the other's worktree.
+
+## Proven defect and actual trigger
+
+`putOptsFromHeaders` parses the trusted source tag timestamp before selecting encryption. The SSE-KMS branch constructs and returns another `ObjectOptions` carrying mtime, ETag, replication trust and both Object Lock timestamps, but omits `ReplicationSourceTaggingTimestamp`. The normal path retains it. The parser accepts and preserves RFC3339 fractional seconds even though its layout is `time.RFC3339`; the reproduction uses nanoseconds.
+
+`CopyObjectHandler` applies local destination encryption configuration before `copyDstOpts` → `putOptsFromReq` → `putOpts` → `putOptsFromHeaders`. The omission is reached by:
+
+1. Explicit destination SSE-KMS request headers (with or without a key ID/context).
+2. A destination bucket with default SSE-KMS, when the request has no explicit SSE choice.
+3. Global automatic encryption with no bucket SSE override and no explicit SSE choice.
+
+Explicit AES256/SSE-C takes its existing branch; source encryption alone does not select the destination KMS branch. Remote federation skips local destination defaults. The relevant trigger is trusted metadata entering the destination KMS branch, not every SSE-KMS object or every tag operation.
+
+At `CopyObjectHandler`'s tag decision, a zero source timestamp skips the tag update. Current under-lock reconciliation can preserve the stored tag/timestamp when metadata REPLACE reconstructs the map with no timestamp. In the observed same-version replica COPY, the request succeeds, destination encryption is valid, and the old tags/timestamp remain. A missing field in the options layer is not itself proof of a content-read failure.
+
+PUT and multipart consumers' independent failure to persist a parsed tag timestamp remain R5's responsibility. R4 does not claim to fix all tag replication by correcting this constructor.
+
+## Source and reproduction evidence
+
+- `cmd/object-api-options.go`: trusted parsing at 383–426; KMS construction at 433–460; normal assignments at 469–475.
+- `cmd/object-handlers.go`: destination default encryption at 1425–1435; `copyDstOpts` at 1454; tag timestamp consumption at 1807–1834; replica reconciliation enabled at 1851; encryption metadata merge at 1910.
+- `internal/bucket/encryption/bucket-sse-config.go:135`: explicit request wins, absent config + auto encryption selects KMS, otherwise configured bucket algorithm/key ID applies.
+- `cmd/erasure-server-pool-consistency.go:232`: stored valid timestamp wins over absent, older or equal incoming timestamp; writes preserve the stored tag value alongside its timestamp.
+- `cmd/erasure-object.go:136` and `cmd/erasure-server-pool.go:1509`: same-version replica COPY reaches existing under-lock tag reconciliation, including object-data rewrites.
+- History: the omission exists in `c4373ef290` (2021-09-18); `b2dca43fda` (2026-09-05) added the two Object Lock timestamps but not the tag timestamp. `cfefc049c` fixed KMS context encoding independently and must remain intact.
+- Fresh temporary reproduction: `/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/r4_repro_test.go` and `baseline-repro.log` (overlay; no production edits).
+- Command: `GOMAXPROCS=2 go test -p 2 -overlay /Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/overlay.json ./cmd -run '^TestReviewR4' -count=1 -timeout 5m -v`.
+- Result: expected failure. Unencrypted and AES256 options preserve `2026-09-15T01:00:00.123456789Z`; KMS returns zero. Signed metadata REPLACE COPY on ErasureSD and Erasure (16 disks), across explicit/default/automatic KMS, returns 200 but retains `key=old` and the old timestamp for newer events. Actual encrypted metadata and plaintext GET roundtrips pass. Test deltas are 1–3 nanoseconds.
+- These are in-process signed HTTP router and real local disk tests. `kms.NewStub` replaces the remote key service; the normal server encryption/decryption code still runs. Existing `tagTestCapacityDisk` avoids the host's free-space percentage threshold; it delegates all object data/metadata I/O to real test disks.
+
+## Proposed production change
+
+Add exactly this field to the existing KMS `ObjectOptions` literal:
+
+```go
+ReplicationSourceTaggingTimestamp: taggingtimestmp,
+```
+
+Update the neighboring explanatory comment to include tagging alongside retention/legal hold. Do not refactor the common return paths, change parsing/fallback/equal-timestamp semantics, change encryption context encoding, modify trust decisions, add SDK dependencies, or change storage/wire format. Those changes are unnecessary to restore the missing existing contract.
+
+## Required validation after consensus
+
+1. Add an options regression matrix covering unencrypted, SSE-S3, SSE-KMS with no context, SSE-KMS with a context, and SSE-C. Validate trusted/untrusted requests, missing/valid/malformed tag timestamps, nanosecond and timezone/whitespace handling, all three replication timestamps, mtime/ETag/trust, nonnil metadata, and unchanged SSE header serialization (including KMS key/context).
+2. Promote the temporary COPY reproduction into a named, isolated regression test. Use actual signed same-version metadata COPY with REPLACE metadata and tagging directives, on single-disk and 16-disk backends. For explicit, bucket-default and automatic SSE-KMS, check newer update, older delivery, duplicate replay, and a second newer update. Verify stored tags, exact timestamp, version ID, encryption kind and plaintext GET after each operation. Include an unencrypted/SSE-S3 control if the fixture can do so without expanding implementation scope.
+3. Fail the final regression tests against unmodified baseline using an overlay. Then run them on the fixed source, alongside existing replication-trust/options and bucket-KMS Object Lock tests. Check `gofmt`, `git diff --check`, and `go vet ./cmd`.
+4. Run the new focused tests under `-race`. Use `GOMAXPROCS=2` and `-p 2` while sibling tasks share the host. A one-field pure option fix does not justify concurrent full-repository suites in all five tasks; full Linux CI and multi-site validation remain separate delivery gates.
+5. If a test exposes a separate handler/storage defect, report evidence and coordinate with R5. Do not broaden R4's production patch to make unrelated tests pass.
+
+## Compatibility, existing state, effort and delivery
+
+- Public API, header names, stored key names, KMS context/key handling and supported dependencies remain unchanged. Untrusted source headers stay ignored; malformed trusted timestamps continue to fail; absent timestamp remains zero. Existing non-KMS behavior remains unchanged.
+- No automatic rewrite/backfill. Lost source tag times cannot be reconstructed from the receiver alone. Upgrading permits subsequent properly timestamped events to be consumed. Review source-of-truth and target state before any targeted resync; full historical convergence also depends on R5. Repeated events subject to existing timestamp/tie semantics are not a universal repair guarantee.
+- The source fix can land independently; complete deletion/empty-tag and mixed-encryption convergence needs R5 plus its integration evidence.
+- Expected effort: approximately 0.5–1 engineer-day including reproduction, review and local validation; key-service deployment, multi-site failures and existing-state remediation are separate.
+- After actual Opus 5.0/max agreement on this exact plan hash, implement locally without another user permission prompt. Preserve raw review, assistant model identity, request effort, baseline and plan hash, issue-by-issue disposition and explicit consensus before source edits.
+- Deliver a reviewable local diff, tests and evidence. No main merge, remote publication/release, deployment or existing-state rewrite is authorized by this plan.

docs/investigations/r4/research.md

@@ -0,0 +1,18 @@
+# R4 research log
+
+## Verified baseline
+
+2026-09-15: local clean HEAD and GitHub main both `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`. Branch created as `codex/r4-kms-tag-timestamp`. Live GitHub open PRs #184 (`6addf9eb916b5a4b837480cf534cd1efa5407d3c`) and #187 (`b8f2fdde41dff3dc3b8db669c1d42d30ca5c1d3d`) concern other tasks. Claude Code reports `2.1.270`; Go reports `go1.27.1 darwin/arm64`.
+
+## Coordination
+
+- Parent task: `01a0a5ab-ee43-7911-bddd-1aca6f8afcc8`.
+- R5: `01a0a5b9-602d-7470-9882-4817cf5fdcd1`, `/Users/vonng/.codex/worktrees/77ad/silo`.
+- Parent and R5 acknowledged the ownership boundary: R4 options constructor and nonempty KMS COPY tests; R5 producer/receiver ordering and empty values. R5 will consume R4's minimal patch for combined KMS acceptance.
+- Initial conservative expectation separated metadata COPY from REPLACE ordering. Inspection of current `ReplicaLockReconcile` and `reconcileStoredObjectTags` shows that stored timestamps are also reconciled under the write lock for REPLACE. The temporary reproduction therefore uses REPLACE directly; the fix must demonstrate the actual sender-shaped path without changing the handler.
+
+## Baseline reproduction
+
+Temporary overlay test source and output are in `/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/`. The options and HTTP/disk reproductions fail for the expected missing timestamp. Every KMS HTTP request completed with 200; newer tags remained old; encrypted object metadata and subsequent ordinary plaintext GET succeeded. The result is narrower than claiming all KMS replication fails, and stronger than merely comparing options.
+
+The temporary source is not a production implementation. See [plan v1](plan-v1.md) for exact scope and required acceptance. The plan is frozen by SHA-256 before invoking real Opus.

docs/investigations/r4/verification.json

@@ -0,0 +1,358 @@
+{
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "branch": "codex/r4-kms-tag-timestamp",
+  "source_sha256": {
+    "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+    "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+    "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+  },
+  "source_files_match_all_test_runs": true,
+  "checks": [
+    {
+      "name": "baseline-final",
+      "command": [
+        "go",
+        "test",
+        "-p",
+        "2",
+        "-overlay",
+        "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/baseline-final-overlay.json",
+        "./cmd",
+        "-run",
+        "^Test(PutOptsFromHeadersReplicationTimestamps|APICopyObjectReplicaTaggingTimestampUnderKMS)$",
+        "-count=1",
+        "-timeout=5m",
+        "-v"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2"
+      },
+      "started_at": "2026-09-15T15:50:49.430860+00:00",
+      "finished_at": "2026-09-15T15:51:20.637630+00:00",
+      "exit_code": 1,
+      "log": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/baseline-final.log",
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+        "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+      },
+      "log_sha256": "f849c2082235213764e3db7a314d52af75c4859102478a1e8f837afcaa8f1ea8",
+      "overlay_sha256": "f4cbc16e4ffccaf63191de2e8476162876055796adb4c38cda2d8c569a3bbabc",
+      "overlay_sources": {
+        "/Users/vonng/.codex/worktrees/a9cb/silo/cmd/object-api-options.go": {
+          "path": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/object-api-options.baseline.go",
+          "sha256": "16a560d0990ae929393f682f22b32ecd2e7f4d9390484b03b54e176fcd00cff5"
+        }
+      },
+      "assessment": "Expected baseline regression failure; KMS timestamp loss. Non-KMS controls pass."
+    },
+    {
+      "name": "focused",
+      "command": [
+        "go",
+        "test",
+        "-p",
+        "2",
+        "./cmd",
+        "-run",
+        "^Test(PutOptsFromHeadersReplicationTimestamps|APICopyObjectReplicaTaggingTimestampUnderKMS|ReplicationTrustControlsInternalOptionsAndEvents|GetAndValidateAttributesOpts.*|APICopyObjectReplicaRetentionRemovalUnderBucketKMS)$",
+        "-count=1",
+        "-timeout=5m",
+        "-v"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2"
+      },
+      "started_at": "2026-09-15T15:51:20.638633+00:00",
+      "finished_at": "2026-09-15T15:51:48.382212+00:00",
+      "exit_code": 1,
+      "log": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/focused.log",
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+        "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+      },
+      "log_sha256": "9a64cbc46e9fd6186b1d3031034b720857851ce70b1466b1da5da6d1a52b76ba",
+      "assessment": "New tests and options/trust pass; pre-existing KMS lock fixture blocked by host disk free-space percentage."
+    },
+    {
+      "name": "focused-capacity-adapted",
+      "command": [
+        "go",
+        "test",
+        "-p",
+        "2",
+        "-overlay",
+        "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/capacity-overlay.json",
+        "./cmd",
+        "-run",
+        "^Test(PutOptsFromHeadersReplicationTimestamps|APICopyObjectReplicaTaggingTimestampUnderKMS|ReplicationTrustControlsInternalOptionsAndEvents|GetAndValidateAttributesOpts.*|APICopyObjectReplicaRetentionRemovalUnderBucketKMS)$",
+        "-count=1",
+        "-timeout=5m",
+        "-v"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2"
+      },
+      "started_at": "2026-09-15T15:52:20.113475+00:00",
+      "finished_at": "2026-09-15T15:52:50.842773+00:00",
+      "exit_code": 0,
+      "log": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/focused-capacity-adapted.log",
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+        "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+      },
+      "log_sha256": "13f7aa54564a433bef4dddcf2c5ad1fe46fa03869527fb902a255ce4c3263bc9",
+      "overlay_sha256": "76a7e6fb364bdaaa3469b1dc79f9ea318059f287a4888f322639920c76dfe53a",
+      "overlay_sources": {
+        "/Users/vonng/.codex/worktrees/a9cb/silo/cmd/replication-trust_test.go": {
+          "path": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/replication-trust-capacity_test.go",
+          "sha256": "c81b526ae51983881bbf464199e6b90f074fa695300fa8ff005e427e4d3c8208"
+        }
+      },
+      "assessment": "PASS"
+    },
+    {
+      "name": "race",
+      "command": [
+        "go",
+        "test",
+        "-p",
+        "2",
+        "-race",
+        "./cmd",
+        "-run",
+        "^Test(PutOptsFromHeadersReplicationTimestamps|APICopyObjectReplicaTaggingTimestampUnderKMS)$",
+        "-count=1",
+        "-timeout=5m",
+        "-v"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2"
+      },
+      "started_at": "2026-09-15T15:52:50.843898+00:00",
+      "finished_at": "2026-09-15T15:53:43.789325+00:00",
+      "exit_code": 0,
+      "log": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/race.log",
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+        "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+      },
+      "log_sha256": "ae66a8c9e569a5c4b57ae56e75afc85c06a8b76f1567187519da0726605a4de4",
+      "assessment": "PASS"
+    },
+    {
+      "name": "vet",
+      "command": [
+        "go",
+        "vet",
+        "-p",
+        "2",
+        "./cmd"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2"
+      },
+      "started_at": "2026-09-15T15:53:43.790197+00:00",
+      "finished_at": "2026-09-15T15:53:51.129773+00:00",
+      "exit_code": 0,
+      "log": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/vet.log",
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+        "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+      },
+      "log_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "assessment": "PASS"
+    },
+    {
+      "name": "lint",
+      "command": [
+        "/Users/vonng/pgsty/silo/.bin/golangci/v2.13.1/golangci-lint",
+        "run",
+        "--build-tags",
+        "kqueue",
+        "--timeout=10m",
+        "--config",
+        "./.golangci.yml",
+        "./cmd/..."
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/a9cb/silo",
+      "env_override": {
+        "GOMAXPROCS": "2"
+      },
+      "started_at": "2026-09-15T15:53:51.130456+00:00",
+      "finished_at": "2026-09-15T15:55:39.114303+00:00",
+      "exit_code": 0,
+      "log": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/lint.log",
+      "source_sha256": {
+        "cmd/object-api-options.go": "25e2e9484fafd94d1b2c857b94373758e481893ad93fb1a063edf7746277accc",
+        "cmd/object-api-options-replication_test.go": "1ea2a060987e32a4c76fce96ee974df475944c2d6ab482a4893e33daf7bca849",
+        "cmd/object-copy-replication-tagging_test.go": "5437a77e68736b4ce69de9c777675251fef24b0352dfe30bd8a836fc7ee810e3"
+      },
+      "log_sha256": "e92606b0bf483111dff0a120c315ea165821348f31365020e2468a0059095c47",
+      "assessment": "PASS"
+    }
+  ],
+  "format_checks": [
+    {
+      "command": [
+        "gofmt",
+        "-l",
+        "cmd/object-api-options.go",
+        "cmd/object-api-options-replication_test.go",
+        "cmd/object-copy-replication-tagging_test.go"
+      ],
+      "exit_code": 0,
+      "output": ""
+    },
+    {
+      "command": [
+        "git",
+        "diff",
+        "--check"
+      ],
+      "exit_code": 0,
+      "output": ""
+    }
+  ],
+  "evidence_directory": "/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb",
+  "evidence_files": {
+    "baseline-final-overlay.json": {
+      "size": 177,
+      "sha256": "f4cbc16e4ffccaf63191de2e8476162876055796adb4c38cda2d8c569a3bbabc"
+    },
+    "baseline-final.json": {
+      "size": 1584,
+      "sha256": "5bf7be51e5a0eb41a40dc5fc5d3a1aba5df7733ad5dcb001f8d870a01c4233ba"
+    },
+    "baseline-final.log": {
+      "size": 22493,
+      "sha256": "f849c2082235213764e3db7a314d52af75c4859102478a1e8f837afcaa8f1ea8"
+    },
+    "baseline-identity.txt": {
+      "size": 1676,
+      "sha256": "9b21841e19a0cbb8ded18c2597488a527a27bedc65109d05d4ff598103073b68"
+    },
+    "baseline-options.log": {
+      "size": 11395,
+      "sha256": "d692f0a4bc9c58e2ac0087afa356ddf48f86e1040ea8138d68ffc4d0992bf3cb"
+    },
+    "baseline-repro.log": {
+      "size": 5693,
+      "sha256": "aa89b76f4723c6a3ce224faa7796403628d978a8707544bd97848b8887de2113"
+    },
+    "capacity-fixture.diff": {
+      "size": 870,
+      "sha256": "8d01e0b0068441f37ecee37125b81424d1f30d7c4fb37d435ea0cfe2e4617e5e"
+    },
+    "capacity-overlay.json": {
+      "size": 185,
+      "sha256": "76a7e6fb364bdaaa3469b1dc79f9ea318059f287a4888f322639920c76dfe53a"
+    },
+    "final_copy_repro_test.go": {
+      "size": 5942,
+      "sha256": "8949e07d96d2949a79f5a9e83c7a7c0473733d77b407e9c51da477d4ab74f1a8"
+    },
+    "focused-capacity-adapted.json": {
+      "size": 1661,
+      "sha256": "1a599b41caabfc5eb44db8d89c8b7e4f4f84f5f036d008369155fd337f65bdf9"
+    },
+    "focused-capacity-adapted.log": {
+      "size": 20384,
+      "sha256": "13f7aa54564a433bef4dddcf2c5ad1fe46fa03869527fb902a255ce4c3263bc9"
+    },
+    "focused.json": {
+      "size": 1253,
+      "sha256": "d9bb4979ea8aeaabb809cdc6e400a8673530bc83abf3dc2b2a06853a8523d0d9"
+    },
+    "focused.log": {
+      "size": 20475,
+      "sha256": "9a64cbc46e9fd6186b1d3031034b720857851ce70b1466b1da5da6d1a52b76ba"
+    },
+    "format-checks.json": {
+      "size": 352,
+      "sha256": "7569260900a799d5efdfb39db1f575ab1dadbbb04ace222e036968e66b6b59e7"
+    },
+    "lint.json": {
+      "size": 991,
+      "sha256": "a7144713b069f470a94b1ebe6fca6683a4b866a891a2756e15f28c280666ca14"
+    },
+    "lint.log": {
+      "size": 10,
+      "sha256": "e92606b0bf483111dff0a120c315ea165821348f31365020e2468a0059095c47"
+    },
+    "object-api-options.baseline.go": {
+      "size": 16653,
+      "sha256": "16a560d0990ae929393f682f22b32ecd2e7f4d9390484b03b54e176fcd00cff5"
+    },
+    "options-overlay.json": {
+      "size": 185,
+      "sha256": "5506b9c3b998b32f01c45af3cf01605eae9e4fb262c9ff3a6b0040abe719d4d9"
+    },
+    "options_repro_test.go": {
+      "size": 4192,
+      "sha256": "1a57a47bdd370042fa0f0d2d90efe447abedee9b9ef48a938d4bed631d83ec0b"
+    },
+    "opus-review-v1.exit": {
+      "size": 2,
+      "sha256": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa"
+    },
+    "opus-review-v1.jsonl": {
+      "size": 410466,
+      "sha256": "eb0918d8a6185b180dddcfc664a96682f05502ecf3b686b08a0547f09879d57d"
+    },
+    "opus-review-v1.stderr.log": {
+      "size": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    },
+    "overlay.json": {
+      "size": 158,
+      "sha256": "95f7c3f7e206fe36731e6d7e4a48f90c7403f07ae8155c125c6b86d2f1c2d487"
+    },
+    "r4-kms-tag-timestamp.patch": {
+      "size": 874,
+      "sha256": "2d4806d986bbd94ba4bc3951f3aeee48401ee1921c28ded0988fa09ca76ca26f"
+    },
+    "r4_repro_test.go": {
+      "size": 5508,
+      "sha256": "9bcefb6da2416b577b58085485cad60f677c2265e9dfa84d02e465e1b203766b"
+    },
+    "race.json": {
+      "size": 1026,
+      "sha256": "50b73e4acbc2426f3dcfadde78d0f0a86f10702345d2939a30204600bc750a13"
+    },
+    "race.log": {
+      "size": 18566,
+      "sha256": "ae66a8c9e569a5c4b57ae56e75afc85c06a8b76f1567187519da0726605a4de4"
+    },
+    "replication-trust-capacity_test.go": {
+      "size": 61271,
+      "sha256": "c81b526ae51983881bbf464199e6b90f074fa695300fa8ff005e427e4d3c8208"
+    },
+    "review-prompt-v1.md": {
+      "size": 2770,
+      "sha256": "07c225beff1523e056c154b3a387cf1ae345def4b4d0173065b882140ac5abdf"
+    },
+    "run-checks.py": {
+      "size": 2266,
+      "sha256": "ddecea5220bc9c286df18c0e9eca101f3d8ab731c307cd4acef937f3ecd11e65"
+    },
+    "vet.json": {
+      "size": 853,
+      "sha256": "e0964444bc91640ed6cf78229050bad5b94af4210bdf44b11d1a848f1ee930a6"
+    },
+    "vet.log": {
+      "size": 0,
+      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    }
+  },
+  "scope": "Darwin arm64; real signed HTTP + disk I/O; KMS service stub; single-pool single/16-disk fixtures; no remote CI, multi-site, release or deployment."
+}

docs/investigations/r4/verification.md

@@ -0,0 +1,52 @@
+# R4 修复与本地验收
+
+这是 2026-09-15 的本地验收快照。用户后续授权的实现级复核、提交规范调整与合并流程见 [合并前复核](merge-verification.md)；以下原始测试记录及哈希保留当时状态。
+
+## 结果
+
+在 `putOptsFromHeaders` 的 SSE-KMS 选项构造中补齐 `ReplicationSourceTaggingTimestamp`。目的端使用显式 SSE-KMS、桶默认 KMS 或自动加密时，可信复制 COPY 现在能消费来源标签时间戳，并在现有存储锁内完成排序。
+
+生产修改只有一个字段和相邻注释。API、存储格式、KMS key/context、信任判断和既有排序规则保持兼容。R5 的删除/空标签及 PUT/multipart 时间戳传播独立交付。
+
+## 方案与 Opus 共识
+
+- 基线：`9ebe81c1b3611f9cc73e676b5b741c2be62c467a`，已重新查询 GitHub main。
+- 分支：`codex/r4-kms-tag-timestamp`。
+- [冻结方案 v1](plan-v1.md)：SHA-256 `ad539f2071155de6955b583991684ed33c4bfe2e29660005840cdc97d7e1a754`。
+- 真实评审为本机 Claude Code 2.1.270，实际 assistant 模型 `claude-opus-5`，显式 `--effort max`。结论 **GO_WITH_NONBLOCKING_NOTES，0 个阻断项**。
+- [逐条意见处置与双方共识](consensus.md)、[原始返回评审正文](opus-v1-review.md)、[模型与哈希记录](opus-v1.metadata.json) 已保存。先保存共识，再修改生产源码。
+
+## 变更与测试
+
+| 文件 | 内容 |
+|---|---|
+| `cmd/object-api-options.go` | 在 KMS 字面量中保留已解析的来源标签时间戳。 |
+| `cmd/object-api-options-replication_test.go` | 无加密、SSE-S3、SSE-KMS、带 key/context 的 KMS、SSE-C；可信/非可信；缺失、有效、无效标签时间；纳秒、时区与空格；mtime/ETag/三个时间戳、metadata 与 SSE 序列化。 |
+| `cmd/object-copy-replication-tagging_test.go` | 两种单池后端 × 五种目的端加密模式 × 五个有序事件，共 50 次签名 COPY 和 50 次普通 GET。每步检查最终标签、精确时间戳、对象版本、加密类型及明文内容。 |
+
+COPY 使用 `metadata=REPLACE`、`tagging=REPLACE` 和可信复制身份。事件为较新更新、乱序旧更新、重复事件、再次更新，以及不带来源标签时间戳的请求。更新间隔仅 1–3 纳秒，防止时间精度退化被秒级测试掩盖。无加密与 AES256 是对照；KMS 覆盖显式、桶默认和自动加密入口。
+
+## 验证状态
+
+| 检查 | 结果 | 证据文件 |
+|---|---|---|
+| 最终测试 + 未修复基线 constructor overlay | 预期失败；只有可信 KMS 有效标签时间戳及 KMS COPY 更新失败，对照通过 | `baseline-final.log/json` |
+| 修复后最终新增测试与既有 trust/options 测试 | 通过；未使用生产源码 overlay | `focused.log` |
+| 既有 KMS Object Lock 回归 | 首次受宿主机磁盘余量阈值阻挡；仅适配测试容量报告后，与上述定向测试一起通过 | `focused-capacity-adapted.log/json`、`capacity-fixture.diff` |
+| 新增测试 `-race` | 通过 | `race.log/json` |
+| `go vet -p 2 ./cmd` | 通过 | `vet.log/json` |
+| 仓库配置的 golangci-lint，范围 `./cmd/...`、`kqueue` build tag | 通过，0 issues | `lint.log/json` |
+| gofmt、git diff --check | 通过 | `format-checks.json` |
+
+原始日志和每条命令的运行记录位于 `/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/`。每份检查 JSON 都记录命令、退出码、时间和三个源码/测试文件的 SHA-256；最终交付已逐一确认文件哈希一致。[验证清单](verification.json) 另记录 overlay 的实际替换文件哈希，避免混淆基线与修复版执行代码。
+
+测试使用真实签名 HTTP 路由、实际本地对象数据/元数据读写、服务器加解密代码；远程密钥服务由 `kms.NewStub` 代替。ErasureSD 与 16 盘 Erasure 均为单池。容量适配只使用已有 `tagTestCapacityDisk`，避免本机磁盘使用比例触发防写阈值，所有对象 I/O 仍由真实测试磁盘承担；未调整生产容量保护。
+
+## 交付与剩余边界
+
+- 本地实现和要求的定向验证均已完成，将源码、回归、研究、共识和验收记录作为一个本地提交交付。
+- R4 的独立生产补丁已提供给 R5：`/Users/vonng/tmp/silo-r4-evidence-20260915-a9cb/r4-kms-tag-timestamp.patch`，SHA-256 `2d4806d986bbd94ba4bc3951f3aeee48401ee1921c28ded0988fa09ca76ca26f`。
+- Opus 共识为方案级共识；本地测试结论来自实际运行，不把它记作 Opus 执行了测试。
+- 多池/多站点故障恢复、外部 KMS 服务、完整 Linux CI、主干合并、远端发布和部署尚未执行。
+- 没有改写存量。丢失的来源时间戳不能仅从接收端推导；后续重放/重同步须核对来源权威性及 R5 的全链路处理，不保证旧事件重放可以修复全部历史状态。
+- 另登记 KMS 字面量缺少 Proxy/Speedtest 标志的范围外观察，已交父任务单独核验，本次未扩大修复。

docs/investigations/r7/README.md

@@ -0,0 +1,74 @@
+# R7：可信复制 Content-Encoding 修复
+
+## 交付摘要
+
+生产修复只改 `cmd/handler-utils.go`：可信复制只恢复六个复制专用字段，保留调用方已规范化的普通元数据。采用 [PR #187](https://github.com/pgsty/silo/pull/187) 的生产逻辑，增加准确说明 Snowball 调用方的注释。PR 原作者：Mikhail Khadarenka；本地新增回归与调查记录由本任务提供。
+
+- `aws-chunked`：对象元数据和 GET/HEAD 不包含 Content-Encoding。
+- `aws-chunked,gzip`：只保留 `gzip`，原始 gzip 字节不变。
+- `gzip`：保持原值与原字节。
+- 六个复制字段保留，包括空 multipart 标记和 SSE-C checksum；认证/权限门控保持原语义。
+- 普通提取删除的旧 unencrypted length/MD5 用户元数据不会被复制恢复阶段重新注入。
+
+## 方案和真实 Opus 共识
+
+- [调查与基线复现](research.md)
+- [冻结方案 v1](plan-v1.md) 与 [逐项处置附录](plan-v1.dispositions.md)
+- [最终共识](consensus.md)：真实 Claude Code 2.1.270，两轮显式 `claude-opus-5 --effort max`；所有实际评审 assistant 消息均为 `claude-opus-5`。第二轮 `APPROVE`，阻断 0。
+- [首轮原文](review/opus-v1.md)、[第二轮原文](review/opus-v1-confirmation.md)；相邻 metadata 文件记录模型、命令、源 SHA、方案/原文哈希与原始 JSONL 路径。
+
+共识在产品代码修改前记录；Opus 审阅代码和方案，测试由本任务执行，二者分别留证。
+
+## 最终实现复核与合并准备
+
+用户随后追加授权：使用 Opus 5 Max 核实，确认无误后合并 main。已对提交 `4fcdf37ce656152b32ad0f615d47f5e3f9748c3a`（基线 `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`）执行新的独立实现复核，实际评审模型为 `claude-opus-5`，显式 `--effort max`。结论为 **APPROVE_WITH_NONBLOCKING_NOTES，阻塞 0**；适用 PR CI 实际通过后才可合并。
+
+[实现复核原文](implementation-review/opus-implementation.md)、[身份与哈希](implementation-review/opus-implementation.metadata.json)、[逐项处置](implementation-review/dispositions.md) 分别记录审阅与执行方验证边界。本次补充仅为文档；生产代码与测试维持已审阅、已本地验证版本。
+
+## 兼容性与边界
+
+Snowball 无 PAX 的可信复制条目不再继承外层归档的 content-type/cache-control/用户元数据，与普通 Snowball 一致。外层的六个复制专用字段仍可按既有规则作用于已授权条目。相同 tar 的普通和 replica 写入已纳入条目元数据一致性回归。
+
+现有精确 token 裁剪规则保持不变：`aws-chunked, gzip` 留下带前导空格的 ` gzip`；`gzip, aws-chunked` 中带空格的 token 仍不会被去掉。这两条记录现状的断言不代表它们已被修复。POST 表单低层元数据提取行为也保持原样。
+
+旧对象不会因升级自动修正；普通 COPY 保留来源已有元数据。若权威来源仍受污染，后续对账可能继续认为目标不一致并再次选择元数据复制。先核实来源版本、再协调副本的操作提案见 [存量处理设计](stored-metadata-remediation.md)。本任务未扫描或改写现网对象。
+
+## 复验命令
+
+在有充足空闲比例的普通测试机器上，正式测试不需要容量 overlay：
+
+```sh
+go test ./cmd -run '^Test(ExtractReplicationMetadata.*|APIReplicaContentEncoding|APISnowballReplicaContentEncoding)$' -count=1
+go test -race ./cmd -run '^Test(ExtractReplicationMetadata.*|APIReplicaContentEncoding|APISnowballReplicaContentEncoding|APISnowballReplicationTrustIsPerEntry|APISSECReplicaSkipsDestinationTransforms|APISSECMultipartReplicaRoundTripWithCompression)$' -count=1
+make verifiers
+make build
+```
+
+本机实际命令见下述每次运行的 JSON；其中包含容量 overlay、并行度和使用的本地 golangci-lint 路径。
+
+## 验证证据
+
+完整命令、源文件/方案哈希、构建身份和检查结果汇总于 [verification.json](verification.json)。构建发生在本地提交前，二进制嵌入基线提交号；代码内容以验证清单中的文件哈希为准，不作为发布制品。
+
+原始材料目录：`/Users/vonng/tmp/silo-r7-20260915-ad51/`。除原始发现阶段外，每次正式验证的 `.json` 记录命令、退出码、时间、日志哈希和四个代码文件的 SHA-256。
+
+| 验证 | 状态与材料 |
+| --- | --- |
+| 原始 helper 基线 | `baseline.log`：裸/混合可信恢复失败，gzip 控制通过 |
+| 原始 HTTP 基线 | `http-baseline-v2.log`：单盘及 16 盘，44 个控制通过，20 个已知缺陷失败 |
+| 最终测试回退原始 helper | `exact-baseline-regression.{json,log}`：测试不变，只覆盖回基线产品文件；44 控制通过、36 预期失败（20 HTTP + 16 helper） |
+| 修复后的定向测试 | `fixed-targeted.{json,log}`：9 个顶层测试、80 个具名子用例全部通过 |
+| 既有 SSE 与信任边界 | `fixed-sse-trust.{json,log}`：SSE-C 单段/多段、SSE multipart trust、PUT/COPY 投毒、普通/复制权限、Snowball per-entry、默认桶加密、streaming trailer 等全部通过 |
+| Race | `fixed-race.{json,log}`：新增 helper/HTTP/Snowball、既有 Snowball per-entry 与 SSE-C 单段/多段全部通过 |
+| 仓库 verifiers | `verifiers.{json,log}`：make verifiers 通过，golangci-lint 0 issues，生成文件与兼容标识检查通过；typos 未安装，按 Makefile 跳过 |
+| 构建 | `build.{json,log}`：make build 通过；本地 silo --version 已核对，二进制身份见 verification.json |
+
+### 本机容量条件
+
+未调整的 HTTP 夹具返回 507 / XMinioStorageFull，原始日志为 `http-baseline-unadapted.log`。宿主 APFS 接近满盘，触发相对空闲阈值。HTTP/既有 SSE/race 验证使用临时 Go overlay 复用仓库的 `tagTestCapacityDisk`，只改变 API 测试夹具看到的容量比率，实际对象和元数据仍读写测试磁盘。该临时文件在仓库外，不进入交付；生产容量策略没有变化。
+
+证据为本机认证请求处理链路及实际存储、读取和既有 SSE 往返，不是双站点调度器、进程重启、网络故障或线上验收。
+
+## 初始交付状态
+
+初始提交形成时，研究、真实 Opus 方案共识、本地实现与验证均完成，结果保存在 `codex/r7-replication-content-encoding` 分支；当时尚未推送或合并。随后按用户追加授权进行上述最终实现复核与合并准备。实际 PR、CI 与主干合并状态以对应远端记录和 `/Users/vonng/tmp/silo-r7-merge-20260916-ad51/` 中的执行回执为准。

docs/investigations/r7/consensus.md

@@ -0,0 +1,25 @@
+# R7 最终方案共识
+
+记录时间：2026-09-15T15:53:09.813317+00:00。此记录写入时产品代码仍为基线，只有调查文件和仓库外的临时测试。
+
+## 同一版方案
+
+- 基线：`9ebe81c1b3611f9cc73e676b5b741c2be62c467a`。
+- [plan-v1.md](plan-v1.md)：`7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96`。
+- [plan-v1.dispositions.md](plan-v1.dispositions.md)：`d32d30f8a420f904da6ee039f0461d6281e7da44bd805d416b4970d071f0af32`。
+- Codex 重新计算并确认上述两个哈希未变。Opus 只读源码，明确未计算哈希、未运行测试。
+
+## 实际讨论结果
+
+两轮均使用 Claude Code 2.1.270，显式 `--model claude-opus-5 --effort max`。原始记录中两轮所有评审 assistant 消息均为 `claude-opus-5`；辅助模型用量与主评审模型分开记录。
+
+1. [首轮独立评审](review/opus-v1.md)：APPROVE_WITH_NONBLOCKING_NOTES，阻断 0，9 项非阻断意见。
+2. Codex 逐项核验：采纳验证/文档建议；纠正 N1 的单包权限比较方式、收窄 N5 的重试风险表述、以源码反驳 N6 的容量适配器不存在判断。详见绑定处置附录。
+3. [第二轮确认](review/opus-v1-confirmation.md)：**APPROVE，阻断 0**；Opus 明确接受 N1/N5 的纠正，撤回 N6 的事实判断，并同意这两个哈希所标识的 v1 组合直接进入实现。
+4. Codex 同意该方案及全部最终处置。没有剩余阻断分歧；共识完成，现在开始本地实现与验证。
+
+评审原始 JSONL、stderr、实际模型、命令、耗时及输出哈希均由 `review/*.metadata.json` 指向 `/Users/vonng/tmp/silo-r7-20260915-ad51/` 中的原始记录。首轮 Claude plan 模式尝试写自己的 plan 文件但 Write 工具被禁用，最后只以文本返回评审；未写产品文件。没有把失败、限流或别的模型当成通过。
+
+## 授权及证据边界
+
+共识是源代码与修复方案的认可。实现、测试、合并、发布和部署仍分别记录。本地常规修复已获工作流授权，无需再次询问；主干合并、远端发布、部署和现网存量改写不在此次范围。

docs/investigations/r7/implementation-review/dispositions.md

@@ -0,0 +1,27 @@
+# R7 最终实现复核意见处置
+
+## 审阅身份与范围
+
+用户追加指令：使用 Opus 5 Max 核实，确认无误后合并 main。该指令授权此次推送、PR 与主干合并。
+
+候选提交 `4fcdf37ce656152b32ad0f615d47f5e3f9748c3a`，基线 `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`。实际评审 assistant 消息均为 `claude-opus-5`，命令显式指定 `--effort max`；辅助 Haiku 用量与评审模型分开记录。结果为 **APPROVE_WITH_NONBLOCKING_NOTES，阻塞 0**。
+
+完整独立意见见 [Opus 原文](opus-implementation.md)，模型、源文件、diff 和原始日志哈希见 [metadata](opus-implementation.metadata.json)。原始 JSONL 保存在 `/Users/vonng/tmp/silo-r7-merge-20260916-ad51/opus-implementation.jsonl`。
+
+## 七项非阻塞意见
+
+| 意见 | 处置与证据 |
+| --- | --- |
+| 1. 必须等待本分支自己的真实 CI | 接受。合并前核验适用检查全部通过，尤其是不使用本机容量 overlay 的完整 `cmd` 测试。现有 main CI 通过不能替代候选 PR 的 CI。 |
+| 2. 自检提交对象、署名和 diff | 已核对实际提交：树 `d5cc904bf2a8b57aea1aac53999b5007522279fc`，包含 Mikhail Khadarenka 的 Co-authored-by 与提交作者匹配的 DCO Signed-off-by。相对基线仅一个生产文件、三个测试文件及调查文档变化；四个 Go 文件哈希与通过的验证日志一致。 |
+| 3. POST 表单路径仍未归一化 | 确认是原有低层调用路径，本修复不改变它。作为独立后续研究项记录；没有把本次结果宣称为所有上传方式的编码归一化。 |
+| 4. 普通请求还可断言六个 wire 字段不泄漏 | 现有 `TestExtractMetadataHeaders` 已输入全部六个 wire 字段，仅期望 `content-type`，并用 `reflect.DeepEqual` 比较完整 metadata map，任何 wire 或 internal 字段泄漏都会失败。该测试已包含在通过的 `fixed-targeted` 验证中；无需增加重复断言。新增 canonical/lowercase 矩阵进一步覆盖恢复行为。 |
+| 5. 可补充 Snowball untrusted-marker 对照 | 保留为可选增强。当前测试含同一归档 ordinary/replica 元数据一致性，以及既有 Snowball 逐条目权限回归；此次调用点与授权门控未改。 |
+| 6. GET/HEAD 使用 getPutObjectURL 命名不够直观 | 确认 URL 构造等价，不影响方法、签名或断言。无需为命名改动已经通过的测试。 |
+| 7. PR 描述必须说明存量与来源污染限制 | 接受并写入 PR 描述。旧对象不会自动修复；来源仍受污染时，后续 heal/resync/比较可能反复选择元数据复制。参见既有存量处理设计；本次没有现网扫描或改写。 |
+
+## 合并条件
+
+此次处置只增加审阅文档，生产与测试代码维持 Opus 审阅版本。推送前再次核对源文件哈希、DCO 和 main 基线；main 若前进，先检查集成增量，相关行为改变时重新验证和评审。通过正常 PR 合并流程保留 #187 作者署名，不修改贡献者分支。
+
+本记录形成时尚未发布本分支的 PR，不能作为 CI 通过或已合并的证据。实际 PR、CI 与合并回执另行核验并保存于 `/Users/vonng/tmp/silo-r7-merge-20260916-ad51/`。发布、部署、线上对象改写和双站点故障验收不在此次执行范围。

docs/investigations/r7/implementation-review/opus-implementation.md

@@ -0,0 +1,82 @@
+## R7 实现与可合并性独立复审（Opus 5，只读会话）
+
+### 1. 审阅对象
+
+| 项 | 值 |
+| --- | --- |
+| 候选提交 | `4fcdf37ce656152b32ad0f615d47f5e3f9748c3a` |
+| 候选树 | `d5cc904bf2a8b57aea1aac53999b5007522279fc` |
+| 现场 main/base | `9ebe81c1b3611f9cc73e676b5b741c2be62c467a` |
+| 审阅 diff | `/Users/vonng/tmp/silo-r7-merge-20260916-ad51/reviewed-code.diff` |
+| diff SHA-256（按调用方声明采信） | `5af32dc2cb4acc34ac91a3e48c5decfcac0d25c70bfa7d6399934c66f629df2a` |
+| 原始 PR | #187，head `b8f2fdde41dff3dc3b8db669c1d42d30ca5c1d3d`，未合并 |
+
+本会话无 shell，未运行任何测试、未计算任何哈希、未做任何网络操作，未修改任何文件。
+
+### 2. 结论
+
+**APPROVE_WITH_NONBLOCKING_NOTES。阻塞问题 0。**
+
+**这份代码在其自身 PR 的 CI 实际变绿后可以合入 main。** 合并前提是 CI 真实通过，而不是"预期会通过"（理由见 §5.1）。
+
+### 3. 我实际核验的内容（非采信）
+
+逐行对读了 diff 与工作区当前源码，确认两者一致（`cmd/handler-utils.go:247-308` 就是 diff 后的内容；三个测试文件的 grep 行号与 diff 偏移量精确对应，如 `replication-content-encoding_test.go:108` = diff:246−138，`replication-metadata_test.go:68/90` = diff:488/510−420）。
+
+**3.1 等价性（普通路径零行为变化）**
+基线 `allowReplication=false` 分支是"命中 supportedHeader → 若属复制映射则 continue"；新代码是"若属复制映射则 continue → 再查表写入"。对 `nv` 的查找、`strings.Join(value, ",")` 的拼接、user-metadata 前缀循环全部逐字保留（`handler-utils.go:262-281`）。普通提取路径字节级等价。
+
+**3.2 六个 SSE 字段与空标记**
+`supportedHeaders` 的后六项（`handler-utils.go:86-91`）与 `replicationToInternalHeaders` 的键集完全相同（`:106-114`），且六→六为单射（含 `ReplicationSsecChecksumHeader` 的恒等映射，`bucket-replication.go:81`），因此由"遍历 slice"改为"遍历 map"后迭代顺序无关，无覆盖歧义。空标记 `X-Minio-Replication-Encrypted-Multipart: ""` 走 `ok=true` 分支、`Join([""])==""`，与基线一致，且 `internal/crypto` 按键存在性消费——测试 `replication-metadata_test.go:82-86` 的 want 映射显式固定了这一点。
+
+**3.3 信任边界（PUT/COPY/MPU/Snowball）**
+diff **未触碰任何调用点与信任判定**。我复核了全部五处调用与门控：
+- PUT：`object-handlers.go:2199` 先提取 → `:2263` `evaluateReplicationTrust` → `:2278-2280` `applyReplicationTrust` → `:2281-2282` 仅 `replicaTrusted` 恢复。注意提取发生在剥离之前，因此普通提取器**必须**无条件跳过复制专用头——新代码正是无条件 `continue`（`:263-265`），这道纵深防御被完整保留。
+- COPY：`object-handlers.go:1144-1152`，仅 REPLACE 分支且 `allowReplication` 为真时恢复；COPY 指令分支语义不变。
+- MPU：`object-multipart-handlers.go:233` + `:246`，仅初始化阶段恢复；Part/Complete 不再提取元数据，测试里的 `Content-Encoding: br` 因此是未来回归护栏。
+- Snowball：`replication-trust.go:78-90` 的整包判定 + `object-handlers.go:2786-2793` 的逐条目 `ReplicateObjectAction` 复核未变；`replicationRequestHeaders`（`replication-trust.go:96-112`）仍覆盖六个 wire 头与 REPLICA 状态。
+无新增投毒面。
+
+**3.4 Snowball 普通/replica parity（最实质的行为变化，判定为修复而非回归）**
+`object-handlers.go:2802-2804` 的 `metadata` 只含 storage class。基线在 `:2836` 的恢复调用会把**外层 archive 请求**的 supportedHeaders + 用户元数据整体灌进每个条目（普通条目则完全没有），补丁后只剩六个映射；PAX 分支 `:2864-2872` 同理不再用原始 wire 值覆盖 `extractMetadata` 已归一化的 `m`。我另行确认仓库内**没有生产代码发送** `X-Amz-Meta-Snowball-Auto-Extract`（只有 `api-router.go:391` 接收），即不存在依赖旧继承行为的内部生产者。
+
+**3.5 空格 token 行为**
+`trimAwsChunkedContentEncoding`（`handler-utils.go:367-378`）按 `,` 切分后做**精确等值**比较、不做 TrimSpace。因此 `"aws-chunked, gzip" → " gzip"`（保留前导空格）、`"gzip, aws-chunked" → "gzip, aws-chunked"`（整串不变）。测试 `replication-metadata_test.go:44-51` 的两条期望与代码一致，属于**现状固化**，不得对外宣称为本次修复。
+
+**3.6 原始普通元数据的删除（GHSA 相关，正面收益）**
+`extractMetadata:219-223` 删除 `X-Amz-Meta-X-Amz-Unencrypted-Content-Length/-Md5`。基线的恢复函数会通过 `x-amz-meta-` 前缀循环把它们**重新注入**，这是对该 advisory 缓解的实际回退（仅限可信 replica 写）。补丁消除了该路径，测试在 canonical/lowercase 两种写法下都做了断言（`:100-105`）。
+
+**3.7 GET/HEAD 与原始字节**
+`ObjectInfo.ContentEncoding` 来自 `fi.Metadata["content-encoding"]`（`erasure-metadata.go:138`），`setObjectHeaders` 仅在非空时下发（`api-headers.go:129-131`）。所以测试同时断言"落盘 UserDefined 无该键"和"响应头不存在该键"是有意义且互相独立的。GET 分支比较原始字节，replica/gzip 用例写入的是真实 gzip 字节。
+
+**3.8 签名与请求体**
+`replicaEncodingStream` 的顺序正确：`newTestStreamingRequest` → 设置全部头 → `signStreamingRequest` → `assembleStreamingChunks`，chunk 签名逐块校验；非 chunked 分支 `newTestSignedRequestV4` 对 payload 计算 `x-amz-content-sha256` 并走 `authTypeSigned` 校验。测试确实经过签名验证链路，不是绕过。无权 replica 用例断言 XML `Code == AccessDenied` 并复核对象未被创建，能区分"签名失败"与"授权拒绝"。
+
+**3.9 生产代码与 PR #187 的关系**
+逐行比对 `pr187.diff` 与候选 diff：`cmd/handler-utils.go` 与 `cmd/handler-utils_test.go` **完全一致**，唯一差异是恢复函数的三行注释（pr187.diff:76-78 vs 候选:76-78），候选版补充了 Snowball 语义。"仅澄清注释"的说法属实。`commit-message.txt` 含 `Co-authored-by: Mikhail Khadarenka` 与 PR #187 归属声明。
+
+**3.10 容量夹具是否削弱证据：不削弱**
+`capacity-overlay.json` 只替换 `cmd/test-utils_test.go`；夹具唯一的功能性改动是 `ExecObjectLayerAPITest` 开头用仓库**既有**的 `tagTestCapacityDisk`（`cmd/erasure-server-pool-tags_test.go:258-264`，`DiskInfo` 返回 `Total=Free, Used=0`）包装 set 磁盘。对象字节与 xl.meta 仍写真实临时盘。未适配时的 `507/XMinioStorageFull` 原始日志保留（`http-baseline-unadapted.log`）。该文件在仓库外，**不在交付 diff 中**。
+反事实同样成立：`exact-baseline-overlay.json` 只额外把**生产文件**换成 `baseline-handler-utils.go`（我核对该文件确含 `extractMetadataFromMimeWithReplication`/`allowReplication` 布尔开关），测试文件一字未改。失败点精确落在 `replica/bare`、`replica/mixed`、全部 8 个 Snowball 用例与全部 16 个 helper 子用例，`ordinary`/`untrusted-marker`/`gzip`/`unauthorized-replica` 全通过——44 通过 / 36 失败与声明吻合。
+
+### 4. 我采信而未独立验证的部分
+
+- 四个源文件与各日志的 SHA-256、diff SHA-256、提交/树对象哈希（无 shell，无法计算）。
+- 所有测试的**执行事实**：`fixed-targeted`（80 叶子）、`fixed-sse-trust`、`fixed-race`、`make verifiers`、`make build` 的通过是读日志所得（我确认了 `fixed-targeted.log:179-180` 的 `PASS/ok`、`fixed-race.log:209-210`、`fixed-sse-trust.log:125-126`，以及四个 fixed 日志中无 `--- FAIL`），但这些进程不是本会话运行的。
+- 提交对象实际携带的 author/trailer（`commit-message.txt` 内容正确，但我无法确认它就是 `4fcdf37ce` 的提交信息）。
+- PR #187 远端当前状态（读取的是本地缓存 `pr187.json` / `pr187.diff`）。
+- 无任何两站点调度器 / 进程重启 / 网络故障 / 线上验收；本次结论只覆盖本机认证 handler 接收链路与既有 SSE 回归。
+
+### 5. 非阻塞意见
+
+1. **CI 是真正的门，不是形式。** 这两个 HTTP 级测试从未在"无容量 overlay"的环境里跑过（本机盘满，未适配时直接 507）。CI 机器盘空间正常时不需要 overlay，预期可过，但这一点**本地无证据**。请等该分支自己的 CI 变绿再合，不要用本地日志替代。
+2. **提交对象自检**（推送前，一条命令）：`git log -1 --format='%H %T %an %ae%n%b' 4fcdf37ce` 与 `git diff --stat 9ebe81c1b..4fcdf37ce`，确认树哈希、`Co-authored-by` 与"只动 2 个产品/测试文件 + 2 个新测试 + docs"。
+3. **POST 表单路径仍未归一化**：`bucket-handlers.go:1262` 直接调用低层 `extractMetadataFromMime`，不走 `extractMetadata` 的 `aws-chunked` 裁剪与默认 content-type。本补丁按 N9 刻意不动它，但这意味着 POST policy 上传仍可能落盘原始 `aws-chunked`。建议单独开 issue，不要在本 PR 顺手统一。
+4. **测试可加一条断言**：`replication-metadata_test.go:68-72` 只断言普通路径不出现六个 **internal** 键；再断言六个 **wire** 键名也不在 metadata 中，可完整封死这一面（当前实现确实不会写入，属加固）。
+5. **Snowball 可选对照**：现有 parity 只比 ordinary vs replica，可再加一条 untrusted-marker archive 对照（Opus v1 的可选建议，非必须）。
+6. **小瑕疵**：`assertReplicaEncodingObject` 用 `getPutObjectURL` 构造 GET/HEAD 目标，功能等价但读起来别扭，`getGetObjectURL` 更清楚。
+7. **存量限制需在 PR 描述里写明**：本补丁只阻止目的端再次污染，不修复既有对象；若来源端仍存错误编码，`bucket-replication.go` 的逐字符串比较会在后续 heal/resync/比较时反复选择元数据复制。措辞按 N5 收窄（"可能反复选中"，不是"不间断热循环"）。设计见 `stored-metadata-remediation.md`，本次不授权任何现网扫描或改写。
+
+### 6. 明确许可声明
+
+**本次审阅的这份代码（候选 `4fcdf37ce…`，基线 `9ebe81c1b…`，即上述 diff）在其 PR 的适用 CI 实际通过后，允许合入 main。** 不得修改原贡献者的 #187 分支；不授权发布、部署或线上对象改写。若在此期间 main 前进，需重新查看集成增量，并在相关行为改变时重跑相应测试并重新评审。

docs/investigations/r7/implementation-review/opus-implementation.metadata.json

@@ -0,0 +1,39 @@
+{
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "candidate_commit": "4fcdf37ce656152b32ad0f615d47f5e3f9748c3a",
+  "candidate_tree": "d5cc904bf2a8b57aea1aac53999b5007522279fc",
+  "code_diff_sha256": "5af32dc2cb4acc34ac91a3e48c5decfcac0d25c70bfa7d6399934c66f629df2a",
+  "source_sha256": {
+    "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+    "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+    "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+    "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+  },
+  "prior_validation_log_hashes_verified": true,
+  "requested_model": "claude-opus-5",
+  "requested_effort": "max",
+  "cli_version": "2.1.270",
+  "started_at": "2026-09-15T16:03:50.465537+00:00",
+  "raw_output": "/Users/vonng/tmp/silo-r7-merge-20260916-ad51/opus-implementation.jsonl",
+  "stderr": "/Users/vonng/tmp/silo-r7-merge-20260916-ad51/opus-implementation.stderr.log",
+  "status": "completed",
+  "command": "claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --output-format stream-json --verbose --add-dir /Users/vonng/tmp/silo-r7-20260915-ad51 --add-dir /Users/vonng/tmp/silo-r7-merge-20260916-ad51",
+  "exit_code": 0,
+  "actual_review_models": [
+    "claude-opus-5"
+  ],
+  "auxiliary_usage_models": [
+    "claude-haiku-4-5-20251001"
+  ],
+  "session_id": "80cab374-7884-47ab-a83f-4cc6d4aa4303",
+  "duration_ms": 179439,
+  "num_turns": 40,
+  "verdict": "APPROVE_WITH_NONBLOCKING_NOTES",
+  "blocking_issues": 0,
+  "merge_condition": "The exact candidate must pass its own applicable PR CI. Inspect main integration changes before merge.",
+  "raw_sha256": "d77c1f1ad308640aefa035a6ebefbe0e15f376613561aa38aa8890cded320485",
+  "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "review_sha256": "cf0f5d4c19c0477b3742dd2de24387886cdd77123b1ca8186768319674e41573",
+  "prompt_sha256": "c95202843857c6b4d079820189473119d8911b09715d012fae4e2862c9a2f780",
+  "permission_denials": []
+}

docs/investigations/r7/implementation-review/opus-implementation.prompt.md

@@ -0,0 +1,17 @@
+Perform a new independent IMPLEMENTATION and merge-readiness review for R7, not another plan approval. User explicitly requests real "opus 5 max" verification and authorizes merging main only if correct. Do not edit files or attempt Write, do not make any network mutations or claim tests you did not run. Return review text only.
+
+Exact candidate commit: 4fcdf37ce656152b32ad0f615d47f5e3f9748c3a
+Exact live main/base fetched now: 9ebe81c1b3611f9cc73e676b5b741c2be62c467a
+Candidate git tree: d5cc904bf2a8b57aea1aac53999b5007522279fc
+Reviewed implementation diff: /Users/vonng/tmp/silo-r7-merge-20260916-ad51/reviewed-code.diff
+Diff SHA-256: 5af32dc2cb4acc34ac91a3e48c5decfcac0d25c70bfa7d6399934c66f629df2a
+Source hashes and full local test evidence: /Users/vonng/.codex/worktrees/ad51/silo/docs/investigations/r7/verification.json
+Caller just recomputed all four code hashes and previous validation log hashes: all match. No source changes since tests passed. The prior evidence includes 80 focused cases, existing SSE-C and trust/poisoning/trailer/Snowball regressions, race, make verifiers and build. Capacity adaptation is explicit, temporary, and real storage I/O still runs. Exact baseline counterfactual: same final tests, old product extractor, 44 controls pass/36 fail. Review logs are available at /Users/vonng/tmp/silo-r7-20260915-ad51/ and permitted via add-dir.
+
+Read the complete actual diff and relevant current call sites/source/tests. Prior agreed plan: docs/investigations/r7/plan-v1.md; prior dispositions and final Opus agreement: plan-v1.dispositions.md and review/opus-v1-confirmation.md. Treat them as context, not a reason to rubber-stamp implementation. Production change adopts PR #187, with only a clarified Snowball comment; candidate also adds stronger tests and documents. PR #187 head remains b8f2fdde41dff3dc3b8db669c1d42d30ca5c1d3d and unmerged; its checks were awaiting outside-contributor workflow approval, not successful CI.
+
+Assess correctness, unintended regressions, trust boundary (PUT/COPY/MPU/Snowball), six SSE metadata fields including empty marker/checksum, request-body signature handling, GET/HEAD/raw-byte expectations, whether tests actually exercise promised paths, and whether capacity fixtures or evidence fail to justify the narrow fix. Inspect especially Snowball ordinary-vs-replica parity, source polluted-metadata limitation, exact token whitespace behavior, and raw ordinary metadata redaction. Do not broaden to R4/R5/R8 or claim full distributed testing. Identify real merge blockers with file/line/evidence and smallest repair; separate optional improvements.
+
+Intended integration: publish this complete reviewed local branch to a normal pgsty/silo PR, wait for its applicable CI, and merge into main. This includes the originally credited PR #187 implementation plus local tests/docs. Do not mutate the original contributor's branch. If main advances first, inspect integration delta, revalidate necessary tests and review again if relevant behavior changed. Merely adding this review report changes documentation only. Merging main is now explicitly authorized by the user, superseding the earlier workflow's no-merge boundary; no release/deploy/live object rewrite is authorized.
+
+Return in Chinese: exact reviewed candidate/base/hash; APPROVE / APPROVE_WITH_NONBLOCKING_NOTES / REQUEST_CHANGES; each blocking finding and optional note; what you inspected vs trusted as supplied test evidence; and explicit whether this exact code may merge once CI is verified. Be concise but evidence-based. No model substitution, no fake test execution.

docs/investigations/r7/plan-v1.dispositions.md

@@ -0,0 +1,23 @@
+# R7 v1 评审意见处置与验收补充
+
+- 冻结方案仍为 `plan-v1.md`，SHA-256 `7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96`。
+- Opus 首轮：`APPROVE_WITH_NONBLOCKING_NOTES`，阻断 0；原始评审见 `review/opus-v1.md`。
+- 本文件只澄清兼容边界与验收，不改变生产补丁范围；作为 v1 的绑定附录交给 Opus 再确认。确认前仍不修改产品代码。
+
+| 意见 | Codex 处置与证据 |
+| --- | --- |
+| N1：Snowball 无 PAX 行为变化与 parity | 接受。可信 replica 的无 PAX 条目不再继承外层 archive 的 ordinary content-type/content-encoding/cache-control/user metadata；这一可见变化使它与普通 Snowball 一致，纳入报告。六个复制专用字段仍能由外层传给每个已授权条目，PAX 的专用字段可按现有次序覆盖。parity 测试用相同 tar 分别执行 ordinary 与 replica 请求并比较条目元数据（排除 replica 状态/时间/ETag）；不能按建议字面在一个 REPLICA 请求中混入无 ReplicateObject 权限的条目并期待它成功，因为 `object-handlers.go:2788-2791` 会拒绝该条目。现有 per-entry 权限回归另行保持。 |
+| N2：HTTP baseline 回归护栏 | 接受，已实测。`/Users/vonng/tmp/silo-r7-20260915-ad51/http-baseline-v2.log` 包含单盘及 16 盘的真实 streaming PUT -> ObjectInfo -> GET/HEAD 失败，同期 ordinary/gzip 控制通过；还覆盖 COPY/multipart/Snowball。64 个叶子：44 控制通过，20 缺陷失败。 |
+| N3：签名前注入所有头 | 接受，已落实在临时测试 `replicaEncodingStream`：先 newTestStreamingRequest、设置所有头，再 signStreamingRequest 和 assembleStreamingChunks。拒绝测试还应断言 XML 错误码为 AccessDenied，区分签名失败。 |
+| N4：空格 token 现状 | 接受。helper 增加 `aws-chunked, gzip -> " gzip"`、`gzip, aws-chunked -> "gzip, aws-chunked"`，仅固定现有精确 token 规则，生产 normalizer 不改。后一例属于既有 token 语法限制，不能宣传为本次已修复。 |
+| N5：历史污染来源反复不一致 | 接受风险并限定措辞。`bucket-replication.go:987-997` 的逐字符串比较可让仍有错误编码的来源与修正后目的对象持续不一致；在再次 heal/resync/比较时可再次选择 metadata 复制。源码证据不单独证明一个不间断热循环。存量提案应先确认并处理权威源版本，再协调各副本；记录重复元数据复制/不一致，而不是只修目的端。自动清理历史来源不进入本次生产补丁。 |
+| N6：容量 adapter 不存在 | 不采纳此事实判断，但接受“临时调整不入交付”的要求。请直接读取基线 `cmd/erasure-server-pool-tags_test.go:258-265`：`type tagTestCapacityDisk struct{ StorageAPI }` 的 DiskInfo 返回当前 Free 作为 Total、Used=0；该文件 :131 有现有调用。此前按字符串 adapter 搜索漏掉了该类型。临时 `capacity-test-utils_test.go` 仅复用它来包装 API 夹具；原始 507 和适配日志均保留，产品容量策略不改。 |
+| N7：map 迭代等价性 | 接受。现有六个 wire key 到六个 internal key 为单射，遍历顺序无关；重复不同大小写头的 canonical map 碰撞行为沿用基线，不引入新的解析规则。 |
+| N8：被删除的旧加密用户字段 | 接受，根因和测试均包含 `X-Amz-Meta-X-Amz-Unencrypted-Content-Length/-Md5` 的再次注入。历史污染可能继续从来源传来；此次目标端普通提取删除后不会恢复这些字段。helper 对 canonical/lowercase 均断言不存在。 |
+| N9：POST 表单路径 | 接受边界说明。POST 表单直接调用低层 extractMetadataFromMime，原本就不执行 extractMetadata 的完整归一化；本补丁保持它的现状，不顺带统一逻辑。 |
+
+## 最终验收范围补充
+
+正式回归保留真实分块签名、有效 gzip 字节、GET 原始字节比较；没有两站点调度器/进程重启/网络故障验收时，就只报告本地复制接收链路和既有 SSE 测试的结论。
+
+存量修复有单独可审阅文件 `stored-metadata-remediation.md`，须按 N5 增补“权威来源优先”的顺序；没有扫描/改写现网对象的授权或动作。

docs/investigations/r7/plan-v1.md

@@ -0,0 +1,71 @@
+# R7 plan v1: preserve normalized replica object metadata
+
+## Frozen scope and source
+
+- Date: 2026-09-15. Worktree: `/Users/vonng/.codex/worktrees/ad51/silo`.
+- Local branch: `codex/r7-replication-content-encoding`.
+- Baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`, also returned by current `gh api repos/pgsty/silo/commits/main` and fetched `origin/main`.
+- PR [#187](https://github.com/pgsty/silo/pull/187): OPEN, unmerged, no reviews/checks returned; head `b8f2fdde41dff3dc3b8db669c1d42d30ca5c1d3d`. GraphQL's PR baseRefOid is `89637554d60c27cfc51d2281d0a4fe15e415f06d`; it is not the live main checked above. Snapshot and exact diff: `/Users/vonng/tmp/silo-r7-20260915-ad51/pr187.{json,diff}`.
+- Introduction: `56fa63bfd155154157cd7e1fb6dc295a3b3104ed` (2026-04-15), replication-header injection hardening. Keep its trust protections intact.
+- Governing scope: PGSTY maintained stack, minimal compatible fix. No dependency, wire-format, credential, encryption algorithm or API changes.
+
+## Root cause and observable contract
+
+`extractMetadata` calls the ordinary extractor, removes disallowed unencrypted-length/MD5 user metadata, and trims the exact `aws-chunked` transport token from `content-encoding`. The trusted-replica restoration currently calls the same broad extractor with `allowReplication=true`. That replays all supported headers and user metadata, reversing normalization and redaction.
+
+Expected mappings are `aws-chunked` -> absent Content-Encoding, `aws-chunked,gzip` -> `gzip`, and `gzip` -> `gzip`. Object bytes are not transformed by this fix. AWS documents this behavior in [SigV4 streaming](https://docs.aws.amazon.com/AmazonS3/latest/developerguide/sigv4-streaming.html). The helper reproduction is `/Users/vonng/tmp/silo-r7-20260915-ad51/baseline_test.go` with Go overlay and `baseline.log`; its failing expectations are evidence of the current defect, not implementation validation.
+
+Persistence/read path: `erasure-metadata.go` reads `fi.Metadata["content-encoding"]` into ObjectInfo.ContentEncoding; `api-headers.go` exposes it on GET/HEAD. Outbound `putReplicationOpts` and metadata-only replication copy also use the object's content encoding. Preventing raw request metadata from being replayed at ingress is sufficient for this defect and avoids read-path masking.
+
+## Input and trust boundary audit
+
+The helper does not authenticate; callers own authentication and authorization. `evaluateReplicationTrust` requires an authenticated principal, the exact single replication marker `true`, and `s3:ReplicateObject`; restoring replica-only metadata also requires `REPLICA`. Unauthorized declared replicas are rejected; marker-only/untrusted requests retain their existing sanitized behavior. Do not move restoration earlier or make headers themselves establish trust.
+
+| Actual caller | Metadata before restoration | Trust gate and intended result |
+| --- | --- | --- |
+| PutObjectHandler | extractMetadataFromReq before trust evaluation | after successful signature verification, evaluate/apply trust; only replicaTrusted restores six fields |
+| CopyObjectHandler via getCpObjMetadataFromHeader | REPLACE calls extractMetadataFromReq; COPY uses source metadata | authenticated source/destination checks; allowReplicationMetadata=replicaTrusted; REPLACE preserves normalization, COPY retains existing semantics |
+| NewMultipartUploadHandler | extractMetadataFromReq after trust/sanitization | replicaTrusted restores six fields into initiation metadata; parts and completion reuse saved metadata |
+| PutObjectExtractHandler, outer Snowball headers | only storage class and per-entry transform metadata, not generic extractMetadata | per-entry PutObject and ReplicateObject authorization; only replicaTrusted restores six fields. No-PAX entries must not inherit ordinary outer archive metadata |
+| PutObjectExtractHandler, PAX entry metadata | extractMetadata on minio.metadata.* records | reuse per-entry trust; merge normalized entry metadata plus six allowed fields. Outer ordinary archive encoding must not leak even if the PAX map omits it |
+
+PutObjectPart, CopyObjectPart and CompleteMultipartUpload do not call this helper; no additional restoration is needed there. Validate completion persistence to catch assumptions at this boundary. POST form upload does not restore replication metadata. Metadata COPY does not normalize historical source metadata; that is deliberately outside this preventive fix.
+
+## Proposed production patch
+
+Adopt the production change in PR #187, adjusted only if current-context application requires it:
+
+1. Remove the `extractMetadataFromMimeWithReplication` boolean-mode helper.
+2. Ordinary `extractMetadataFromMime` keeps header canonicalization and supported/user metadata extraction, always skips the replication-only mapping keys.
+3. `extractReplicationMetadataFromMime` keeps nil-input error behavior and canonical header lookup; loops only over `replicationToInternalHeaders` and joins multi-values exactly as before.
+4. Never re-read ordinary supported or user metadata in the restoration helper. Preserve keys already normalized, defaulted, redacted, or set by the caller.
+5. Preserve all six mappings: sealed SSE-C key, seal algorithm, IV, encrypted-multipart marker (including its empty value), actual object size, and ReplicationSsecChecksumHeader (identity mapping). Preserve canonicalized/lowercase input header compatibility.
+6. Clarify the comment to cover Snowball: ordinary metadata is owned by the caller; the common normalizing path runs before restoration, while outer archive metadata is not per-entry object metadata.
+
+No normalizer/token grammar rewrite. The current exact-token trimming semantics, malformed duplicate-cased headers, and validation of SSE field payloads are outside this bug; retain existing behavior rather than expanding accepted formats or validation rules.
+
+## Verification matrix and acceptance
+
+Use temporary overlay reproductions before consensus. Promote focused regressions only after recorded Opus agreement. Run targeted tests with bounded Go parallelism because sibling tasks share this host.
+
+1. Helper pipeline: absent encoding, bare aws-chunked, aws-chunked,gzip, gzip, gzip,aws-chunked, multi-valued encoding; ordinary vs restoration; key absence for bare encoding; legitimate gzip retained; ordinary/user metadata sentinel values and redacted unencrypted metadata not restored. Exact expected six-field map, canonical/lowercase headers, empty multipart marker, nil input handling. Repeat restoration should not change ordinary metadata.
+2. Real signed HTTP PUT -> persisted ObjectInfo -> GET and HEAD on the existing single-drive and 16-drive erasure fixtures. Use real streaming chunk signatures for transport cases and a valid gzip payload for gzip cases. Compare raw response bytes and content encoding; bare transport must have no header. Test authenticated ordinary, trusted replica, and marker without replication permission; declared replica without permission returns 403 and creates nothing.
+3. Signed COPY REPLACE and multipart initiate/part/complete -> persisted ObjectInfo -> GET/HEAD for bare, mixed, and plain gzip. Include ordinary controls. Initiation carries object metadata; part/completion carry contrasting content encoding to prove they cannot replace it. COPY preserves existing source metadata semantics.
+4. Snowball trusted entry tests with and without PAX, including PAX no Content-Encoding and PAX mixed encoding; outer aws-chunked never leaks. Existing per-entry trust test stays green.
+5. Existing SSE-C single PUT and multipart replication round trips plus replication-header poisoning regressions. Helper matrix verifies all six field mappings; actual SSE-C tests verify readable ciphertext replicas, encryption metadata, checksum and multipart layout. Preserve bucket default encryption/compression behavior. Run relevant SSE-KMS/SSE-S3 option/replica tests if available without broadening R4 scope.
+6. Targeted package tests, focused race run, build and repository verifiers. If environmental failures (e.g. disk free-space threshold) prevent existing tests from reaching the path, retain the original failure and use an explicitly documented temporary capacity adapter already used by the repository, keeping actual object I/O on test disks. Do not mislabel that as an unmodified pass.
+7. Baseline regression must fail for raw/mixed trusted metadata and fixed code must pass identical expectations. Record exact commands, exit status, baseline/diff hashes and fixture limits. No full distributed sites/deployment acceptance claim from local handler tests.
+
+## Stored-object remediation proposal (separate, no execution)
+
+Upgrade only prevents new pollution. Existing source/COPY metadata can remain wrong, and rollback reopens ingress pollution without undoing repairs. Do not rewrite production objects or private xl.meta files.
+
+A separate operator-reviewed job must inventory bucket/key/version, original Content-Encoding and complete metadata, source/replica provenance, version/ETag/size/checksum and encryption/retention settings. Identify exact aws-chunked tokens and preserve other encodings/order. Verify source bytes/encoding before deciding; gzip must not be guessed or decompressed merely from the broken label. Keep an immutable manifest and metadata backup. Test a version-preserving supported metadata operation on a local replica of the relevant setup; ordinary S3 self-COPY can create a new version/change metadata timestamps and is not a universal version-preserving repair. Resolve object-lock, SSE-C keys, concurrent changes and replication ordering before approving the concrete write plan. Apply a small approved batch with concurrency guards, re-read exact versions, verify GET/HEAD and raw bytes/checksums, then reconcile replicas. Skips/conflicts need explicit reporting and a tested rollback. This task supplies the reviewable design only.
+
+## Effort, delivery and gate
+
+Expected 0.5-1 engineer-day for patch, targeted tests and evidence on a familiar checkout; stored-object repair and release are separate work. Production patch is about 30 added/20 removed lines in one helper file; tests provide most of the new code.
+
+Before implementation: actual Claude Code `--model claude-opus-5 --effort max`, read-only tools, same frozen plan hash + baseline + PR snapshot. Record raw review, actual assistant model(s), objections and dispositions. Any model mismatch/error/rate-limit is not consensus. Resolve substantive findings and get explicit approval of the same plan version before production changes. After consensus, implement and verify without another user permission request.
+
+Deliver research, versioned plan, consensus/dispositions, minimal production diff, tests and verification summary. Local commit may package the reviewable result. No main merge, remote PR mutation, push, release, deployment, or existing-object rewrite is included.

docs/investigations/r7/research.md

@@ -0,0 +1,43 @@
+# R7 调查与复现
+
+## 结论
+
+基线 `9ebe81c1b3611f9cc73e676b5b741c2be62c467a` 中，可信复制恢复过程把已规范化的普通元数据从原始请求中重新提取。`aws-chunked` 是传输编码，重新落盘后会被 GET/HEAD 返回。SILO 在 2026-04-15 的 `56fa63bfd155154157cd7e1fb6dc295a3b3104ed` 中引入此回归；该提交修复的复制头信任边界仍需保留。
+
+实时核对 [PR #187](https://github.com/pgsty/silo/pull/187)：OPEN、未合并，head `b8f2fdde41dff3dc3b8db669c1d42d30ca5c1d3d`；只恢复复制专用字段的方向与根因吻合。没有将 PR 自报测试当成本轮验证。
+
+## 当前调用链
+
+五处调用：PUT、COPY REPLACE、NewMultipartUpload、Snowball 外层请求、Snowball PAX 条目。
+
+- PUT/COPY/multipart 使用规范化普通元数据；可信分支不应再覆盖它。
+- Snowball 外层只有 storage class/条目转换信息，没有通用提取；归档外层 Content-Type/Content-Encoding 不是条目元数据。
+- PAX 元数据经过普通提取；无 Content-Encoding 的 PAX 映射也不能留下先前泄漏的外层编码。
+- multipart 的 Part/CopyPart/Complete 不调用该恢复函数；完成后必须检验初始化元数据确实被保留。
+- 所有可信恢复均需通过认证、精确复制标记、ReplicateObject 权限和 REPLICA 状态的组合判断；Snowball 对每个条目分别鉴权。
+
+`erasure-metadata.go` 将落盘 `content-encoding` 读入 ObjectInfo；`api-headers.go` 在 GET/HEAD 返回该值。对象字节并非因此一定受损。
+
+AWS [SigV4 streaming 规范](https://docs.aws.amazon.com/AmazonS3/latest/developerguide/sigv4-streaming.html) 要求保存对象时去掉 aws-chunked，只保留实际的内容编码；只有 aws-chunked 时读取响应不应有 Content-Encoding。
+
+## 实测记录
+
+原始证据根目录：`/Users/vonng/tmp/silo-r7-20260915-ad51/`。
+
+| 记录 | 结果与边界 |
+| --- | --- |
+| `baseline_test.go` / `baseline-overlay.json` / `baseline.log` | 原始产品代码，临时 Go 测试覆盖：ordinary bare/mixed 正常，trusted bare/mixed 重新污染，纯 gzip 正常 |
+| `http-baseline-unadapted.log` | 未调整夹具的本机单盘 HTTP 上传被 507 / XMinioStorageFull 拒绝；不是 R7 结果 |
+| `http-baseline.log` | 首个容量适配 HTTP 运行；PUT/COPY/Snowball 可复现；multipart 夹具错误用 Header.Get 读取了仓库直接写入的 ETag 键，完成时 InvalidPart，不能用于 multipart 结论 |
+| `http_test.go` / `http-capacity-overlay.json` / `http-baseline-v2.log` | 修正 ETag 读取后：PUT、COPY REPLACE、multipart 的普通和 untrusted-marker 对照通过；trusted bare/mixed 在落盘和 GET/HEAD 中失败；trusted gzip 通过；无权 replica 返回 403 且不建对象 |
+| 同一最终 HTTP 基线日志的 Snowball 部分 | no-PAX、PAX 无编码、PAX 裸编码、PAX 混合编码均复现 GET/HEAD 污染 |
+
+最终 HTTP 基线总计 64 个叶子用例：44 个控制通过，20 个预期缺陷失败。每个用例均使用认证 HTTP handler + 实际对象磁盘。PUT 传输案例和 Snowball 使用真实流式分块签名，gzip 案例使用实际 gzip 字节；读取比较原始字节并检查 Content-Encoding。
+
+## 容量适配的准确含义
+
+宿主 APFS 报告约 100% 使用率，触发 SILO 的相对空闲阈值。临时 `capacity-test-utils_test.go` 使用仓库既有 `tagTestCapacityDisk` 包装 API 测试夹具，将总量/已用量报告为当前空闲量/0；实际对象写入、读取和元数据持久化仍落在测试磁盘。该 overlay 不进入产品 diff，也不调整生产容量策略。结果属于明确容量适配后的本地验证，不是未改夹具的通过。
+
+## 方案与存量
+
+冻结方案见 [plan-v1.md](plan-v1.md)。存量对象处理见 [stored-metadata-remediation.md](stored-metadata-remediation.md)，只提供设计，本任务不执行现网对象改写。

docs/investigations/r7/review/opus-v1-confirmation.md

@@ -0,0 +1,28 @@
+本轮只做回应核验，未重复独立审计；无 shell，未计算哈希、未运行任何测试。
+
+# R7 v1 第二轮确认（Opus 5）
+
+## 1. 版本与哈希
+
+- 冻结方案：`docs/investigations/r7/plan-v1.md`，任务声明 SHA-256 **`7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96`**。
+- 处置附录：`docs/investigations/r7/plan-v1.dispositions.md`，任务声明 SHA-256 **`d32d30f8a420f904da6ee039f0461d6281e7da44bd805d416b4970d071f0af32`**。
+- 两个哈希均按任务声明采纳，本会话无法计算，校验仍属调用方责任。产品基线 `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`，工作区确认未改产品代码（`docs/investigations/r7/` 为 untracked）。生产补丁仍读 `pr187.diff`，作为提议代码而非我运行的证据。
+
+## 2. 三个争议点的裁定
+
+**N6：我错了，撤回。** `cmd/erasure-server-pool-tags_test.go:258-264` 确有 `type tagTestCapacityDisk struct{ StorageAPI }`，其 `DiskInfo` 把 `Total=Free, Used=0`，并已在同文件 `:131` 被 `TestReplicaWritesPreserveTagOrdering` 使用。我上轮按 “adapter” 字面 grep 漏掉了该类型，所以 plan 第 56 行 “repository already used 的临时容量适配器” 属实。撤回 N6 的事实判断，保留其被接受的部分：临时 `capacity-test-utils_test.go` 不进交付 diff，原始 507 与适配过程需记录。补充一句非阻塞：若正式回归仍需容量适配，直接复用同包内已有类型即可（不算临时旁路），产品容量策略不得改动。
+
+**N1：接受纠正。** `object-handlers.go:2740` 的 `rawReplica` 是整包级判定，`:2788-2791` 对任何缺 `ReplicateObject` 的条目直接 `ErrAccessDenied` 并中止该条目；因此我建议的“同一 REPLICA 包内 trusted 与 untrusted 条目对比”在机制上不可能成立。Codex 的做法正确：同一 tar 分别发 ordinary 与 replica 两次请求，比较条目元数据（排除 replica 状态/时间戳/ETag）。实质结论不变且已被证据坐实——补丁前 `:2835-2838` 的恢复分支会把外层 `supportedHeaders` + 用户元数据整体灌进仅含 storage class 的 `metadata`（`:2802-2804`），补丁后只剩六个映射；`http-baseline-v2.log:195-201` 的 `Erasure/no-pax` 正是外层 `aws-chunked` 泄漏。可选增补（非必须）：再加一条 untrusted-marker 归档对照。
+
+**N5：接受收窄措辞。** 源码只能支撑到：`bucket-replication.go:987-997` 的逐字符串比较 → `replicationActionForTarget:1131` → 仅在 `replicateObjectToTarget:1598` 的复制任务里求值，且成功后置 `Completed`、不自我重排队。所以“每次 heal/resync/重放对账都会再次选中 `replicateMetadata`” 成立，“不间断热循环” 我上轮说过头，撤回该措辞。补救顺序“先确认并修权威源版本、再协调副本”，以及把重复元数据复制/不一致记为已知影响，均予保留。
+
+## 3. 其余处置确认
+
+- **N2 已用实测兑现**：`http-baseline-v2.log` 是 handler 级证据，单盘 `ErasureSD` 与 16 盘 `Erasure` 均覆盖 put/copy-replace/multipart，失败精确落在 `replica/bare`（persisted/GET/HEAD 均为 `aws-chunked`，期望空）与 `replica/mixed`（`aws-chunked,gzip` vs `gzip`），ordinary、untrusted-marker、gzip、unauthorized-replica 全通过；连同 Snowball 8 例，64 叶 = 44 通过 / 20 失败，与附录计数一致。护栏诉求已满足（该日志由本会话之外产生，我只读未跑）。
+- **N3 / N7 / N8 / N9**：接受无异议。N3 额外要求断言 XML 码为 `AccessDenied` 以区分签名失败，正确。
+- **N4**：接受，且两条期望值与代码一致——`handler-utils.go:357-368` 按 `,` 精确等值比较，故 `"aws-chunked, gzip" → " gzip"`（保留前导空格）、`"gzip, aws-chunked" → "gzip, aws-chunked"`。仅作现状记录，不得宣传为本次修复。
+- **最终验收范围补充**：同意。无双站点调度器/重启/网络故障验收时只报告本地接收链路与既有 SSE 结论；存量修复只出设计文件，不授权扫描或改写现网对象。
+
+## 4. 结论
+
+**APPROVE。阻塞问题 0。** plan-v1 + 本处置附录构成的 v1 组合可直接进入实现，无需 v2；N1/N5 采用 Codex 的修正表述，N6 以我撤回告结。实现时请把 N1（无 PAX 行为变化）、N4（空格现状）、N5（收敛风险与补救顺序）落为断言或文档，并确保临时容量适配文件不出现在交付 diff 中。

docs/investigations/r7/review/opus-v1-confirmation.metadata.json

@@ -0,0 +1,36 @@
+{
+  "requested_model": "claude-opus-5",
+  "requested_effort": "max",
+  "cli_version": "2.1.270",
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "plan": "plan-v1.md",
+  "plan_sha256": "7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96",
+  "addendum": "plan-v1.dispositions.md",
+  "addendum_sha256": "d32d30f8a420f904da6ee039f0461d6281e7da44bd805d416b4970d071f0af32",
+  "status": "completed",
+  "started_at": "2026-09-15T15:51:25.514239+00:00",
+  "raw_output": "/Users/vonng/tmp/silo-r7-20260915-ad51/opus-v1-confirmation.jsonl",
+  "stderr": "/Users/vonng/tmp/silo-r7-20260915-ad51/opus-v1-confirmation.stderr.log",
+  "command": "claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --output-format stream-json --verbose --add-dir /Users/vonng/tmp/silo-r7-20260915-ad51",
+  "actual_assistant_models": [
+    "claude-opus-5"
+  ],
+  "subtype": "success",
+  "is_error": false,
+  "duration_ms": 57910,
+  "num_turns": 18,
+  "session_id": "9b4140a8-140f-4d94-9a59-9983346709fd",
+  "raw_sha256": "2e2db44411a0b367607b73a2f7fe38c5125f49294f496d603c8ed4975c549bba",
+  "review_sha256": "a471ca03b6e68d77d01dede8c6d1a8f989bceed9f7eeb99fc36634c025eb3541",
+  "verdict": "APPROVE",
+  "blocking_findings": 0,
+  "completed_at": "2026-09-15T15:53:09.806502+00:00",
+  "auxiliary_model_ids": [
+    "claude-haiku-4-5-20251001",
+    "claude-opus-5"
+  ],
+  "observed_tool_attempts": [
+    "Grep",
+    "Read"
+  ]
+}

docs/investigations/r7/review/opus-v1-confirmation.prompt.md

@@ -0,0 +1,7 @@
+This is round 2 of the actual R7 Opus review discussion. Product baseline remains 9ebe81c1b3611f9cc73e676b5b741c2be62c467a and NO product code has been changed. Your first actual review is /Users/vonng/.codex/worktrees/ad51/silo/docs/investigations/r7/review/opus-v1.md. It approved v1 with 9 nonblocking notes and zero blockers.
+
+Please read the SAME frozen plan /Users/vonng/.codex/worktrees/ad51/silo/docs/investigations/r7/plan-v1.md (SHA-256 7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96) and Codex's numbered replies / binding acceptance addendum /Users/vonng/.codex/worktrees/ad51/silo/docs/investigations/r7/plan-v1.dispositions.md (SHA-256 d32d30f8a420f904da6ee039f0461d6281e7da44bd805d416b4970d071f0af32). The plan+addendum is the exact v1 consensus bundle; production patch remains /Users/vonng/tmp/silo-r7-20260915-ad51/pr187.diff.
+
+Focus this round on replies, not repeating the whole independent audit. Check the factual disagreement N6 by reading cmd/erasure-server-pool-tags_test.go lines 258-265 (tagTestCapacityDisk does exist; Grep for adapter missed it). Check N1's parity testing correction: a single REPLICA archive with an unauthorized entry rejects that entry, so equal ordinary vs replica archive uploads is the correct comparison. Check N5's narrower statement about repeated reconciliation opportunities rather than claiming an automatic continuous hot loop. N2 is now backed by /Users/vonng/tmp/silo-r7-20260915-ad51/http-baseline-v2.log. The other notes are accepted and will become assertions/docs.
+
+Return a concise Chinese confirmation that (a) explicitly names both hashes, (b) agrees/disagrees with each disputed point N1/N5/N6 and remaining dispositions, and (c) states APPROVE or REQUEST_CHANGES and whether there are any blockers to implementing this exact v1 bundle. Do not simulate tests. No shell is available: hash checking remains the caller's responsibility. Do not attempt Write or write a Claude plan file; tools are restricted to Read/Grep/Glob and your review is the final text.

docs/investigations/r7/review/opus-v1.md

@@ -0,0 +1,51 @@
+Write 工具在本会话被禁用，因此评审直接以回复形式交付（本次任务本就是只读评审，未改动任何文件、未运行任何测试）。
+
+---
+
+# R7 修复方案独立评审
+
+## 1. 审阅版本与哈希
+
+- **审阅对象**：`/Users/vonng/.codex/worktrees/ad51/silo/docs/investigations/r7/plan-v1.md`，72 行，标题 `R7 plan v1: preserve normalized replica object metadata`（`docs/investigations/r7/` 在 git status 中为 untracked）。
+- **任务给定 SHA-256**：`7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96`。本会话只读且无 shell，**我未重新计算该哈希**；逐字审阅的是该路径当前内容。
+- 基线 `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`、引入提交 `56fa63bf…`、PR #187 的 OPEN/head 状态均按任务声明采纳，**未用 git/gh 复核**。
+- `pr187.diff` 被当作“提议的代码”阅读，不视为已运行的测试；`baseline.log` 由本会话之外产生，我只读未跑。
+- **性质**：源码与方案验证，非我执行的测试。
+
+## 2. 结论
+
+**APPROVE_WITH_NONBLOCKING_NOTES**
+
+## 3. 阻塞问题
+
+**无阻塞问题。** 以下为逐项核验依据。
+
+**最小补丁充分性（充分）**：恢复辅助函数是 replica 路径上唯一能把未归一化的 ordinary 头写进对象元数据的入口。`putOptsFromHeaders`/`getDefaultOpts`（`cmd/object-api-options.go:388-477`）只读 SSE 与 source-* 时间戳；`completeMultipartOpts:542-548` 只取 actual-object-size 与 ssec-crc。五处调用点（`object-handlers.go:1149/2282/2836/2869`、`object-multipart-handlers.go:246`）全部被覆盖。
+
+**信任边界（未改变）**：`evaluateReplicationTrust`（`cmd/replication-trust.go:78-90`）要求已认证主体 + 精确单值 `true` 标记 + `s3:ReplicateObject`，`REPLICA` 声明无权限直接 403；Snowball 走等价的 per-entry 内联判定（`object-handlers.go:2784-2793`）。补丁不前移恢复点、不让头部本身产生信任。
+
+**调用链（方案描述与代码一致）**：PUT 在签名校验后评估信任、元数据在 `:2199` 已归一化；COPY REPLACE 用 `extractMetadataFromReq`、COPY 保留源语义（`:1143-1156`、`:1411`、`:1801`）；多段初始化在 sanitization 之后提取（`object-multipart-handlers.go:179/233`）；**分片与完成确实不需要恢复**——分片从 `mi.UserDefined` 取加密状态（`:885-886`、`:957-987`），完成从 `completeMultipartOpts` 取两个字段。
+
+**六个映射与空标记**：补丁遍历 `replicationToInternalHeaders`（`handler-utils.go:106-114`），与基线遍历 `supportedHeaders` 的键集完全相同，且六→六为单射，故 map 迭代顺序无关；空值 multipart 标记按 key 存在性消费（`internal/crypto/metadata.go:27`），`strings.Join([]string{""}, ",")==""` 行为与基线一致。
+
+**归一化与冗余用户元数据**：基线恢复分支会把刚被 `extractMetadata`（`:218-241`）删除的 `X-Amz-Meta-X-Amz-Unencrypted-Content-Length/-Md5`（`internal/http/headers.go:138-139`，GHSA-76wf-9vgp-pj7w）按原始大小写写回，补丁一并消除。
+
+**额外独立验证（支持方案的关键事实）**：本仓库固定的 minio-go（`go.mod:71` → `pkg/signer/utils.go:70-87 setAwsChunkedContentEncoding`）**保留调用方已设编码**并生成 `aws-chunked` 或 `aws-chunked,gzip`（无空格）。因此方案声明的 `aws-chunked→无`、`aws-chunked,gzip→gzip` 与真实复制线路一致，修复后目标端存储值将等于源端 `objInfo.ContentEncoding`（`bucket-replication.go:838`），读路径 `erasure-metadata.go:138` → `api-headers.go:129-131` 也成立。
+
+## 4. 非阻塞意见
+
+1. **Snowball 无 PAX 条目的行为变化必须显式承认并加断言**。证据：`object-handlers.go:2802-2842` 的 `metadata` 只有 storage class 与压缩键，基线恢复会把外层 tar 请求的 content-type / `x-amz-meta-*` / cache-control 复制进每个 entry；补丁后不再复制，与普通 Snowball（`:2874-2877` 分支从不做 `extractMetadata`）一致。我同意这个选择，但它超出“只去掉 aws-chunked”。最小修正：在 §Proposed patch 第 6 条写明“trusted replica 无 PAX 条目不再继承外层归档 ordinary 元数据”，并在 §Verification 4 增加断言：同一 tar 中 trusted 与 untrusted 无 PAX 条目的 UserDefined（除 replica 状态/时间戳/ETag 外）相等；同时注明“外层请求的六个字段仍套用到所有条目”是既有且有意保留的行为。
+2. **回归护栏应绑定 handler 级用例**。`baseline_test.go` 只覆盖 helper、绕过信任门；真正会退化的是调用点。最小修正：§Verification 7 的“基线必须失败”至少绑定一条 HTTP 用例（trusted replica streaming PUT → `GetObjectInfo().ContentEncoding`）。
+3. **流式签名测试必须在签名前注入 replication 头**。`newTestStreamingSignedCustomEncodingRequest`（`test-utils_test.go:817-834`）先 Set 编码再签名；若签名后再加 `x-amz-bucket-replication-status`，得到的是 403 SignatureDoesNotMatch，容易被误读成“未恢复元数据”。最小修正：在 §Verification 2/3 补一句，并要求区分签名失败与权限拒绝。
+4. **精确 token 裁剪的空格限制未被测试固定**。`handler-utils.go:357-368` 按 `,` 分割做精确等值比较，`"gzip, aws-chunked"` 不会被裁剪。同意不改语法；最小修正：§Verification 1 增加两条“记录现状”的断言用例。
+5. **已污染对象的收敛性风险应进入补救段**。`bucket-replication.go:987-997` 用源端 `ContentEncoding` 与目标端元数据比较，而 minio-go 在已含 `aws-chunked` token 时直接早退（`utils.go:78-80`）：源端仍为 `aws-chunked` 的历史对象在目标端修复后将永久 mismatch 并反复触发 metadata 复制。最小修正：补救顺序明确为“先修源端版本、再 reconcile 副本”，并把该 churn 列为已知影响与监控项。
+6. **“仓库已有的容量适配器”与现状不符**：`cmd/` 下不存在此类测试旁路（`object-api-utils.go:1302 hasSpaceFor` 无测试开关；`cmd` 内 grep `adapter` 仅命中 `object-api-interface.go:325` 注释）。最小修正：改写为“显式记录所用临时本地改动并保证其不进入交付 diff”。
+7. **map 迭代等价性建议写入方案**（第 3 条补一句“顺序无关性来自单射映射”），避免未来新增映射时被误改；`nv` 对重复大小写头仍是“最后写入者胜”，与基线相同，不是本次引入。
+8. **GHSA 冗余键回写的修复建议在根因段显式成句**；并注意 `putReplicationOpts`（`bucket-replication.go:797-813`）不像 `getCopyObjMetadata`（`:733-735`）那样过滤这两个键，故历史对象可能确实携带它们，修复后目标端会与普通 PUT 一样丢弃。
+9. **POST 表单路径的不对称**：`bucket-handlers.go:1260-1266` 直接调用 `extractMetadataFromMime`，本就不做裁剪/默认 content-type/冗余键删除；补丁未改变它，建议在方案中补半句，避免未来“顺手统一”而改变 POST 行为。
+
+## 5. 是否同意同一 v1 方案进入实现
+
+**同意。** 该 v1 方案的根因、信任边界、调用链与补丁范围均与当前基线源码相符，不存在阻塞缺陷；上述 9 条均为记录/测试/文档层面的处置，不需要改动补丁本体。在 Codex 逐条记录处置（尤其第 1、2、5 条）后，**同一 v1 方案可直接进入实现，无需 v2**。
+
+最后重申两点事实边界：PR #187 diff 中的测试改动是提议代码而非已运行证据；`baseline.log` 的失败断言是既有缺陷的证据，非实现验证——两者都不是我在本会话中运行的测试。

docs/investigations/r7/review/opus-v1.metadata.json

@@ -0,0 +1,37 @@
+{
+  "requested_model": "claude-opus-5",
+  "requested_effort": "max",
+  "cli_version": "2.1.270",
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "plan": "plan-v1.md",
+  "plan_sha256": "7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96",
+  "status": "completed",
+  "started_at": "2026-09-15T15:44:49.861232+00:00",
+  "raw_output": "/Users/vonng/tmp/silo-r7-20260915-ad51/opus-v1.jsonl",
+  "stderr": "/Users/vonng/tmp/silo-r7-20260915-ad51/opus-v1.stderr.log",
+  "command": "claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --output-format stream-json --verbose --add-dir /Users/vonng/tmp/silo-r7-20260915-ad51",
+  "actual_assistant_models": [
+    "claude-opus-5"
+  ],
+  "observed_tool_attempts": [
+    "Glob",
+    "Grep",
+    "Read",
+    "Write"
+  ],
+  "subtype": "success",
+  "is_error": false,
+  "duration_ms": 309465,
+  "num_turns": 62,
+  "session_id": "10f1e9b3-7120-4a1f-94d9-e53745849cd1",
+  "raw_sha256": "b749be2acbec45b978907849c741e1f71994b0ad4e13d5b5204f429c9ba408b3",
+  "review_sha256": "d832e461291f0ec32455ade97c9bf9cd02d31812642a4b9254dd5e06ab1cc82b",
+  "verdict": "APPROVE_WITH_NONBLOCKING_NOTES",
+  "blocking_findings": 0,
+  "completed_at": "2026-09-15T15:51:03.390714+00:00",
+  "auxiliary_model_ids": [
+    "claude-haiku-4-5-20251001",
+    "claude-opus-5"
+  ],
+  "note": "Actual reviewer assistant messages all identify claude-opus-5. CLI plan mode attempted Write to a Claude plan artifact, but Write was disabled; final review was returned as text. No product changes."
+}

docs/investigations/r7/review/opus-v1.prompt.md

@@ -0,0 +1,12 @@
+Independently review the R7 repair plan. Do not edit files or run implementation. The user requires real Opus 5.0 discussion and explicit consensus before product code changes. Disagree when evidence warrants it; do not assume PR author claims are tests we ran.
+
+Repository baseline: 9ebe81c1b3611f9cc73e676b5b741c2be62c467a.
+Plan v1: /Users/vonng/.codex/worktrees/ad51/silo/docs/investigations/r7/plan-v1.md
+Plan SHA-256: 7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96
+Current upstream PR snapshot and proposed production patch: /Users/vonng/tmp/silo-r7-20260915-ad51/pr187.json and /Users/vonng/tmp/silo-r7-20260915-ad51/pr187.diff
+Direct current-baseline helper reproduction: /Users/vonng/tmp/silo-r7-20260915-ad51/baseline_test.go, /Users/vonng/tmp/silo-r7-20260915-ad51/baseline.log (expected assertions fail).
+Read the full plan and then independently inspect relevant current code including cmd/handler-utils.go, cmd/replication-trust.go, all five restoration call sites in cmd/object-handlers.go and cmd/object-multipart-handlers.go, existing test fixtures and SSE replication consumers. The Snowball no-PAX caller has NOT already performed generic ordinary metadata extraction; explicitly evaluate the proposed behavior there.
+
+Check: minimal patch sufficiency; ordinary and trusted/replica trust boundary; PUT/COPY/multipart/parts/completion/Snowball call chains; six SSE-only mappings and empty multipart marker/checksum; normalization and removed unsafe ordinary user metadata; correct actual HTTP tests; stored-object remedial design risks. Do not expand this into independent R4/R5 issues unless this proposed fix depends on them.
+
+Return a review in Chinese with (1) the reviewed version and exact hash, (2) verdict APPROVE / APPROVE_WITH_NONBLOCKING_NOTES / REQUEST_CHANGES, (3) each blocking issue with severity, exact code/plan evidence and smallest correction, (4) separately numbered nonblocking notes, (5) explicit whether you agree this SAME v1 plan can proceed to implementation after Codex records its dispositions. If no blocking issues, say so. Your review is source/plan validation, not actual tests run by you.

docs/investigations/r7/stored-metadata-remediation.md

@@ -0,0 +1,31 @@
+# R7 存量错误元数据处理提案（待单独批准）
+
+本次代码修复只阻止可信复制再次从传输头写入 `aws-chunked`。升级不会扫描或改写旧对象；普通 COPY 继续保留来源对象的既有元数据。本文件是后续操作的设计，不是已执行的迁移。
+
+## 1. 只读清单
+
+按桶、键和精确 version ID 记录候选；包含非当前版本，不能只检查最新版本。条件是 Content-Encoding 的逗号分隔 token 中包含 `aws-chunked`。若大小写、空格或重复值异常，单列人工核验，不能凭字符串子串匹配改写。
+
+清单至少保存：来源及目标站点、bucket/key/version ID、完整原始 Content-Encoding、拟保留编码、ETag、对象大小、可用内容校验值、修改时间、完整普通及用户元数据、标签、Object Lock/保留期/法律保留、SSE 模式及必要密钥的可用性、复制状态。清单不保存 SSE-C 密钥或凭据。读取响应时禁用客户端自动 gzip 解码，以便核对原始字节。
+
+仅凭错误响应头不能判断字节是什么。比对可信来源版本或独立的原始内容校验值；有 gzip 的对象确认其字节确为 gzip 并保持原字节，不重新压缩。来源也受污染、来源版本不存在或校验依据不足时，标记为需调查，不自动修复。
+
+## 2. 制作具体变更清单
+
+优先确认并处理权威来源的精确版本，再协调副本。若来源仍保存错误编码，复制比较器会把规范化后的目标判断为不一致；在后续 heal/resync/比较时可能反复选择元数据复制。多向复制须核对整组来源和副本，记录持续不一致及重复元数据复制。这个风险不等于已证明存在不间断重试热循环。
+
+原则上只删除被证实属于传输层的 `aws-chunked` token：仅有该 token 时移除 Content-Encoding，有其他编码时保留顺序和值。每条候选给出前后值和可回滚的元数据快照，其余内容、元数据和对象标识的保持条件逐项列出。
+
+普通 S3 自 COPY 可能创建新版本、更新修改时间和复制排序；它不能被当作通用的原版本元数据修复 API。先在本地同配置克隆中验证可用的受支持管理/元数据操作，再选择方案。如果必须创建替代版本，要在清单中明确 version ID、当前版本关系和调用方影响。如果没有受支持的安全路径，停止该项，不编辑 `xl.meta` 或内部盘文件。
+
+## 3. 审批与小批执行条件
+
+批准的是具体清单和已经验证过的写入方式。执行前再次核对 version ID、ETag、大小、原元数据和时间等并发保护；ETag 单独不足以检测元数据更新。明确写入协调或维护窗口，发生冲突则跳过。Object Lock、SSE-C、生命周期和双向复制等条件分别验证；不得为了修元数据绕过保留限制。
+
+先在可回滚的小批次验证：精确版本 HEAD 的 Content-Encoding 正确、GET 原始字节/校验值一致、对象锁和其他元数据没有被丢弃、目标站点版本与复制状态最终一致。留下逐项结果、冲突、跳过和失败日志，再决定扩大批次。
+
+## 4. 回滚边界
+
+保留不可变清单和完整元数据备份；为选用的具体操作验证回滚步骤。若写入创建了新版本，回滚必须考虑 version ID 和当前版本关系，不能用“再 COPY 一次”代替证明。代码回滚会重新开放新污染入口，并不会自动还原任何已修过的元数据。
+
+本任务未对现网执行清单扫描、对象写入、版本调整、部署或存量修复。

docs/investigations/r7/verification.json

@@ -0,0 +1,230 @@
+{
+  "recorded_at": "2026-09-15T15:59:49.705292+00:00",
+  "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+  "branch": "codex/r7-replication-content-encoding",
+  "source_sha256": {
+    "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+    "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+    "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+    "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+  },
+  "plan_sha256": {
+    "plan-v1.md": "7af5705ebbfb0a375956d38dba059095dc16e24558290b1bd35a6ce85b9e1f96",
+    "plan-v1.dispositions.md": "d32d30f8a420f904da6ee039f0461d6281e7da44bd805d416b4970d071f0af32"
+  },
+  "validation": {
+    "fixed-targeted": {
+      "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+      "files": {
+        "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+        "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+        "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+        "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+      },
+      "exact_baseline_overlay": "/Users/vonng/tmp/silo-r7-20260915-ad51/exact-baseline-overlay.json",
+      "recorded_at": "2026-09-15T15:55:15.454079+00:00",
+      "exit_code": 0,
+      "log": "/Users/vonng/tmp/silo-r7-20260915-ad51/fixed-targeted.log",
+      "log_sha256": "a30add5762881d604acffd63bca5880050b92a3ae125d208c53545ac78c9b212",
+      "command": [
+        "go",
+        "test",
+        "-p",
+        "2",
+        "-overlay",
+        "/Users/vonng/tmp/silo-r7-20260915-ad51/capacity-overlay.json",
+        "./cmd",
+        "-run",
+        "^Test(ExtractMetadataHeaders|ExtractMetadataFromRequest.*|ExtractReplicationMetadata.*|GetCopyObjectMetadataFromHeaderReplication|APIReplicaContentEncoding|APISnowballReplicaContentEncoding)$",
+        "-count=1",
+        "-v"
+      ],
+      "GOMAXPROCS": "4",
+      "top_level_tests_passed": 9,
+      "leaf_subtests_passed": 80
+    },
+    "fixed-sse-trust": {
+      "command": [
+        "go",
+        "test",
+        "-p",
+        "2",
+        "-overlay",
+        "/Users/vonng/tmp/silo-r7-20260915-ad51/capacity-overlay.json",
+        "./cmd",
+        "-run",
+        "^Test(API(SSECReplicaSkipsDestinationTransforms|SSECMultipartReplicaRoundTripWithCompression|SSECMultipartReplicationTrust|PutObjectReplicationHeaderPoisoning|CopyObjectReplicationHeaderPoisoning|PutObjectReplicationTrust|SnowballReplicationTrustIsPerEntry|SnowballInheritsBucketEncryption|StreamingTrailerWithUntrustedReplicationHeaders)|CloneRequestWithoutReplicationHeaders|PutReplicationOpts.*|ReplicationTrustControlsInternalOptionsAndEvents)$",
+        "-count=1",
+        "-v"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/ad51/silo",
+      "started_at": "2026-09-15T15:54:14.556575+00:00",
+      "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+      "source_sha256": {
+        "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+        "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+        "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+        "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+      },
+      "GOMAXPROCS": "4",
+      "GOFLAGS": null,
+      "status": "completed",
+      "log": "/Users/vonng/tmp/silo-r7-20260915-ad51/fixed-sse-trust.log",
+      "exit_code": 0,
+      "duration_seconds": 11.364,
+      "completed_at": "2026-09-15T15:54:25.939958+00:00",
+      "log_sha256": "09222aaea6f0504dce7a801d5f392df77728ebd9875f4fef9dcdb28f71ce7b5a"
+    },
+    "fixed-race": {
+      "command": [
+        "go",
+        "test",
+        "-race",
+        "-p",
+        "2",
+        "-overlay",
+        "/Users/vonng/tmp/silo-r7-20260915-ad51/capacity-overlay.json",
+        "./cmd",
+        "-run",
+        "^Test(ExtractReplicationMetadata.*|APIReplicaContentEncoding|APISnowballReplicaContentEncoding|APISnowballReplicationTrustIsPerEntry|APISSECReplicaSkipsDestinationTransforms|APISSECMultipartReplicaRoundTripWithCompression)$",
+        "-count=1",
+        "-v"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/ad51/silo",
+      "started_at": "2026-09-15T15:55:27.787466+00:00",
+      "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+      "source_sha256": {
+        "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+        "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+        "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+        "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+      },
+      "GOMAXPROCS": "4",
+      "GOFLAGS": null,
+      "status": "completed",
+      "log": "/Users/vonng/tmp/silo-r7-20260915-ad51/fixed-race.log",
+      "exit_code": 0,
+      "duration_seconds": 48.566,
+      "completed_at": "2026-09-15T15:56:16.372187+00:00",
+      "log_sha256": "b68af50412a8c1c03ea91aa40e72314ba9026798ac5dafb0d1580bc9824526b2"
+    },
+    "verifiers": {
+      "command": [
+        "make",
+        "verifiers",
+        "GOLANGCI=/Users/vonng/pgsty/silo/.bin/golangci/v2.13.1/golangci-lint"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/ad51/silo",
+      "started_at": "2026-09-15T15:56:40.045654+00:00",
+      "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+      "source_sha256": {
+        "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+        "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+        "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+        "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+      },
+      "GOMAXPROCS": "4",
+      "GOFLAGS": "-p=2",
+      "status": "completed",
+      "log": "/Users/vonng/tmp/silo-r7-20260915-ad51/verifiers.log",
+      "exit_code": 0,
+      "duration_seconds": 102.527,
+      "completed_at": "2026-09-15T15:58:22.585543+00:00",
+      "log_sha256": "e42a5bb55f5c1ebfcf02cebebf6d82cf1ec5a2d74590cdf838deba16dd80bfdf"
+    },
+    "build": {
+      "command": [
+        "make",
+        "build"
+      ],
+      "cwd": "/Users/vonng/.codex/worktrees/ad51/silo",
+      "started_at": "2026-09-15T15:58:33.660305+00:00",
+      "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+      "source_sha256": {
+        "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+        "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+        "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+        "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+      },
+      "GOMAXPROCS": "4",
+      "GOFLAGS": "-p=2",
+      "status": "completed",
+      "log": "/Users/vonng/tmp/silo-r7-20260915-ad51/build.log",
+      "exit_code": 0,
+      "duration_seconds": 24.007,
+      "completed_at": "2026-09-15T15:58:57.722704+00:00",
+      "log_sha256": "6ba9b545236be964861749c72e7609edf12b8f470df30d1ede8fd62f497e629b"
+    }
+  },
+  "counterfactual": {
+    "command": [
+      "go",
+      "test",
+      "-p",
+      "2",
+      "-overlay",
+      "/Users/vonng/tmp/silo-r7-20260915-ad51/exact-baseline-overlay.json",
+      "./cmd",
+      "-run",
+      "^Test(ExtractReplicationMetadataPreservesNormalizedMetadata|APIReplicaContentEncoding|APISnowballReplicaContentEncoding)$",
+      "-count=1",
+      "-v"
+    ],
+    "cwd": "/Users/vonng/.codex/worktrees/ad51/silo",
+    "started_at": "2026-09-15T15:54:49.754711+00:00",
+    "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a",
+    "source_sha256": {
+      "cmd/handler-utils.go": "76d6f98a8c9b04fcf1ae7c79d5832396bbf234bd5b5efc32a7f80dec64179b1a",
+      "cmd/handler-utils_test.go": "f8a53dd29170280eb007c4e43773187710f027e2aa49aadda91d2777bd0e034e",
+      "cmd/replication-content-encoding_test.go": "c71c4cf79a3ea6336febaf14bcb8ddfd68fcbef7625ff18830c14e9143749ee7",
+      "cmd/replication-metadata_test.go": "2cd3c45687d7333a9466906a278d7953fe4eeb008e07e2c7e2d167480c8b64bb"
+    },
+    "GOMAXPROCS": "4",
+    "GOFLAGS": null,
+    "status": "completed",
+    "log": "/Users/vonng/tmp/silo-r7-20260915-ad51/exact-baseline-regression.log",
+    "exit_code": 1,
+    "duration_seconds": 27.026,
+    "completed_at": "2026-09-15T15:55:16.823655+00:00",
+    "log_sha256": "75d42b2a711c6d9a9f6d277449d1428e768e04ab10a2a331103b554d75ab17f8",
+    "expected_negative": true,
+    "reason": "Only production handler-utils.go reverted to baseline through an overlay; final regression tests and capacity fixture unchanged",
+    "passed_leaf_count": 44,
+    "failed_leaf_count": 36,
+    "effective_production_override_sha256": {
+      "cmd/handler-utils.go": "e647b53e5288d57ddbaacc5549d76ced0123b5e0cb630450cf071ffc78818398"
+    }
+  },
+  "binary": {
+    "path": "/Users/vonng/.codex/worktrees/ad51/silo/silo",
+    "sha256": "c346ee4edf575bf6232689d25a7e38caa44ddd69dfa919df96c05aba4bb04c77",
+    "size": 93054290,
+    "version_output": "silo version DEVELOPMENT.2026-09-15T15-14-22Z (commit-id=9ebe81c1b3611f9cc73e676b5b741c2be62c467a)\nRuntime: go1.27.1 darwin/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-2025 MinIO, Inc.\nModifications: Copyright 2025-2026 PGSTY\nSource compatibility: based on MinIO technology\n",
+    "note": "Built and validated from the recorded uncommitted candidate file hashes; embedded commit ID is the baseline. This is not a release artifact."
+  },
+  "capacity_overlay": {
+    "path": "/Users/vonng/tmp/silo-r7-20260915-ad51/capacity-overlay.json",
+    "sha256": "aa20dd65d323f09542d2317fc1c7e24fff67cf45e831057ed0e2b0be0a4628dd",
+    "fixture_path": "/Users/vonng/tmp/silo-r7-20260915-ad51/capacity-test-utils_test.go",
+    "fixture_sha256": "d6bf8cc58b651c86ca02063fecab14c59c950b345acf739d979e553c6c2a0897",
+    "tracked": false
+  },
+  "remote_pr_state": {
+    "headRefOid": "b8f2fdde41dff3dc3b8db669c1d42d30ca5c1d3d",
+    "mergedAt": null,
+    "reviews": [],
+    "state": "OPEN",
+    "statusCheckRollup": [],
+    "updatedAt": "2026-09-15T07:07:26Z"
+  },
+  "remote_main": {
+    "message": "Merge pull request #192 from pgsty/codex/iam-revision-tombstones\n\nfix(iam): retain revocation versions through replay and recovery",
+    "sha": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a"
+  },
+  "limits": [
+    "HTTP/SSE/race tests use a temporary capacity overlay, with real disk I/O.",
+    "typos was not installed and Makefile skipped it.",
+    "No full distributed two-site scheduler/restart/network-fault acceptance.",
+    "No merge, remote push, release, deployment or live stored-object rewrite."
+  ]
+}

docs/site-replication/iam-revocations.md

@@ -0,0 +1,224 @@
+# Durable IAM revocations in site replication
+
+SILO retains the version of a deleted IAM record so an offline site cannot
+restore an older identity or grant when it reconnects. This covers built-in
+users, service accounts, groups, policy documents, and policy mappings in their
+actual user/STS-parent/group namespaces. It also revokes the deleted built-in
+user's older service accounts, STS credentials and group grants across deliberate
+same-name recreation.
+
+## Ordering and persistence
+
+Deletion records occupy the original IAM configuration paths. They contain the
+originating timestamp, `Deleted`, and, where required, `RevokedBefore`; identity
+deletion records contain no secret key or session token. Normal IAM listings and
+authorization hide deleted records. Object storage and etcd both serialize each
+path's version comparison and write with a distributed lock. The source timestamp
+is persisted without replacing it with the receiving node's clock.
+
+Older events cannot overwrite a newer revision. At an identical timestamp, a
+deletion wins over a live record. A local deliberate recreation receives a
+version newer than the stored deletion. An older user/group deletion arriving
+after recreation retains its revocation boundary while preserving the newer
+live record. Receiving an already-applied tombstone does not rewrite or advance
+it. These rules also apply after cold loading persistent IAM state.
+
+The user or group revision is the commit point of deletion. Cleanup of mappings
+and children follows that commit; cleanup failure cannot undo it. The API returns
+the cleanup error and still notifies sibling nodes to reload the committed
+state. Such an error does **not** mean the identity is still active. Retrying the
+operation is safe. When the API returns a committed-cleanup error, the admin
+handler does not send its immediate cross-site hook; cross-site propagation
+relies on the normal deletion-healing retry. Sibling deletion notifications reload current shared state,
+so a delayed notification cannot delete a subsequently recreated identity.
+Etcd siblings also receive persistent changes through watches. Failed
+notifications/watches remain eventual propagation, not a distributed
+instantaneous revocation transaction.
+
+Keep node clocks synchronized and monitor offsets. Ordering uses source wall
+clock timestamps, with monotonic advancement for local writes to the same path.
+It does not establish causal order between concurrent writers at different sites
+or resolve conflicting live updates with identical timestamps deterministically.
+
+## Users, child credentials and groups
+
+A recreated user retains `RevokedBefore`. New service accounts and built-in STS
+issuances carry a signed `siloParentRevocation` claim identifying the parent
+boundary known at issuance. Editing or replaying an old child does not update
+this claim. Old children remain invalid even when their own update timestamp is
+newer than the parent deletion; children issued for the recreated parent remain
+valid. Claims are read from verified tokens on credential load/write.
+
+A newer replicated service-account snapshot can replace an older service with
+the same access key, including a deliberate change of owner or secret. Local
+duplicate creates remain rejected. Snapshots preserve disabled status and the
+service's own revocation boundary, so earlier mappings cannot attach to the new
+service. Equal-version retries reload the committed identity without rewriting
+it. Periodic live healing compares source versions even when the public status
+summary is unchanged, and includes disabled identities as healing sources.
+Service snapshots also preserve their absolute expiration. The receiving site
+does not reapply the local minimum lifetime for a newly issued credential. A
+newer already-expired snapshot still supersedes the old key, is denied by
+authentication, and is collected into a durable service tombstone by normal
+loading. Cache/claims loading failures are returned for retry, not acknowledged;
+after a committed replacement the stale cached secret is evicted immediately.
+Collisions with an existing built-in or cached STS identity report an error and
+require an explicit administrative resolution; replication cannot change its
+credential kind. Concurrent conflicting service updates with exactly the same
+timestamp can retain different winners at different sites; the status summary
+does not resolve that case.
+
+Each group member has its own `MemberGrants` timestamp. Changing another member
+or the group's enabled status does not reissue everyone else's grants. Effective
+membership requires the grant to be newer than both the user's and the group's
+retained boundaries. Listings and policy evaluation use the same effective
+membership. Peer snapshots preserve grant times, including unknown legacy grant
+times; they cannot treat a recent snapshot time as a fresh grant to a revoked
+identity. A new explicit administrative group grant can restore access.
+
+This does not implement a general conflict-resolution protocol for all group
+membership edits. In particular, the inherited live-group snapshot merge adds
+members and does not reconcile a missed ordinary member removal. Removing a
+member from a live group during a site outage is a separate known limitation;
+do not infer that this change resolves it. User/group deletion boundaries and
+same-name recreation are covered here.
+
+## Retention and expiration
+
+Permanent identities, groups, policy documents and mappings have no automatic
+tombstone TTL. A disconnected peer or an old backup may return arbitrarily late.
+Successful replay acknowledgements are an optimization, **not** permission to
+garbage-collect this history.
+
+Natural expiration of an immutable STS token physically removes its token-key
+record and any legacy token-key mapping, without generating a permanent
+tombstone. An early STS revocation is retained until that token's expiration plus
+the existing clock-skew allowance. Replaying the same revoked token with a later
+event timestamp cannot recreate it. Etcd uses an expiration lease; object storage
+collects expired STS tombstones during its existing credential loading/purge.
+A record with unknown expiration is retained conservatively. Cleanup writes are
+best effort and use a short lock budget. If one fails, that load stops optional
+reclamation, reports the error and still loads healthy users; expired credentials
+stay denied and retain their existing durable version. The next load retries.
+Healthy cleanup has no fixed record quota. The reusable STS
+parent policy mapping is not assigned the token's TTL by deletion cleanup.
+External-IDP disablement is an early revocation, not natural token expiration,
+and its cached STS and service accounts are included in cleanup. Expiring service
+accounts retain a durable revision because their access keys are reusable and
+an older version might have no expiration.
+
+Direct per-token `RevokeTokens` delivery between sites is not a new guarantee of
+this change. The guarantee for built-in parent deletion follows from the durable
+parent boundary, including children not currently present in the deleting node's
+cache.
+
+## Healing, failures and operational cost
+
+Each process starts one healing loop. Losing its distributed leadership lease
+pauses work until leadership is reacquired; it does not permanently terminate
+healing. Configuration reloads do not create extra loops. The 30-second interval
+starts after leadership is acquired and after each completed pass. Initial lock
+retries and endpoint recovery can add further delay; it is not a convergence SLA.
+
+The normal IAM loaders maintain an in-memory index of deletion records and
+retained boundaries, without secrets. Healing uses this index; it does not add a
+second full walk of `config/iam/` every cycle. Existing full IAM loading still
+scans persistent records, including tombstones, at startup and on refresh.
+
+Each peer receives batches of at most 128 records. The sender remembers which
+path/version each peer acknowledged. Unrelated new changes at either site do not
+reset that progress. A failed batch remains pending while later independent
+batches can progress; a lost response may cause safe idempotent replay. A pass
+has a bounded duration, and its successful acknowledgements survive that timeout.
+The protocol reports each node name and process instance. Switching between
+known node instances behind a load balancer preserves acknowledgements. A new
+node instance conservatively invalidates prior acknowledgements once; repeated
+switches among those known instances do not reset progress. Restore persistent state only with the affected processes
+stopped, so a restore cannot reuse an old process acknowledgement.
+
+Steady-state healing still traverses/sorts the retained in-memory set and checks
+the peer's protocol status. It suppresses repeated deletion PUTs once acknowledged.
+Memory use scales with retained paths and peers; startup storage reads scale with
+history. This release does not provide general history compaction.
+After upgrading, older live records whose receiving sites originally assigned
+different timestamps can require an initial reconciliation wave. Allow for its
+storage writes and sibling notifications when planning the maintenance window.
+
+The cluster IAM metrics include `revocation_records`,
+`revocation_heal_failures`, `revocation_heal_duration_millis`, and
+`revocation_heal_last_success_timestamp_seconds`. Errors are also logged. A
+nominal 30-second scheduler interval is not a convergence deadline: outages,
+large backlogs, lock contention and failed requests can require more passes.
+
+Cached credential lookup checks the in-memory parent revision index and performs
+no additional storage read. STS issuance and cold credential loading still consult
+the persistent parent revision. Revision I/O and distributed lock waits release
+the IAM cache lock while a separate local writer mutex preserves write order.
+These operations have bounded contexts, including etcd lock and lease cleanup.
+
+## Protocol and supported upgrade
+
+The server-owned versioned route is
+`/minio/admin/v3/site-replication/peer/iam-revisions`. It carries source versions,
+member grant times and distinct user/group revocation items without changing the
+admin client SDK or S3 API. Older servers reject this route. The sender reports
+the failure and does not fall back to a route that would discard the metadata.
+The existing legacy IAM route remains readable for best-effort compatibility;
+this does not confer the new guarantees on an older peer.
+
+All participating servers must be upgraded for the guarantee in this document.
+Mixed old/new nodes sharing an IAM backend and rolling downgrade are unsupported:
+older binaries do not interpret tombstones or signed parent boundaries correctly.
+Use a maintenance window for coordinated upgrade:
+
+1. Pause IAM changes and isolate any offline site or backup whose state is unknown.
+2. Back up each site's complete IAM storage and required encryption material. A
+   live IAM admin export omits deletion history and is not an adequate backup.
+3. Stop all nodes sharing each site's IAM backend, replace their binaries, and
+   restart them on the upgraded version. Complete this for every participating
+   site before relying on the new revocation semantics.
+4. Check IAM loading, site-replication errors and revocation convergence. Verify
+   representative old credentials are denied and deliberately reissued ones work.
+5. Resolve pre-upgrade revocations explicitly. Absence cannot reconstruct an
+   already-lost deletion version: remove surviving old records on the sites that
+   still have them, and rebuild stale offline peers from approved state before
+   admitting them. Do not reconnect an unknown old snapshot just to discover its
+   deleted credentials.
+
+Credentials issued by an older server for a recreated parent lack the required
+signed boundary and must be reissued by an upgraded server. Parents without any
+retained revocation history preserve existing credential behavior.
+
+A pristine built-in policy remains protected from local deletion. If an
+administrator explicitly overrides that policy and later deletes the override,
+the durable deletion now suppresses automatic recreation of the built-in policy
+on reload. This prevents reload from undoing the deletion. Restore the policy by
+an explicit policy-create operation if desired. Local deletion of a nonexistent
+policy remains idempotent and does not create a new tombstone; replicated
+unknown deletions retain their version.
+
+For rollback, stop and isolate the affected sites and assess changes since the
+backup before restoring compatible state. Restoring an older backup can itself
+lose later revocations and requires reconciliation/rekeying before access is
+reopened. Do not delete tombstones online or convert only live IAM records to
+make an older binary start. Server, client, Console, package and deployment
+acceptance remain separate delivery gates of the maintained PGSTY stack.
+
+## Regression and performance checks
+
+Focused coverage is in `iam-revocation_test.go`, `iam-revision_test.go`,
+`iam-revision-lock_test.go`, `iam-revision-boundary_test.go`,
+`iam-replication-protocol_test.go`, `iam-credential-retention_test.go`,
+`iam-peer-reload_test.go`, and `iam-replay_test.go`. Set
+`SILO_TEST_IAM_REVOCATION_ETCD` to a disposable etcd endpoint to include backend
+lifecycle/locking/boundary tests; they use isolated key namespaces.
+
+`BenchmarkIAMCachedCredential` and `BenchmarkIAMSetTempUser` can be run against
+the pre-change source for a comparable local baseline.
+`BenchmarkIAMRevisionConvergedHealing` covers 1,000 and 10,000 retained records;
+it measures steady-state index/network work and asserts zero repeated PUTs. Its
+fake peer does not measure durable catch-up throughput.
+`BenchmarkIAMColdLoadExpiredServices` measures loading and cleanup with 100 or
+1,000 expired reusable credentials; run it with `-benchtime=1x`. Use actual multi-site
+signed S3/STS tests and deployment-specific latency/scale measurements in
+addition to these component tests.