fix(iam): persist revocations across site replay and recovery

Retain source-ordered tombstones and parent grant boundaries across both IAM backends, cache reloads, and deliberate identity recreation. Reconcile deletions through a versioned, bounded replication protocol with restart-aware acknowledgements. Cover inherited group grants, STS retention, same-key service recreation, absolute expiration, and failures after the durable commit. Document coordinated upgrades and the remaining consistency boundaries. Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-09-26 04:45:59 +03:00 · 2026-09-15 22:06:56 +08:00
parent dff81f293b
commit 709d50a916
27 changed files with 4721 additions and 407 deletions
@@ -0,0 +1,749 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
+	etcd "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+var errIAMStaleUpdate = errors.New("IAM update predates a stored revision or revocation")
+
+// The parent is still live; callers must not broadcast a user deletion when
+// only its revocation boundary was retained.
+var errIAMRevocationRetained = errors.New("IAM revocation recorded without deleting the record")
+
+// A revocation advances the boundary even when a newer identity already
+// exists. Keep this operation distinct from replacing/deleting that identity.
+type iamUserRevocation struct {
+	UserIdentity
+	retained bool
+}
+
+type iamGroupRevocation struct {
+	GroupInfo
+	retained     bool
+	requireEmpty bool
+}
+
+// Natural expiration is distinct from revoking a live credential. An expired
+// immutable STS token can be removed; a reusable service-account key retains
+// its revision so an older non-expiring credential cannot return.
+type iamExpireIdentity struct{}
+
+// The authoritative revocation is durable even if dependent cleanup fails.
+// Callers must publish it to sibling caches before returning the error.
+type iamCommittedCleanupError struct {
+	err      error
+	retained bool
+}
+
+func (e *iamCommittedCleanupError) Error() string {
+	return "IAM revocation committed; cleanup failed: " + e.err.Error()
+}
+func (e *iamCommittedCleanupError) Unwrap() error { return e.err }
+
+type iamReplicationTimeKey struct{}
+
+func withIAMReplicationTime(ctx context.Context, at time.Time) context.Context {
+	return context.WithValue(ctx, iamReplicationTimeKey{}, at)
+}
+
+func iamReplicationTime(ctx context.Context) (time.Time, bool) {
+	at, ok := ctx.Value(iamReplicationTimeKey{}).(time.Time)
+	return at, ok
+}
+
+func iamReplicationError(err error) error {
+	if errors.Is(err, errIAMStaleUpdate) {
+		// Retrying an obsolete event cannot change the result.
+		return nil
+	}
+	return wrapSRErr(err)
+}
+
+// Deletions occupy the original IAM config path. They contain no secret or
+// grant and are hidden by the normal loaders, but remain available to heal
+// and to timestamp comparisons after a restart. Do not age them out: a peer
+// can be offline indefinitely.
+type iamRevision struct {
+	UpdatedAt     time.Time        `json:"updatedAt"`
+	UpdateDate    time.Time        `json:"UpdateDate"`
+	Deleted       bool             `json:"deleted"`
+	RevokedBefore time.Time        `json:"revokedBefore"`
+	ExpiresAt     time.Time        `json:"expiresAt,omitempty"`
+	Credentials   auth.Credentials `json:"credentials"`
+}
+
+func (r iamRevision) timestamp() time.Time {
+	if r.UpdateDate.After(r.UpdatedAt) {
+		return r.UpdateDate
+	}
+	return r.UpdatedAt
+}
+
+func loadIAMRevision(ctx context.Context, store IAMStorageAPI, path string) (iamRevision, error) {
+	var r iamRevision
+	err := store.loadIAMConfig(ctx, &r, path)
+	if errors.Is(err, errConfigNotFound) {
+		err = nil
+	}
+	return r, err
+}
+
+func (store *IAMStoreSys) checkIAMRevision(ctx context.Context, path string, deleting bool) error {
+	at, replicated := iamReplicationTime(ctx)
+	if !replicated {
+		return nil
+	}
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		r, err := loadIAMRevision(ctx, store.IAMStorageAPI, path)
+		if err != nil {
+			return err
+		}
+		if r.timestamp().After(at) || (r.Deleted && !deleting && !at.After(r.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		return nil
+	})
+}
+
+// This signed claim records the parent's revocation boundary at issuance.
+// Unlike UpdatedAt, it cannot advance when an offline site edits an old child.
+// It travels in the existing service-account Claims and STS SessionToken fields.
+const iamParentRevocationClaim = "siloParentRevocation"
+
+func setIAMParentRevocationClaim(ctx context.Context, store IAMStorageAPI, parent string, claims map[string]any) error {
+	delete(claims, iamParentRevocationClaim)
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted {
+		return errIAMStaleUpdate
+	}
+	if !r.RevokedBefore.IsZero() {
+		claims[iamParentRevocationClaim] = r.RevokedBefore.Format(time.RFC3339Nano)
+	}
+	return nil
+}
+
+func iamCredentialSurvivesRevocation(cred auth.Credentials, at time.Time) bool {
+	if at.IsZero() {
+		return true
+	}
+	s, _ := cred.Claims[iamParentRevocationClaim].(string)
+	issuedAfter, err := time.Parse(time.RFC3339Nano, s)
+	return err == nil && !issuedAfter.Before(at)
+}
+
+// Parent revocations delete old children even if an offline peer has edited
+// them later. Preserve children that prove issuance after this revocation.
+func iamChildDeletionContext(ctx context.Context, child UserIdentity) (context.Context, bool) {
+	if at, replicated := iamReplicationTime(ctx); replicated {
+		if !at.IsZero() && iamCredentialSurvivesRevocation(child.Credentials, at) {
+			return ctx, false
+		}
+		if child.UpdatedAt.After(at) {
+			ctx = withIAMReplicationTime(ctx, child.UpdatedAt)
+		}
+	}
+	return ctx, true
+}
+
+// A delayed service account or STS event must not outlive deletion of its
+// built-in parent. The caller must populate Claims from the verified token.
+func checkIAMParentRevision(ctx context.Context, store IAMStorageAPI, cred auth.Credentials) error {
+	parent := cred.ParentUser
+	if parent == "" || parent == globalActiveCred.AccessKey {
+		return nil
+	}
+	r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
+	if err != nil {
+		return err
+	}
+	if r.Deleted || !iamCredentialSurvivesRevocation(cred, r.RevokedBefore) {
+		return errIAMStaleUpdate
+	}
+	return nil
+}
+
+// Called with the IAM writer mutex and cache lock held. Persistence only
+// touches the caller's record, not the cache. Keep writers serialized while
+// allowing cached authentication reads throughout storage and lock waits.
+func (store *IAMStoreSys) withIAMStorage(ctx context.Context, fn func(context.Context) error) error {
+	store.IAMStorageAPI.unlock()
+	defer store.IAMStorageAPI.lock()
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	return fn(ctx)
+}
+
+func (store *IAMStoreSys) saveIAMRevision(ctx context.Context, path string, item any, opts ...options) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return saveIAMRevision(ctx, store.IAMStorageAPI, path, item, opts...)
+	})
+}
+
+func (store *IAMStoreSys) checkIAMParentRevision(ctx context.Context, cred auth.Credentials) error {
+	return store.withIAMStorage(ctx, func(ctx context.Context) error {
+		return checkIAMParentRevision(ctx, store.IAMStorageAPI, cred)
+	})
+}
+
+// Update the caller's record with the persisted revision before it is cached.
+func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item any, opts ...options) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	// Serialize compare-and-write across nodes, as well as goroutines. Use a
+	// separate lock name so saving the config does not reacquire this lock.
+	switch s := store.(type) {
+	case *IAMObjectStore:
+		lock := s.objAPI.NewNSLock(minioMetaBucket, path+".revision-lock")
+		lc, err := lock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		defer lock.Unlock(lc)
+		ctx = lc.Context()
+	case *IAMEtcdStore:
+		// Mutex.Lock also uses Client.Ctx() for cleanup after cancellation.
+		// Borrow the existing services with the operation's bounded context;
+		// never close this facade, which does not own those services.
+		client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(s.client.GetLogger()))
+		client.KV, client.Lease, client.Watcher = s.client.KV, s.client.Lease, s.client.Watcher
+		session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
+		if err != nil {
+			return err
+		}
+		defer func() {
+			session.Orphan()
+			// A canceled operation must still release its lease when etcd is
+			// reachable. If it is unavailable, stop waiting and let it expire.
+			cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultContextTimeout)
+			defer cancel()
+			_, _ = s.client.Revoke(cleanupCtx, session.Lease())
+		}()
+		lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
+		if err = lock.Lock(ctx); err != nil {
+			return err
+		}
+		// Revoking the session lease releases the lock, including on cancellation.
+	}
+	previous, err := loadIAMRevision(ctx, store, path)
+	if err != nil {
+		return err
+	}
+	if _, expiring := item.(*iamExpireIdentity); expiring {
+		sts := strings.HasPrefix(path, iamConfigSTSPrefix)
+		if previous.Deleted {
+			if sts && !previous.ExpiresAt.IsZero() && UTCNow().After(previous.ExpiresAt) {
+				return expireIAMSTSConfig(ctx, store, path)
+			}
+			return nil
+		}
+		if previous.timestamp().IsZero() || !previous.Credentials.IsExpired() {
+			return nil
+		}
+		if sts {
+			return expireIAMSTSConfig(ctx, store, path)
+		}
+		item = &UserIdentity{Version: 1, Deleted: true}
+		ctx = withIAMReplicationTime(ctx, previous.timestamp())
+	}
+	var revocation *iamUserRevocation
+	if op, ok := item.(*iamUserRevocation); ok {
+		revocation = op
+		op.UserIdentity = UserIdentity{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.UserIdentity = UserIdentity{Version: 1, Credentials: previous.Credentials, UpdatedAt: previous.timestamp(), RevokedBefore: origin}
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.UserIdentity
+	}
+	var groupRevocation *iamGroupRevocation
+	if op, ok := item.(*iamGroupRevocation); ok {
+		groupRevocation = op
+		var group GroupInfo
+		if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+			return err
+		}
+		if op.requireEmpty && !group.Deleted {
+			for _, member := range group.Members {
+				r := store.revisionIndex().get(getUserIdentityPath(member, regUser))
+				at := group.MemberGrants[member]
+				if !r.Deleted && (r.RevokedBefore.IsZero() || at.After(r.RevokedBefore)) && (group.RevokedBefore.IsZero() || at.After(group.RevokedBefore)) {
+					return errGroupNotEmpty
+				}
+			}
+		}
+		op.GroupInfo = GroupInfo{Version: 1, Deleted: true}
+		if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
+			if previous.Deleted || !origin.After(previous.RevokedBefore) {
+				return errIAMStaleUpdate
+			}
+			op.retained = true
+			op.GroupInfo = group
+			op.RevokedBefore = origin
+			ctx = withIAMReplicationTime(ctx, previous.timestamp())
+		}
+		item = &op.GroupInfo
+	}
+	var at *time.Time
+	var deleted bool
+	switch v := item.(type) {
+	case *UserIdentity:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *GroupInfo:
+		at, deleted = &v.UpdatedAt, v.Deleted
+		if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
+			v.RevokedBefore = boundary
+		}
+		if !deleted {
+			var group GroupInfo
+			if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
+				return err
+			}
+			mergeIAMGroupMutation(ctx, group, v)
+		}
+		if previous.RevokedBefore.After(v.RevokedBefore) {
+			v.RevokedBefore = previous.RevokedBefore
+		}
+	case *MappedPolicy:
+		at, deleted = &v.UpdatedAt, v.Deleted
+	case *PolicyDoc:
+		at, deleted = &v.UpdateDate, v.Deleted
+	default:
+		return errInvalidArgument
+	}
+	if strings.HasPrefix(path, iamConfigSTSPrefix) && previous.Deleted && !deleted {
+		// STS access keys identify immutable tokens, not reusable user names.
+		return errIAMStaleUpdate
+	}
+	if origin, replicated := iamReplicationTime(ctx); replicated {
+		*at = origin
+		if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
+			return errIAMStaleUpdate
+		}
+		if strings.HasPrefix(path, iamConfigServiceAccountsPrefix) && !deleted && previous.Credentials.AccessKey != "" && previous.timestamp().Equal(origin) {
+			// Duplicate service snapshots are acknowledgements, not new creates
+			// or edits. Reload the winner without writing, so even a stale
+			// sibling cache is refreshed by the retry before acknowledging it.
+			return store.loadIAMConfig(ctx, item, path)
+		}
+		if previous.Deleted && deleted && !origin.After(previous.timestamp()) {
+			// An already-applied tombstone needs no further persistent write.
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+	} else {
+		if previous.Deleted && deleted {
+			// A peer notification without an originating revision must not
+			// advance a tombstone past a subsequent deliberate recreation.
+			*at = previous.timestamp()
+			if v, ok := item.(*UserIdentity); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			if v, ok := item.(*GroupInfo); ok {
+				v.RevokedBefore = previous.RevokedBefore
+			}
+			return nil
+		}
+		if at.IsZero() {
+			*at = UTCNow()
+		}
+		if !at.After(previous.timestamp()) {
+			*at = previous.timestamp().Add(time.Nanosecond)
+		}
+	}
+	if v, ok := item.(*UserIdentity); ok {
+		if deleted {
+			// Retain only the parent name for root-account exclusion during heal.
+			v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
+			v.RevokedBefore = *at
+			if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) {
+				// The signed STS token cannot authorize beyond this time, even
+				// if an offline site replays it with a newer event timestamp.
+				v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime)
+				opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
+			}
+		} else {
+			if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
+				claims, err := extractJWTClaims(*v)
+				if err != nil {
+					return err
+				}
+				v.Credentials.Claims = claims.Map()
+			}
+			if err = checkIAMParentRevision(ctx, store, v.Credentials); err != nil {
+				return err
+			}
+		}
+	}
+	if v, ok := item.(*GroupInfo); ok && deleted {
+		v.RevokedBefore = *at
+		v.Members, v.MemberGrants = nil, nil
+	}
+	if _, ok := item.(*MappedPolicy); ok && !deleted {
+		if parentPath := iamMappingParentPath(path); parentPath != "" {
+			parent, err := loadIAMRevision(ctx, store, parentPath)
+			if err != nil {
+				return err
+			}
+			if parent.Deleted {
+				return errIAMStaleUpdate
+			}
+			if !parent.RevokedBefore.IsZero() && !at.After(parent.RevokedBefore) {
+				if _, replicated := iamReplicationTime(ctx); replicated {
+					return errIAMStaleUpdate
+				}
+				*at = parent.RevokedBefore.Add(time.Nanosecond)
+			}
+		}
+	}
+	if err := store.saveIAMConfig(ctx, item, path, opts...); err != nil {
+		return err
+	}
+	if revocation != nil && revocation.retained {
+		return errIAMRevocationRetained
+	}
+	if groupRevocation != nil && groupRevocation.retained {
+		return errIAMRevocationRetained
+	}
+	return nil
+}
+
+func (iamOS *IAMObjectStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var paths []string
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+"/") {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+		paths = append(paths, iamConfigPrefix+"/"+item.Item)
+	}
+	return paths, nil
+}
+
+func (ies *IAMEtcdStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	r, err := ies.client.Get(ctx, iamConfigPrefix+"/", etcd.WithPrefix(), etcd.WithKeysOnly())
+	if err != nil {
+		return nil, err
+	}
+	paths := make([]string, 0, len(r.Kvs))
+	for _, kv := range r.Kvs {
+		paths = append(paths, string(kv.Key))
+	}
+	return paths, nil
+}
+
+func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool) {
+	if (strings.HasPrefix(path, iamConfigUsersPrefix) || strings.HasPrefix(path, iamConfigGroupsPrefix)) && !r.RevokedBefore.IsZero() {
+		// Recreating a parent does not cancel its older revocation of derived
+		// credentials. Replay this boundary even after the parent is live again.
+		r.Deleted = true
+		r.UpdatedAt, r.UpdateDate = r.RevokedBefore, time.Time{}
+	}
+	if !r.Deleted {
+		return item, false
+	}
+	item.UpdatedAt = r.timestamp()
+	switch {
+	case strings.HasPrefix(path, iamConfigUsersPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
+		item.Type = madmin.SRIAMItemIAMUser
+		item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
+	case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
+		if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
+			return item, false
+		}
+		item.Type = madmin.SRIAMItemSvcAcc
+		item.SvcAccChange = &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: name}}
+	case strings.HasPrefix(path, iamConfigGroupsPrefix):
+		name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigGroupsPrefix), "/"+iamGroupMembersFile)
+		item.Type = madmin.SRIAMItemGroupInfo
+		item.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: name, IsRemove: true}}
+	case strings.HasPrefix(path, iamConfigPoliciesPrefix):
+		item.Type = madmin.SRIAMItemPolicy
+		item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigPoliciesPrefix), "/"+iamPolicyFile)
+	case strings.HasPrefix(path, iamConfigPolicyDBPrefix):
+		prefix, name, found := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
+		if !found {
+			return item, false
+		}
+		typ := regUser
+		switch prefix {
+		case "sts-users":
+			typ = stsUser
+		case "service-accounts":
+			typ = svcUser
+		}
+		item.Type = madmin.SRIAMItemPolicyMapping
+		item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
+	default:
+		// Expired STS credentials are not replayed. Parent revocations and
+		// their retained timestamp reject delayed copies of derived tokens.
+		return item, false
+	}
+	return item, true
+}
+
+func iamDeletionPath(item madmin.SRIAMItem) string {
+	switch item.Type {
+	case madmin.SRIAMItemIAMUser:
+		if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
+			return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
+		}
+	case madmin.SRIAMItemSvcAcc:
+		if item.SvcAccChange != nil && item.SvcAccChange.Delete != nil {
+			return getUserIdentityPath(item.SvcAccChange.Delete.AccessKey, svcUser)
+		}
+	case madmin.SRIAMItemGroupInfo:
+		if item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
+			return getGroupInfoPath(item.GroupInfo.UpdateReq.Group)
+		}
+	case madmin.SRIAMItemPolicy:
+		if len(item.Policy) == 0 {
+			return getPolicyDocPath(item.Name)
+		}
+	case madmin.SRIAMItemPolicyMapping:
+		if p := item.PolicyMapping; p != nil && p.Policy == "" {
+			return getMappedPolicyPath(p.UserOrGroup, IAMUserType(p.UserType), p.IsGroup)
+		}
+	}
+	return ""
+}
+
+func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
+	started := time.Now()
+	defer func() {
+		c.iamRevisionMetrics.healDurationMillis.Store(time.Since(started).Milliseconds())
+		if err != nil {
+			c.iamRevisionMetrics.healFailures.Add(1)
+		} else {
+			c.iamRevisionMetrics.healLastSuccess.Store(time.Now().Unix())
+		}
+	}()
+	c.iamHealMu.Lock()
+	defer c.iamHealMu.Unlock()
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	snapshot := globalIAMSys.store.revisionIndex().snapshot()
+	paths := make([]string, 0, len(snapshot))
+	for path := range snapshot {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	byType := make(map[string][]iamReplicationItem)
+	for _, path := range paths {
+		r := snapshot[path]
+		item, ok := iamDeletionItem(path, r)
+		if !ok {
+			continue
+		}
+		out := iamReplicationItem{SRIAMItem: item}
+		if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
+			out.Type, out.IAMUser = iamUserBoundaryType, nil
+			out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
+		}
+		if item.Type == madmin.SRIAMItemGroupInfo && !r.Deleted {
+			out.Type, out.GroupInfo = iamGroupBoundaryType, nil
+			out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
+		}
+		byType[item.Type] = append(byType[item.Type], out)
+	}
+	var items []iamReplicationItem
+	for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
+		items = append(items, byType[typ]...)
+	}
+	if len(items) == 0 {
+		return nil
+	}
+	if c.iamRevisionProgress == nil {
+		c.iamRevisionProgress = make(map[string]iamRevisionProgress)
+	}
+	for id := range c.iamRevisionProgress {
+		if _, present := c.state.Peers[id]; !present {
+			delete(c.iamRevisionProgress, id)
+		}
+	}
+	var progressMu sync.Mutex
+	cerr := c.concDo(nil, func(id string, p madmin.PeerInfo) error {
+		// Bound each pass, but retain acknowledgements independently of the
+		// pass deadline or unrelated changes at either site.
+		peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+		defer cancel()
+		client, err := c.getAdminClient(peerCtx, id)
+		if err != nil {
+			return err
+		}
+		remote, err := executeIAMRevisionRequest(peerCtx, client, http.MethodGet, nil)
+		if err != nil {
+			return err
+		}
+		progressMu.Lock()
+		progress := c.iamRevisionProgress[id]
+		progressMu.Unlock()
+		progress.observePeer(remote)
+		defer func() {
+			progressMu.Lock()
+			c.iamRevisionProgress[id] = progress
+			progressMu.Unlock()
+		}()
+		var pending []iamReplicationItem
+		for _, item := range items {
+			path, version := iamReplicationMarker(item)
+			if progress.Acknowledged[path] != version {
+				pending = append(pending, item)
+			}
+		}
+		// Acknowledgements are only a replay optimization, never GC proof.
+		for path := range progress.Acknowledged {
+			if _, retained := snapshot[path]; !retained {
+				delete(progress.Acknowledged, path)
+			}
+		}
+		var failures []error
+		for next := 0; next < len(pending); {
+			end := min(next+maxIAMRevisionBatch, len(pending))
+			batch := pending[next:end]
+			remote, err = executeIAMRevisionRequest(peerCtx, client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: batch})
+			if err != nil {
+				var batchErr *iamRevisionBatchError
+				if !errors.As(err, &batchErr) {
+					return errors.Join(append(failures, err)...)
+				}
+				failures = append(failures, err)
+			} else {
+				progress.observePeer(remote)
+				for _, item := range batch {
+					path, version := iamReplicationMarker(item)
+					progress.Acknowledged[path] = version
+				}
+			}
+			next = end
+		}
+		return errors.Join(failures...)
+	}, "IAM revision convergence")
+	return errors.Unwrap(cerr)
+}
+
+func iamReplicationMarker(item iamReplicationItem) (path, version string) {
+	path = iamDeletionPath(item.SRIAMItem)
+	if item.UserRevocation != nil {
+		path = getUserIdentityPath(item.UserRevocation.User, regUser)
+	}
+	if item.GroupRevocation != nil {
+		path = getGroupInfoPath(item.GroupRevocation.Group)
+	}
+	return path, item.Type + ":" + item.UpdatedAt.UTC().Format(time.RFC3339Nano)
+}
+
+func (store *IAMStoreSys) savePolicyDoc(ctx context.Context, policyName string, p *PolicyDoc) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(policyName), p)
+}
+
+func (store *IAMStoreSys) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp *MappedPolicy, opts ...options) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), mp, opts...)
+}
+
+func (store *IAMStoreSys) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u *UserIdentity, opts ...options) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), u, opts...)
+}
+
+func (store *IAMStoreSys) saveGroupInfo(ctx context.Context, name string, gi *GroupInfo) error {
+	return store.saveIAMRevision(ctx, getGroupInfoPath(name), gi)
+}
+
+func (store *IAMStoreSys) deletePolicyDoc(ctx context.Context, name string) error {
+	return store.saveIAMRevision(ctx, getPolicyDocPath(name), &PolicyDoc{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
+	return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), &MappedPolicy{Version: 1, Deleted: true})
+}
+
+func (store *IAMStoreSys) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
+	return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), &UserIdentity{Version: 1, Deleted: true})
+}
+
+// Called under the identity's distributed revision lock, after verifying that
+// its immutable STS token (or early-revocation retention) has expired. Only the
+// old token-key mapping is removed; the reusable parent mapping is unaffected.
+func expireIAMSTSConfig(ctx context.Context, store IAMStorageAPI, path string) error {
+	key := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
+	if err := store.deleteIAMConfig(ctx, getMappedPolicyPath(key, stsUser, false)); err != nil && !errors.Is(err, errConfigNotFound) {
+		return err
+	}
+	return store.deleteIAMConfig(ctx, path)
+}
+
+type (
+	iamExpirationCleanupKey   struct{}
+	iamExpirationCleanupState struct{ failed atomic.Bool }
+)
+
+func withIAMExpirationCleanup(ctx context.Context) context.Context {
+	if _, ok := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState); ok {
+		return ctx
+	}
+	return context.WithValue(ctx, iamExpirationCleanupKey{}, &iamExpirationCleanupState{})
+}
+
+func bestEffortIAMExpiration(ctx context.Context, store IAMStorageAPI, path string) {
+	state, _ := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState)
+	if ctx.Err() != nil || (state != nil && state.failed.Load()) {
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	// Failure leaves the expired record and its existing version intact.
+	// Stop optional reclamation for this load, while still loading healthy
+	// users. Healthy cleanup has no per-scan quota that could build a backlog.
+	if err := saveIAMRevision(ctx, store, path, &iamExpireIdentity{}); err != nil {
+		if state != nil {
+			state.failed.Store(true)
+		}
+		iamLogIf(ctx, err)
+	}
+}