astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-09-15 23:14:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(iam): persist revocations across site replay and recovery

Browse Source 
Retain source-ordered tombstones and parent grant boundaries across both IAM backends, cache reloads, and deliberate identity recreation. Reconcile deletions through a versioned, bounded replication protocol with restart-aware acknowledgements.

Cover inherited group grants, STS retention, same-key service recreation, absolute expiration, and failures after the durable commit. Document coordinated upgrades and the remaining consistency boundaries.

Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-09-15 22:06:56 +08:00
parent dff81f293b
commit 709d50a916
27 changed files with 4721 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
buildscripts/rebrand-guard/compat-baseline.json
+2
View File
@@ -829,6 +829,7 @@
"/site-replication/peer/bucket-ops",
"/site-replication/peer/edit",
"/site-replication/peer/iam-item",
"/site-replication/peer/iam-revisions",
"/site-replication/peer/idp-settings",
"/site-replication/peer/join",
"/site-replication/peer/remove",
@@ -877,6 +878,7 @@
"/v2/metrics/cluster",
"/v2/metrics/node",
"/v2/metrics/resource",
"/v3/site-replication/peer/iam-revisions",
"/var/vcap/bosh",
"/verifybinary",
"/version",
cmd/admin-router.go
+1
View File
@@ -388,6 +388,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
cmd/iam-credential-retention_test.go
+111
View File
@@ -0,0 +1,111 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"errors"
"testing"
"time"
"github.com/minio/minio/internal/auth"
)
func TestIAMCredentialRetention(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
secret, err := getTokenSigningKey()
mustIAM(t, err)
parent := "external-idp-parent"
credential := func(exp time.Time) auth.Credentials {
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": exp.Unix(), parentClaim: parent}, secret)
mustIAM(t, err)
cred.ParentUser = parent
return cred
}
// Disablement of an external identity must include cached STS,
// which are kept separately from regular and service accounts.
cred := credential(UTCNow().Add(time.Hour))
_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
mustIAM(t, sys.store.DeleteUsers(ctx, []string{parent}))
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
mustIAM(t, err)
if !r.Deleted || !r.ExpiresAt.Equal(cred.Expiration.Add(globalMaxSkewTime)) || r.Credentials.SessionToken != "" || r.Credentials.SecretKey != "" {
t.Fatal("early STS revocation lost its retention boundary or retained a secret")
}
if _, ok := sys.store.GetUser(cred.AccessKey); ok {
t.Fatal("external disablement left the STS cache live")
}
_, err = sys.SetTempUser(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), cred.AccessKey, cred, "")
if !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("same revoked token was reissued by replay: %v", err)
}
var mp MappedPolicy
err = sys.store.loadIAMConfig(ctx, &mp, getMappedPolicyPath(cred.AccessKey, stsUser, false))
if !errors.Is(err, errConfigNotFound) {
t.Fatalf("random STS key produced a permanent mapping: %v", err)
}
// Seed genuinely expired immutable tokens, as an ordinary startup
// loader sees them. Natural expiry leaves no permanent tombstone.
expired := credential(UTCNow().Add(-time.Hour))
path := getUserIdentityPath(expired.AccessKey, stsUser)
mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: expired, UpdatedAt: UTCNow().Add(-2 * time.Hour)}, path))
_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
var u UserIdentity
if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
t.Fatalf("natural expiration retained a random key: %v", err)
}
// A retained early-revocation record is collectable only after the
// immutable token's expiration plus the skew allowance.
tomb := UserIdentity{Version: 1, Deleted: true, UpdatedAt: UTCNow().Add(-2 * time.Hour), ExpiresAt: expired.Expiration.Add(globalMaxSkewTime)}
mustIAM(t, sys.store.saveIAMConfig(ctx, &tomb, path))
_ = sys.store.loadUser(ctx, expired.AccessKey, stsUser, make(map[string]UserIdentity))
if err := sys.store.loadIAMConfig(ctx, &u, path); !errors.Is(err, errConfigNotFound) {
t.Fatalf("expired STS revocation not collected: %v", err)
}
if _, ok := sys.store.revisionIndex().snapshot()[path]; ok {
t.Fatal("expired STS retained an index entry")
}
})
}
}
func TestIAMPolicyDeletionRemainsExplicit(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
mustIAM(t, sys.DeletePolicy(ctx, "misspelled-policy", true))
r, err := loadIAMRevision(ctx, sys.store, getPolicyDocPath("misspelled-policy"))
mustIAM(t, err)
if r.Deleted {
t.Fatal("local nonexistent policy created a tombstone")
}
p, err := sys.store.GetPolicy("readwrite")
mustIAM(t, err)
if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil {
t.Fatal("local pristine builtin policy became deletable")
}
_, err = sys.SetPolicy(ctx, "readwrite", p)
mustIAM(t, err)
mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true))
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
if _, err := sys.store.GetPolicy("readwrite"); !errors.Is(err, errNoSuchPolicy) {
t.Fatalf("reload restored an explicitly deleted override: %v", err)
}
_, err = sys.SetPolicy(ctx, "readwrite", p)
mustIAM(t, err)
if _, err := sys.store.GetPolicy("readwrite"); err != nil {
t.Fatal("explicit policy recreation failed", err)
}
mustIAM(t, globalSiteReplicationSys.PeerAddPolicyHandler(ctx, "remote-unknown-policy", nil, UTCNow()))
r, err = loadIAMRevision(ctx, sys.store, getPolicyDocPath("remote-unknown-policy"))
mustIAM(t, err)
if !r.Deleted {
t.Fatal("replicated unknown deletion lost its version")
}
})
}
}
cmd/iam-etcd-store.go
+65 -71
View File
@@ -26,7 +26,6 @@ import (
"sync"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/kms"
@@ -62,6 +61,7 @@ type IAMEtcdStore struct {
sync.RWMutex
*iamCache
index iamRevisionIndex
usersSysType UsersSysType
@@ -69,13 +69,17 @@ type IAMEtcdStore struct {
}
func newIAMEtcdStore(client *etcd.Client, usersSysType UsersSysType) *IAMEtcdStore {
return &IAMEtcdStore{
store := &IAMEtcdStore{
iamCache: newIamCache(),
client: client,
usersSysType: usersSysType,
}
store.revisions = &store.index
return store
}
func (ies *IAMEtcdStore) revisionIndex() *iamRevisionIndex { return &ies.index }
func (ies *IAMEtcdStore) rlock() *iamCache {
ies.RLock()
return ies.iamCache
@@ -103,6 +107,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
if err != nil {
return err
}
plain := data
if GlobalKMS != nil {
data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
minioMetaBucket: path.Join(minioMetaBucket, itemPath),
@@ -111,24 +116,28 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
return err
}
}
return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
if err := saveKeyEtcd(ctx, ies.client, itemPath, data, opts...); err != nil {
return err
}
ies.index.observe(itemPath, plain)
return nil
}
func getIAMConfig(item any, data []byte, itemPath string) error {
data, err := decryptData(data, itemPath)
func (ies *IAMEtcdStore) decodeIAMConfig(item any, data []byte, path string) error {
data, err := decryptData(data, path)
if err != nil {
return err
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
ies.index.observe(path, data)
return json.Unmarshal(data, item)
}
func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item any, path string) error {
data, err := readKeyEtcd(ctx, ies.client, path)
data, err := ies.loadIAMConfigBytes(ctx, path)
if err != nil {
return err
}
return getIAMConfig(item, data, path)
return json.Unmarshal(data, item)
}
func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) {
@@ -136,11 +145,19 @@ func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([
if err != nil {
return nil, err
}
return decryptData(data, path)
data, err = decryptData(data, path)
if err == nil {
ies.index.observe(path, data)
}
return data, err
}
func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error {
return deleteKeyEtcd(ctx, ies.client, path)
if err := deleteKeyEtcd(ctx, ies.client, path); err != nil {
return err
}
ies.index.forget(path)
return nil
}
func (ies *IAMEtcdStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, _ int) error {
@@ -162,6 +179,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
return err
}
if p.Deleted {
return errNoSuchPolicy
}
m[policy] = p
return nil
}
@@ -181,7 +201,11 @@ func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValu
return err
}
ies.index.observe(string(kvs.Key), data)
policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key)))
if p.Deleted {
return errNoSuchPolicy
}
m[policy] = p
return nil
}
@@ -207,7 +231,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]Policy
func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
var u UserIdentity
err := getIAMConfig(&u, userkv.Value, string(userkv.Key))
err := ies.decodeIAMConfig(&u, userkv.Value, string(userkv.Key))
if err != nil {
if err == errConfigNotFound {
return errNoSuchUser
@@ -219,10 +243,14 @@ func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue,
}
func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error {
if u.Deleted {
if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
}
return errNoSuchUser
}
if u.Credentials.IsExpired() {
// Delete expired identity.
deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
bestEffortIAMExpiration(ctx, ies, getUserIdentityPath(user, userType))
return nil
}
if u.Credentials.AccessKey == "" {
@@ -231,16 +259,17 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
if u.Credentials.SessionToken != "" {
jwtClaims, err := extractJWTClaims(u)
if err != nil {
if u.Credentials.IsTemp() {
// We should delete such that the client can re-request
// for the expiring credentials.
deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
}
// A temporarily unavailable signing key is not proof of expiration.
return nil
}
u.Credentials.Claims = jwtClaims.Map()
}
if err := checkIAMParentRevision(ctx, ies, u.Credentials); err != nil {
if errors.Is(err, errIAMStaleUpdate) {
return errNoSuchUser
}
return err
}
if u.Credentials.Description == "" {
u.Credentials.Description = u.Credentials.Comment
}
@@ -258,6 +287,9 @@ func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userTyp
}
return "", err
}
if u.Deleted {
return "", errNoSuchUser
}
return u.Credentials.SecretKey, nil
}
@@ -274,6 +306,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
}
func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
ctx = withIAMExpirationCleanup(ctx)
var basePrefix string
switch userType {
case svcUser:
@@ -312,6 +345,9 @@ func (ies *IAMEtcdStore) loadGroup(ctx context.Context, group string, m map[stri
}
return err
}
if gi.Deleted {
return errNoSuchGroup
}
m[group] = gi
return nil
}
@@ -349,13 +385,16 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
}
return err
}
if !ies.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
return errNoSuchPolicy
}
m.Store(name, p)
return nil
}
func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
func (ies *IAMEtcdStore) getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
var p MappedPolicy
err := getIAMConfig(&p, kv.Value, string(kv.Key))
err := ies.decodeIAMConfig(&p, kv.Value, string(kv.Key))
if err != nil {
if err == errConfigNotFound {
return errNoSuchPolicy
@@ -363,6 +402,9 @@ func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy],
return err
}
name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json")
if !ies.index.mappingAllowed(string(kv.Key), p) {
return errNoSuchPolicy
}
m.Store(name, p)
return nil
}
@@ -392,61 +434,13 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
// Parse all policies mapping to create the proper data model
for _, kv := range r.Kvs {
if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
if err = ies.getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
return err
}
}
return nil
}
func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
}
func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
}
func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
}
func (ies *IAMEtcdStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
return ies.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
}
func (ies *IAMEtcdStore) deletePolicyDoc(ctx context.Context, name string) error {
err := ies.deleteIAMConfig(ctx, getPolicyDocPath(name))
if err == errConfigNotFound {
err = errNoSuchPolicy
}
return err
}
func (ies *IAMEtcdStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
err := ies.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
if err == errConfigNotFound {
err = errNoSuchPolicy
}
return err
}
func (ies *IAMEtcdStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
err := ies.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
if err == errConfigNotFound {
err = errNoSuchUser
}
return err
}
func (ies *IAMEtcdStore) deleteGroupInfo(ctx context.Context, name string) error {
err := ies.deleteIAMConfig(ctx, getGroupInfoPath(name))
if err == errConfigNotFound {
err = errNoSuchGroup
}
return err
}
func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWatchEvent {
ch := make(chan iamWatchEvent)
cmd/iam-group-revision.go
+164
View File
@@ -0,0 +1,164 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"maps"
"slices"
"time"
"github.com/minio/minio-go/v7/pkg/set"
)
type (
iamGroupGrantsKey struct{}
iamGroupMutationKey struct{}
iamGroupMutation struct {
Members []string
Remove bool
StatusOnly bool
}
)
// Merge the intended mutation with the record read under the distributed
// revision lock, not the older cache used to prepare the request.
func mergeIAMGroupMutation(ctx context.Context, previous GroupInfo, next *GroupInfo) {
op, ok := ctx.Value(iamGroupMutationKey{}).(iamGroupMutation)
if !ok || previous.Deleted || previous.Version == 0 {
return
}
members := set.CreateStringSet(previous.Members...)
grants := maps.Clone(previous.MemberGrants)
if grants == nil {
grants = make(map[string]time.Time)
}
switch {
case op.StatusOnly:
// Only the status changes.
case op.Remove:
for _, member := range op.Members {
members.Remove(member)
delete(grants, member)
}
next.Status = previous.Status
default:
requested := set.CreateStringSet(next.Members...)
for _, member := range op.Members {
if !requested.Contains(member) {
continue
}
at := next.MemberGrants[member]
if at.Before(grants[member]) {
continue
}
members.Add(member)
grants[member] = at
}
next.Status = previous.Status
}
next.Members, next.MemberGrants = members.ToSlice(), grants
slices.Sort(next.Members)
}
// A non-nil map is supplied by the versioned peer envelope, including for
// snapshots. Missing times are unknown, never the snapshot's newer timestamp.
func withIAMGroupGrants(ctx context.Context, grants map[string]time.Time) context.Context {
return context.WithValue(ctx, iamGroupGrantsKey{}, grants)
}
func (c *iamCache) effectiveGroupMembers(gi GroupInfo) []string {
var members []string
for _, member := range gi.Members {
if c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
members = append(members, member)
}
}
return members
}
func (c *iamCache) effectiveUserGroups(user string) []string {
var groups []string
for group := range c.iamUserGroupMemberships[user] {
gi, ok := c.iamGroupsMap[group]
r := c.revisions.get(getGroupInfoPath(group))
if r.RevokedBefore.After(gi.RevokedBefore) {
gi.RevokedBefore = r.RevokedBefore
}
if ok && !r.Deleted && c.groupMemberAllowed(user, gi.MemberGrants[user], gi.RevokedBefore) {
groups = append(groups, group)
}
}
return groups
}
func (c *iamCache) addGroupMembers(ctx context.Context, gi GroupInfo, members []string) (GroupInfo, error) {
grants, versioned := ctx.Value(iamGroupGrantsKey{}).(map[string]time.Time)
if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(gi.RevokedBefore) {
gi.RevokedBefore = boundary
}
origin, replicated := iamReplicationTime(ctx)
gi.Members = slices.Clone(gi.Members)
gi.MemberGrants = maps.Clone(gi.MemberGrants)
if gi.MemberGrants == nil {
gi.MemberGrants = make(map[string]time.Time)
}
current := set.CreateStringSet(gi.Members...)
gi.UpdatedAt = UTCNow()
if replicated {
gi.UpdatedAt = origin
}
for _, member := range members {
at := gi.UpdatedAt
r := c.userRevocation(member)
if replicated {
switch {
case versioned:
at = grants[member]
if at.After(origin) {
return gi, errInvalidArgument
}
case !r.RevokedBefore.IsZero() || r.Deleted || !gi.RevokedBefore.IsZero():
// Legacy snapshots cannot prove a post-revocation grant.
continue
case current.Contains(member):
continue
}
if !c.groupMemberAllowed(member, at, gi.RevokedBefore) {
continue
}
} else {
if current.Contains(member) && c.groupMemberAllowed(member, gi.MemberGrants[member], gi.RevokedBefore) {
continue // Editing the group is not reissuing every grant.
}
if !at.After(gi.RevokedBefore) {
at = gi.RevokedBefore.Add(time.Nanosecond)
}
if !at.After(r.RevokedBefore) {
at = r.RevokedBefore.Add(time.Nanosecond)
}
if !at.After(gi.MemberGrants[member]) {
at = gi.MemberGrants[member].Add(time.Nanosecond)
}
if at.After(gi.UpdatedAt) {
gi.UpdatedAt = at
}
}
u, ok := c.iamUsersMap[member]
if !ok {
return gi, errNoSuchUser
}
if u.Credentials.IsTemp() || u.Credentials.IsServiceAccount() {
return gi, errIAMActionNotAllowed
}
if previous := gi.MemberGrants[member]; previous.After(at) {
continue
}
current.Add(member)
gi.MemberGrants[member] = at
}
gi.Members = current.ToSlice()
slices.Sort(gi.Members)
return gi, nil
}
cmd/iam-healing-performance_test.go
+60
View File
@@ -0,0 +1,60 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
)
// Measures steady-state index traversal, sorting and the capability request.
// The network peer acknowledges real batches but performs no disk I/O; this
// benchmark deliberately does not claim durable catch-up throughput.
func BenchmarkIAMRevisionConvergedHealing(b *testing.B) {
for _, n := range []int{1000, 10000} {
b.Run(fmt.Sprint(n), func(b *testing.B) {
ctx, sys, _ := prepareIAMRevisionFixture(b)
_, err := sys.CreateUser(ctx, "benchmark-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
mustIAM(b, err)
for i := range n {
at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
mustIAM(b, err)
sys.store.revisionIndex().observe(getUserIdentityPath(fmt.Sprintf("deleted-%06d", i), regUser), data)
}
var puts atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/minio/health/live" {
w.WriteHeader(http.StatusOK)
return
}
if r.Method == http.MethodPut {
puts.Add(1)
}
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: "benchmark-peer", Digest: "constant"}})
}))
defer server.Close()
c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "benchmark-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
mustIAM(b, c.healIAMDeletions(ctx))
before := puts.Load()
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
mustIAM(b, c.healIAMDeletions(ctx))
}
b.StopTimer()
b.ReportMetric(float64(puts.Load()-before)/float64(b.N), "PUT/op")
if puts.Load() != before {
b.Fatal("steady-state healing replayed acknowledged records")
}
})
}
}
cmd/iam-object-store.go
+56 -61
View File
@@ -45,6 +45,7 @@ type IAMObjectStore struct {
sync.RWMutex
*iamCache
index iamRevisionIndex
usersSysType UsersSysType
@@ -52,13 +53,17 @@ type IAMObjectStore struct {
}
func newIAMObjectStore(objAPI ObjectLayer, usersSysType UsersSysType) *IAMObjectStore {
return &IAMObjectStore{
store := &IAMObjectStore{
iamCache: newIamCache(),
objAPI: objAPI,
usersSysType: usersSysType,
}
store.revisions = &store.index
return store
}
func (iamOS *IAMObjectStore) revisionIndex() *iamRevisionIndex { return &iamOS.index }
func (iamOS *IAMObjectStore) rlock() *iamCache {
iamOS.RLock()
return iamOS.iamCache
@@ -87,6 +92,7 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
if err != nil {
return err
}
plain := data
if GlobalKMS != nil {
data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
minioMetaBucket: path.Join(minioMetaBucket, objPath),
@@ -95,7 +101,11 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
return err
}
}
return saveConfig(ctx, iamOS.objAPI, objPath, data)
if err := saveConfig(ctx, iamOS.objAPI, objPath, data); err != nil {
return err
}
iamOS.index.observe(objPath, plain)
return nil
}
func decryptData(data []byte, objPath string) ([]byte, error) {
@@ -133,6 +143,7 @@ func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context,
if err != nil {
return nil, meta, err
}
iamOS.index.observe(objPath, data)
return data, meta, nil
}
@@ -146,7 +157,11 @@ func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item any, objPat
}
func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) error {
return deleteConfig(ctx, iamOS.objAPI, path)
if err := deleteConfig(ctx, iamOS.objAPI, path); err != nil {
return err
}
iamOS.index.forget(path)
return nil
}
func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error {
@@ -171,6 +186,10 @@ func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy
return err
}
if p.Deleted {
return errNoSuchPolicy
}
if p.Version == 0 {
// This means that policy was in the old version (without any
// timestamp info). We fetch the mod time of the file and save
@@ -200,6 +219,10 @@ func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (Pol
return p, err
}
if p.Deleted {
return PolicyDoc{}, errNoSuchPolicy
}
if p.Version == 0 {
// This means that policy was in the old version (without any
// timestamp info). We fetch the mod time of the file and save
@@ -245,6 +268,9 @@ func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, use
}
return "", err
}
if u.Deleted {
return "", errNoSuchUser
}
return u.Credentials.SecretKey, nil
}
@@ -258,10 +284,15 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
return u, err
}
if u.Deleted {
if userType == stsUser && !u.ExpiresAt.IsZero() && UTCNow().After(u.ExpiresAt) {
bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
}
return UserIdentity{}, errNoSuchUser
}
if u.Credentials.IsExpired() {
// Delete expired identity - ignoring errors here.
iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
bestEffortIAMExpiration(ctx, iamOS, getUserIdentityPath(user, userType))
return u, errNoSuchUser
}
@@ -272,16 +303,18 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
if u.Credentials.SessionToken != "" {
jwtClaims, err := extractJWTClaims(u)
if err != nil {
if u.Credentials.IsTemp() {
// We should delete such that the client can re-request
// for the expiring credentials.
iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
}
return u, errNoSuchUser
// During startup the site signing key may not be available yet.
// Reject this load without deleting a credential that has not expired.
return UserIdentity{}, errNoSuchUser
}
u.Credentials.Claims = jwtClaims.Map()
}
if err := checkIAMParentRevision(ctx, iamOS, u.Credentials); err != nil {
if errors.Is(err, errIAMStaleUpdate) {
return UserIdentity{}, errNoSuchUser
}
return UserIdentity{}, err
}
if u.Credentials.Description == "" {
u.Credentials.Description = u.Credentials.Comment
@@ -320,6 +353,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
}
func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
ctx = withIAMExpirationCleanup(ctx)
var basePrefix string
switch userType {
case svcUser:
@@ -354,6 +388,9 @@ func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[
}
return err
}
if g.Deleted {
return errNoSuchGroup
}
m[group] = g
return nil
}
@@ -391,6 +428,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
goto retry
}
if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
return errNoSuchPolicy
}
m.Store(name, p)
return nil
}
@@ -405,6 +445,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name
}
return p, err
}
if !iamOS.index.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), p) {
return MappedPolicy{}, errNoSuchPolicy
}
return p, nil
}
@@ -824,54 +867,6 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
return nil
}
func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
}
func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
}
func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
}
func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
return iamOS.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
}
func (iamOS *IAMObjectStore) deletePolicyDoc(ctx context.Context, name string) error {
err := iamOS.deleteIAMConfig(ctx, getPolicyDocPath(name))
if err == errConfigNotFound {
err = errNoSuchPolicy
}
return err
}
func (iamOS *IAMObjectStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
err := iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
if err == errConfigNotFound {
err = errNoSuchPolicy
}
return err
}
func (iamOS *IAMObjectStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
err := iamOS.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
if err == errConfigNotFound {
err = errNoSuchUser
}
return err
}
func (iamOS *IAMObjectStore) deleteGroupInfo(ctx context.Context, name string) error {
err := iamOS.deleteIAMConfig(ctx, getGroupInfoPath(name))
if err == errConfigNotFound {
err = errNoSuchGroup
}
return err
}
// Lists objects in the minioMetaBucket at the given path prefix. All returned
// items have the pathPrefix removed from their names.
func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr[string] {
cmd/iam-performance_test.go
+131
View File
@@ -0,0 +1,131 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
)
// Uses APIs shared with the pre-revision tree so the same benchmark can be
// overlaid on that tree for a comparable local baseline.
func prepareIAMPerformanceFixture(b *testing.B) (context.Context, *IAMSys) {
b.Helper()
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
disks, err := getRandomDisks(1)
if err != nil {
b.Fatal(err)
}
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
if err != nil {
b.Fatal(err)
}
initAllSubsystems(ctx)
globalIAMSys.initStore(obj, nil)
if err := globalIAMSys.Load(ctx, true); err != nil {
b.Fatal(err)
}
b.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
return ctx, globalIAMSys
}
func BenchmarkIAMCachedCredential(b *testing.B) {
for _, kind := range []string{"user", "service", "sts"} {
b.Run(kind, func(b *testing.B) {
ctx, sys := prepareIAMPerformanceFixture(b)
const parent = "benchmark-parent"
_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
if err != nil {
b.Fatal(err)
}
key := parent
if kind == "service" {
c, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "benchmark-service", secretKey: "benchmark-service-password"})
if err != nil {
b.Fatal(err)
}
key = c.AccessKey
}
if kind == "sts" {
secret, err := getTokenSigningKey()
if err != nil {
b.Fatal(err)
}
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
if err != nil {
b.Fatal(err)
}
c.ParentUser = parent
if _, err := sys.SetTempUser(ctx, c.AccessKey, c, ""); err != nil {
b.Fatal(err)
}
key = c.AccessKey
}
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
if _, ok := sys.store.GetUser(key); !ok {
b.Fatal("credential missing")
}
}
})
}
}
func BenchmarkIAMSetTempUser(b *testing.B) {
ctx, sys := prepareIAMPerformanceFixture(b)
const parent = "benchmark-sts-parent"
_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "benchmark-user-password", Status: madmin.AccountEnabled})
if err != nil {
b.Fatal(err)
}
secret, err := getTokenSigningKey()
if err != nil {
b.Fatal(err)
}
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
if err != nil {
b.Fatal(err)
}
cred.ParentUser = parent
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
if _, err := sys.SetTempUser(ctx, cred.AccessKey, cred, "readwrite"); err != nil {
b.Fatal(err)
}
}
}
// Run with -benchtime=1x. Preparation is outside the timer; each measured load
// sees a fresh set of expired reusable service-account records.
func BenchmarkIAMColdLoadExpiredServices(b *testing.B) {
for _, count := range []int{100, 1000} {
b.Run(fmt.Sprint(count), func(b *testing.B) {
ctx, sys := prepareIAMPerformanceFixture(b)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
b.StopTimer()
for j := 0; j < count; j++ {
key := fmt.Sprintf("expired-benchmark-%d-%d", i, j)
u := UserIdentity{Version: 1, UpdatedAt: UTCNow().Add(-2 * time.Hour), Credentials: auth.Credentials{AccessKey: key, SecretKey: "expired-benchmark-password", ParentUser: "absent-idp-parent", Expiration: UTCNow().Add(-time.Hour), Status: auth.AccountOn}}
if err := sys.store.saveIAMConfig(ctx, &u, getUserIdentityPath(key, svcUser)); err != nil {
b.Fatal(err)
}
}
b.StartTimer()
if err := sys.store.LoadIAMCache(ctx, true); err != nil {
b.Fatal(err)
}
}
})
}
}
cmd/iam-replay_test.go
+168
View File
@@ -0,0 +1,168 @@
package cmd
import (
"context"
"errors"
"os"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/pgsty/silo-pkg/v3/policy"
)
func TestReviewIAMRevokedUserReplay(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
obj, disk, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
user := "review-revoked-user"
req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
created, err := globalIAMSys.CreateUser(ctx, user, req)
if err != nil {
t.Fatal(err)
}
policyAt, err := globalIAMSys.PolicyDBSet(ctx, user, "readwrite", regUser, false)
if err != nil {
t.Fatal(err)
}
args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "review-bucket", ObjectName: "review-object"}
if !globalIAMSys.IsAllowed(args) {
t.Fatal("seed must allow object read")
}
if err := globalIAMSys.DeleteUser(ctx, user, false); err != nil {
t.Fatal(err)
}
if err := globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
t.Fatal(err)
}
if globalIAMSys.IsAllowed(args) {
t.Fatal("deletion did not remove initial permission")
}
if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
t.Fatal(err)
}
if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
t.Errorf("revoked user restored by an older replicated create, GetUserInfo error = %v", err)
}
if err := globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: user, UserType: int(regUser), Policy: "readwrite"}, policyAt); err != nil {
t.Fatal(err)
}
if globalIAMSys.IsAllowed(args) {
t.Error("older replicated identity and policy events restored revoked S3 read permission")
}
}
func TestReviewIAMSourceTimestampOrder(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
obj, disk, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
user := "review-ordered-user"
req := madmin.AddOrUpdateUserReq{SecretKey: "review-valid-password", Status: madmin.AccountEnabled}
origin := UTCNow().Add(-time.Hour)
if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin); err != nil {
t.Fatal(err)
}
if err := globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if _, err := globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
t.Fatalf("newer source deletion skipped after delayed creation, GetUserInfo error = %v", err)
}
}
// A user's old group grant must not return after deletion and deliberate recreation.
func TestR3CandidateOldGroupReplayAfterRecreation(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
obj, disk, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
user, group := "r3-group-member", "r3-granting-group"
origin := UTCNow().Add(-time.Hour)
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
peer := &globalSiteReplicationSys
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
if !globalIAMSys.IsAllowed(args) {
t.Fatal("fixture must grant through group")
}
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
must(globalIAMSys.store.LoadIAMCache(ctx, false))
if globalIAMSys.IsAllowed(args) {
t.Fatal("recreation must start without deleted group membership")
}
must(peer.PeerGroupInfoChangeHandler(ctx, add, origin.Add(time.Minute)))
must(globalIAMSys.store.LoadIAMCache(ctx, false))
if globalIAMSys.IsAllowed(args) {
t.Fatal("old group event restored the deleted user's read grant after recreation and durable reload")
}
}
// The user delete is also a revocation of its earlier group memberships.
func TestR3CandidateLateDeleteRetainsOldGroupGrant(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
obj, disk, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
user, group := "r3-late-member", "r3-late-group"
origin := UTCNow().Add(-time.Hour)
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-r3-user-password", Status: madmin.AccountEnabled}
peer := &globalSiteReplicationSys
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin))
must(peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin.Add(time.Minute)))
must(peer.PeerPolicyMappingHandler(ctx, &madmin.SRPolicyMapping{UserOrGroup: group, IsGroup: true, UserType: int(regUser), Policy: "readwrite"}, origin.Add(time.Minute)))
args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "r3-bucket", ObjectName: "probe"}
if !globalIAMSys.IsAllowed(args) {
t.Fatal("fixture must grant through group")
}
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, origin.Add(3*time.Minute)))
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
must(globalIAMSys.store.LoadIAMCache(ctx, false))
if _, ok := globalIAMSys.GetUser(ctx, user); !ok {
t.Fatal("newer identity must survive")
}
if globalIAMSys.IsAllowed(args) {
t.Fatal("late user deletion retained the older group grant on the recreated identity")
}
}
cmd/iam-replication-protocol.go
+321
View File
@@ -0,0 +1,321 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/minio/madmin-go/v3"
xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/policy"
)
const (
iamRevisionProtocol = 1
iamRevisionPeerPath = "/v3/site-replication/peer/iam-revisions"
iamUserBoundaryType = "silo-user-revocation"
iamGroupBoundaryType = "silo-group-revocation"
maxIAMRevisionBatch = 128
)
var iamRevisionInstance = mustGetUUID()
type iamUserBoundary struct {
User string `json:"user"`
Before time.Time `json:"before"`
}
type iamGroupBoundary struct {
Group string `json:"group"`
Before time.Time `json:"before"`
}
// The server owns this additive protocol, without changing the client SDK or
// overloading a policy/document field. Old servers reject the dedicated route
// before applying any change that would lose revocation or member metadata.
type iamReplicationItem struct {
madmin.SRIAMItem
GroupGrants map[string]time.Time `json:"groupGrants,omitempty"`
GroupSnapshot bool `json:"groupSnapshot,omitempty"`
UserRevocation *iamUserBoundary `json:"userRevocation,omitempty"`
GroupRevocation *iamGroupBoundary `json:"groupRevocation,omitempty"`
RevokedBefore time.Time `json:"revokedBefore,omitempty"`
}
type iamRevisionBatch struct {
Version int `json:"version"`
Items []iamReplicationItem `json:"items"`
}
type iamRevisionStatus struct {
Version int `json:"version"`
Node string `json:"node"`
Instance string `json:"instance"`
Digest string `json:"digest"`
}
type iamRevisionResponse struct {
iamRevisionStatus
Errors []string `json:"errors,omitempty"`
}
type iamRevisionBatchError struct{ failures []string }
func (e *iamRevisionBatchError) Error() string {
return "IAM revision batch: " + strings.Join(e.failures, "; ")
}
type iamRevisionProgress struct {
Instances map[string]string
Acknowledged map[string]string
}
type iamRevisionMetrics struct {
healFailures atomic.Uint64
healLastSuccess atomic.Int64
healDurationMillis atomic.Int64
}
func iamRevisionDigest(items map[string]iamRevision) string {
paths := make([]string, 0, len(items))
for path := range items {
paths = append(paths, path)
}
sort.Strings(paths)
h := sha256.New()
for _, path := range paths {
r := items[path]
fmt.Fprintf(h, "%q %s %t %s\n", path, r.timestamp().UTC().Format(time.RFC3339Nano), r.Deleted, r.RevokedBefore.UTC().Format(time.RFC3339Nano))
}
return hex.EncodeToString(h.Sum(nil))
}
func (store *IAMStoreSys) iamRevisionStatus() iamRevisionStatus {
node := globalLocalNodeName
if node == "" {
node = "local"
}
return iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: iamRevisionInstance, Digest: store.revisionIndex().digest()}
}
func executeIAMRevisionRequest(ctx context.Context, client *madmin.AdminClient, method string, batch *iamRevisionBatch) (status iamRevisionStatus, err error) {
var content []byte
if batch != nil {
content, err = json.Marshal(batch)
if err != nil {
return status, err
}
}
resp, err := client.ExecuteMethod(ctx, method, madmin.RequestData{RelPath: iamRevisionPeerPath, QueryValues: url.Values{"api-version": {madmin.SiteReplAPIVersion}}, Content: content})
if resp != nil {
defer xhttp.DrainBody(resp.Body)
}
if err != nil {
return status, err
}
if resp.StatusCode != http.StatusOK {
var remote madmin.ErrorResponse
if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&remote) == nil && remote.Code != "" {
return status, remote
}
return status, fmt.Errorf("IAM revision protocol requires upgraded peers: %s", resp.Status)
}
var response iamRevisionResponse
if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&response); err != nil {
return status, err
}
status = response.iamRevisionStatus
if status.Version != iamRevisionProtocol || status.Node == "" || status.Instance == "" || status.Digest == "" {
return status, errors.New("peer did not acknowledge the IAM revision protocol")
}
if len(response.Errors) != 0 {
return status, &iamRevisionBatchError{failures: response.Errors}
}
return status, nil
}
type (
iamRecordBoundaryKey struct{}
iamGroupSnapshotKey struct{}
)
func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) {
out := iamReplicationItem{SRIAMItem: item}
if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil {
var key string
if item.SvcAccChange.Create != nil {
key = item.SvcAccChange.Create.AccessKey
} else if item.SvcAccChange.Update != nil {
key = item.SvcAccChange.Update.AccessKey
}
if key != "" {
r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(key, svcUser))
if err != nil {
return out, err
}
if r.Deleted || r.timestamp().After(item.UpdatedAt) {
return out, errIAMStaleUpdate
}
out.RevokedBefore = r.RevokedBefore
}
}
if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && !item.GroupInfo.UpdateReq.IsRemove {
out.GroupSnapshot = true
var gi GroupInfo
if err := globalIAMSys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath(item.GroupInfo.UpdateReq.Group)); err != nil {
return out, err
}
// The matching persisted snapshot carries member grant times. If a
// later write won before sending, propagate that whole newer state.
if gi.Deleted {
return out, errIAMStaleUpdate
}
out.UpdatedAt = gi.UpdatedAt
out.RevokedBefore = gi.RevokedBefore
out.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: item.GroupInfo.UpdateReq.Group, Status: madmin.GroupStatus(gi.Status)}}
cache := globalIAMSys.store.rlock()
out.GroupInfo.UpdateReq.Members = cache.effectiveGroupMembers(gi)
globalIAMSys.store.runlock()
out.GroupGrants = make(map[string]time.Time, len(out.GroupInfo.UpdateReq.Members))
for _, member := range out.GroupInfo.UpdateReq.Members {
out.GroupGrants[member] = gi.MemberGrants[member]
}
}
if item.Type == madmin.SRIAMItemGroupInfo && item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
r, err := loadIAMRevision(ctx, globalIAMSys.store, getGroupInfoPath(item.GroupInfo.UpdateReq.Group))
if err != nil {
return out, err
}
if !r.Deleted && !r.RevokedBefore.IsZero() {
out.Type, out.GroupInfo = iamGroupBoundaryType, nil
out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
out.UpdatedAt = r.RevokedBefore
}
}
if item.Type == madmin.SRIAMItemIAMUser && item.IAMUser != nil {
r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(item.IAMUser.AccessKey, regUser))
if err != nil {
return out, err
}
if item.IAMUser.IsDeleteReq && !r.Deleted && !r.RevokedBefore.IsZero() {
out.Type = iamUserBoundaryType
out.IAMUser = nil
out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
out.UpdatedAt = r.RevokedBefore
} else if !item.IAMUser.IsDeleteReq {
if r.Deleted || r.timestamp().After(item.UpdatedAt) {
return out, errIAMStaleUpdate
}
out.RevokedBefore = r.RevokedBefore
}
}
return out, nil
}
func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error {
if item.GroupInfo != nil {
if item.GroupSnapshot {
ctx = context.WithValue(ctx, iamGroupSnapshotKey{}, true)
}
// A nil map also explicitly denotes unknown legacy grants. Do not
// turn an unrelated group edit into a new grant after a revocation.
ctx = withIAMGroupGrants(ctx, item.GroupGrants)
}
if !item.RevokedBefore.IsZero() {
if item.RevokedBefore.After(item.UpdatedAt) {
return errSRInvalidRequest(errInvalidArgument)
}
ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore)
}
switch item.Type {
case iamUserBoundaryType:
if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() {
return errSRInvalidRequest(errInvalidArgument)
}
return iamReplicationError(globalIAMSys.DeleteUser(withIAMReplicationTime(ctx, item.UserRevocation.Before), item.UserRevocation.User, true))
case iamGroupBoundaryType:
if item.GroupRevocation == nil || item.GroupRevocation.Group == "" || item.GroupRevocation.Before.IsZero() {
return errSRInvalidRequest(errInvalidArgument)
}
_, err := globalIAMSys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, item.GroupRevocation.Before), item.GroupRevocation.Group, nil)
return iamReplicationError(err)
case madmin.SRIAMItemPolicy:
if len(item.Policy) == 0 {
return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
}
p, err := policy.ParseConfig(bytes.NewReader(item.Policy))
if err != nil {
return err
}
if p.IsEmpty() {
p = nil
}
return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, p, item.UpdatedAt)
case madmin.SRIAMItemSvcAcc:
return globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
case madmin.SRIAMItemPolicyMapping:
return globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
case madmin.SRIAMItemSTSAcc:
return globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
case madmin.SRIAMItemIAMUser:
return globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
case madmin.SRIAMItemGroupInfo:
return globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
default:
return errSRInvalidRequest(errInvalidArgument)
}
}
func (a adminAPIHandlers) SRPeerIAMRevisions(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if obj, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction); obj == nil {
return
}
var failures []string
if r.Method == http.MethodPut {
var batch iamRevisionBatch
if err := parseJSONBody(ctx, r.Body, &batch, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if batch.Version != iamRevisionProtocol || len(batch.Items) == 0 || len(batch.Items) > maxIAMRevisionBatch {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
return
}
for i, item := range batch.Items {
if err := applyIAMReplicationItem(ctx, item); err != nil {
failures = append(failures, fmt.Sprintf("item %d (%s): %v", i, item.Type, err))
}
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: globalIAMSys.store.iamRevisionStatus(), Errors: failures})
}
// A site endpoint can balance requests across nodes sharing durable IAM state.
// Switching between known node incarnations preserves ACKs; a new incarnation
// conservatively invalidates them so restoring an old backend cannot inherit
// acknowledgements from before the restore.
func (p *iamRevisionProgress) observePeer(status iamRevisionStatus) {
if p.Instances == nil {
p.Instances = make(map[string]string)
}
if p.Instances[status.Node] != status.Instance || p.Acknowledged == nil {
p.Instances[status.Node] = status.Instance
p.Acknowledged = make(map[string]string)
}
}
cmd/iam-replication-protocol_test.go
+161
View File
@@ -0,0 +1,161 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
)
func TestIAMRevisionProtocolDoesNotFallBackToLegacy(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
t.Errorf("unsafe fallback path: %s", r.URL.Path)
}
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"Code":"NotImplemented","Message":"old server"}`))
}))
defer server.Close()
client, err := madmin.New(strings.TrimPrefix(server.URL, "http://"), "test-access", "valid-test-secret", false)
mustIAM(t, err)
_, err = executeIAMRevisionRequest(context.Background(), client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{{SRIAMItem: madmin.SRIAMItem{Type: iamUserBoundaryType}, UserRevocation: &iamUserBoundary{User: "recreated", Before: UTCNow()}}}})
if err == nil || requests.Load() != 1 {
t.Fatalf("old peer must reject without fallback, err=%v requests=%d", err, requests.Load())
}
}
type iamNoHealingScanStore struct{ IAMStorageAPI }
func (s *iamNoHealingScanStore) listIAMConfigPaths(context.Context) ([]string, error) {
panic("healing must use the loaded revision index")
}
func TestIAMRevisionHealingAcknowledgements(t *testing.T) {
for _, balanced := range []bool{false, true} {
t.Run(fmt.Sprintf("load_balanced_%t", balanced), func(t *testing.T) { testIAMRevisionHealingAcknowledgements(t, balanced) })
}
}
func testIAMRevisionHealingAcknowledgements(t *testing.T, balanced bool) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
_, err := sys.CreateUser(ctx, "ack-sync", madmin.AddOrUpdateUserReq{SecretKey: "valid-sync-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
for i := range maxIAMRevisionBatch*2 + 1 {
at := UTCNow().Add(time.Duration(i) * time.Nanosecond)
mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath(fmt.Sprintf("ack-%04d", i), regUser)))
}
sys.store.IAMStorageAPI = &iamNoHealingScanStore{IAMStorageAPI: sys.store.IAMStorageAPI}
var mu sync.Mutex
var puts, gets int
var applied int
instance := "boot-1"
failSecondBatch := true
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/minio/health/live" {
w.WriteHeader(http.StatusOK)
return
}
mu.Lock()
defer mu.Unlock()
if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" {
t.Errorf("unexpected request: %s", r.URL.Path)
w.WriteHeader(404)
return
}
var failures []string
if r.Method == http.MethodGet {
gets++
} else {
puts++
var batch iamRevisionBatch
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
t.Error(err)
w.WriteHeader(400)
return
}
if len(batch.Items) > maxIAMRevisionBatch {
t.Error("batch exceeds limit")
}
if failSecondBatch && puts == 2 {
failures = []string{"injected item error"}
} else {
applied += len(batch.Items)
}
}
node := "node-1"
if balanced {
node = fmt.Sprintf("node-%d", (gets+puts)%2+1)
}
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: node, Instance: node + instance, Digest: fmt.Sprintf("%d", applied)}, Errors: failures})
}))
defer server.Close()
c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "ack-sync", Peers: map[string]madmin.PeerInfo{globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"}, "remote": {DeploymentID: "remote", Name: "remote", Endpoint: server.URL}}}}
if err := c.healIAMDeletions(ctx); err == nil {
t.Fatal("item failure was hidden")
}
mu.Lock()
if puts != 3 || applied != maxIAMRevisionBatch+1 {
t.Errorf("failed middle batch blocked later revocations: puts=%d applied=%d", puts, applied)
}
failSecondBatch = false
applied++ // Unrelated remote mutation changes its digest.
mu.Unlock()
at := UTCNow()
mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, UpdatedAt: at, RevokedBefore: at}, getUserIdentityPath("ack-new-local", regUser)))
mustIAM(t, c.healIAMDeletions(ctx))
mu.Lock()
if puts != 5 {
t.Errorf("did not resume at unacknowledged batch: puts=%d", puts)
}
mu.Unlock()
mustIAM(t, c.healIAMDeletions(ctx))
mu.Lock()
if puts != 5 || gets != 3 {
t.Errorf("converged records were replayed: puts=%d gets=%d", puts, gets)
}
instance = "boot-2"
mu.Unlock()
mustIAM(t, c.healIAMDeletions(ctx))
mu.Lock()
defer mu.Unlock()
if puts != 8 {
t.Fatalf("peer restart reused an old acknowledgement: puts=%d", puts)
}
}
func TestIAMRevisionIndexRebuildsFromStorage(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
const user = "index-parent"
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-parent-password", Status: madmin.AccountEnabled}
_, err := sys.CreateUser(ctx, user, req)
mustIAM(t, err)
mustIAM(t, sys.DeleteUser(ctx, user, false))
before := sys.store.revisionIndex().snapshot()
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
mustIAM(t, store.LoadIAMCache(ctx, true))
if iamRevisionDigest(before) != iamRevisionDigest(store.revisionIndex().snapshot()) {
t.Fatal("ordinary IAM loading did not restore the deletion index")
}
_, err = store.AddUser(ctx, user, req)
mustIAM(t, err)
r := store.revisionIndex().get(getUserIdentityPath(user, regUser))
if r.Deleted || r.RevokedBefore.IsZero() {
t.Fatal("recreation discarded the retained boundary")
}
if r.Credentials.SecretKey != "" || r.Credentials.SessionToken != "" {
t.Fatal("index retained credentials")
}
}
cmd/iam-revision-boundary_test.go
+393
View File
@@ -0,0 +1,393 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"errors"
"fmt"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/grid"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/policy"
etcd "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/namespace"
)
func prepareIAMRevisionFixture(t testing.TB, backend ...string) (context.Context, *IAMSys, ObjectLayer) {
t.Helper()
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
disks, err := getRandomDisks(1)
mustIAM(t, err)
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
mustIAM(t, err)
initAllSubsystems(ctx)
// Deliberately omit the periodic refresh goroutine. Fault injection can
// replace this fixture's storage interface without racing initialization.
var client *etcd.Client
if len(backend) != 0 && backend[0] == "etcd" {
endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
if endpoint == "" {
cancel()
obj.Shutdown(context.Background())
os.RemoveAll(disks[0])
t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
}
client, err = etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
mustIAM(t, err)
prefix := fmt.Sprintf("/silo-boundary-test/%d/", time.Now().UnixNano())
client.KV = namespace.NewKV(client.KV, prefix)
client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
t.Cleanup(func() { client.Delete(context.Background(), "", etcd.WithPrefix()); client.Close() })
}
globalIAMSys.initStore(obj, client)
mustIAM(t, globalIAMSys.Load(ctx, true))
t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); os.RemoveAll(disks[0]); resetTestGlobals() })
return ctx, globalIAMSys, obj
}
func mustIAM(t testing.TB, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
var errIAMInjectedWrite = errors.New("injected IAM persistence failure")
type iamFailingCleanupStore struct {
IAMStorageAPI
parentPath string
beforeCommit bool
}
func (s *iamFailingCleanupStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
if s.beforeCommit || path != s.parentPath {
return errIAMInjectedWrite
}
return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
}
func TestIAMRevocationCommitBoundary(t *testing.T) {
for _, before := range []bool{true, false} {
name := "after_identity_commit"
if before {
name = "before_identity_commit"
}
t.Run(name, func(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
const user = "commit-boundary-user"
origin := UTCNow().Add(-time.Hour)
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
mustIAM(t, err)
_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(time.Minute)), user, "readwrite", regUser, false)
mustIAM(t, err)
_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(time.Minute)), "commit-group", []string{user})
mustIAM(t, err)
_, err = sys.PolicyDBSet(ctx, "commit-group", "readwrite", regUser, true)
mustIAM(t, err)
child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), user, nil, newServiceAccountOpts{accessKey: "commit-child", secretKey: "valid-child-password"})
mustIAM(t, err)
args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
if !sys.IsAllowed(args) {
t.Fatal("fixture has no grant")
}
siblingStore := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
mustIAM(t, siblingStore.LoadIAMCache(ctx, true))
sibling := &IAMSys{store: siblingStore, usersSysType: MinIOUsersSysType}
tg, err := grid.SetupTestGrid(2)
mustIAM(t, err)
defer tg.Cleanup()
var notifications atomic.Int32
mustIAM(t, deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
notifications.Add(1)
if err := sibling.LoadUserAfterDelete(ctx, r.Get(peerRESTUser)); err != nil {
return grid.NoPayload{}, grid.NewRemoteErr(err)
}
return grid.NoPayload{}, nil
}))
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
mustIAM(t, err)
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{host: host, gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}}}
original := sys.store.IAMStorageAPI
sys.store.IAMStorageAPI = &iamFailingCleanupStore{IAMStorageAPI: original, parentPath: getUserIdentityPath(user, regUser), beforeCommit: before}
boundary := origin.Add(2 * time.Minute)
err = sys.DeleteUser(withIAMReplicationTime(ctx, boundary), user, true)
if !errors.Is(err, errIAMInjectedWrite) {
t.Fatalf("expected write failure, got %v", err)
}
sys.store.IAMStorageAPI = original
r, err := loadIAMRevision(ctx, original, getUserIdentityPath(user, regUser))
mustIAM(t, err)
if before {
if r.Deleted || !sys.IsAllowed(args) || !sibling.IsAllowed(args) || notifications.Load() != 0 {
t.Fatal("failure before commit changed the identity or grant")
}
return
}
if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
t.Fatal("cleanup failure lost durable revocation")
}
if sys.IsAllowed(args) || sibling.IsAllowed(args) || notifications.Load() != 1 {
t.Fatal("cleanup failure retained old permission")
}
// Subsequent fixture writes need no additional RPC handlers.
globalNotificationSys = &NotificationSys{}
// Recreate after the partial cleanup. The old mapping, group member
// and child still exist in storage; none may authorize this identity.
_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), user, req)
mustIAM(t, err)
reloaded := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
mustIAM(t, reloaded.LoadIAMCache(ctx, true))
fresh := &IAMSys{store: reloaded, usersSysType: MinIOUsersSysType}
if fresh.IsAllowed(args) {
t.Fatal("cold reload restored partially cleaned-up grants")
}
if _, ok := reloaded.GetUser(child.AccessKey); ok {
t.Fatal("cold reload restored the old child")
}
gd, err := reloaded.GetGroupDescription("commit-group")
mustIAM(t, err)
if len(gd.Members) != 0 {
t.Fatalf("listing exposed a revoked group relation: %v", gd.Members)
}
_, err = sys.AddUsersToGroup(ctx, "commit-group", []string{user})
mustIAM(t, err)
if !sys.IsAllowed(args) {
t.Fatal("explicit new group grant was not accepted")
}
})
}
}
func TestIAMGroupGrantVersionsSurviveSnapshotsAndRecreation(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
origin := UTCNow().Add(-time.Hour)
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
for _, user := range []string{"grant-alice", "grant-bob"} {
_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, req)
mustIAM(t, err)
}
grant := origin.Add(time.Minute)
_, err := sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), "grant-group", []string{"grant-alice"})
mustIAM(t, err)
_, err = sys.PolicyDBSet(ctx, "grant-group", "readwrite", regUser, true)
mustIAM(t, err)
boundary := origin.Add(2 * time.Minute)
mustIAM(t, sys.DeleteUser(withIAMReplicationTime(ctx, boundary), "grant-alice", false))
_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), "grant-alice", req)
mustIAM(t, err)
_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), "grant-group", []string{"grant-bob"})
mustIAM(t, err)
_, err = sys.SetGroupStatus(withIAMReplicationTime(ctx, origin.Add(5*time.Minute)), "grant-group", true)
mustIAM(t, err)
var gi GroupInfo
mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
if !gi.MemberGrants["grant-alice"].Equal(grant) {
t.Fatal("unrelated group edits refreshed an old grant")
}
args := policy.Args{AccountName: "grant-alice", Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
for _, stale := range []time.Time{grant, boundary, {}} {
item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(6 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: "grant-group", Members: []string{"grant-alice", "grant-bob"}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{"grant-alice": stale, "grant-bob": origin.Add(4 * time.Minute)}}
mustIAM(t, applyIAMReplicationItem(ctx, item))
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
if sys.IsAllowed(args) {
t.Fatalf("snapshot restored revoked grant %s", stale)
}
gd, err := sys.GetGroupDescription("grant-group")
mustIAM(t, err)
if len(gd.Members) != 1 || gd.Members[0] != "grant-bob" {
t.Fatalf("inconsistent effective members: %v", gd.Members)
}
}
// Only an explicit post-revocation grant restores access.
freshAt, err := sys.AddUsersToGroup(ctx, "grant-group", []string{"grant-alice"})
mustIAM(t, err)
if !sys.IsAllowed(args) {
t.Fatal("explicit regrant rejected")
}
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
mustIAM(t, sys.store.loadIAMConfig(ctx, &gi, getGroupInfoPath("grant-group")))
if !gi.MemberGrants["grant-alice"].Equal(freshAt) {
t.Fatal("new grant version was not persisted")
}
if !gi.MemberGrants["grant-bob"].Equal(origin.Add(4 * time.Minute)) {
t.Fatal("regranting Alice changed Bob's grant")
}
}
func TestIAMGroupRevocationCommitAndRecreation(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) { testIAMGroupRevocationCommitAndRecreation(t, backend) })
}
}
func testIAMGroupRevocationCommitAndRecreation(t *testing.T, backend string) {
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
origin := UTCNow().Add(-time.Hour)
user, group := "group-boundary-user", "group-boundary"
_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
grant, boundary := origin.Add(time.Minute), origin.Add(2*time.Minute)
_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
mustIAM(t, err)
// A newer mapping must not veto the authoritative group deletion.
_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin.Add(3*time.Minute)), group, "readwrite", regUser, true)
mustIAM(t, err)
_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, boundary), group, nil)
mustIAM(t, err)
r, err := loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
mustIAM(t, err)
if !r.Deleted || !r.RevokedBefore.Equal(boundary) {
t.Fatal("newer mapping swallowed group deletion")
}
_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, origin.Add(4*time.Minute)), group, nil)
mustIAM(t, err)
for _, at := range []time.Time{grant, boundary, {}} {
item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, UpdatedAt: origin.Add(5 * time.Minute), GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}}, GroupSnapshot: true, GroupGrants: map[string]time.Time{user: at}}
mustIAM(t, applyIAMReplicationItem(ctx, item))
gd, err := sys.GetGroupDescription(group)
mustIAM(t, err)
if len(gd.Members) != 0 {
t.Fatalf("group recreation restored grant %s", at)
}
}
_, err = sys.AddUsersToGroup(ctx, group, []string{user})
mustIAM(t, err)
args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
if !sys.IsAllowed(args) {
t.Fatal("explicit group regrant was rejected")
}
// The newer live snapshot may arrive before an older group deletion.
lateBoundary := origin.Add(6 * time.Minute)
_, err = sys.RemoveUsersFromGroup(withIAMReplicationTime(ctx, lateBoundary), group, nil)
mustIAM(t, err)
r, err = loadIAMRevision(ctx, sys.store, getGroupInfoPath(group))
mustIAM(t, err)
if r.Deleted || !r.RevokedBefore.Equal(lateBoundary) {
t.Fatal("late deletion lost the live group's revocation boundary")
}
// The old mapping is now revoked; a new explicit mapping restores access.
if sys.IsAllowed(args) {
t.Fatal("late group boundary retained an old mapping")
}
_, err = sys.PolicyDBSet(ctx, group, "readwrite", regUser, true)
mustIAM(t, err)
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
if es, ok := sys.store.IAMStorageAPI.(*IAMEtcdStore); ok {
store.IAMStorageAPI = newIAMEtcdStore(es.client, MinIOUsersSysType)
}
mustIAM(t, store.LoadIAMCache(ctx, true))
fresh := &IAMSys{store: store, usersSysType: MinIOUsersSysType}
if !fresh.IsAllowed(args) {
t.Fatal("reload lost explicit grants after a retained group boundary")
}
item, err := globalSiteReplicationSys.replicationItem(ctx, madmin.SRIAMItem{Type: madmin.SRIAMItemGroupInfo, GroupInfo: &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group}}, UpdatedAt: r.timestamp()})
mustIAM(t, err)
if !item.RevokedBefore.Equal(lateBoundary) || !item.GroupGrants[user].After(lateBoundary) {
t.Fatal("group snapshot lost revision metadata")
}
}
// A committed revision is observable before all cached dependents have been
// cleaned up. Every authorization read must apply that boundary in this window.
func TestIAMCachedMappingHonorsCommittedRevision(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
origin := UTCNow().Add(-time.Hour)
parent := "cached-external-parent"
_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), parent, "readwrite", stsUser, false)
mustIAM(t, err)
policies, err := sys.PolicyDBGet(parent)
mustIAM(t, err)
if len(policies) == 0 {
t.Fatal("fixture has no STS-parent mapping")
}
mustIAM(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Deleted: true, UpdatedAt: origin.Add(time.Minute)}, getMappedPolicyPath(parent, stsUser, false)))
policies, err = sys.PolicyDBGet(parent)
mustIAM(t, err)
if len(policies) != 0 {
t.Fatal("cached STS mapping ignored its own namespace tombstone")
}
user, group := "cached-group-user", "cached-group"
_, err = sys.CreateUser(withIAMReplicationTime(ctx, origin), user, madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
grant := origin.Add(5 * time.Minute)
_, err = sys.AddUsersToGroup(withIAMReplicationTime(ctx, grant), group, []string{user})
mustIAM(t, err)
_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), group, "readwrite", regUser, true)
mustIAM(t, err)
args := policy.Args{AccountName: user, Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}
if !sys.IsAllowed(args) {
t.Fatal("fixture has no group grant")
}
// A late deletion preserves the newer member grant but revokes the older
// policy mapping. Simulate the interval before mapping cleanup completes.
gi := GroupInfo{Version: 1, Status: statusEnabled, Members: []string{user}, MemberGrants: map[string]time.Time{user: grant}, UpdatedAt: grant, RevokedBefore: origin.Add(2 * time.Minute)}
mustIAM(t, sys.store.saveIAMConfig(ctx, &gi, getGroupInfoPath(group)))
if sys.IsAllowed(args) {
t.Fatal("cached group mapping ignored the committed group boundary")
}
gd, err := sys.GetGroupDescription(group)
mustIAM(t, err)
if gd.Policy != "" {
t.Fatal("group listing exposed a revoked mapping")
}
}
type (
iamExpiryLockFailure struct {
ObjectLayer
path string
}
iamFailedExpiryLock struct{ RWLocker }
)
func (o *iamExpiryLockFailure) NewNSLock(bucket string, objects ...string) RWLocker {
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path+".revision-lock" {
return &iamFailedExpiryLock{RWLocker: lock}
}
return lock
}
func (l *iamFailedExpiryLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) {
return LockContext{}, errIAMInjectedWrite
}
func TestIAMExpiredCredentialCleanupDoesNotBlockLoading(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
_, err := sys.CreateUser(ctx, "healthy-user", madmin.AddOrUpdateUserReq{SecretKey: "healthy-user-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
_, err = sys.PolicyDBSet(ctx, "healthy-user", "readwrite", regUser, false)
mustIAM(t, err)
c, _, err := sys.NewServiceAccount(ctx, "healthy-user", nil, newServiceAccountOpts{accessKey: "expired-service", secretKey: "expired-service-password"})
mustIAM(t, err)
c.Expiration = UTCNow().Add(-time.Hour)
path := getUserIdentityPath(c.AccessKey, svcUser)
mustIAM(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: c, UpdatedAt: UTCNow()}, path))
// A cold loader sees the existing version but cannot acquire the cleanup
// write lock. Healthy users must still load; the expired one stays denied.
fresh := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(&iamExpiryLockFailure{ObjectLayer: obj, path: path}, MinIOUsersSysType)}
mustIAM(t, fresh.LoadIAMCache(ctx, true))
if _, ok := fresh.GetUser("healthy-user"); !ok {
t.Fatal("cleanup failure prevented healthy IAM state from loading")
}
if _, ok := fresh.GetUser(c.AccessKey); ok {
t.Fatal("cleanup failure admitted an expired service account")
}
r, err := loadIAMRevision(ctx, fresh, path)
mustIAM(t, err)
if r.Deleted || !r.Credentials.IsExpired() {
t.Fatal("failed cleanup lost the existing expired revision")
}
}
cmd/iam-revision-index.go
+220
View File
@@ -0,0 +1,220 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"encoding/json"
"fmt"
"maps"
"strings"
"sync"
"time"
"github.com/minio/minio/internal/auth"
)
// This index is rebuilt by the existing IAM loaders and updated by successful
// storage operations. It avoids a second full IAM walk during every heal pass.
// It is an optimization of the durable records, never a reason to delete them.
// The index contains no secrets or grants.
type iamParentRevision struct {
deleted bool
before time.Time
}
type iamRevisionIndex struct {
mu sync.RWMutex
items map[string]iamRevision
parents map[string]iamParentRevision
floors map[string]time.Time
generation uint64
}
func (idx *iamRevisionIndex) observe(path string, data []byte) {
if !strings.HasPrefix(path, iamConfigPrefix+"/") {
return
}
var r iamRevision
if json.Unmarshal(data, &r) != nil {
return // The caller reports malformed data using its normal decoder.
}
r.Credentials = auth.Credentials{ParentUser: r.Credentials.ParentUser, Expiration: r.Credentials.Expiration}
idx.mu.Lock()
defer idx.mu.Unlock()
if strings.HasPrefix(path, iamConfigUsersPrefix) {
// Keep a compact name-keyed view for the authentication hot path;
// constructing a config path on every S3 request allocates needlessly.
defer func() {
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
if current, ok := idx.items[path]; ok {
if idx.parents == nil {
idx.parents = make(map[string]iamParentRevision)
}
idx.parents[name] = iamParentRevision{deleted: current.Deleted, before: current.RevokedBefore}
} else {
delete(idx.parents, name)
}
}()
}
if floor, ok := idx.floors[path]; ok && r.timestamp().Before(floor) {
return
}
if previous, ok := idx.items[path]; ok {
// A concurrent read that began before a write must not roll it back.
if previous.timestamp().After(r.timestamp()) || (previous.Deleted && !r.Deleted && !r.timestamp().After(previous.timestamp())) {
return
}
if previous.RevokedBefore.After(r.RevokedBefore) {
r.RevokedBefore = previous.RevokedBefore
}
if previous.timestamp().Equal(r.timestamp()) && previous.Deleted == r.Deleted && previous.RevokedBefore.Equal(r.RevokedBefore) {
return
}
}
if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
if _, tracked := idx.items[path]; tracked {
delete(idx.items, path)
idx.generation++
}
delete(idx.floors, path)
return
}
if !r.Deleted && r.RevokedBefore.IsZero() {
_, tracked := idx.items[path]
_, hasFloor := idx.floors[path]
if tracked || hasFloor {
if idx.floors == nil {
idx.floors = make(map[string]time.Time)
}
idx.floors[path] = r.timestamp()
}
if tracked {
delete(idx.items, path)
idx.generation++
}
return
}
if idx.items == nil {
idx.items = make(map[string]iamRevision)
}
idx.items[path] = r
delete(idx.floors, path)
idx.generation++
}
func (idx *iamRevisionIndex) get(path string) iamRevision {
if idx == nil {
return iamRevision{}
}
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.items[path]
}
func (idx *iamRevisionIndex) snapshot() map[string]iamRevision {
idx.mu.Lock()
defer idx.mu.Unlock()
for path, r := range idx.items {
if r.Deleted && !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
delete(idx.items, path)
delete(idx.floors, path)
idx.generation++
}
}
return maps.Clone(idx.items)
}
func (idx *iamRevisionIndex) count() int {
idx.mu.RLock()
defer idx.mu.RUnlock()
return len(idx.items)
}
// A process-local generation plus the protocol's instance ID is sufficient
// for acknowledgements. Avoid hashing the entire index on every IAM write.
func (idx *iamRevisionIndex) digest() string {
idx.mu.RLock()
defer idx.mu.RUnlock()
return fmt.Sprintf("%x:%x", idx.generation, len(idx.items))
}
func (idx *iamRevisionIndex) forget(path string) {
idx.mu.Lock()
if _, ok := idx.items[path]; ok {
delete(idx.items, path)
idx.generation++
}
delete(idx.floors, path)
if strings.HasPrefix(path, iamConfigUsersPrefix) {
delete(idx.parents, strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile))
}
idx.mu.Unlock()
}
func (c *iamCache) userRevocation(user string) iamRevision {
r := c.revisions.parentRevision(user)
if u, ok := c.iamUsersMap[user]; ok && u.RevokedBefore.After(r.RevokedBefore) {
r.RevokedBefore = u.RevokedBefore
}
return r
}
func (c *iamCache) groupMemberAllowed(member string, grantedAt, groupBoundary time.Time) bool {
r := c.userRevocation(member)
return !r.Deleted && (r.RevokedBefore.IsZero() || grantedAt.After(r.RevokedBefore)) && (groupBoundary.IsZero() || grantedAt.After(groupBoundary))
}
func iamMappingParentPath(path string) string {
kind, name, ok := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
if !ok {
return ""
}
name = strings.TrimSuffix(name, ".json")
switch kind {
case "users", "sts-users":
return getUserIdentityPath(name, regUser)
case "service-accounts":
return getUserIdentityPath(name, svcUser)
case "groups":
return getGroupInfoPath(name)
}
return ""
}
func (idx *iamRevisionIndex) mappingAllowed(path string, mp MappedPolicy) bool {
if mp.Deleted || idx.get(path).Deleted {
return false
}
r := idx.get(iamMappingParentPath(path))
return !r.Deleted && (r.RevokedBefore.IsZero() || mp.UpdatedAt.After(r.RevokedBefore))
}
// Apply the persisted commit boundary even before dependent cache cleanup has
// completed. The map namespace is part of the authorization record's identity.
func (c *iamCache) cachedMappedPolicy(name string, userType IAMUserType, isGroup bool) (MappedPolicy, bool) {
var mp MappedPolicy
var ok bool
switch {
case isGroup:
mp, ok = c.iamGroupPolicyMap.Load(name)
case userType == stsUser:
mp, ok = c.iamSTSPolicyMap.Load(name)
default:
mp, ok = c.iamUserPolicyMap.Load(name)
}
if !ok || !c.revisions.mappingAllowed(getMappedPolicyPath(name, userType, isGroup), mp) {
return MappedPolicy{}, false
}
return mp, true
}
func (idx *iamRevisionIndex) parentRevision(user string) iamRevision {
if idx == nil {
return iamRevision{}
}
idx.mu.RLock()
p := idx.parents[user]
idx.mu.RUnlock()
return iamRevision{Deleted: p.deleted, RevokedBefore: p.before}
}
cmd/iam-revision-lock_test.go
+305
View File
@@ -0,0 +1,305 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"crypto/sha256"
"fmt"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/minio/madmin-go/v3"
etcd "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
"go.etcd.io/etcd/client/v3/namespace"
)
type iamRevisionLockObserver struct {
ObjectLayer
path string
waiting chan struct{}
once sync.Once
}
func (o *iamRevisionLockObserver) NewNSLock(bucket string, objects ...string) RWLocker {
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
if bucket == minioMetaBucket && len(objects) == 1 && objects[0] == o.path {
return &iamRevisionObservedLock{RWLocker: lock, observe: func() { o.once.Do(func() { close(o.waiting) }) }}
}
return lock
}
type iamRevisionObservedLock struct {
RWLocker
observe func()
}
func (l *iamRevisionObservedLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
l.observe()
return l.RWLocker.GetLock(ctx, timeout)
}
type iamRevisionWatchObserver struct {
etcd.Watcher
waiting chan struct{}
once sync.Once
}
func (w *iamRevisionWatchObserver) Watch(ctx context.Context, key string, opts ...etcd.OpOption) etcd.WatchChan {
w.once.Do(func() { close(w.waiting) })
return w.Watcher.Watch(ctx, key, opts...)
}
// Simulate an unavailable cleanup RPC. Mutex.Lock calls Delete after its wait
// is canceled; that RPC must inherit a deadline too, not Client.Ctx() forever.
type iamRevisionCleanupBlocker struct {
etcd.KV
release chan struct{}
}
func (b *iamRevisionCleanupBlocker) Delete(ctx context.Context, key string, opts ...etcd.OpOption) (*etcd.DeleteResponse, error) {
if strings.Contains(key, "/iam-revision-locks/") {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-b.release:
}
}
return b.KV.Delete(ctx, key, opts...)
}
type iamRevisionReadBlocker struct {
IAMStorageAPI
path string
after int
waiting chan struct{}
}
func (b *iamRevisionReadBlocker) loadIAMConfig(ctx context.Context, item any, path string) error {
if path == b.path {
b.after--
if b.after == 0 {
close(b.waiting)
<-ctx.Done()
return ctx.Err()
}
}
return b.IAMStorageAPI.loadIAMConfig(ctx, item, path)
}
func TestIAMRevisionReadDoesNotBlockAuthentication(t *testing.T) {
for _, stage := range []struct {
name string
offset time.Duration
}{{"deletion", time.Minute}, {"retained_revocation", -time.Minute}} {
t.Run(stage.name, func(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
disks, err := getRandomDisks(1)
if err != nil {
t.Fatal(err)
}
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
obj.Shutdown(context.Background())
os.RemoveAll(disks[0])
})
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
const user = "read-blocked-parent"
created, err := store.AddUser(ctx, user, madmin.AddOrUpdateUserReq{SecretKey: "original-password", Status: madmin.AccountEnabled})
if err != nil {
t.Fatal(err)
}
blocked := &iamRevisionReadBlocker{IAMStorageAPI: store.IAMStorageAPI, path: getUserIdentityPath(user, regUser), after: 1, waiting: make(chan struct{})}
store.IAMStorageAPI = blocked
done := make(chan error, 1)
go func() {
done <- store.DeleteUser(withIAMReplicationTime(ctx, created.Add(stage.offset)), user, regUser)
}()
defer func() { cancel(); <-done }()
select {
case <-blocked.waiting:
case <-time.After(5 * time.Second):
t.Fatal("revision read was not attempted")
}
read := make(chan bool, 1)
go func() {
u, ok := store.GetUser(user)
read <- ok && u.Credentials.SecretKey == "original-password"
}()
select {
case ok := <-read:
if !ok {
t.Fatal("pending revision read changed the cached identity")
}
case <-time.After(time.Second):
t.Fatal("revision read blocked cached authentication")
}
})
}
}
func TestIAMRevisionLockContention(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
if backend == "etcd" && endpoint == "" {
t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
}
for _, outcome := range []string{"release", "cancel", "default_timeout"} {
t.Run(outcome, func(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
oldTimeout := defaultContextTimeout
defaultContextTimeout = 2 * time.Second
t.Cleanup(func() { defaultContextTimeout = oldTimeout })
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
const user = "contended-user"
path := getUserIdentityPath(user, regUser)
waiting := make(chan struct{})
var store *IAMStoreSys
var hold func() func()
unblockCleanup := func() {}
if backend == "object" {
disks, err := getRandomDisks(1)
must(err)
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
must(err)
t.Cleanup(func() {
obj.Shutdown(context.Background())
os.RemoveAll(disks[0])
})
observed := &iamRevisionLockObserver{ObjectLayer: obj, path: path + ".revision-lock", waiting: waiting}
store = &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
hold = func() func() {
lock := obj.NewNSLock(minioMetaBucket, observed.path)
lc, err := lock.GetLock(ctx, newDynamicTimeout(time.Second, time.Second))
must(err)
store.IAMStorageAPI.(*IAMObjectStore).objAPI = observed
return func() { lock.Unlock(lc) }
}
} else {
client, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: time.Second})
must(err)
t.Cleanup(func() { client.Close() })
prefix := fmt.Sprintf("/silo-lock-test/%d/", time.Now().UnixNano())
client.KV = namespace.NewKV(client.KV, prefix)
client.Watcher = namespace.NewWatcher(client.Watcher, prefix)
store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(client, MinIOUsersSysType)}
hold = func() func() {
session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
must(err)
lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
must(lock.Lock(ctx))
client.Watcher = &iamRevisionWatchObserver{Watcher: client.Watcher, waiting: waiting}
blocker := &iamRevisionCleanupBlocker{KV: client.KV, release: make(chan struct{})}
client.KV = blocker
unblockCleanup = sync.OnceFunc(func() { close(blocker.release) })
t.Cleanup(unblockCleanup)
return func() { session.Close() }
}
}
request := func(secret string) madmin.AddOrUpdateUserReq {
return madmin.AddOrUpdateUserReq{SecretKey: secret, Status: madmin.AccountEnabled}
}
_, err := store.AddUser(ctx, user, request("original-password"))
must(err)
release := sync.OnceFunc(hold())
t.Cleanup(release)
writeCtx, cancelWrite := context.WithCancel(ctx)
defer cancelWrite()
first, second := make(chan error, 1), make(chan error, 1)
var writers sync.WaitGroup
t.Cleanup(func() {
cancelWrite()
unblockCleanup()
release()
writers.Wait()
})
writers.Go(func() {
_, err := store.AddUser(writeCtx, user, request("first-password"))
first <- err
})
select {
case <-waiting:
case <-time.After(5 * time.Second):
t.Fatal("writer did not attempt the held revision lock")
}
// A second writer must queue without taking the cache's RWMutex:
// Go's writer preference would otherwise block every new reader.
writers.Go(func() {
_, err := store.AddUser(ctx, user, request("second-password"))
second <- err
})
select {
case err := <-second:
t.Fatalf("second writer bypassed the first: %v", err)
case <-time.After(50 * time.Millisecond):
}
read := make(chan UserIdentity, 1)
go func() {
u, _ := store.GetUser(user)
read <- u
}()
select {
case u := <-read:
if u.Credentials.SecretKey != "original-password" {
t.Fatal("pending write changed the cached credential")
}
case <-time.After(time.Second):
t.Fatal("distributed lock contention blocked cached authentication")
}
switch outcome {
case "release":
release()
case "cancel":
cancelWrite()
}
select {
case err := <-first:
if outcome == "release" {
must(err)
} else if err == nil {
t.Fatal("canceled or timed-out write succeeded")
}
case <-time.After(5 * time.Second):
t.Fatal("lock wait or cancellation cleanup exceeded its deadline")
}
release()
select {
case err := <-second:
must(err)
case <-time.After(5 * time.Second):
t.Fatal("queued writer did not recover after the first completed")
}
cached, ok := store.GetUser(user)
if !ok || cached.Credentials.SecretKey != "second-password" {
t.Fatal("cached write order was lost")
}
var persisted UserIdentity
must(store.loadIAMConfig(ctx, &persisted, path))
if persisted.Credentials.SecretKey != cached.Credentials.SecretKey || !persisted.UpdatedAt.Equal(cached.UpdatedAt) {
t.Fatal("persistent and cached revisions differ")
}
})
}
})
}
}
cmd/iam-revision.go
+749
View File
@@ -0,0 +1,749 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"net/http"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
etcd "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
)
var errIAMStaleUpdate = errors.New("IAM update predates a stored revision or revocation")
// The parent is still live; callers must not broadcast a user deletion when
// only its revocation boundary was retained.
var errIAMRevocationRetained = errors.New("IAM revocation recorded without deleting the record")
// A revocation advances the boundary even when a newer identity already
// exists. Keep this operation distinct from replacing/deleting that identity.
type iamUserRevocation struct {
UserIdentity
retained bool
}
type iamGroupRevocation struct {
GroupInfo
retained bool
requireEmpty bool
}
// Natural expiration is distinct from revoking a live credential. An expired
// immutable STS token can be removed; a reusable service-account key retains
// its revision so an older non-expiring credential cannot return.
type iamExpireIdentity struct{}
// The authoritative revocation is durable even if dependent cleanup fails.
// Callers must publish it to sibling caches before returning the error.
type iamCommittedCleanupError struct {
err error
retained bool
}
func (e *iamCommittedCleanupError) Error() string {
return "IAM revocation committed; cleanup failed: " + e.err.Error()
}
func (e *iamCommittedCleanupError) Unwrap() error { return e.err }
type iamReplicationTimeKey struct{}
func withIAMReplicationTime(ctx context.Context, at time.Time) context.Context {
return context.WithValue(ctx, iamReplicationTimeKey{}, at)
}
func iamReplicationTime(ctx context.Context) (time.Time, bool) {
at, ok := ctx.Value(iamReplicationTimeKey{}).(time.Time)
return at, ok
}
func iamReplicationError(err error) error {
if errors.Is(err, errIAMStaleUpdate) {
// Retrying an obsolete event cannot change the result.
return nil
}
return wrapSRErr(err)
}
// Deletions occupy the original IAM config path. They contain no secret or
// grant and are hidden by the normal loaders, but remain available to heal
// and to timestamp comparisons after a restart. Do not age them out: a peer
// can be offline indefinitely.
type iamRevision struct {
UpdatedAt time.Time `json:"updatedAt"`
UpdateDate time.Time `json:"UpdateDate"`
Deleted bool `json:"deleted"`
RevokedBefore time.Time `json:"revokedBefore"`
ExpiresAt time.Time `json:"expiresAt,omitempty"`
Credentials auth.Credentials `json:"credentials"`
}
func (r iamRevision) timestamp() time.Time {
if r.UpdateDate.After(r.UpdatedAt) {
return r.UpdateDate
}
return r.UpdatedAt
}
func loadIAMRevision(ctx context.Context, store IAMStorageAPI, path string) (iamRevision, error) {
var r iamRevision
err := store.loadIAMConfig(ctx, &r, path)
if errors.Is(err, errConfigNotFound) {
err = nil
}
return r, err
}
func (store *IAMStoreSys) checkIAMRevision(ctx context.Context, path string, deleting bool) error {
at, replicated := iamReplicationTime(ctx)
if !replicated {
return nil
}
return store.withIAMStorage(ctx, func(ctx context.Context) error {
r, err := loadIAMRevision(ctx, store.IAMStorageAPI, path)
if err != nil {
return err
}
if r.timestamp().After(at) || (r.Deleted && !deleting && !at.After(r.timestamp())) {
return errIAMStaleUpdate
}
return nil
})
}
// This signed claim records the parent's revocation boundary at issuance.
// Unlike UpdatedAt, it cannot advance when an offline site edits an old child.
// It travels in the existing service-account Claims and STS SessionToken fields.
const iamParentRevocationClaim = "siloParentRevocation"
func setIAMParentRevocationClaim(ctx context.Context, store IAMStorageAPI, parent string, claims map[string]any) error {
delete(claims, iamParentRevocationClaim)
if parent == "" || parent == globalActiveCred.AccessKey {
return nil
}
r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
if err != nil {
return err
}
if r.Deleted {
return errIAMStaleUpdate
}
if !r.RevokedBefore.IsZero() {
claims[iamParentRevocationClaim] = r.RevokedBefore.Format(time.RFC3339Nano)
}
return nil
}
func iamCredentialSurvivesRevocation(cred auth.Credentials, at time.Time) bool {
if at.IsZero() {
return true
}
s, _ := cred.Claims[iamParentRevocationClaim].(string)
issuedAfter, err := time.Parse(time.RFC3339Nano, s)
return err == nil && !issuedAfter.Before(at)
}
// Parent revocations delete old children even if an offline peer has edited
// them later. Preserve children that prove issuance after this revocation.
func iamChildDeletionContext(ctx context.Context, child UserIdentity) (context.Context, bool) {
if at, replicated := iamReplicationTime(ctx); replicated {
if !at.IsZero() && iamCredentialSurvivesRevocation(child.Credentials, at) {
return ctx, false
}
if child.UpdatedAt.After(at) {
ctx = withIAMReplicationTime(ctx, child.UpdatedAt)
}
}
return ctx, true
}
// A delayed service account or STS event must not outlive deletion of its
// built-in parent. The caller must populate Claims from the verified token.
func checkIAMParentRevision(ctx context.Context, store IAMStorageAPI, cred auth.Credentials) error {
parent := cred.ParentUser
if parent == "" || parent == globalActiveCred.AccessKey {
return nil
}
r, err := loadIAMRevision(ctx, store, getUserIdentityPath(parent, regUser))
if err != nil {
return err
}
if r.Deleted || !iamCredentialSurvivesRevocation(cred, r.RevokedBefore) {
return errIAMStaleUpdate
}
return nil
}
// Called with the IAM writer mutex and cache lock held. Persistence only
// touches the caller's record, not the cache. Keep writers serialized while
// allowing cached authentication reads throughout storage and lock waits.
func (store *IAMStoreSys) withIAMStorage(ctx context.Context, fn func(context.Context) error) error {
store.IAMStorageAPI.unlock()
defer store.IAMStorageAPI.lock()
ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
return fn(ctx)
}
func (store *IAMStoreSys) saveIAMRevision(ctx context.Context, path string, item any, opts ...options) error {
return store.withIAMStorage(ctx, func(ctx context.Context) error {
return saveIAMRevision(ctx, store.IAMStorageAPI, path, item, opts...)
})
}
func (store *IAMStoreSys) checkIAMParentRevision(ctx context.Context, cred auth.Credentials) error {
return store.withIAMStorage(ctx, func(ctx context.Context) error {
return checkIAMParentRevision(ctx, store.IAMStorageAPI, cred)
})
}
// Update the caller's record with the persisted revision before it is cached.
func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item any, opts ...options) error {
ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
// Serialize compare-and-write across nodes, as well as goroutines. Use a
// separate lock name so saving the config does not reacquire this lock.
switch s := store.(type) {
case *IAMObjectStore:
lock := s.objAPI.NewNSLock(minioMetaBucket, path+".revision-lock")
lc, err := lock.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
}
defer lock.Unlock(lc)
ctx = lc.Context()
case *IAMEtcdStore:
// Mutex.Lock also uses Client.Ctx() for cleanup after cancellation.
// Borrow the existing services with the operation's bounded context;
// never close this facade, which does not own those services.
client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(s.client.GetLogger()))
client.KV, client.Lease, client.Watcher = s.client.KV, s.client.Lease, s.client.Watcher
session, err := concurrency.NewSession(client, concurrency.WithContext(ctx))
if err != nil {
return err
}
defer func() {
session.Orphan()
// A canceled operation must still release its lease when etcd is
// reachable. If it is unavailable, stop waiting and let it expire.
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), defaultContextTimeout)
defer cancel()
_, _ = s.client.Revoke(cleanupCtx, session.Lease())
}()
lock := concurrency.NewMutex(session, fmt.Sprintf("%s/iam-revision-locks/%x", minioConfigPrefix, sha256.Sum256([]byte(path))))
if err = lock.Lock(ctx); err != nil {
return err
}
// Revoking the session lease releases the lock, including on cancellation.
}
previous, err := loadIAMRevision(ctx, store, path)
if err != nil {
return err
}
if _, expiring := item.(*iamExpireIdentity); expiring {
sts := strings.HasPrefix(path, iamConfigSTSPrefix)
if previous.Deleted {
if sts && !previous.ExpiresAt.IsZero() && UTCNow().After(previous.ExpiresAt) {
return expireIAMSTSConfig(ctx, store, path)
}
return nil
}
if previous.timestamp().IsZero() || !previous.Credentials.IsExpired() {
return nil
}
if sts {
return expireIAMSTSConfig(ctx, store, path)
}
item = &UserIdentity{Version: 1, Deleted: true}
ctx = withIAMReplicationTime(ctx, previous.timestamp())
}
var revocation *iamUserRevocation
if op, ok := item.(*iamUserRevocation); ok {
revocation = op
op.UserIdentity = UserIdentity{Version: 1, Deleted: true}
if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
if previous.Deleted || !origin.After(previous.RevokedBefore) {
return errIAMStaleUpdate
}
op.retained = true
op.UserIdentity = UserIdentity{Version: 1, Credentials: previous.Credentials, UpdatedAt: previous.timestamp(), RevokedBefore: origin}
ctx = withIAMReplicationTime(ctx, previous.timestamp())
}
item = &op.UserIdentity
}
var groupRevocation *iamGroupRevocation
if op, ok := item.(*iamGroupRevocation); ok {
groupRevocation = op
var group GroupInfo
if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
return err
}
if op.requireEmpty && !group.Deleted {
for _, member := range group.Members {
r := store.revisionIndex().get(getUserIdentityPath(member, regUser))
at := group.MemberGrants[member]
if !r.Deleted && (r.RevokedBefore.IsZero() || at.After(r.RevokedBefore)) && (group.RevokedBefore.IsZero() || at.After(group.RevokedBefore)) {
return errGroupNotEmpty
}
}
}
op.GroupInfo = GroupInfo{Version: 1, Deleted: true}
if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
if previous.Deleted || !origin.After(previous.RevokedBefore) {
return errIAMStaleUpdate
}
op.retained = true
op.GroupInfo = group
op.RevokedBefore = origin
ctx = withIAMReplicationTime(ctx, previous.timestamp())
}
item = &op.GroupInfo
}
var at *time.Time
var deleted bool
switch v := item.(type) {
case *UserIdentity:
at, deleted = &v.UpdatedAt, v.Deleted
if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
v.RevokedBefore = boundary
}
if previous.RevokedBefore.After(v.RevokedBefore) {
v.RevokedBefore = previous.RevokedBefore
}
case *GroupInfo:
at, deleted = &v.UpdatedAt, v.Deleted
if boundary, ok := ctx.Value(iamRecordBoundaryKey{}).(time.Time); ok && boundary.After(v.RevokedBefore) {
v.RevokedBefore = boundary
}
if !deleted {
var group GroupInfo
if err := store.loadIAMConfig(ctx, &group, path); err != nil && !errors.Is(err, errConfigNotFound) {
return err
}
mergeIAMGroupMutation(ctx, group, v)
}
if previous.RevokedBefore.After(v.RevokedBefore) {
v.RevokedBefore = previous.RevokedBefore
}
case *MappedPolicy:
at, deleted = &v.UpdatedAt, v.Deleted
case *PolicyDoc:
at, deleted = &v.UpdateDate, v.Deleted
default:
return errInvalidArgument
}
if strings.HasPrefix(path, iamConfigSTSPrefix) && previous.Deleted && !deleted {
// STS access keys identify immutable tokens, not reusable user names.
return errIAMStaleUpdate
}
if origin, replicated := iamReplicationTime(ctx); replicated {
*at = origin
if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
return errIAMStaleUpdate
}
if strings.HasPrefix(path, iamConfigServiceAccountsPrefix) && !deleted && previous.Credentials.AccessKey != "" && previous.timestamp().Equal(origin) {
// Duplicate service snapshots are acknowledgements, not new creates
// or edits. Reload the winner without writing, so even a stale
// sibling cache is refreshed by the retry before acknowledging it.
return store.loadIAMConfig(ctx, item, path)
}
if previous.Deleted && deleted && !origin.After(previous.timestamp()) {
// An already-applied tombstone needs no further persistent write.
if v, ok := item.(*UserIdentity); ok {
v.RevokedBefore = previous.RevokedBefore
}
if v, ok := item.(*GroupInfo); ok {
v.RevokedBefore = previous.RevokedBefore
}
return nil
}
} else {
if previous.Deleted && deleted {
// A peer notification without an originating revision must not
// advance a tombstone past a subsequent deliberate recreation.
*at = previous.timestamp()
if v, ok := item.(*UserIdentity); ok {
v.RevokedBefore = previous.RevokedBefore
}
if v, ok := item.(*GroupInfo); ok {
v.RevokedBefore = previous.RevokedBefore
}
return nil
}
if at.IsZero() {
*at = UTCNow()
}
if !at.After(previous.timestamp()) {
*at = previous.timestamp().Add(time.Nanosecond)
}
}
if v, ok := item.(*UserIdentity); ok {
if deleted {
// Retain only the parent name for root-account exclusion during heal.
v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
v.RevokedBefore = *at
if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) {
// The signed STS token cannot authorize beyond this time, even
// if an offline site replays it with a newer event timestamp.
v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime)
opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
}
} else {
if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
claims, err := extractJWTClaims(*v)
if err != nil {
return err
}
v.Credentials.Claims = claims.Map()
}
if err = checkIAMParentRevision(ctx, store, v.Credentials); err != nil {
return err
}
}
}
if v, ok := item.(*GroupInfo); ok && deleted {
v.RevokedBefore = *at
v.Members, v.MemberGrants = nil, nil
}
if _, ok := item.(*MappedPolicy); ok && !deleted {
if parentPath := iamMappingParentPath(path); parentPath != "" {
parent, err := loadIAMRevision(ctx, store, parentPath)
if err != nil {
return err
}
if parent.Deleted {
return errIAMStaleUpdate
}
if !parent.RevokedBefore.IsZero() && !at.After(parent.RevokedBefore) {
if _, replicated := iamReplicationTime(ctx); replicated {
return errIAMStaleUpdate
}
*at = parent.RevokedBefore.Add(time.Nanosecond)
}
}
}
if err := store.saveIAMConfig(ctx, item, path, opts...); err != nil {
return err
}
if revocation != nil && revocation.retained {
return errIAMRevocationRetained
}
if groupRevocation != nil && groupRevocation.retained {
return errIAMRevocationRetained
}
return nil
}
func (iamOS *IAMObjectStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var paths []string
for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+"/") {
if item.Err != nil {
return nil, item.Err
}
paths = append(paths, iamConfigPrefix+"/"+item.Item)
}
return paths, nil
}
func (ies *IAMEtcdStore) listIAMConfigPaths(ctx context.Context) ([]string, error) {
ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
r, err := ies.client.Get(ctx, iamConfigPrefix+"/", etcd.WithPrefix(), etcd.WithKeysOnly())
if err != nil {
return nil, err
}
paths := make([]string, 0, len(r.Kvs))
for _, kv := range r.Kvs {
paths = append(paths, string(kv.Key))
}
return paths, nil
}
func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool) {
if (strings.HasPrefix(path, iamConfigUsersPrefix) || strings.HasPrefix(path, iamConfigGroupsPrefix)) && !r.RevokedBefore.IsZero() {
// Recreating a parent does not cancel its older revocation of derived
// credentials. Replay this boundary even after the parent is live again.
r.Deleted = true
r.UpdatedAt, r.UpdateDate = r.RevokedBefore, time.Time{}
}
if !r.Deleted {
return item, false
}
item.UpdatedAt = r.timestamp()
switch {
case strings.HasPrefix(path, iamConfigUsersPrefix):
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
item.Type = madmin.SRIAMItemIAMUser
item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
return item, false
}
item.Type = madmin.SRIAMItemSvcAcc
item.SvcAccChange = &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: name}}
case strings.HasPrefix(path, iamConfigGroupsPrefix):
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigGroupsPrefix), "/"+iamGroupMembersFile)
item.Type = madmin.SRIAMItemGroupInfo
item.GroupInfo = &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: name, IsRemove: true}}
case strings.HasPrefix(path, iamConfigPoliciesPrefix):
item.Type = madmin.SRIAMItemPolicy
item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigPoliciesPrefix), "/"+iamPolicyFile)
case strings.HasPrefix(path, iamConfigPolicyDBPrefix):
prefix, name, found := strings.Cut(strings.TrimPrefix(path, iamConfigPolicyDBPrefix), "/")
if !found {
return item, false
}
typ := regUser
switch prefix {
case "sts-users":
typ = stsUser
case "service-accounts":
typ = svcUser
}
item.Type = madmin.SRIAMItemPolicyMapping
item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
default:
// Expired STS credentials are not replayed. Parent revocations and
// their retained timestamp reject delayed copies of derived tokens.
return item, false
}
return item, true
}
func iamDeletionPath(item madmin.SRIAMItem) string {
switch item.Type {
case madmin.SRIAMItemIAMUser:
if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
}
case madmin.SRIAMItemSvcAcc:
if item.SvcAccChange != nil && item.SvcAccChange.Delete != nil {
return getUserIdentityPath(item.SvcAccChange.Delete.AccessKey, svcUser)
}
case madmin.SRIAMItemGroupInfo:
if item.GroupInfo != nil && item.GroupInfo.UpdateReq.IsRemove && len(item.GroupInfo.UpdateReq.Members) == 0 {
return getGroupInfoPath(item.GroupInfo.UpdateReq.Group)
}
case madmin.SRIAMItemPolicy:
if len(item.Policy) == 0 {
return getPolicyDocPath(item.Name)
}
case madmin.SRIAMItemPolicyMapping:
if p := item.PolicyMapping; p != nil && p.Policy == "" {
return getMappedPolicyPath(p.UserOrGroup, IAMUserType(p.UserType), p.IsGroup)
}
}
return ""
}
func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
started := time.Now()
defer func() {
c.iamRevisionMetrics.healDurationMillis.Store(time.Since(started).Milliseconds())
if err != nil {
c.iamRevisionMetrics.healFailures.Add(1)
} else {
c.iamRevisionMetrics.healLastSuccess.Store(time.Now().Unix())
}
}()
c.iamHealMu.Lock()
defer c.iamHealMu.Unlock()
c.RLock()
defer c.RUnlock()
if !c.enabled {
return nil
}
snapshot := globalIAMSys.store.revisionIndex().snapshot()
paths := make([]string, 0, len(snapshot))
for path := range snapshot {
paths = append(paths, path)
}
sort.Strings(paths)
byType := make(map[string][]iamReplicationItem)
for _, path := range paths {
r := snapshot[path]
item, ok := iamDeletionItem(path, r)
if !ok {
continue
}
out := iamReplicationItem{SRIAMItem: item}
if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
out.Type, out.IAMUser = iamUserBoundaryType, nil
out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
}
if item.Type == madmin.SRIAMItemGroupInfo && !r.Deleted {
out.Type, out.GroupInfo = iamGroupBoundaryType, nil
out.GroupRevocation = &iamGroupBoundary{Group: item.GroupInfo.UpdateReq.Group, Before: r.RevokedBefore}
}
byType[item.Type] = append(byType[item.Type], out)
}
var items []iamReplicationItem
for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
items = append(items, byType[typ]...)
}
if len(items) == 0 {
return nil
}
if c.iamRevisionProgress == nil {
c.iamRevisionProgress = make(map[string]iamRevisionProgress)
}
for id := range c.iamRevisionProgress {
if _, present := c.state.Peers[id]; !present {
delete(c.iamRevisionProgress, id)
}
}
var progressMu sync.Mutex
cerr := c.concDo(nil, func(id string, p madmin.PeerInfo) error {
// Bound each pass, but retain acknowledgements independently of the
// pass deadline or unrelated changes at either site.
peerCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel()
client, err := c.getAdminClient(peerCtx, id)
if err != nil {
return err
}
remote, err := executeIAMRevisionRequest(peerCtx, client, http.MethodGet, nil)
if err != nil {
return err
}
progressMu.Lock()
progress := c.iamRevisionProgress[id]
progressMu.Unlock()
progress.observePeer(remote)
defer func() {
progressMu.Lock()
c.iamRevisionProgress[id] = progress
progressMu.Unlock()
}()
var pending []iamReplicationItem
for _, item := range items {
path, version := iamReplicationMarker(item)
if progress.Acknowledged[path] != version {
pending = append(pending, item)
}
}
// Acknowledgements are only a replay optimization, never GC proof.
for path := range progress.Acknowledged {
if _, retained := snapshot[path]; !retained {
delete(progress.Acknowledged, path)
}
}
var failures []error
for next := 0; next < len(pending); {
end := min(next+maxIAMRevisionBatch, len(pending))
batch := pending[next:end]
remote, err = executeIAMRevisionRequest(peerCtx, client, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: batch})
if err != nil {
var batchErr *iamRevisionBatchError
if !errors.As(err, &batchErr) {
return errors.Join(append(failures, err)...)
}
failures = append(failures, err)
} else {
progress.observePeer(remote)
for _, item := range batch {
path, version := iamReplicationMarker(item)
progress.Acknowledged[path] = version
}
}
next = end
}
return errors.Join(failures...)
}, "IAM revision convergence")
return errors.Unwrap(cerr)
}
func iamReplicationMarker(item iamReplicationItem) (path, version string) {
path = iamDeletionPath(item.SRIAMItem)
if item.UserRevocation != nil {
path = getUserIdentityPath(item.UserRevocation.User, regUser)
}
if item.GroupRevocation != nil {
path = getGroupInfoPath(item.GroupRevocation.Group)
}
return path, item.Type + ":" + item.UpdatedAt.UTC().Format(time.RFC3339Nano)
}
func (store *IAMStoreSys) savePolicyDoc(ctx context.Context, policyName string, p *PolicyDoc) error {
return store.saveIAMRevision(ctx, getPolicyDocPath(policyName), p)
}
func (store *IAMStoreSys) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp *MappedPolicy, opts ...options) error {
return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), mp, opts...)
}
func (store *IAMStoreSys) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u *UserIdentity, opts ...options) error {
return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), u, opts...)
}
func (store *IAMStoreSys) saveGroupInfo(ctx context.Context, name string, gi *GroupInfo) error {
return store.saveIAMRevision(ctx, getGroupInfoPath(name), gi)
}
func (store *IAMStoreSys) deletePolicyDoc(ctx context.Context, name string) error {
return store.saveIAMRevision(ctx, getPolicyDocPath(name), &PolicyDoc{Version: 1, Deleted: true})
}
func (store *IAMStoreSys) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
return store.saveIAMRevision(ctx, getMappedPolicyPath(name, userType, isGroup), &MappedPolicy{Version: 1, Deleted: true})
}
func (store *IAMStoreSys) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
return store.saveIAMRevision(ctx, getUserIdentityPath(name, userType), &UserIdentity{Version: 1, Deleted: true})
}
// Called under the identity's distributed revision lock, after verifying that
// its immutable STS token (or early-revocation retention) has expired. Only the
// old token-key mapping is removed; the reusable parent mapping is unaffected.
func expireIAMSTSConfig(ctx context.Context, store IAMStorageAPI, path string) error {
key := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
if err := store.deleteIAMConfig(ctx, getMappedPolicyPath(key, stsUser, false)); err != nil && !errors.Is(err, errConfigNotFound) {
return err
}
return store.deleteIAMConfig(ctx, path)
}
type (
iamExpirationCleanupKey struct{}
iamExpirationCleanupState struct{ failed atomic.Bool }
)
func withIAMExpirationCleanup(ctx context.Context) context.Context {
if _, ok := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState); ok {
return ctx
}
return context.WithValue(ctx, iamExpirationCleanupKey{}, &iamExpirationCleanupState{})
}
func bestEffortIAMExpiration(ctx context.Context, store IAMStorageAPI, path string) {
state, _ := ctx.Value(iamExpirationCleanupKey{}).(*iamExpirationCleanupState)
if ctx.Err() != nil || (state != nil && state.failed.Load()) {
return
}
ctx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
// Failure leaves the expired record and its existing version intact.
// Stop optional reclamation for this load, while still loading healthy
// users. Healthy cleanup has no per-scan quota that could build a backlog.
if err := saveIAMRevision(ctx, store, path, &iamExpireIdentity{}); err != nil {
if state != nil {
state.failed.Store(true)
}
iamLogIf(ctx, err)
}
}
cmd/iam-revision_test.go
+330
View File
@@ -0,0 +1,330 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"encoding/json"
"errors"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/grid"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// Count physical saves: comparing timestamps alone would miss identical
// tombstones being rewritten on every heal pass.
type iamRevisionWriteCounter struct {
IAMStorageAPI
data []byte
writes int
}
func (s *iamRevisionWriteCounter) loadIAMConfig(_ context.Context, item any, _ string) error {
return json.Unmarshal(s.data, item)
}
func (s *iamRevisionWriteCounter) saveIAMConfig(_ context.Context, item any, _ string, _ ...options) error {
data, err := json.Marshal(item)
if err == nil {
s.data = data
s.writes++
}
return err
}
func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
at := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC)
for _, record := range []struct {
name string
new func(bool) any
}{
{"user", func(deleted bool) any { return &UserIdentity{Version: 1, Deleted: deleted} }},
{"group", func(deleted bool) any { return &GroupInfo{Version: 1, Deleted: deleted} }},
{"policy", func(deleted bool) any { return &PolicyDoc{Version: 1, Deleted: deleted} }},
{"mapping", func(deleted bool) any { return &MappedPolicy{Version: 1, Deleted: deleted} }},
} {
t.Run(record.name, func(t *testing.T) {
data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
if err != nil {
t.Fatal(err)
}
store := &iamRevisionWriteCounter{data: data}
ctx := context.Background()
for range 3 {
// Site heal carries the original timestamp. Sibling notifications
// have no timestamp; both must leave an applied deletion untouched.
for _, replay := range []context.Context{withIAMReplicationTime(ctx, at), ctx} {
if err := saveIAMRevision(replay, store, record.name, record.new(true)); err != nil {
t.Fatal(err)
}
}
}
if store.writes != 0 || string(store.data) != string(data) {
t.Fatalf("replayed tombstone changed storage: writes=%d, record=%s", store.writes, store.data)
}
for _, deleted := range []bool{false, true} {
err := saveIAMRevision(withIAMReplicationTime(ctx, at.Add(-time.Second)), store, record.name, record.new(deleted))
if !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("older event accepted, deleted=%t: %v", deleted, err)
}
}
if err := saveIAMRevision(withIAMReplicationTime(ctx, at), store, record.name, record.new(false)); !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("equal-time recreation accepted: %v", err)
}
newer := at.Add(time.Minute)
if err := saveIAMRevision(withIAMReplicationTime(ctx, newer), store, record.name, record.new(true)); err != nil {
t.Fatal(err)
}
r, err := loadIAMRevision(ctx, store, record.name)
if err != nil || store.writes != 1 || !r.timestamp().Equal(newer) || !r.Deleted {
t.Fatalf("newer deletion did not advance storage: writes=%d, revision=%+v, error=%v", store.writes, r, err)
}
if err := saveIAMRevision(withIAMReplicationTime(ctx, newer.Add(time.Minute)), store, record.name, record.new(false)); err != nil {
t.Fatalf("newer recreation rejected: %v", err)
}
})
}
}
// Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
// The object-store case uses the real peer RPC and deletion handler, so a
// spurious notification actually destroys the parent instead of only counting it.
func testIAMRevocationReplayAfterRecreation(ctx context.Context, t *testing.T, sys *IAMSys) {
t.Helper()
peer := &globalSiteReplicationSys
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
user := "heal-recreated-parent"
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
create := func(at time.Time) {
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
}
revoke := func(at time.Time) {
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
}
create(origin)
revoke(deleted)
create(recreated)
child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
accessKey: "heal-recreated-child", secretKey: "valid-service-password",
})
must(err)
tg, err := grid.SetupTestGrid(2)
must(err)
t.Cleanup(tg.Cleanup)
var deletes atomic.Int32
server := &peerRESTServer{}
must(deleteUserRPC.Register(tg.Managers[1], func(req *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
deletes.Add(1)
return server.DeleteUserHandler(req)
}))
// Future user updates still use the normal peer reload notification.
must(loadUserRPC.Register(tg.Managers[1], server.LoadUserHandler))
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
must(err)
previousNotifications := globalNotificationSys
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
host: host,
gridConn: func() *grid.Connection {
return tg.Managers[0].Connection(tg.Hosts[1])
},
}}}
t.Cleanup(func() { globalNotificationSys = previousNotifications })
assertLive := func(key string) {
t.Helper()
if _, ok := sys.GetUser(ctx, key); !ok {
t.Fatalf("live credential %s lost during deletion replay", key)
}
}
assertNoDelete := func() {
t.Helper()
if n := deletes.Load(); n != 0 {
t.Fatalf("retained revocation sent %d destructive sibling notifications", n)
}
}
for range 3 {
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
must(err)
item, ok := iamDeletionItem(getUserIdentityPath(user, regUser), r)
if !ok || item.IAMUser == nil || !item.UpdatedAt.Equal(deleted) {
t.Fatal("recreated user lost its durable revocation replay")
}
must(peer.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt))
must(sys.store.LoadIAMCache(ctx, false))
assertLive(user)
assertLive(child.AccessKey)
assertNoDelete()
}
// A divergent site sends a previously unseen revocation between our old
// boundary and recreation. Retain it and revoke old children, but never
// turn it into an unversioned delete of the recreated parent.
delayed := deleted.Add(time.Minute)
revoke(delayed)
assertNoDelete()
must(sys.store.LoadIAMCache(ctx, false))
assertLive(user)
if _, ok := sys.GetUser(ctx, child.AccessKey); ok {
t.Fatal("child from before the delayed revocation remains usable")
}
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
must(err)
if r.Deleted || !r.RevokedBefore.Equal(delayed) || !r.timestamp().Equal(recreated) {
t.Fatalf("retained revocation damaged the recreated identity: %+v", r)
}
// A genuinely newer deletion must still reach siblings and remove the
// parent plus credentials issued under its latest revocation boundary.
fresh, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
accessKey: "heal-fresh-child", secretKey: "valid-service-password",
})
must(err)
latest := recreated.Add(time.Minute)
revoke(latest)
wantDeletes := int32(1)
if sys.HasWatcher() {
wantDeletes = 0
}
if n := deletes.Load(); n != wantDeletes {
t.Fatalf("new deletion notifications=%d, want %d", n, wantDeletes)
}
for _, key := range []string{user, fresh.AccessKey} {
if _, ok := sys.GetUser(ctx, key); ok {
t.Fatalf("newer deletion left credential %s usable", key)
}
}
// Exercise the actual sibling handler again against the already persisted
// tombstone. Its context has no revision; it must not re-stamp the record.
_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
if remoteErr != nil {
t.Fatal(remoteErr)
}
r, err = loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
must(err)
if !r.Deleted || !r.timestamp().Equal(latest) {
t.Fatalf("sibling re-stamped the tombstone: got %s, want %s", r.timestamp(), latest)
}
create(latest.Add(time.Minute))
_, remoteErr = server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
if remoteErr != nil {
t.Fatal(remoteErr)
}
assertLive(user)
revoke(latest)
must(sys.store.LoadIAMCache(ctx, false))
assertLive(user)
}
// Counts what a retained revocation actually sends to sibling nodes.
func TestIAMRevocationRetainedReloadsSibling(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
disks, err := getRandomDisks(1)
if err != nil {
t.Fatal(err)
}
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
if err != nil {
t.Fatal(err)
}
initAllSubsystems(ctx)
globalIAMSys.Init(ctx, obj, nil, 2*time.Second)
defer os.RemoveAll(disks[0])
defer obj.Shutdown(ctx)
defer resetTestGlobals()
sys, peer := globalIAMSys, &globalSiteReplicationSys
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
user := "retained-parent"
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
create := func(at time.Time) {
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
}
revoke := func(at time.Time) {
must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
}
create(origin)
revoke(deleted)
create(recreated)
child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
accessKey: "retained-child", secretKey: "valid-service-password",
})
must(err)
// A sibling shares persistent state but has an independent IAM cache.
sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, sys.usersSysType)}
must(sibling.LoadIAMCache(ctx, false))
if _, ok := sibling.GetUser(child.AccessKey); !ok {
t.Fatal("sibling fixture did not load child")
}
tg, err := grid.SetupTestGrid(2)
must(err)
t.Cleanup(tg.Cleanup)
var deletes, loads atomic.Int32
server := &peerRESTServer{}
must(deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
deletes.Add(1)
return server.DeleteUserHandler(r)
}))
must(loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
loads.Add(1)
// LoadUserHandler delegates to this same cache reload method.
if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), regUser); err != nil {
return grid.NoPayload{}, grid.NewRemoteErr(err)
}
return grid.NoPayload{}, nil
}))
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
must(err)
prev := globalNotificationSys
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
host: host,
gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
}}}
t.Cleanup(func() { globalNotificationSys = prev })
delayed := deleted.Add(time.Minute)
revoke(delayed)
t.Logf("sibling notifications after a retained revocation: destructive=%d reload=%d", deletes.Load(), loads.Load())
if deletes.Load() != 0 {
t.Errorf("destructive sibling delete sent: %d", deletes.Load())
}
if loads.Load() == 0 {
t.Errorf("retained revocation did not notify the sibling")
}
if _, ok := sibling.GetUser(child.AccessKey); ok {
t.Error("sibling still resolves revoked child")
}
if _, ok := sibling.GetUser(user); !ok {
t.Error("sibling lost live parent")
}
if _, ok := sys.store.GetUser(user); !ok {
t.Error("live parent lost")
}
if _, ok := sys.store.GetUser(child.AccessKey); ok {
t.Error("revoked child still resolves on the receiving node")
}
}
cmd/iam-revocation_test.go
+535
View File
@@ -0,0 +1,535 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
etcd "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/namespace"
)
// Exercise the persisted IAM store and the same peer handler used by site heal.
// A delete must survive a cache reload and an older create arriving afterwards.
func TestIAMRevocationRejectsOfflineUser(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
obj, disk, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
user := "offline-revoked-user"
req := madmin.AddOrUpdateUserReq{SecretKey: "test-password-valid", Status: madmin.AccountEnabled}
created, err := globalIAMSys.CreateUser(ctx, user, req)
if err != nil {
t.Fatal(err)
}
if err = globalIAMSys.DeleteUser(ctx, user, false); err != nil {
t.Fatal(err)
}
if err = globalIAMSys.store.LoadIAMCache(ctx, false); err != nil {
t.Fatal(err)
}
if err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, created); err != nil {
t.Fatal(err)
}
if _, err = globalIAMSys.GetUserInfo(ctx, user); !errors.Is(err, errNoSuchUser) {
t.Fatalf("revoked user restored by old peer event: %v", err)
}
}
func TestIAMRevocationHealingContinuesAfterPeerRejectsDelete(t *testing.T) {
resetTestGlobals()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
obj, disk, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
if _, err := globalIAMSys.CreateUser(ctx, "heal-sync", req); err != nil {
t.Fatal(err)
}
if _, err := globalIAMSys.CreateUser(ctx, "heal-deleted", req); err != nil {
t.Fatal(err)
}
if err := globalIAMSys.DeleteUser(ctx, "heal-deleted", false); err != nil {
t.Fatal(err)
}
p, err := globalIAMSys.store.GetPolicy("readwrite")
if err != nil {
t.Fatal(err)
}
if _, err := globalIAMSys.SetPolicy(ctx, "heal-new-policy", p); err != nil {
t.Fatal(err)
}
var liveUpdates atomic.Int32
peer := func(id string, rejectDelete bool) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case strings.HasSuffix(r.URL.Path, "/metainfo"):
_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: id})
case r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions":
if r.Method == http.MethodGet {
_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
return
}
var batch iamRevisionBatch
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
t.Error(err)
w.WriteHeader(http.StatusBadRequest)
return
}
for _, item := range batch.Items {
if rejectDelete && iamDeletionPath(item.SRIAMItem) != "" {
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"Code":"AccessDenied","Message":"delete rejected"}`))
return
}
if id == "healthy" && item.Type == madmin.SRIAMItemPolicy && item.Name == "heal-new-policy" && len(item.Policy) > 0 {
liveUpdates.Add(1)
}
}
_ = json.NewEncoder(w).Encode(iamRevisionStatus{Version: iamRevisionProtocol, Node: "node-1", Instance: id, Digest: "fixture"})
default:
t.Errorf("unexpected peer request %s", r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}
}))
}
healthy, rejected := peer("healthy", false), peer("rejected", true)
defer healthy.Close()
defer rejected.Close()
c := &SiteReplicationSys{enabled: true, state: srState{
ServiceAccountAccessKey: "heal-sync",
Peers: map[string]madmin.PeerInfo{
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
"healthy": {Name: "healthy", DeploymentID: "healthy", Endpoint: healthy.URL},
"rejected": {Name: "rejected", DeploymentID: "rejected", Endpoint: rejected.URL},
},
}}
if err := c.healIAMSystem(ctx, obj); err == nil {
t.Fatal("deletion failure was not reported")
}
if liveUpdates.Load() == 0 {
t.Fatal("one peer rejecting a deletion blocked unrelated live IAM healing to a healthy peer")
}
}
func TestIAMRevocationLifecycle(t *testing.T) {
testIAMRevocationLifecycle(t, nil)
}
func TestIAMRevocationEtcdLifecycle(t *testing.T) {
endpoint := os.Getenv("SILO_TEST_IAM_REVOCATION_ETCD")
if endpoint == "" {
t.Skip("set SILO_TEST_IAM_REVOCATION_ETCD to a disposable etcd endpoint")
}
connection, err := etcd.New(etcd.Config{Endpoints: strings.Split(endpoint, ","), DialTimeout: 5 * time.Second})
if err != nil {
t.Fatal(err)
}
defer connection.Close()
// The facade borrows the connection's services. Close the owning client,
// not namespace.Watcher while IAM's canceled watch loop is winding down.
ctx, cancel := context.WithCancel(connection.Ctx())
defer cancel()
client := etcd.NewCtxClient(ctx, etcd.WithZapLogger(connection.GetLogger()))
prefix := fmt.Sprintf("/silo-revocation-test/%d/", time.Now().UnixNano())
client.KV = namespace.NewKV(connection.KV, prefix)
client.Watcher = namespace.NewWatcher(connection.Watcher, prefix)
client.Lease = connection.Lease
testIAMRevocationLifecycle(t, client)
}
func testIAMRevocationLifecycle(t *testing.T, client *etcd.Client) {
resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
disks, err := getRandomDisks(1)
if err != nil {
t.Fatal(err)
}
disk := disks[0]
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
if err == nil {
initAllSubsystems(ctx)
globalIAMSys.Init(ctx, obj, client, 2*time.Second)
}
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(disk)
defer obj.Shutdown(ctx)
defer resetTestGlobals()
sys, peer := globalIAMSys, &globalSiteReplicationSys
must := func(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
reload := func(t *testing.T) { t.Helper(); must(t, sys.store.LoadIAMCache(ctx, false)) }
req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
createUser := func(t *testing.T, name string) {
t.Helper()
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, origin))
}
assertAbsent := func(t *testing.T, name string) {
t.Helper()
if _, ok := sys.GetUser(ctx, name); ok {
t.Fatalf("revoked credential %s is usable", name)
}
}
t.Run("replay after recreation", func(t *testing.T) {
testIAMRevocationReplayAfterRecreation(ctx, t, sys)
})
t.Run("origin timestamp and recreation", func(t *testing.T) {
name := "revocation-recreate"
createUser(t, name)
ui, ok := sys.store.GetUser(name)
if !ok || !ui.UpdatedAt.Equal(origin) {
t.Fatalf("origin time changed: %v", ui.UpdatedAt)
}
must(t, sys.DeleteUser(ctx, name, false))
reload(t)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, time.Time{}))
assertAbsent(t, name)
newTime := UTCNow().Add(time.Minute)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, UserReq: &req}, newTime))
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, origin.Add(time.Second)))
reload(t)
ui, ok = sys.store.GetUser(name)
if !ok || !ui.UpdatedAt.Equal(newTime) || ui.RevokedBefore.IsZero() {
t.Fatalf("newer recreation lost, or deletion boundary missing: present=%v", ok)
}
})
t.Run("groups policies and mappings", func(t *testing.T) {
user, group, name := "revocation-member", "revocation-group", "revocation-policy"
createUser(t, user)
p, err := sys.store.GetPolicy("readwrite")
must(t, err)
must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
add := &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}
must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
for _, isGroup := range []bool{false, true} {
entity := user
if isGroup {
entity = group
}
mp := &madmin.SRPolicyMapping{UserOrGroup: entity, Policy: name, UserType: int(regUser), IsGroup: isGroup}
must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
_, err = sys.PolicyDBSet(ctx, entity, "", regUser, isGroup)
must(t, err)
must(t, peer.PeerPolicyMappingHandler(ctx, mp, origin))
if _, ok := sys.store.GetMappedPolicy(entity, isGroup); ok {
t.Fatal("old grant restored")
}
}
// This receiver never saw the member-removal event preceding deletion.
must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, UTCNow()))
must(t, sys.DeletePolicy(ctx, name, true))
reload(t)
must(t, peer.PeerGroupInfoChangeHandler(ctx, add, origin))
must(t, peer.PeerAddPolicyHandler(ctx, name, &p, origin))
if _, err = sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
t.Fatalf("group restored: %v", err)
}
if _, err = sys.store.GetPolicyDoc(name); !errors.Is(err, errNoSuchPolicy) {
t.Fatalf("policy restored: %v", err)
}
paths, err := sys.store.listIAMConfigPaths(ctx)
must(t, err)
found := make(map[string]bool)
for _, path := range paths {
r, err := loadIAMRevision(ctx, sys.store, path)
must(t, err)
if item, ok := iamDeletionItem(path, r); ok {
found[iamDeletionPath(item)] = true
if item.UpdatedAt.IsZero() {
t.Fatal("undated delete replay")
}
}
}
for _, path := range []string{getGroupInfoPath(group), getPolicyDocPath(name), getMappedPolicyPath(user, regUser, false), getMappedPolicyPath(group, regUser, true)} {
if !found[path] {
t.Errorf("deletion missing from heal: %s", path)
}
}
})
t.Run("parent revokes service accounts and STS", func(t *testing.T) {
parent := "revocation-parent"
createUser(t, parent)
svc, svcAt, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "revocation-service", secretKey: "valid-service-password"})
must(t, err)
secret, err := getTokenSigningKey()
must(t, err)
sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
must(t, err)
sts.ParentUser = parent
_, err = sys.SetTempUser(withIAMReplicationTime(ctx, origin), sts.AccessKey, sts, "readwrite")
must(t, err)
must(t, sys.DeleteUser(ctx, parent, false))
reload(t)
assertAbsent(t, parent)
assertAbsent(t, svc.AccessKey)
assertAbsent(t, sts.AccessKey)
// Recreate the parent, then deliver old child events from the offline site.
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, UTCNow()))
must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: svc.AccessKey, SecretKey: svc.SecretKey}}, svcAt))
must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken, ParentPolicyMapping: "readwrite"}, origin))
reload(t)
assertAbsent(t, svc.AccessKey)
assertAbsent(t, sts.AccessKey)
// A freshly issued credential is still supported after deliberate recreation.
_, _, err = sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-service", secretKey: "valid-service-password"})
must(t, err)
if _, ok := sys.GetUser(ctx, "new-service"); !ok {
t.Fatal("fresh service account rejected")
}
})
t.Run("delete before first create", func(t *testing.T) {
name := "revocation-unseen"
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}, UTCNow()))
createUser(t, name)
assertAbsent(t, name)
svc := "unseen-service"
must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: svc}}, UTCNow()))
must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "revocation-recreate", AccessKey: svc, SecretKey: "valid-service-password"}}, origin))
assertAbsent(t, svc)
})
t.Run("recreation arrives before revocation", func(t *testing.T) {
parent := "reordered-parent"
createUser(t, parent)
child, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: "reordered-child", secretKey: "valid-service-password"})
must(t, err)
newTime, deleteTime := UTCNow(), origin.Add(time.Minute)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, newTime))
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
assertAbsent(t, child.AccessKey)
reload(t)
assertAbsent(t, child.AccessKey)
u, ok := sys.GetUser(ctx, parent)
if !ok || !u.UpdatedAt.Equal(newTime) || !u.RevokedBefore.Equal(deleteTime) {
t.Fatal("reordered revocation damaged the new parent or lost its boundary")
}
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(parent, regUser))
must(t, err)
item, ok := iamDeletionItem(getUserIdentityPath(parent, regUser), r)
if !ok || !item.UpdatedAt.Equal(deleteTime) {
t.Fatal("recreation erased deletion replay")
}
})
t.Run("user cleanup does not supersede group deletion", func(t *testing.T) {
user, group := "cascade-user", "cascade-group"
createUser(t, user)
must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, Members: []string{user}}}, origin))
// On the origin site the group was removed before the user, but the
// recovering receiver processes those independent events in reverse.
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, origin.Add(2*time.Minute)))
must(t, peer.PeerGroupInfoChangeHandler(ctx, &madmin.SRGroupInfo{UpdateReq: madmin.GroupAddRemove{Group: group, IsRemove: true}}, origin.Add(time.Minute)))
reload(t)
if _, err := sys.GetGroupDescription(group); !errors.Is(err, errNoSuchGroup) {
t.Fatalf("deleted group survived reordered cleanup: %v", err)
}
groups, err := sys.ListGroups(ctx)
must(t, err)
for _, name := range groups {
if name == group {
t.Fatal("deleted group listed")
}
}
})
t.Run("parent revocation covers later updates to existing children", func(t *testing.T) {
parent, key := "late-update-parent", "late-update-child"
createUser(t, parent)
_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
must(t, err)
// This site missed the deletion and subsequently edited an old child.
_, err = sys.UpdateServiceAccount(withIAMReplicationTime(ctx, origin.Add(2*time.Minute)), key, updateServiceAccountOpts{description: "edited while the peer was offline"})
must(t, err)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, origin.Add(time.Minute)))
assertAbsent(t, key)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(3*time.Minute)))
reload(t)
assertAbsent(t, key)
})
t.Run("old generation cannot return with a newer event timestamp", func(t *testing.T) {
parent := "generation-parent"
createUser(t, parent)
deleteTime := origin.Add(time.Minute)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, origin.Add(2*time.Minute)))
// Another offline site issued this child under the original parent,
// after this site's delete/recreate. Wall-clock ordering cannot identify it.
late := origin.Add(3 * time.Minute)
must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: "old-gen-service", SecretKey: "valid-service-password"}}, late))
secret, err := getTokenSigningKey()
must(t, err)
sts, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
must(t, err)
must(t, peer.PeerSTSAccHandler(ctx, &madmin.SRSTSCredential{AccessKey: sts.AccessKey, SecretKey: sts.SecretKey, ParentUser: parent, SessionToken: sts.SessionToken}, late))
reload(t)
assertAbsent(t, "old-gen-service")
assertAbsent(t, sts.AccessKey)
// A local issuer knows the new boundary and signs it into both kinds
// of child. Untrusted inherited claims cannot select that boundary.
child, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: "new-gen-service", secretKey: "valid-service-password", claims: map[string]any{iamParentRevocationClaim: "forged"}})
must(t, err)
newClaims := map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}
must(t, setIAMParentRevocationClaim(ctx, sys.store, parent, newClaims))
fresh, err := auth.GetNewCredentialsWithMetadata(newClaims, secret)
must(t, err)
fresh.ParentUser = parent
_, err = sys.SetTempUser(ctx, fresh.AccessKey, fresh, "")
must(t, err)
reload(t)
// A periodic reload retains the STS cache. Explicitly clear it to
// exercise the cold credential load performed after process restart.
cache := sys.store.lock()
cache.iamSTSAccountsMap = make(map[string]UserIdentity)
sys.store.unlock()
for _, key := range []string{child.AccessKey, fresh.AccessKey} {
u, ok := sys.GetUser(ctx, key)
if !ok || !iamCredentialSurvivesRevocation(u.Credentials, deleteTime) {
t.Fatalf("new-generation credential %s rejected", key)
}
}
})
t.Run("late revocation preserves proven new-generation children", func(t *testing.T) {
for _, recreateFirst := range []bool{false, true} {
parent := fmt.Sprintf("gen-parent-%t", recreateFirst)
createUser(t, parent)
deleteTime, createTime := origin.Add(time.Minute), origin.Add(2*time.Minute)
if recreateFirst {
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
}
key := fmt.Sprintf("gen-child-%t", recreateFirst)
must(t, peer.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: parent, AccessKey: key, SecretKey: "valid-service-password", Claims: map[string]any{iamParentRevocationClaim: deleteTime.Format(time.RFC3339Nano)}}}, createTime.Add(time.Second)))
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, IsDeleteReq: true}, deleteTime))
if !recreateFirst {
assertAbsent(t, key)
must(t, peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: parent, UserReq: &req}, createTime))
}
reload(t)
if _, ok := sys.GetUser(ctx, key); !ok {
t.Fatal("late revocation deleted a child issued by the recreated parent")
}
}
})
t.Run("cold loading preserves site-signed STS", func(t *testing.T) {
parent := "cold-sts-parent"
createUser(t, parent)
secret := "site-signing-key-valid"
_, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{
accessKey: siteReplicatorSvcAcc, secretKey: secret, allowSiteReplicatorAccount: true,
})
must(t, err)
setReplication := func(enabled bool) {
globalSiteReplicationSys.Lock()
globalSiteReplicationSys.enabled = enabled
globalSiteReplicationSys.Unlock()
globalSiteReplicatorCred.Set("")
}
setReplication(true)
defer setReplication(false)
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
must(t, err)
cred.ParentUser = parent
_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
must(t, err)
// IAM can load before the site replication manager during startup.
// A signing key that is not available yet must not delete live tokens.
setReplication(false)
for range 3 {
unverified := make(map[string]UserIdentity)
_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, unverified)
if _, ok := unverified[cred.AccessKey]; ok {
t.Fatal("accepted STS before the signing key became available")
}
}
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
must(t, err)
if r.Credentials.SessionToken == "" {
t.Fatal("cold IAM load physically deleted a non-expired site-signed STS credential")
}
setReplication(true)
loaded := make(map[string]UserIdentity)
must(t, sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded))
if _, ok := loaded[cred.AccessKey]; !ok {
t.Fatal("STS credential did not recover when the signing key became available")
}
})
t.Run("unverifiable STS stay denied and expired STS are removed", func(t *testing.T) {
parent := "invalid-sts-parent"
createUser(t, parent)
// Keep the etcd watcher from cleaning half of the fixture before the
// second record is seeded; this subtest exercises the loader directly.
sys.store.lock()
defer sys.store.unlock()
for _, expired := range []bool{false, true} {
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, "unavailable-test-signing-key")
must(t, err)
cred.ParentUser = parent
if expired {
cred.Expiration = UTCNow().Add(-time.Minute)
}
identityPath := getUserIdentityPath(cred.AccessKey, stsUser)
mappingPath := getMappedPolicyPath(cred.AccessKey, stsUser, false)
// Seed disk directly to exercise loading, including existing records
// whose key is unknown. The write API should not accept such tokens.
must(t, sys.store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()}, identityPath))
must(t, sys.store.saveIAMConfig(ctx, &MappedPolicy{Version: 1, Policies: "readwrite"}, mappingPath))
loaded := make(map[string]UserIdentity)
_ = sys.store.loadUser(ctx, cred.AccessKey, stsUser, loaded)
if _, ok := loaded[cred.AccessKey]; ok {
t.Fatalf("invalid STS accepted, expired=%t", expired)
}
for _, path := range []string{identityPath, mappingPath} {
var record map[string]any
err := sys.store.loadIAMConfig(ctx, &record, path)
if expired {
if !errors.Is(err, errConfigNotFound) {
t.Fatalf("expired STS data not cleaned up at %s: %v", path, err)
}
} else {
must(t, err)
}
}
}
})
}
cmd/iam-service-recreation_test.go
+239
View File
@@ -0,0 +1,239 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-only
package cmd
import (
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
)
// The receiver missed a deletion and still has the previous service key.
// A later full snapshot must replace it, including when the owner changed.
func TestIAMServiceAccountRecreation(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
origin := UTCNow().Add(-time.Hour)
for _, parent := range []string{"old-owner", "new-owner"} {
_, err := sys.CreateUser(withIAMReplicationTime(ctx, origin), parent, madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
}
const key = "reusable-service"
old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "old-owner", AccessKey: key, SecretKey: "old-service-password"}}
mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
oldIdentity, _ := sys.store.GetUser(key)
_, err := sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
mustIAM(t, err)
boundary, newer := origin.Add(time.Minute), origin.Add(2*time.Minute)
fresh := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: madmin.SRIAMItemSvcAcc, UpdatedAt: newer, SvcAccChange: &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "new-owner", AccessKey: key, SecretKey: "new-service-password", Status: auth.AccountOff}}}, RevokedBefore: boundary}
mustIAM(t, applyIAMReplicationItem(ctx, fresh))
// A sibling may have missed the notification of the committed
// replacement. An equal-version retry must refresh that cache too.
staleCache := sys.store.lock()
staleCache.iamUsersMap[key] = oldIdentity
sys.store.unlock()
mustIAM(t, applyIAMReplicationItem(ctx, fresh)) // duplicate delivery is acknowledged
if current, _ := sys.store.GetUser(key); current.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey {
t.Fatal("duplicate snapshot acknowledged without refreshing the stale cache")
}
mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, &madmin.SRSvcAccChange{Delete: &madmin.SRSvcAccDelete{AccessKey: key}}, boundary))
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
u, ok := sys.store.GetUser(key)
if !ok || u.Credentials.SecretKey != fresh.SvcAccChange.Create.SecretKey || u.Credentials.ParentUser != "new-owner" || u.Credentials.Status != auth.AccountOff || !u.UpdatedAt.Equal(newer) || !u.RevokedBefore.Equal(boundary) {
t.Fatal("recreation did not retain the new identity, disabled status, source version and revocation")
}
if _, ok := sys.GetUser(ctx, key); ok {
t.Fatal("replicated disabled service can authenticate")
}
cache := sys.store.rlock()
_, mapped := cache.cachedMappedPolicy(key, svcUser, false)
sys.store.runlock()
if mapped {
t.Fatal("recreated service inherited an older mapping")
}
_, err = sys.PolicyDBSet(withIAMReplicationTime(ctx, origin), key, "readwrite", svcUser, false)
if !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("old service mapping replay was accepted: %v", err)
}
_, _, err = sys.NewServiceAccount(ctx, "new-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "local-service-password"})
if !errors.Is(err, errIAMServiceAccountNotAllowed) {
t.Fatalf("local duplicate creation must remain rejected: %v", err)
}
// Outbound snapshots must carry the retained service boundary too.
out, err := globalSiteReplicationSys.replicationItem(ctx, fresh.SRIAMItem)
mustIAM(t, err)
if !out.RevokedBefore.Equal(boundary) {
t.Fatal("outbound service snapshot lost its revocation")
}
})
}
}
func TestIAMServiceAccountReplicationRejectsOtherCredentialKinds(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
_, err := sys.CreateUser(ctx, "builtin-collision", madmin.AddOrUpdateUserReq{SecretKey: "valid-user-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
secret, err := getTokenSigningKey()
mustIAM(t, err)
token, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "builtin-collision"}, secret)
mustIAM(t, err)
token.ParentUser = "builtin-collision"
_, err = sys.SetTempUser(ctx, token.AccessKey, token, "")
mustIAM(t, err)
for _, key := range []string{"builtin-collision", token.AccessKey} {
_, _, err := sys.NewServiceAccount(withIAMReplicationTime(ctx, UTCNow().Add(time.Minute)), "another-owner", nil, newServiceAccountOpts{accessKey: key, secretKey: "valid-service-password"})
if !errors.Is(err, errIAMServiceAccountNotAllowed) {
t.Fatalf("service replication replaced another credential kind: %v", err)
}
}
}
// SR configuration can be temporarily unreadable even though a service token
// is signed with its own valid secret. Do not acknowledge a failed cache load.
func TestIAMServiceAccountRetryReportsClaimLoadFailure(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
globalSiteReplicatorCred.RLock()
previousSigningKey := globalSiteReplicatorCred.secretKey
globalSiteReplicatorCred.RUnlock()
globalSiteReplicatorCred.Set("")
t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSigningKey) })
_, err := sys.CreateUser(ctx, "retry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
opts := newServiceAccountOpts{accessKey: "retry-service", secretKey: "valid-service-password"}
_, _, err = sys.NewServiceAccount(ctx, "retry-owner", nil, opts)
mustIAM(t, err)
old, _ := sys.store.GetUser(opts.accessKey)
opts.secretKey = "replacement-service-password"
at, err := sys.UpdateServiceAccount(ctx, opts.accessKey, updateServiceAccountOpts{secretKey: opts.secretKey})
mustIAM(t, err)
cache := sys.store.lock()
cache.iamUsersMap[opts.accessKey] = old // Missed sibling notification.
sys.store.unlock()
globalSiteReplicationSys.Lock()
globalSiteReplicationSys.enabled = true // No site-replicator credential is installed.
globalSiteReplicationSys.Unlock()
_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
if err == nil {
t.Fatal("acknowledged service retry despite failed claims loading")
}
if _, ok := sys.store.GetUser(opts.accessKey); ok {
t.Fatal("failed cache refresh retained the superseded service secret")
}
globalSiteReplicationSys.Lock()
globalSiteReplicationSys.enabled = false
globalSiteReplicationSys.Unlock()
_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, at), "retry-owner", nil, opts)
mustIAM(t, err)
}
// A delayed snapshot still has its original absolute expiration. Reapplying
// the local minimum issuance lifetime would leave the old unexpired key alive.
func TestIAMServiceAccountReplicationPreservesExpiration(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
for _, action := range []string{"create", "update"} {
for _, expired := range []bool{false, true} {
name := backend + "/" + action + "/near_expiry"
if expired {
name = backend + "/" + action + "/expired"
}
t.Run(name, func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
origin := UTCNow().Add(-time.Hour)
_, err := sys.CreateUser(ctx, "expiry-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
old := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "old-service-password"}}
mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
expires := UTCNow().Add(time.Minute)
if expired {
expires = UTCNow().Add(-time.Minute)
}
change := &madmin.SRSvcAccChange{Create: &madmin.SRSvcAccCreate{Parent: "expiry-owner", AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
if action == "update" {
change = &madmin.SRSvcAccChange{Update: &madmin.SRSvcAccUpdate{AccessKey: "expiry-service", SecretKey: "new-service-password", Expiration: &expires}}
}
mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, change, origin.Add(2*time.Minute)))
u, ok := sys.store.GetUser("expiry-service")
if !ok || u.Credentials.SecretKey != "new-service-password" || !u.Credentials.Expiration.Equal(expires) {
t.Fatal("delayed snapshot lost its new secret or absolute expiration")
}
_, allowed := sys.GetUser(ctx, "expiry-service")
if allowed == expired {
t.Fatal("credential validity disagrees with its absolute expiration")
}
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
mustIAM(t, globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, old, origin))
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath("expiry-service", svcUser))
mustIAM(t, err)
if r.Credentials.SecretKey == old.Create.SecretKey || (expired && !r.Deleted) {
t.Fatal("old non-expiring credential returned after reload")
}
_, _, err = sys.NewServiceAccount(ctx, "expiry-owner", nil, newServiceAccountOpts{accessKey: "local-expiry", secretKey: "valid-service-password", expiration: &expires})
if !errors.Is(err, errInvalidSvcAcctExpiration) {
t.Fatalf("local issuance lifetime check changed: %v", err)
}
})
}
}
}
}
// Status-only summaries intentionally omit secrets. Different revisions must
// still trigger live healing, and disabled identities must be eligible sources.
func TestIAMServiceAccountHealingNewerSnapshot(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
origin := UTCNow().Add(-time.Hour)
_, err := sys.CreateUser(ctx, "heal-svc-owner", madmin.AddOrUpdateUserReq{SecretKey: "valid-owner-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
_, _, err = sys.NewServiceAccount(withIAMReplicationTime(ctx, origin), "heal-svc-owner", nil, newServiceAccountOpts{accessKey: "heal-service", secretKey: "valid-service-password"})
mustIAM(t, err)
at, err := sys.UpdateServiceAccount(ctx, "heal-service", updateServiceAccountOpts{status: auth.AccountOff})
mustIAM(t, err)
var sent atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/minio/health/live" {
return
}
if r.URL.Path != "/minio/admin/v3/site-replication/peer/iam-revisions" || r.Method != http.MethodPut {
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
var batch iamRevisionBatch
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
t.Error(err)
w.WriteHeader(http.StatusBadRequest)
return
}
for _, item := range batch.Items {
if item.SvcAccChange != nil && item.SvcAccChange.Create != nil && item.SvcAccChange.Create.AccessKey == "heal-service" && item.SvcAccChange.Create.Status == auth.AccountOff && item.UpdatedAt.Equal(at) {
sent.Add(1)
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "node", Instance: "boot", Digest: "fixture"}})
}))
defer server.Close()
peers := map[string]madmin.PeerInfo{globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()}, "remote": {Name: "remote", DeploymentID: "remote", Endpoint: server.URL}}
c := &SiteReplicationSys{enabled: true, state: srState{ServiceAccountAccessKey: "heal-svc-owner", Peers: peers}}
local := madmin.UserInfo{Status: madmin.AccountStatus(auth.AccountOff), UpdatedAt: at}
remote := local
remote.UpdatedAt = origin
if isUserInfoReplicated(2, 2, []madmin.UserInfo{local, remote}) {
t.Fatal("status-only summaries concealed different service revisions")
}
info := srStatusInfo{Sites: peers, UserStats: map[string]map[string]srUserStatsSummary{"heal-service": {globalDeploymentID(): {userInfo: srUserInfo{UserInfo: local}}, "remote": {SRUserStatsSummary: madmin.SRUserStatsSummary{UserInfoMismatch: true}, userInfo: srUserInfo{UserInfo: remote}}}}}
mustIAM(t, c.healUsers(ctx, obj, "heal-service", info))
if sent.Load() == 0 {
t.Fatal("disabled newer service snapshot was not healed")
}
}
cmd/iam-store.go
+337 -221
View File
File diff suppressed because it is too large Load Diff
cmd/iam.go
+56 -21
View File
@@ -605,10 +605,22 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
return errServerNotInitialized
}
for _, v := range policy.DefaultPolicies {
if v.Name == policyName {
if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound {
return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
if _, replicated := iamReplicationTime(ctx); !replicated && notifyPeers {
for _, v := range policy.DefaultPolicies {
if v.Name == policyName {
var err error
if objectStore, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
err = checkConfig(ctx, objectStore.objAPI, getPolicyDocPath(policyName))
} else {
var r iamRevision
err = sys.store.loadIAMConfig(ctx, &r, getPolicyDocPath(policyName))
}
if errors.Is(err, errConfigNotFound) {
return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
}
if err != nil {
return err
}
}
}
}
@@ -714,20 +726,30 @@ func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers
return errServerNotInitialized
}
if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil {
err := sys.store.DeleteUser(ctx, accessKey, regUser)
var cleanupErr *iamCommittedCleanupError
retained := errors.Is(err, errIAMRevocationRetained)
if errors.As(err, &cleanupErr) {
retained = cleanupErr.retained
} else if err != nil && !retained {
return err
}
// Notify all other MinIO peers to delete user.
// Publish the committed state even when dependent cleanup must be retried.
if notifyPeers && !sys.HasWatcher() {
for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
if nerr.Err != nil {
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
iamLogIf(ctx, nerr.Err)
if retained {
sys.notifyForUser(ctx, accessKey, false)
} else {
for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) {
if nerr.Err != nil {
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
iamLogIf(ctx, nerr.Err)
}
}
}
}
if cleanupErr != nil {
return cleanupErr
}
return nil
}
@@ -1061,6 +1083,7 @@ type newServiceAccountOpts struct {
sessionPolicy *policy.Policy
accessKey string
secretKey string
status string // Used by replication snapshots; local creates default to enabled.
name, description string
expiration *time.Time
allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
@@ -1125,6 +1148,11 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
m[k] = v
}
}
if _, replicated := iamReplicationTime(ctx); !replicated {
if err := setIAMParentRevocationClaim(ctx, sys.store, parentUser, m); err != nil {
return auth.Credentials{}, time.Time{}, err
}
}
var accessKey, secretKey string
var err error
@@ -1143,12 +1171,19 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
cred.ParentUser = parentUser
cred.Groups = groups
cred.Status = string(auth.AccountOn)
switch opts.status {
case "", auth.AccountOn, string(madmin.AccountEnabled):
case auth.AccountOff, string(madmin.AccountDisabled):
cred.Status = auth.AccountOff
default:
return auth.Credentials{}, time.Time{}, errInvalidArgument
}
cred.Name = opts.name
cred.Description = opts.description
if opts.expiration != nil {
expirationInUTC := opts.expiration.UTC()
if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
if err := validateSvcExpirationInUTC(ctx, expirationInUTC); err != nil {
return auth.Credentials{}, time.Time{}, err
}
cred.Expiration = expirationInUTC
@@ -1379,7 +1414,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
}
sa, ok := sys.store.GetUser(accessKey)
if !ok || !sa.Credentials.IsServiceAccount() {
if _, replicated := iamReplicationTime(ctx); (!ok || !sa.Credentials.IsServiceAccount()) && !replicated {
return nil
}
@@ -1483,8 +1518,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
}
}
// We ignore any errors
_ = sys.store.DeleteUsers(ctx, expiredUsers)
// Keep failed revocations visible so the next purge can retry.
iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
}
// purgeExpiredCredentialsForLDAP - validates if local credentials are still
@@ -1512,8 +1547,8 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
return
}
// We ignore any errors
_ = sys.store.DeleteUsers(ctx, expiredUsers)
// Keep failed revocations visible so the next purge can retry.
iamLogIf(ctx, sys.store.DeleteUsers(ctx, expiredUsers))
}
// updateGroupMembershipsForLDAP - updates the list of groups associated with the credential.
@@ -1934,12 +1969,12 @@ func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, membe
}
updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members)
if err != nil {
var cleanupErr *iamCommittedCleanupError
if err != nil && !errors.As(err, &cleanupErr) {
return updatedAt, err
}
sys.notifyForGroup(ctx, group)
return updatedAt, nil
return updatedAt, err
}
// SetGroupStatus - enable/disabled a group
cmd/metrics-v3-cluster-iam.go
+14
View File
@@ -34,6 +34,10 @@ const (
sinceLastSyncMillis = "since_last_sync_millis"
syncFailures = "sync_failures"
syncSuccesses = "sync_successes"
revocationRecords = "revocation_records"
revocationHealFailures = "revocation_heal_failures"
revocationHealDurationMillis = "revocation_heal_duration_millis"
revocationHealLastSuccess = "revocation_heal_last_success_timestamp_seconds"
)
var (
@@ -47,10 +51,20 @@ var (
sinceLastSyncMillisMD = NewCounterMD(sinceLastSyncMillis, "Time (in milliseconds) since last successful IAM data sync.")
syncFailuresMD = NewCounterMD(syncFailures, "Number of failed IAM data syncs since server start.")
syncSuccessesMD = NewCounterMD(syncSuccesses, "Number of successful IAM data syncs since server start.")
revocationRecordsMD = NewGaugeMD(revocationRecords, "Retained IAM deletion records and revocation boundaries in this node's index.")
revocationHealFailuresMD = NewCounterMD(revocationHealFailures, "Failed IAM revocation convergence passes since server start.")
revocationHealDurationMillisMD = NewGaugeMD(revocationHealDurationMillis, "Duration of the last IAM revocation convergence pass in milliseconds.")
revocationHealLastSuccessMD = NewGaugeMD(revocationHealLastSuccess, "Unix timestamp of the last successful IAM revocation convergence pass.")
)
// loadClusterIAMMetrics - `MetricsLoaderFn` for cluster IAM metrics.
func loadClusterIAMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
if globalIAMSys.Initialized() {
m.Set(revocationRecords, float64(globalIAMSys.store.revisionIndex().count()))
}
m.Set(revocationHealFailures, float64(globalSiteReplicationSys.iamRevisionMetrics.healFailures.Load()))
m.Set(revocationHealDurationMillis, float64(globalSiteReplicationSys.iamRevisionMetrics.healDurationMillis.Load()))
m.Set(revocationHealLastSuccess, float64(globalSiteReplicationSys.iamRevisionMetrics.healLastSuccess.Load()))
m.Set(lastSyncDurationMillis, float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)))
pluginAuthNMetrics := globalAuthNPlugin.Metrics()
m.Set(pluginAuthnServiceFailedRequestsMinute, float64(pluginAuthNMetrics.FailedRequests))
cmd/metrics-v3.go
+4
View File
@@ -323,6 +323,10 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
sinceLastSyncMillisMD,
syncFailuresMD,
syncSuccessesMD,
revocationRecordsMD,
revocationHealFailuresMD,
revocationHealDurationMillisMD,
revocationHealLastSuccessMD,
},
loadClusterIAMMetrics,
)
cmd/peer-rest-server.go
+6 -2
View File
@@ -204,7 +204,9 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(mss *grid.MSS) (np grid.NoP
return np, grid.NewRemoteErr(errors.New("service account name is missing"))
}
if err := globalIAMSys.DeleteServiceAccount(context.Background(), accessKey, false); err != nil {
ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
defer cancel()
if err := globalIAMSys.LoadServiceAccount(ctx, accessKey); err != nil {
return np, grid.NewRemoteErr(err)
}
@@ -274,7 +276,9 @@ func (s *peerRESTServer) LoadUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr
userType = stsUser
}
if err = globalIAMSys.LoadUser(context.Background(), objAPI, accessKey, userType); err != nil {
ctx, cancel := context.WithTimeout(GlobalContext, defaultContextTimeout)
defer cancel()
if err = globalIAMSys.LoadUser(ctx, objAPI, accessKey, userType); err != nil {
return np, grid.NewRemoteErr(err)
}
cmd/site-replication-metadata-gate_test.go
+4
View File
@@ -274,6 +274,10 @@ func TestBucketMetadataInitialSyncPhysicalCreated(t *testing.T) {
}
events = append(events, event)
}
if r.URL.Path == "/minio/admin/v3/site-replication/peer/iam-revisions" {
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "initial-peer", Instance: "initial-boot", Digest: "ack"}})
return
}
w.WriteHeader(http.StatusOK)
}))
defer peer.Close()
cmd/site-replication.go
+67 -31
View File
@@ -28,6 +28,7 @@ import (
"fmt"
"maps"
"math/rand"
"net/http"
"net/url"
"reflect"
"runtime"
@@ -209,7 +210,10 @@ type SiteReplicationSys struct {
// In-memory and persisted multi-site replication state.
state srState
iamMetaCache srIAMCache
iamMetaCache srIAMCache
iamHealMu sync.Mutex
iamRevisionProgress map[string]iamRevisionProgress
iamRevisionMetrics iamRevisionMetrics
}
type srState srStateV1
@@ -1257,13 +1261,36 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
return nil
}
if path := iamDeletionPath(item); path != "" {
r, err := loadIAMRevision(ctx, globalIAMSys.store, path)
if err != nil {
return err
}
if !r.Deleted && ((item.Type != madmin.SRIAMItemIAMUser && item.Type != madmin.SRIAMItemGroupInfo) || r.RevokedBefore.IsZero()) {
// A concurrent recreation has already superseded this delete.
return nil
}
item.UpdatedAt = r.timestamp()
if !r.Deleted {
item.UpdatedAt = r.RevokedBefore
}
}
versioned, err := c.replicationItem(ctx, item)
if errors.Is(err, errIAMStaleUpdate) {
return nil
}
if err != nil {
return err
}
cerr := c.concDo(nil, func(d string, p madmin.PeerInfo) error {
admClient, err := c.getAdminClient(ctx, d)
if err != nil {
return wrapSRErr(err)
}
return c.annotatePeerErr(p.Name, replicateIAMItem, admClient.SRPeerReplicateIAMItem(ctx, item))
_, err = executeIAMRevisionRequest(ctx, admClient, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{versioned}})
return c.annotatePeerErr(p.Name, replicateIAMItem, err)
},
replicateIAMItem,
)
@@ -1273,6 +1300,7 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
// PeerAddPolicyHandler - copies IAM policy to local. A nil policy argument,
// causes the named policy to be deleted.
func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyName string, p *policy.Policy, updatedAt time.Time) error {
ctx = withIAMReplicationTime(ctx, updatedAt)
var err error
// skip overwrite of local update if peer sent stale info
if !updatedAt.IsZero() {
@@ -1286,18 +1314,19 @@ func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyNam
_, err = globalIAMSys.SetPolicy(ctx, policyName, *p)
}
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
return nil
}
// PeerIAMUserChangeHandler - copies IAM user to local.
func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, change *madmin.SRIAMUser, updatedAt time.Time) error {
ctx = withIAMReplicationTime(ctx, updatedAt)
if change == nil {
return errSRInvalidRequest(errInvalidArgument)
}
// skip overwrite of local update if peer sent stale info
if !updatedAt.IsZero() {
if !change.IsDeleteReq && !updatedAt.IsZero() {
if ui, err := globalIAMSys.GetUserInfo(ctx, change.AccessKey); err == nil && ui.UpdatedAt.After(updatedAt) {
return nil
}
@@ -1326,13 +1355,14 @@ func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, chang
}
}
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
return nil
}
// PeerGroupInfoChangeHandler - copies group changes to local.
func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, change *madmin.SRGroupInfo, updatedAt time.Time) error {
ctx = withIAMReplicationTime(ctx, updatedAt)
if change == nil {
return errSRInvalidRequest(errInvalidArgument)
}
@@ -1340,7 +1370,7 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
var err error
// skip overwrite of local update if peer sent stale info
if !updatedAt.IsZero() {
if !updatedAt.IsZero() && (!updReq.IsRemove || len(updReq.Members) != 0) {
if gd, err := globalIAMSys.GetGroupDescription(updReq.Group); err == nil && gd.UpdatedAt.After(updatedAt) {
return nil
}
@@ -1349,7 +1379,8 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
if updReq.IsRemove {
_, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
} else {
if updReq.Status != "" && len(updReq.Members) == 0 {
snapshot, _ := ctx.Value(iamGroupSnapshotKey{}).(bool)
if !snapshot && updReq.Status != "" && len(updReq.Members) == 0 {
_, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
} else {
if globalIAMSys.LDAPConfig.Enabled() {
@@ -1365,13 +1396,14 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
}
}
if err != nil && !errors.Is(err, errNoSuchGroup) {
return wrapSRErr(err)
return iamReplicationError(err)
}
return nil
}
// PeerSvcAccChangeHandler - copies service-account change to local.
func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange, updatedAt time.Time) error {
ctx = withIAMReplicationTime(ctx, updatedAt)
if change == nil {
return errSRInvalidRequest(errInvalidArgument)
}
@@ -1382,7 +1414,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
if len(change.Create.SessionPolicy) > 0 {
sp, err = policy.ParseConfig(bytes.NewReader(change.Create.SessionPolicy))
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
}
// skip overwrite of local update if peer sent stale info
@@ -1394,6 +1426,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
opts := newServiceAccountOpts{
accessKey: change.Create.AccessKey,
secretKey: change.Create.SecretKey,
status: change.Create.Status,
sessionPolicy: sp,
claims: change.Create.Claims,
name: change.Create.Name,
@@ -1402,7 +1435,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
}
_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
case change.Update != nil:
@@ -1411,7 +1444,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
if len(change.Update.SessionPolicy) > 0 {
sp, err = policy.ParseConfig(bytes.NewReader(change.Update.SessionPolicy))
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
}
// skip overwrite of local update if peer sent stale info
@@ -1431,7 +1464,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
_, err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts)
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
case change.Delete != nil:
@@ -1442,7 +1475,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
}
}
if err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true); err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
}
@@ -1451,24 +1484,17 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
// PeerPolicyMappingHandler - copies policy mapping to local.
func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping, updatedAt time.Time) error {
ctx = withIAMReplicationTime(ctx, updatedAt)
if mapping == nil {
return errSRInvalidRequest(errInvalidArgument)
}
// skip overwrite of local update if peer sent stale info
if !updatedAt.IsZero() {
mp, ok := globalIAMSys.store.GetMappedPolicy(mapping.Policy, mapping.IsGroup)
if ok && mp.UpdatedAt.After(updatedAt) {
return nil
}
}
// When LDAP is enabled, we verify that the user or group exists in LDAP and
// use the normalized form of the entityName (which will be an LDAP DN).
userType := IAMUserType(mapping.UserType)
isGroup := mapping.IsGroup
entityName := mapping.UserOrGroup
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser {
if mapping.Policy != "" && globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser {
// Validate that the user or group exists in LDAP and use the normalized
// form of the entityName (which will be an LDAP DN).
var err error
@@ -1491,19 +1517,20 @@ func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mappi
entityName = foundUserDN.NormDN
}
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
}
_, err := globalIAMSys.PolicyDBSet(ctx, entityName, mapping.Policy, userType, isGroup)
if err != nil {
return wrapSRErr(err)
return iamReplicationError(err)
}
return nil
}
// PeerSTSAccHandler - replicates STS credential locally.
func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential, updatedAt time.Time) error {
ctx = withIAMReplicationTime(ctx, updatedAt)
if stsCred == nil {
return errSRInvalidRequest(errInvalidArgument)
}
@@ -1555,7 +1582,7 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
// Set these credentials to IAM.
if _, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, stsCred.ParentPolicyMapping); err != nil {
return fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err)
return iamReplicationError(fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err))
}
return nil
@@ -4193,7 +4220,8 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
}
info.UserInfoMap[k] = madmin.UserInfo{
Status: madmin.AccountStatus(v.Credentials.Status),
Status: madmin.AccountStatus(v.Credentials.Status),
UpdatedAt: v.UpdatedAt,
}
}
}
@@ -5170,13 +5198,16 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
}
func (c *SiteReplicationSys) healIAMSystem(ctx context.Context, objAPI ObjectLayer) error {
// A peer rejecting a deletion must not stop unrelated live updates from
// reaching healthy peers. Retain and report the error for the next retry.
deletionErr := c.healIAMDeletions(ctx)
info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
Users: true,
Policies: true,
Groups: true,
})
if err != nil {
return err
return errors.Join(deletionErr, err)
}
for policy := range info.PolicyStats {
c.healPolicies(ctx, objAPI, policy, info)
@@ -5194,7 +5225,7 @@ func (c *SiteReplicationSys) healIAMSystem(ctx context.Context, objAPI ObjectLay
c.healGroupPolicies(ctx, objAPI, group, info)
}
return nil
return deletionErr
}
// heal iam policies present on this site to peers, provided current cluster has the most recent update.
@@ -5429,8 +5460,10 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
peerName := info.Sites[dID].Name
u, ok := globalIAMSys.GetUser(ctx, user)
if !ok {
// Disabled identities are valid replication sources. CheckKey returns
// their stored record even though authentication is denied.
u, _, err := globalIAMSys.CheckKey(ctx, user)
if err != nil || u.Credentials.AccessKey == "" || u.Credentials.IsExpired() {
continue
}
creds := u.Credentials
@@ -5629,7 +5662,10 @@ func isGroupDescEqual(g1, g2 madmin.GroupDesc) bool {
}
func isUserInfoEqual(u1, u2 madmin.UserInfo) bool {
if u1.PolicyName != u2.PolicyName ||
// Full-site summaries omit secrets and claims. Equal status alone cannot
// distinguish a recreated identity or an edited service-account policy.
if !u1.UpdatedAt.Equal(u2.UpdatedAt) ||
u1.PolicyName != u2.PolicyName ||
u1.Status != u2.Status ||
u1.SecretKey != u2.SecretKey {
return false
cmd/sts-handlers.go
+4
View File
@@ -624,6 +624,10 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
claims[expClaim] = UTCNow().Add(duration).Unix()
claims[parentClaim] = user.AccessKey
if err := setIAMParentRevocationClaim(ctx, globalIAMSys.store, user.AccessKey, claims); err != nil {
writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
return
}
tokenRevokeType := r.Form.Get(stsRevokeTokenType)
if tokenRevokeType != "" {
docs/site-replication/iam-revocations.md
+218
View File
@@ -0,0 +1,218 @@
# Durable IAM revocations in site replication
SILO retains the version of a deleted IAM record so an offline site cannot
restore an older identity or grant when it reconnects. This covers built-in
users, service accounts, groups, policy documents, and policy mappings in their
actual user/STS-parent/group namespaces. It also revokes the deleted built-in
user's older service accounts, STS credentials and group grants across deliberate
same-name recreation.
## Ordering and persistence
Deletion records occupy the original IAM configuration paths. They contain the
originating timestamp, `Deleted`, and, where required, `RevokedBefore`; identity
deletion records contain no secret key or session token. Normal IAM listings and
authorization hide deleted records. Object storage and etcd both serialize each
path's version comparison and write with a distributed lock. The source timestamp
is persisted without replacing it with the receiving node's clock.
Older events cannot overwrite a newer revision. At an identical timestamp, a
deletion wins over a live record. A local deliberate recreation receives a
version newer than the stored deletion. An older user/group deletion arriving
after recreation retains its revocation boundary while preserving the newer
live record. Receiving an already-applied tombstone does not rewrite or advance
it. These rules also apply after cold loading persistent IAM state.
The user or group revision is the commit point of deletion. Cleanup of mappings
and children follows that commit; cleanup failure cannot undo it. The API returns
the cleanup error and still notifies sibling nodes to reload the committed
state. Such an error does **not** mean the identity is still active. Retrying the
operation is safe. When the API returns a committed-cleanup error, the admin
handler does not send its immediate cross-site hook; cross-site propagation
relies on the normal deletion-healing retry. Sibling deletion notifications reload current shared state,
so a delayed notification cannot delete a subsequently recreated identity.
Etcd siblings also receive persistent changes through watches. Failed
notifications/watches remain eventual propagation, not a distributed
instantaneous revocation transaction.
Keep node clocks synchronized and monitor offsets. Ordering uses source wall
clock timestamps, with monotonic advancement for local writes to the same path.
It does not establish causal order between concurrent writers at different sites
or resolve conflicting live updates with identical timestamps deterministically.
## Users, child credentials and groups
A recreated user retains `RevokedBefore`. New service accounts and built-in STS
issuances carry a signed `siloParentRevocation` claim identifying the parent
boundary known at issuance. Editing or replaying an old child does not update
this claim. Old children remain invalid even when their own update timestamp is
newer than the parent deletion; children issued for the recreated parent remain
valid. Claims are read from verified tokens on credential load/write.
A newer replicated service-account snapshot can replace an older service with
the same access key, including a deliberate change of owner or secret. Local
duplicate creates remain rejected. Snapshots preserve disabled status and the
service's own revocation boundary, so earlier mappings cannot attach to the new
service. Equal-version retries reload the committed identity without rewriting
it. Periodic live healing compares source versions even when the public status
summary is unchanged, and includes disabled identities as healing sources.
Service snapshots also preserve their absolute expiration. The receiving site
does not reapply the local minimum lifetime for a newly issued credential. A
newer already-expired snapshot still supersedes the old key, is denied by
authentication, and is collected into a durable service tombstone by normal
loading. Cache/claims loading failures are returned for retry, not acknowledged;
after a committed replacement the stale cached secret is evicted immediately.
Collisions with an existing built-in or cached STS identity report an error and
require an explicit administrative resolution; replication cannot change its
credential kind. Concurrent conflicting service updates with exactly the same
timestamp can retain different winners at different sites; the status summary
does not resolve that case.
Each group member has its own `MemberGrants` timestamp. Changing another member
or the group's enabled status does not reissue everyone else's grants. Effective
membership requires the grant to be newer than both the user's and the group's
retained boundaries. Listings and policy evaluation use the same effective
membership. Peer snapshots preserve grant times, including unknown legacy grant
times; they cannot treat a recent snapshot time as a fresh grant to a revoked
identity. A new explicit administrative group grant can restore access.
This does not implement a general conflict-resolution protocol for all group
membership edits. In particular, the inherited live-group snapshot merge adds
members and does not reconcile a missed ordinary member removal. Removing a
member from a live group during a site outage is a separate known limitation;
do not infer that this change resolves it. User/group deletion boundaries and
same-name recreation are covered here.
## Retention and expiration
Permanent identities, groups, policy documents and mappings have no automatic
tombstone TTL. A disconnected peer or an old backup may return arbitrarily late.
Successful replay acknowledgements are an optimization, **not** permission to
garbage-collect this history.
Natural expiration of an immutable STS token physically removes its token-key
record and any legacy token-key mapping, without generating a permanent
tombstone. An early STS revocation is retained until that token's expiration plus
the existing clock-skew allowance. Replaying the same revoked token with a later
event timestamp cannot recreate it. Etcd uses an expiration lease; object storage
collects expired STS tombstones during its existing credential loading/purge.
A record with unknown expiration is retained conservatively. Cleanup writes are
best effort and use a short lock budget. If one fails, that load stops optional
reclamation, reports the error and still loads healthy users; expired credentials
stay denied and retain their existing durable version. The next load retries.
Healthy cleanup has no fixed record quota. The reusable STS
parent policy mapping is not assigned the token's TTL by deletion cleanup.
External-IDP disablement is an early revocation, not natural token expiration,
and its cached STS and service accounts are included in cleanup. Expiring service
accounts retain a durable revision because their access keys are reusable and
an older version might have no expiration.
Direct per-token `RevokeTokens` delivery between sites is not a new guarantee of
this change. The guarantee for built-in parent deletion follows from the durable
parent boundary, including children not currently present in the deleting node's
cache.
## Healing, failures and operational cost
The normal IAM loaders maintain an in-memory index of deletion records and
retained boundaries, without secrets. Healing uses this index; it does not add a
second full walk of `config/iam/` every cycle. Existing full IAM loading still
scans persistent records, including tombstones, at startup and on refresh.
Each peer receives batches of at most 128 records. The sender remembers which
path/version each peer acknowledged. Unrelated new changes at either site do not
reset that progress. A failed batch remains pending while later independent
batches can progress; a lost response may cause safe idempotent replay. A pass
has a bounded duration, and its successful acknowledgements survive that timeout.
The protocol reports each node name and process instance. Switching between
known node instances behind a load balancer preserves acknowledgements. A new
node instance conservatively invalidates prior acknowledgements once; repeated
switches among those known instances do not reset progress. Restore persistent state only with the affected processes
stopped, so a restore cannot reuse an old process acknowledgement.
Steady-state healing still traverses/sorts the retained in-memory set and checks
the peer's protocol status. It suppresses repeated deletion PUTs once acknowledged.
Memory use scales with retained paths and peers; startup storage reads scale with
history. This release does not provide general history compaction.
After upgrading, older live records whose receiving sites originally assigned
different timestamps can require an initial reconciliation wave. Allow for its
storage writes and sibling notifications when planning the maintenance window.
The cluster IAM metrics include `revocation_records`,
`revocation_heal_failures`, `revocation_heal_duration_millis`, and
`revocation_heal_last_success_timestamp_seconds`. Errors are also logged. A
nominal 30-second scheduler interval is not a convergence deadline: outages,
large backlogs, lock contention and failed requests can require more passes.
Cached credential lookup checks the in-memory parent revision index and performs
no additional storage read. STS issuance and cold credential loading still consult
the persistent parent revision. Revision I/O and distributed lock waits release
the IAM cache lock while a separate local writer mutex preserves write order.
These operations have bounded contexts, including etcd lock and lease cleanup.
## Protocol and supported upgrade
The server-owned versioned route is
`/minio/admin/v3/site-replication/peer/iam-revisions`. It carries source versions,
member grant times and distinct user/group revocation items without changing the
admin client SDK or S3 API. Older servers reject this route. The sender reports
the failure and does not fall back to a route that would discard the metadata.
The existing legacy IAM route remains readable for best-effort compatibility;
this does not confer the new guarantees on an older peer.
All participating servers must be upgraded for the guarantee in this document.
Mixed old/new nodes sharing an IAM backend and rolling downgrade are unsupported:
older binaries do not interpret tombstones or signed parent boundaries correctly.
Use a maintenance window for coordinated upgrade:
1. Pause IAM changes and isolate any offline site or backup whose state is unknown.
2. Back up each site's complete IAM storage and required encryption material. A
live IAM admin export omits deletion history and is not an adequate backup.
3. Stop all nodes sharing each site's IAM backend, replace their binaries, and
restart them on the upgraded version. Complete this for every participating
site before relying on the new revocation semantics.
4. Check IAM loading, site-replication errors and revocation convergence. Verify
representative old credentials are denied and deliberately reissued ones work.
5. Resolve pre-upgrade revocations explicitly. Absence cannot reconstruct an
already-lost deletion version: remove surviving old records on the sites that
still have them, and rebuild stale offline peers from approved state before
admitting them. Do not reconnect an unknown old snapshot just to discover its
deleted credentials.
Credentials issued by an older server for a recreated parent lack the required
signed boundary and must be reissued by an upgraded server. Parents without any
retained revocation history preserve existing credential behavior.
A pristine built-in policy remains protected from local deletion. If an
administrator explicitly overrides that policy and later deletes the override,
the durable deletion now suppresses automatic recreation of the built-in policy
on reload. This prevents reload from undoing the deletion. Restore the policy by
an explicit policy-create operation if desired. Local deletion of a nonexistent
policy remains idempotent and does not create a new tombstone; replicated
unknown deletions retain their version.
For rollback, stop and isolate the affected sites and assess changes since the
backup before restoring compatible state. Restoring an older backup can itself
lose later revocations and requires reconciliation/rekeying before access is
reopened. Do not delete tombstones online or convert only live IAM records to
make an older binary start. Server, client, Console, package and deployment
acceptance remain separate delivery gates of the maintained PGSTY stack.
## Regression and performance checks
Focused coverage is in `iam-revocation_test.go`, `iam-revision_test.go`,
`iam-revision-lock_test.go`, `iam-revision-boundary_test.go`,
`iam-replication-protocol_test.go`, `iam-credential-retention_test.go`,
`iam-peer-reload_test.go`, and `iam-replay_test.go`. Set
`SILO_TEST_IAM_REVOCATION_ETCD` to a disposable etcd endpoint to include backend
lifecycle/locking/boundary tests; they use isolated key namespaces.
`BenchmarkIAMCachedCredential` and `BenchmarkIAMSetTempUser` can be run against
the pre-change source for a comparable local baseline.
`BenchmarkIAMRevisionConvergedHealing` covers 1,000 and 10,000 retained records;
it measures steady-state index/network work and asserts zero repeated PUTs. Its
fake peer does not measure durable catch-up throughput.
`BenchmarkIAMColdLoadExpiredServices` measures loading and cleanup with 100 or
1,000 expired reusable credentials; run it with `-benchtime=1x`. Use actual multi-site
signed S3/STS tests and deployment-specific latency/scale measurements in
addition to these component tests.