minio/cmd/iam-revision_test.go

// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later

package cmd

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/minio/madmin-go/v3"
	"github.com/minio/minio/internal/grid"
	xnet "github.com/pgsty/silo-pkg/v3/net"
)

// Count physical saves: comparing timestamps alone would miss identical
// tombstones being rewritten on every heal pass.
type iamRevisionWriteCounter struct {
	IAMStorageAPI
	data   []byte
	writes int
}

func (s *iamRevisionWriteCounter) loadIAMConfig(_ context.Context, item any, _ string) error {
	return json.Unmarshal(s.data, item)
}

func (s *iamRevisionWriteCounter) saveIAMConfig(_ context.Context, item any, _ string, _ ...options) error {
	data, err := json.Marshal(item)
	if err == nil {
		s.data = data
		s.writes++
	}
	return err
}

func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
	at := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC)
	for _, record := range []struct {
		name string
		new  func(bool) any
	}{
		{"user", func(deleted bool) any { return &UserIdentity{Version: 1, Deleted: deleted} }},
		{"group", func(deleted bool) any { return &GroupInfo{Version: 1, Deleted: deleted} }},
		{"policy", func(deleted bool) any { return &PolicyDoc{Version: 1, Deleted: deleted} }},
		{"mapping", func(deleted bool) any { return &MappedPolicy{Version: 1, Deleted: deleted} }},
	} {
		t.Run(record.name, func(t *testing.T) {
			data, err := json.Marshal(iamRevision{Deleted: true, UpdatedAt: at, RevokedBefore: at})
			if err != nil {
				t.Fatal(err)
			}
			store := &iamRevisionWriteCounter{data: data}
			ctx := context.Background()
			for range 3 {
				// Site heal carries the original timestamp. Sibling notifications
				// have no timestamp; both must leave an applied deletion untouched.
				for _, replay := range []context.Context{withIAMReplicationTime(ctx, at), ctx} {
					if err := saveIAMRevision(replay, store, record.name, record.new(true)); err != nil {
						t.Fatal(err)
					}
				}
			}
			if store.writes != 0 || string(store.data) != string(data) {
				t.Fatalf("replayed tombstone changed storage: writes=%d, record=%s", store.writes, store.data)
			}
			for _, deleted := range []bool{false, true} {
				err := saveIAMRevision(withIAMReplicationTime(ctx, at.Add(-time.Second)), store, record.name, record.new(deleted))
				if !errors.Is(err, errIAMStaleUpdate) {
					t.Fatalf("older event accepted, deleted=%t: %v", deleted, err)
				}
			}
			if err := saveIAMRevision(withIAMReplicationTime(ctx, at), store, record.name, record.new(false)); !errors.Is(err, errIAMStaleUpdate) {
				t.Fatalf("equal-time recreation accepted: %v", err)
			}
			newer := at.Add(time.Minute)
			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer), store, record.name, record.new(true)); err != nil {
				t.Fatal(err)
			}
			r, err := loadIAMRevision(ctx, store, record.name)
			if err != nil || store.writes != 1 || !r.timestamp().Equal(newer) || !r.Deleted {
				t.Fatalf("newer deletion did not advance storage: writes=%d, revision=%+v, error=%v", store.writes, r, err)
			}
			if err := saveIAMRevision(withIAMReplicationTime(ctx, newer.Add(time.Minute)), store, record.name, record.new(false)); err != nil {
				t.Fatalf("newer recreation rejected: %v", err)
			}
		})
	}
}

// Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
// The object-store case uses the real peer RPC and deletion handler, so a
// spurious notification actually destroys the parent instead of only counting it.
func testIAMRevocationReplayAfterRecreation(ctx context.Context, t *testing.T, sys *IAMSys) {
	t.Helper()
	peer := &globalSiteReplicationSys
	must := func(err error) {
		t.Helper()
		if err != nil {
			t.Fatal(err)
		}
	}
	user := "heal-recreated-parent"
	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
	create := func(at time.Time) {
		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
	}
	revoke := func(at time.Time) {
		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
	}
	create(origin)
	revoke(deleted)
	create(recreated)
	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
		accessKey: "heal-recreated-child", secretKey: "valid-service-password",
	})
	must(err)

	tg, err := grid.SetupTestGrid(2)
	must(err)
	t.Cleanup(tg.Cleanup)
	var deletes atomic.Int32
	server := &peerRESTServer{}
	must(deleteUserRPC.Register(tg.Managers[1], func(req *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
		deletes.Add(1)
		return server.DeleteUserHandler(req)
	}))
	// Future user updates still use the normal peer reload notification.
	must(loadUserRPC.Register(tg.Managers[1], server.LoadUserHandler))
	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
	must(err)
	previousNotifications := globalNotificationSys
	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
		host: host,
		gridConn: func() *grid.Connection {
			return tg.Managers[0].Connection(tg.Hosts[1])
		},
	}}}
	t.Cleanup(func() { globalNotificationSys = previousNotifications })
	assertLive := func(key string) {
		t.Helper()
		if _, ok := sys.GetUser(ctx, key); !ok {
			t.Fatalf("live credential %s lost during deletion replay", key)
		}
	}
	assertNoDelete := func() {
		t.Helper()
		if n := deletes.Load(); n != 0 {
			t.Fatalf("retained revocation sent %d destructive sibling notifications", n)
		}
	}
	for range 3 {
		r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
		must(err)
		item, ok := iamDeletionItem(getUserIdentityPath(user, regUser), r)
		if !ok || item.IAMUser == nil || !item.UpdatedAt.Equal(deleted) {
			t.Fatal("recreated user lost its durable revocation replay")
		}
		must(peer.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt))
		must(sys.store.LoadIAMCache(ctx, false))
		assertLive(user)
		assertLive(child.AccessKey)
		assertNoDelete()
	}

	// A divergent site sends a previously unseen revocation between our old
	// boundary and recreation. Retain it and revoke old children, but never
	// turn it into an unversioned delete of the recreated parent.
	delayed := deleted.Add(time.Minute)
	revoke(delayed)
	assertNoDelete()
	must(sys.store.LoadIAMCache(ctx, false))
	assertLive(user)
	if _, ok := sys.GetUser(ctx, child.AccessKey); ok {
		t.Fatal("child from before the delayed revocation remains usable")
	}
	r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
	must(err)
	if r.Deleted || !r.RevokedBefore.Equal(delayed) || !r.timestamp().Equal(recreated) {
		t.Fatalf("retained revocation damaged the recreated identity: %+v", r)
	}

	// A genuinely newer deletion must still reach siblings and remove the
	// parent plus credentials issued under its latest revocation boundary.
	fresh, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
		accessKey: "heal-fresh-child", secretKey: "valid-service-password",
	})
	must(err)
	latest := recreated.Add(time.Minute)
	revoke(latest)
	wantDeletes := int32(1)
	if sys.HasWatcher() {
		wantDeletes = 0
	}
	if n := deletes.Load(); n != wantDeletes {
		t.Fatalf("new deletion notifications=%d, want %d", n, wantDeletes)
	}
	for _, key := range []string{user, fresh.AccessKey} {
		if _, ok := sys.GetUser(ctx, key); ok {
			t.Fatalf("newer deletion left credential %s usable", key)
		}
	}
	// Exercise the actual sibling handler again against the already persisted
	// tombstone. Its context has no revision; it must not re-stamp the record.
	_, remoteErr := server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
	if remoteErr != nil {
		t.Fatal(remoteErr)
	}
	r, err = loadIAMRevision(ctx, sys.store, getUserIdentityPath(user, regUser))
	must(err)
	if !r.Deleted || !r.timestamp().Equal(latest) {
		t.Fatalf("sibling re-stamped the tombstone: got %s, want %s", r.timestamp(), latest)
	}
	create(latest.Add(time.Minute))
	_, remoteErr = server.DeleteUserHandler(grid.NewMSSWith(map[string]string{peerRESTUser: user}))
	if remoteErr != nil {
		t.Fatal(remoteErr)
	}
	assertLive(user)
	revoke(latest)
	must(sys.store.LoadIAMCache(ctx, false))
	assertLive(user)
}

// Counts what a retained revocation actually sends to sibling nodes.
func TestIAMRevocationRetainedReloadsSibling(t *testing.T) {
	resetTestGlobals()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	disks, err := getRandomDisks(1)
	if err != nil {
		t.Fatal(err)
	}
	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
	if err != nil {
		t.Fatal(err)
	}
	initAllSubsystems(ctx)
	globalIAMSys.Init(ctx, obj, nil, 2*time.Second)
	defer os.RemoveAll(disks[0])
	defer obj.Shutdown(ctx)
	defer resetTestGlobals()

	sys, peer := globalIAMSys, &globalSiteReplicationSys
	must := func(err error) {
		t.Helper()
		if err != nil {
			t.Fatal(err)
		}
	}
	user := "retained-parent"
	req := madmin.AddOrUpdateUserReq{SecretKey: "valid-test-password", Status: madmin.AccountEnabled}
	origin := UTCNow().Add(-time.Hour).Truncate(time.Millisecond)
	deleted, recreated := origin.Add(time.Minute), origin.Add(3*time.Minute)
	create := func(at time.Time) {
		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, UserReq: &req}, at))
	}
	revoke := func(at time.Time) {
		must(peer.PeerIAMUserChangeHandler(ctx, &madmin.SRIAMUser{AccessKey: user, IsDeleteReq: true}, at))
	}
	create(origin)
	revoke(deleted)
	create(recreated)
	child, _, err := sys.NewServiceAccount(ctx, user, nil, newServiceAccountOpts{
		accessKey: "retained-child", secretKey: "valid-service-password",
	})
	must(err)

	// A sibling shares persistent state but has an independent IAM cache.
	sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, sys.usersSysType)}
	must(sibling.LoadIAMCache(ctx, false))
	if _, ok := sibling.GetUser(child.AccessKey); !ok {
		t.Fatal("sibling fixture did not load child")
	}
	tg, err := grid.SetupTestGrid(2)
	must(err)
	t.Cleanup(tg.Cleanup)
	var deletes, loads atomic.Int32
	server := &peerRESTServer{}
	must(deleteUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
		deletes.Add(1)
		return server.DeleteUserHandler(r)
	}))
	must(loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
		loads.Add(1)
		// LoadUserHandler delegates to this same cache reload method.
		if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), regUser); err != nil {
			return grid.NoPayload{}, grid.NewRemoteErr(err)
		}
		return grid.NoPayload{}, nil
	}))
	host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
	must(err)
	prev := globalNotificationSys
	globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{
		host:     host,
		gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) },
	}}}
	t.Cleanup(func() { globalNotificationSys = prev })

	delayed := deleted.Add(time.Minute)
	revoke(delayed)

	t.Logf("sibling notifications after a retained revocation: destructive=%d reload=%d", deletes.Load(), loads.Load())
	if deletes.Load() != 0 {
		t.Errorf("destructive sibling delete sent: %d", deletes.Load())
	}
	if loads.Load() == 0 {
		t.Errorf("retained revocation did not notify the sibling")
	}
	if _, ok := sibling.GetUser(child.AccessKey); ok {
		t.Error("sibling still resolves revoked child")
	}
	if _, ok := sibling.GetUser(user); !ok {
		t.Error("sibling lost live parent")
	}
	if _, ok := sys.store.GetUser(user); !ok {
		t.Error("live parent lost")
	}
	if _, ok := sys.store.GetUser(child.AccessKey); ok {
		t.Error("revoked child still resolves on the receiving node")
	}
}