mirror of
https://github.com/pgsty/minio.git
synced 2026-11-09 17:21:51 +03:00
Feng Ruohang
326 lines
11 KiB
Go
326 lines
11 KiB
Go
// Copyright (c) 2026 PGSTY
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"slices"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pgsty/silo-pkg/v3/policy"
|
|
)
|
|
|
|
// Hold a lazy user load after it has read storage. This makes the ordering
|
|
// between an in-flight read and a completed credential edit deterministic.
|
|
type iamLoadUserBlocker struct {
|
|
IAMStorageAPI
|
|
key string
|
|
waiting chan struct{}
|
|
release chan struct{}
|
|
once sync.Once
|
|
loads atomic.Int32
|
|
retryWaiting chan struct{}
|
|
retryRelease chan struct{}
|
|
retryErr error
|
|
}
|
|
|
|
func (s *iamLoadUserBlocker) loadUser(ctx context.Context, key string, userType IAMUserType, out map[string]UserIdentity) error {
|
|
if key == s.key && userType == regUser && s.loads.Add(1) == 2 {
|
|
if s.retryWaiting != nil {
|
|
close(s.retryWaiting)
|
|
select {
|
|
case <-s.retryRelease:
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
if s.retryErr != nil {
|
|
return s.retryErr
|
|
}
|
|
}
|
|
err := s.IAMStorageAPI.loadUser(ctx, key, userType, out)
|
|
if err == nil && key == s.key && userType == regUser {
|
|
s.once.Do(func() { close(s.waiting) })
|
|
select {
|
|
case <-s.release:
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
func TestIAMLoadUserConcurrentUpdates(t *testing.T) {
|
|
for _, change := range []string{"unrelated-edit", "unrelated-load", "delete", "mapping", "policy"} {
|
|
t.Run(change, func(t *testing.T) {
|
|
ctx, sys, _ := prepareIAMRevisionFixture(t)
|
|
const policyName = "lazy-policy"
|
|
batchAuthzPolicy(ctx, t, sys, policyName, `{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::bucket/*"]}`)
|
|
cold := iamAdminUser(ctx, t, sys, "lazy-cold", policyName)
|
|
other := iamAdminUser(ctx, t, sys, "lazy-other", "readwrite")
|
|
cache := sys.store.lock()
|
|
delete(cache.iamUsersMap, cold.AccessKey)
|
|
sys.store.unlock()
|
|
original := sys.store.IAMStorageAPI
|
|
blocker := &iamLoadUserBlocker{
|
|
IAMStorageAPI: original, key: cold.AccessKey,
|
|
waiting: make(chan struct{}), release: make(chan struct{}),
|
|
}
|
|
sys.store.IAMStorageAPI = blocker
|
|
unblock := sync.OnceFunc(func() { close(blocker.release) })
|
|
done := make(chan struct{})
|
|
t.Cleanup(func() {
|
|
unblock()
|
|
<-done
|
|
sys.store.IAMStorageAPI = original
|
|
})
|
|
var valid bool
|
|
var loadErr error
|
|
go func() {
|
|
defer close(done)
|
|
_, valid, loadErr = sys.CheckKey(ctx, cold.AccessKey)
|
|
}()
|
|
select {
|
|
case <-blocker.waiting:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("cold load did not read the identity")
|
|
}
|
|
switch change {
|
|
case "unrelated-edit":
|
|
mustIAM(t, sys.SetUserSecretKey(ctx, other.AccessKey, "unrelated-rotated-secret"))
|
|
case "unrelated-load":
|
|
mustIAM(t, sys.store.LoadUser(ctx, other.AccessKey))
|
|
case "delete":
|
|
mustIAM(t, sys.DeleteUser(ctx, cold.AccessKey, false))
|
|
case "mapping":
|
|
_, err := sys.PolicyDBSet(ctx, cold.AccessKey, "readonly", regUser, false)
|
|
mustIAM(t, err)
|
|
case "policy":
|
|
batchAuthzPolicy(ctx, t, sys, policyName, `{"Effect":"Deny","Action":["s3:*"],"Resource":["arn:aws:s3:::bucket/*"]}`)
|
|
}
|
|
unblock()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("cold load did not finish")
|
|
}
|
|
if loadErr != nil || valid != (change != "delete") {
|
|
t.Fatalf("valid=%v err=%v after %s", valid, loadErr, change)
|
|
}
|
|
if blocker.loads.Load() != 2 {
|
|
t.Fatalf("conflicting load read the identity %d times, want 2", blocker.loads.Load())
|
|
}
|
|
if change == "mapping" {
|
|
names, err := sys.store.PolicyDBGet(cold.AccessKey)
|
|
mustIAM(t, err)
|
|
if !slices.Equal(names, []string{"readonly"}) {
|
|
t.Fatalf("restored stale mapping: %v", names)
|
|
}
|
|
}
|
|
if change == "policy" {
|
|
current, err := sys.store.GetPolicy(policyName)
|
|
mustIAM(t, err)
|
|
if current.IsAllowed(policy.Args{Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) {
|
|
t.Fatal("restored stale policy")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIAMLoadUserRetryKeepsCachedReadsAvailable(t *testing.T) {
|
|
for _, unavailable := range []bool{false, true} {
|
|
t.Run(map[bool]string{false: "loaded", true: "unavailable"}[unavailable], func(t *testing.T) {
|
|
ctx, sys, _ := prepareIAMRevisionFixture(t)
|
|
cold := iamAdminUser(ctx, t, sys, "retry-cold", "readwrite")
|
|
other := iamAdminUser(ctx, t, sys, "retry-cached", "readwrite")
|
|
cache := sys.store.lock()
|
|
delete(cache.iamUsersMap, cold.AccessKey)
|
|
sys.store.unlock()
|
|
original := sys.store.IAMStorageAPI
|
|
blocker := &iamLoadUserBlocker{
|
|
IAMStorageAPI: original, key: cold.AccessKey,
|
|
waiting: make(chan struct{}), release: make(chan struct{}),
|
|
retryWaiting: make(chan struct{}), retryRelease: make(chan struct{}),
|
|
}
|
|
if unavailable {
|
|
blocker.retryErr = errErasureReadQuorum
|
|
}
|
|
sys.store.IAMStorageAPI = blocker
|
|
unblock := sync.OnceFunc(func() { close(blocker.release) })
|
|
unblockRetry := sync.OnceFunc(func() { close(blocker.retryRelease) })
|
|
done := make(chan struct{})
|
|
t.Cleanup(func() {
|
|
unblock()
|
|
unblockRetry()
|
|
<-done
|
|
sys.store.IAMStorageAPI = original
|
|
})
|
|
var valid bool
|
|
var loadErr error
|
|
go func() {
|
|
defer close(done)
|
|
_, valid, loadErr = sys.CheckKey(ctx, cold.AccessKey)
|
|
}()
|
|
awaitIAMPolicyReload(t, blocker.waiting)
|
|
mustIAM(t, sys.SetUserSecretKey(ctx, other.AccessKey, "retry-rotated-secret"))
|
|
unblock()
|
|
awaitIAMPolicyReload(t, blocker.retryWaiting)
|
|
readDone := make(chan struct{})
|
|
go func() {
|
|
defer close(readDone)
|
|
user, ok, err := sys.CheckKey(ctx, other.AccessKey)
|
|
if err != nil || !ok || user.Credentials.SecretKey != "retry-rotated-secret" {
|
|
t.Errorf("cached read failed: valid=%v err=%v", ok, err)
|
|
}
|
|
}()
|
|
select {
|
|
case <-readDone:
|
|
case <-time.After(time.Second):
|
|
unblockRetry()
|
|
<-readDone
|
|
t.Fatal("cold-load retry blocked cached authentication")
|
|
}
|
|
unblockRetry()
|
|
<-done
|
|
if unavailable {
|
|
if valid || !errors.Is(loadErr, errIAMNotInitialized) {
|
|
t.Fatalf("retry outage was not retryable: valid=%v err=%v", valid, loadErr)
|
|
}
|
|
} else if !valid || loadErr != nil {
|
|
t.Fatalf("retry lost the cold identity: valid=%v err=%v", valid, loadErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIAMLoadUserRefreshesAssociatedRecords(t *testing.T) {
|
|
t.Setenv("_MINIO_IAM_GROUP_REFRESH", "on")
|
|
for _, change := range []string{"policy", "mapping", "group", "parent"} {
|
|
t.Run(change, func(t *testing.T) {
|
|
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
|
const policyName = "associated-policy"
|
|
const groupName = "associated-group"
|
|
batchAuthzPolicy(ctx, t, sys, policyName, `{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::bucket/*"]}`)
|
|
parent := iamAdminUser(ctx, t, sys, "associated-parent", policyName)
|
|
_, err := sys.AddUsersToGroup(ctx, groupName, []string{parent.AccessKey})
|
|
mustIAM(t, err)
|
|
key := parent.AccessKey
|
|
if change == "parent" {
|
|
key = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "associated-service", "").AccessKey
|
|
}
|
|
gate := &iamPolicyRefreshReadGate{
|
|
ObjectLayer: obj, userPath: getGroupInfoPath(groupName),
|
|
refreshEntered: make(chan struct{}), refreshRelease: make(chan struct{}),
|
|
}
|
|
sys.store.IAMStorageAPI.(*IAMObjectStore).objAPI = gate
|
|
unblock := sync.OnceFunc(func() { close(gate.refreshRelease) })
|
|
done := make(chan struct{})
|
|
var loadErr error
|
|
t.Cleanup(func() {
|
|
unblock()
|
|
<-done
|
|
})
|
|
go func() {
|
|
defer close(done)
|
|
loadErr = sys.store.LoadUser(context.WithValue(ctx, iamPolicyRefreshContext{}, "refresh"), key)
|
|
}()
|
|
awaitIAMPolicyReload(t, gate.refreshEntered)
|
|
switch change {
|
|
case "policy":
|
|
batchAuthzPolicy(ctx, t, sys, policyName, `{"Effect":"Deny","Action":["s3:*"],"Resource":["arn:aws:s3:::bucket/*"]}`)
|
|
case "mapping":
|
|
_, err = sys.PolicyDBSet(ctx, parent.AccessKey, "readonly", regUser, false)
|
|
mustIAM(t, err)
|
|
case "group":
|
|
_, err = sys.RemoveUsersFromGroup(ctx, groupName, []string{parent.AccessKey})
|
|
mustIAM(t, err)
|
|
case "parent":
|
|
mustIAM(t, sys.SetUserSecretKey(ctx, parent.AccessKey, "associated-rotated-secret"))
|
|
}
|
|
unblock()
|
|
select {
|
|
case <-done:
|
|
mustIAM(t, loadErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("associated-record reload did not finish")
|
|
}
|
|
switch change {
|
|
case "policy":
|
|
current, err := sys.store.GetPolicy(policyName)
|
|
mustIAM(t, err)
|
|
if current.IsAllowed(policy.Args{Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) {
|
|
t.Fatal("partial snapshot restored old policy")
|
|
}
|
|
case "mapping":
|
|
mapping, ok := sys.store.GetMappedPolicy(parent.AccessKey, false)
|
|
if !ok || mapping.Policies != "readonly" {
|
|
t.Fatalf("partial snapshot restored old mapping: %+v", mapping)
|
|
}
|
|
case "group":
|
|
cache := sys.store.rlock()
|
|
member := cache.iamUserGroupMemberships[parent.AccessKey].Contains(groupName)
|
|
sys.store.runlock()
|
|
if member {
|
|
t.Fatal("partial snapshot restored old group membership")
|
|
}
|
|
case "parent":
|
|
current, valid, err := sys.CheckKey(ctx, parent.AccessKey)
|
|
if err != nil || !valid || current.Credentials.SecretKey != "associated-rotated-secret" {
|
|
t.Fatal("child reload restored old parent credential")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIAMLoadUserDoesNotRestoreRotatedCredential(t *testing.T) {
|
|
ctx, sys, _ := prepareIAMRevisionFixture(t)
|
|
user := iamAdminUser(ctx, t, sys, "load-user-race", "readwrite")
|
|
original := sys.store.IAMStorageAPI
|
|
blocker := &iamLoadUserBlocker{
|
|
IAMStorageAPI: original,
|
|
key: user.AccessKey,
|
|
waiting: make(chan struct{}),
|
|
release: make(chan struct{}),
|
|
}
|
|
sys.store.IAMStorageAPI = blocker
|
|
t.Cleanup(func() { sys.store.IAMStorageAPI = original })
|
|
|
|
loadDone := make(chan error, 1)
|
|
go func() { loadDone <- sys.store.LoadUser(ctx, user.AccessKey) }()
|
|
select {
|
|
case <-blocker.waiting:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("LoadUser did not reach the controlled read barrier")
|
|
}
|
|
|
|
const rotatedSecret = "load-user-rotated-secret"
|
|
if err := sys.SetUserSecretKey(ctx, user.AccessKey, rotatedSecret); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, valid, err := sys.CheckKey(ctx, user.AccessKey)
|
|
if err != nil || !valid || current.Credentials.SecretKey != rotatedSecret {
|
|
t.Fatalf("rotation did not update the serving cache: valid=%v secret=%q err=%v", valid, current.Credentials.SecretKey, err)
|
|
}
|
|
|
|
close(blocker.release)
|
|
select {
|
|
case err := <-loadDone:
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("LoadUser did not finish after the read barrier was released")
|
|
}
|
|
loaded, valid, err := sys.CheckKey(ctx, user.AccessKey)
|
|
if err != nil || !valid || loaded.Credentials.SecretKey != rotatedSecret {
|
|
t.Fatalf("an in-flight LoadUser restored the old credential: valid=%v secret=%q err=%v", valid, loaded.Credentials.SecretKey, err)
|
|
}
|
|
}
|