mirror of
https://github.com/pgsty/minio.git
synced 2026-11-11 18:15:29 +03:00
test(batch): cover submitter authorization and locked versions
Submission and start with changed, expired or restricted credentials, session policies with and without an authorization plugin, request context in policy conditions, per-object denies, jobs without a recorded submitter and describe output; expire, pull, push and rotation over versions under retention or legal hold, on one and sixteen drives; cursor and counter persistence; and the cases found in review: destination versions and tags of pulls, encryption values, S3 Tables statements, copies in several pools, claims, prefixes, listing errors and retry accounting. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,287 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func batchPullJob(bucket string) BatchJobReplicateV1 {
|
||||
return BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
|
||||
}
|
||||
}
|
||||
|
||||
// A pull does not read the destination version it may replace, so a policy
|
||||
// that conditions the write on that version's tags refuses it; a version
|
||||
// condition sees the version the pull writes.
|
||||
func TestBatchPullDestinationConditions(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
action policy.Action
|
||||
marker bool
|
||||
condition func(versionID string) string
|
||||
}{
|
||||
{"tags-put", policy.ReplicateObjectAction, false, func(string) string {
|
||||
return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
|
||||
}},
|
||||
{"tags-marker", policy.ReplicateDeleteAction, true, func(string) string {
|
||||
return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
|
||||
}},
|
||||
{"version-put", policy.PutObjectAction, false, func(v string) string {
|
||||
return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v)
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"})
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`,
|
||||
{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID)))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
job := batchPullJob(f.bucket)
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
|
||||
Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag",
|
||||
DeleteMarker: tc.marker, UserDefined: map[string]string{},
|
||||
}, false, sub)
|
||||
if !isBatchJobAuthzError(err) {
|
||||
t.Errorf("pull returned %v, want a refusal", err)
|
||||
}
|
||||
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID})
|
||||
if err != nil || after.ETag != before.ETag {
|
||||
t.Errorf("refused pull changed the destination version: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A pull does not encrypt what it writes; encryption headers reported by the
|
||||
// source do not satisfy a destination policy that requires encryption.
|
||||
func TestBatchPullEncryptionCondition(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", `
|
||||
{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]},
|
||||
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
job := batchPullJob(f.bucket)
|
||||
version := mustGetUUID()
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{
|
||||
Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag",
|
||||
UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"},
|
||||
}, false, sub)
|
||||
if err == nil {
|
||||
oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version})
|
||||
mustIAM(t, gerr)
|
||||
if !crypto.S3KMS.IsEncrypted(oi.UserDefined) {
|
||||
t.Fatal("a pull was allowed by an encryption condition it did not meet")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A conditional Deny on an S3 Tables action covers the S3 data actions it
|
||||
// implies, under the tables resource form; the job refuses rather than skip it.
|
||||
func TestBatchJobTableConditionalDeny(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context")
|
||||
if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) {
|
||||
t.Fatal("control: with the request context the Deny applies")
|
||||
}
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("authorize returned %v, want a refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
type batchArchiveRevocation struct {
|
||||
ObjectLayer
|
||||
revoke func()
|
||||
}
|
||||
|
||||
func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
if object == "archive/second" {
|
||||
a.revoke()
|
||||
}
|
||||
return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
// Each snowball entry is authorized for the submitter as it is then.
|
||||
func TestBatchPushSnowballEntryRevalidation(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/")
|
||||
mustIAM(t, err)
|
||||
previous := newGlobalAuthZPluginFn()
|
||||
setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
|
||||
URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() },
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil
|
||||
}),
|
||||
}))
|
||||
t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
|
||||
entries := []ObjectInfo{
|
||||
batchPutVersion(t, f.obj, f.bucket, "archive/first", nil),
|
||||
batchPutVersion(t, f.obj, f.bucket, "archive/second", nil),
|
||||
}
|
||||
remote, err := minio.New("archive.invalid", &minio.Options{
|
||||
Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
||||
}),
|
||||
})
|
||||
mustIAM(t, err)
|
||||
job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}}
|
||||
job.Source.Snowball.InMemory = ptr(true)
|
||||
job.Source.Snowball.Compress = ptr(false)
|
||||
job.Source.Snowball.SkipErrs = ptr(false)
|
||||
api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }}
|
||||
if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A definition holds one job type; one with several is refused, so that what
|
||||
// is authorized is what runs.
|
||||
func TestBatchJobSeveralSections(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
older := f.putVersions(t, "data/a")
|
||||
job := f.expireYAML("") + `keyrotate:
|
||||
apiVersion: v1
|
||||
bucket: other
|
||||
encryption:
|
||||
type: sse-s3
|
||||
`
|
||||
rec, queued := f.submit(t, globalActiveCred, job)
|
||||
if queued != nil {
|
||||
f.run(t, queued)
|
||||
}
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !f.versionExists(t, "data/a", older) {
|
||||
t.Fatal("a definition with two job types ran")
|
||||
}
|
||||
}
|
||||
|
||||
// Prefixes are walked once each in one order: a cursor in a later prefix is
|
||||
// after every key of the earlier ones, and an overlapping prefix is not
|
||||
// walked twice.
|
||||
func TestBatchExpirePrefixes(t *testing.T) {
|
||||
if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" {
|
||||
t.Fatalf("canonical prefixes %q", got)
|
||||
}
|
||||
if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" {
|
||||
t.Fatalf("canonical prefixes %q", got)
|
||||
}
|
||||
f := newBatchAuthzFixture(t)
|
||||
done := f.putVersions(t, "a/done")
|
||||
todo := f.putVersions(t, "z/todo")
|
||||
batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD"))
|
||||
batchPutVersion(t, f.obj, f.bucket, "x/data", nil)
|
||||
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
|
||||
apiVersion: v1
|
||||
bucket: %s
|
||||
prefix: ["z/", "x/", "a/", "x/"]
|
||||
rules:
|
||||
- type: object
|
||||
purge:
|
||||
retainVersions: 1
|
||||
`, f.bucket))
|
||||
if job == nil {
|
||||
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
// A run that completed a/ and stopped.
|
||||
ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"}
|
||||
mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
|
||||
f.run(t, job)
|
||||
if !f.versionExists(t, "a/done", done) {
|
||||
t.Error("resume did again a key before the cursor")
|
||||
}
|
||||
if f.versionExists(t, "z/todo", todo) {
|
||||
t.Error("resume skipped a later prefix")
|
||||
}
|
||||
if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 {
|
||||
t.Errorf("a locked version under two equal prefixes was counted %d times", got)
|
||||
}
|
||||
}
|
||||
|
||||
type batchPartialWalk struct {
|
||||
ObjectLayer
|
||||
version ObjectInfo
|
||||
cancel context.CancelFunc
|
||||
read chan struct{}
|
||||
}
|
||||
|
||||
func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
|
||||
go func() {
|
||||
defer close(results)
|
||||
results <- itemOrErr[ObjectInfo]{Item: a.version}
|
||||
if a.cancel != nil {
|
||||
<-a.read
|
||||
a.cancel()
|
||||
return
|
||||
}
|
||||
results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
|
||||
if a.read != nil {
|
||||
close(a.read)
|
||||
}
|
||||
return a.version, nil
|
||||
}
|
||||
|
||||
// A rotation whose walk fails or is canceled within a key keeps that key
|
||||
// after its cursor and does not report completion.
|
||||
func TestBatchKeyRotationIncompleteWalk(t *testing.T) {
|
||||
for _, canceled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
defer cancel()
|
||||
meta := batchLockMeta("LEGALHOLD")
|
||||
meta[crypto.MetaSealedKeyS3] = "test-sealed-key"
|
||||
api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}}
|
||||
if canceled {
|
||||
api.cancel, api.read = cancel, make(chan struct{})
|
||||
}
|
||||
job := BatchJobRequest{
|
||||
ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
|
||||
KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}},
|
||||
}
|
||||
_ = job.KeyRotate.Start(ctx, api, job)
|
||||
ri := f.jobInfo(t, &job)
|
||||
if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 {
|
||||
t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user