astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-11-10 01:31:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge batch job authorization and Object Lock enforcement

Browse Source 
Batch jobs run as their submitter, and expiration, replication and key rotation respect Object Lock on every version they touch.
This commit is contained in:
Feng Ruohang
2026-11-06 13:00:04 +08:00
parent 9fefb756ad 77ad2fc12b
commit c45b26a75c
27 changed files with 5786 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/batch-authz-review2_test.go
+356
View File
@@ -0,0 +1,356 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"maps"
"net/http"
"strings"
"testing"
"time"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/pgsty/silo-pkg/v3/policy"
)
// A pull into a suspended bucket replaces the null version. Its policy
// decision must name that version even when the source is unversioned S3.
func TestBatchReviewPullImplicitNullVersionAuthorization(t *testing.T) {
f := newBatchAuthzFixture(t)
_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
mustIAM(t, err)
data := []byte("original null version")
before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: true})
mustIAM(t, err)
batchAuthzPolicy(f.ctx, t, f.sys, "null-deny", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"null"}}}`)
cred := iamAdminUser(f.ctx, t, f.sys, "null-deny-user", "null-deny")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
job := batchPullJob(f.bucket)
job.Source.Type = BatchJobReplicateResourceS3
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
Name: before.Name, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
}, false, sub)
if !isBatchJobAuthzError(err) {
t.Errorf("null version policy must refuse the write, got %v", err)
}
after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: nullVersionID})
if gerr != nil || before.ETag != after.ETag {
t.Errorf("denied null version must remain unchanged: read=%v unchanged=%v", gerr, before.ETag == after.ETag)
}
}
type batchReviewPrefixWalkFailure struct{ ObjectLayer }
func (a batchReviewPrefixWalkFailure) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
if prefix != "a/" {
return a.ObjectLayer.Walk(ctx, bucket, prefix, results, opts)
}
go func() {
defer close(results)
results <- itemOrErr[ObjectInfo]{Err: errors.New("test: incomplete first prefix")}
}()
return nil
}
type batchReviewWalkCutoff struct {
ObjectLayer
cutoff string
}
func (a batchReviewWalkCutoff) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
inner := make(chan itemOrErr[ObjectInfo], 1)
if err := a.ObjectLayer.Walk(ctx, bucket, prefix, inner, opts); err != nil {
close(results)
return err
}
go func() {
defer close(results)
stopped := false
for result := range inner {
if stopped {
continue
}
if result.Err == nil && result.Item.Name >= a.cutoff {
stopped = true
result = itemOrErr[ObjectInfo]{Err: errors.New("test: listing stopped before remaining child keys")}
}
select {
case results <- result:
case <-ctx.Done():
stopped = true
}
}
}()
return nil
}
// Listing a directory prefix returns the directory object before its child
// keys. A durable cursor for that directory must preserve unfinished children.
func TestBatchReviewDirectoryCursorKeepsChildrenPending(t *testing.T) {
f := newBatchAuthzFixture(t)
for range 12 {
batchPutVersion(t, f.obj, f.bucket, "a/", nil)
}
var pending string
for i := 0; i < 13; i++ {
older := f.putVersions(t, fmt.Sprintf("a/key-%02d", i))
if i == 11 {
pending = older
}
}
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
apiVersion: v1
bucket: %s
prefix: ["a/"]
rules:
- type: object
purge:
retainVersions: 1
`, f.bucket))
if job == nil {
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
}
_ = job.Expire.Start(f.ctx, batchReviewWalkCutoff{ObjectLayer: f.obj, cutoff: "a/key-11"}, *job)
ri := f.jobInfo(t, job)
if ri.Object != "a/" {
t.Fatalf("control: the completed directory batch must be the cursor, got %q", ri.Object)
}
mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
if f.versionExists(t, "a/key-11", pending) {
t.Error("a directory cursor must not suppress the unvisited child key on resume")
}
}
// Later prefixes must not advance the completion cursor past an earlier
// prefix whose listing failed. This state is produced by Start itself.
func TestBatchReviewExpireFailedPrefixKeepsWorkPending(t *testing.T) {
f := newBatchAuthzFixture(t)
older := f.putVersions(t, "a/todo")
for i := 0; i < 13; i++ {
f.putVersions(t, fmt.Sprintf("z/key-%02d", i))
}
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
apiVersion: v1
bucket: %s
prefix: ["z/", "a/"]
rules:
- type: object
purge:
retainVersions: 1
`, f.bucket))
if job == nil {
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
}
_ = job.Expire.Start(f.ctx, batchReviewPrefixWalkFailure{f.obj}, *job)
ri := f.jobInfo(t, job)
if ri.Object >= "a/todo" {
t.Errorf("cursor must not cross an incomplete prefix, got %q", ri.Object)
}
mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
if f.versionExists(t, "a/todo", older) {
t.Error("a fresh Start must revisit the key omitted by the failed listing")
}
}
// A repeated marker replication error must remain a failure on the second
// attempt, including after the retry loop resets its counters.
func TestBatchReviewMarkerRetryRemainsFailed(t *testing.T) {
f := newBatchAuthzFixture(t)
batchPutVersion(t, f.obj, f.bucket, "marker", nil)
_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
mustIAM(t, err)
oldTransport := globalRemoteTargetTransport
globalRemoteTargetTransport = batchLockTransport(func(r *http.Request) (*http.Response, error) {
code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
if r.URL.Query().Has("location") {
code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
}
return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
t.Cleanup(func() { globalRemoteTargetTransport = oldTransport })
job := BatchJobRequest{
ID: "replicate-marker-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
Replicate: &BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
},
}
job.Replicate.Flags.Retry.Attempts = 2
job.Replicate.Flags.Retry.Delay = time.Millisecond
mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
ri := f.jobInfo(t, &job)
if ri.Complete || !ri.Failed || ri.DeleteMarkersFailed != 1 {
t.Errorf("both marker attempts failed: complete=%v failed=%v markerFailures=%d attempts=%d", ri.Complete, ri.Failed, ri.DeleteMarkersFailed, ri.RetryAttempts)
}
}
// A current parent mapping suppresses the OpenID policy-name fallback.
// Background-condition validation must inspect the same effective policies.
func TestBatchReviewMappedPolicySuppressesClaimFallback(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
batchAuthzPolicy(f.ctx, t, f.sys, "unused-claim-policy", `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "mapped-full")
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), iamPolicyClaimNameOpenID(): "unused-claim-policy",
}, globalActiveCred.SecretKey)
mustIAM(t, err)
cred.ParentUser = parent.AccessKey
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
p, err := sub.principal(f.ctx)
mustIAM(t, err)
req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "mapped/key", versionID: mustGetUUID()}
if !f.sys.isAllowedLocally(policy.Args{AccountName: p.accessKey, Claims: p.claims, Action: req.action, BucketName: req.bucket, ObjectName: req.object, ConditionValues: p.conditionValues(req)}) {
t.Fatal("control: the current mapped policy must allow this read")
}
if err := sub.authorize(f.ctx, req); err != nil {
t.Errorf("an unused fallback policy must not refuse a mapped-policy grant: %v", err)
}
}
func TestBatchReviewBindingAndExpiringServiceControls(t *testing.T) {
f := newBatchAuthzFixture(t)
root := batchRootSubmitter(t)
priorRoot := globalActiveCred
globalActiveCred.SecretKey = "different-test-root-secret"
_, err := root.principal(f.ctx)
globalActiveCred = priorRoot
if !isBatchJobAuthzError(err) {
t.Errorf("root-secret change must invalidate the binding: %v", err)
}
missing := *root
missing.Binding = nil
if _, err := missing.principal(f.ctx); !isBatchJobAuthzError(err) {
t.Errorf("missing fingerprint must refuse the principal: %v", err)
}
batchAuthzPolicy(f.ctx, t, f.sys, "service-parent-full", batchAuthzFullAccess)
parent := iamAdminUser(f.ctx, t, f.sys, "service-parent", "service-parent-full")
sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`))
mustIAM(t, err)
expires := UTCNow().Add(time.Hour)
cred, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp, expiration: &expires})
mustIAM(t, err)
stored, ok := f.sys.GetUser(f.ctx, cred.AccessKey)
if !ok {
t.Fatal("created service account must resolve")
}
cred = stored.Credentials
if !cred.IsTemp() || !cred.IsServiceAccount() {
t.Fatal("fixture must be an expiring service account")
}
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "allowed"}); err != nil {
t.Errorf("expiring service account must retain its allowed read: %v", err)
}
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectAction, bucket: f.bucket, object: "allowed"}); !isBatchJobAuthzError(err) {
t.Errorf("expiring service account must remain session-policy restricted: %v", err)
}
}
// An interrupted cross-pool copy can leave the same version in both pools.
// A newer legal-hold field in either copy must prevent key rotation.
func TestBatchReviewRotationUsesAuthoritativePoolHold(t *testing.T) {
z, bucket := consistencyPools(t)
const name = "rotation/split-hold"
previousKMS := GlobalKMS
GlobalKMS = kms.NewStub("batch-rotation-key")
t.Cleanup(func() { GlobalKMS = previousKMS })
dek, err := GlobalKMS.GenerateKey(t.Context(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{bucket: bucket + "/" + name}})
mustIAM(t, err)
objectKey := crypto.GenerateKey(dek.Plaintext, nil)
sealed := objectKey.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), bucket, name)
first := crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed)
first[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "OFF"
first[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
second := maps.Clone(first)
second[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
second[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("protected payload"), objectKey))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
logical, err := z.replicaObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if !enforceRetentionForDeletion(t.Context(), logical) {
t.Fatal("control: the authoritative logical version must be held")
}
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
err = r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
if !isErrObjectLocked(err) {
t.Errorf("rotation must refuse the authoritative legal hold, got %v", err)
}
after, gerr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, gerr)
if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
t.Error("held logical version must not have its key rewritten")
}
}
// An encryption condition must describe what rotation actually commits,
// even when the requested encryption type differs from the existing type.
func TestBatchReviewRotationEncryptionConditionMatchesResult(t *testing.T) {
f := newBatchAuthzFixture(t)
previousKMS := GlobalKMS
GlobalKMS = kms.NewStub("required-key")
t.Cleanup(func() { GlobalKMS = previousKMS })
const name = "rotation/encryption-condition"
meta := make(map[string]string)
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, name, meta, nil)
mustIAM(t, err)
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("encrypted payload"), key))
mustIAM(t, err)
before, err := f.obj.PutObject(f.ctx, f.bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, UserDefined: meta})
mustIAM(t, err)
batchAuthzPolicy(f.ctx, t, f.sys, "kms-write-only", `
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]},
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
cred := iamAdminUser(f.ctx, t, f.sys, "kms-write-user", "kms-write-only")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
r := BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "required-key"}}
err = r.KeyRotate(f.ctx, f.obj, before, sub)
after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, gerr)
if err == nil && !crypto.S3KMS.IsEncrypted(after.UserDefined) {
t.Error("rotation allowed by an aws:kms-only write policy must produce SSE-KMS or refuse the operation")
}
if err != nil && before.UserDefined[crypto.MetaSealedKeyS3] != after.UserDefined[crypto.MetaSealedKeyS3] {
t.Error("a refused rotation must leave the version unchanged")
}
}
// An explicit null marker collision must retain a held null data version.
func TestBatchReviewPullMarkerNullLock(t *testing.T) {
f := newBatchAuthzFixture(t)
meta := batchLockMeta("LEGALHOLD")
before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, strings.NewReader("held null"), 9, "", ""), ObjectOptions{VersionID: nullVersionID, Versioned: true, UserDefined: meta})
mustIAM(t, err)
job := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
}
err = job.ReplicateFromSource(f.ctx, f.obj, nil, ObjectInfo{
Name: before.Name, VersionID: nullVersionID, ModTime: UTCNow(), DeleteMarker: true,
}, false, batchRootSubmitter(t))
if !isErrObjectLocked(err) {
t.Errorf("marker collision with protected null data must return ObjectLocked, got %v", err)
}
if !batchVersionExists(t, f.obj, f.bucket, before.Name, nullVersionID) {
t.Fatal("protected null data must remain present")
}
}
cmd/batch-authz-review3_test.go
+140
View File
@@ -0,0 +1,140 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"fmt"
"io"
"net/http"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/policy"
)
// A read policy must use the newest tag state of the logical version even
// when an interrupted movement left an older copy in the preferred pool.
func TestBatchReviewPushUsesAuthoritativePoolTags(t *testing.T) {
z, bucket := consistencyPools(t)
f := newBatchAuthzFixture(t)
const name = "push/tagged"
first := map[string]string{
xhttp.AmzObjectTagging: "classification=public",
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
}
second := map[string]string{
xhttp.AmzObjectTagging: "classification=private",
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
}
before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if logical.UserTags != "classification=private" {
t.Fatal("control: logical version must have the newer private tag")
}
batchAuthzPolicy(f.ctx, t, f.sys, "pooled-push", fmt.Sprintf(`
{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s/*"]},
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
cred := iamAdminUser(f.ctx, t, f.sys, "pooled-push-user", "pooled-push")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: bucket, object: name, versionID: before.VersionID, tags: logical.UserTags}); !isBatchJobAuthzError(err) {
t.Fatal("control: authoritative tags must deny the read")
}
writes := 0
target, err := minio.NewCore("target.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.Copy(io.Discard, r.Body)
_ = r.Body.Close()
}
if r.Method == http.MethodPut {
writes++
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}),
})
mustIAM(t, err)
r := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
}
err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
if !isBatchJobAuthzError(err) || writes != 0 {
t.Errorf("newer authoritative tags must deny push: err=%v writes=%d", err, writes)
}
}
// Verified token claims must not replace the actual version being authorized.
func TestBatchReviewVersionConditionResistsClaimCollision(t *testing.T) {
f := newBatchAuthzFixture(t)
deniedVersion := mustGetUUID()
batchAuthzPolicy(f.ctx, t, f.sys, "version-claims", batchAuthzFullAccess+fmt.Sprintf(`,
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, deniedVersion))
parent := iamAdminUser(f.ctx, t, f.sys, "version-claims-parent", "version-claims")
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), "versionid": "an-unrelated-claim-value",
}, globalActiveCred.SecretKey)
mustIAM(t, err)
cred.ParentUser = parent.AccessKey
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
req := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: deniedVersion}
if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
t.Errorf("a token claim must not replace the actual denied version: %v", err)
}
}
// Retry totals describe a pass, including markers already successful before
// another item caused that pass to retry.
func TestBatchReviewMarkerSuccessCountResetsOnRetry(t *testing.T) {
f := newBatchAuthzFixture(t)
batchPutVersion(t, f.obj, f.bucket, "marker", nil)
_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
mustIAM(t, err)
oldTransport := remoteInstanceTransport
remoteInstanceTransport = atomic.Value{}
setRemoteInstanceTransport(batchLockTransport(func(r *http.Request) (*http.Response, error) {
code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
h := make(http.Header)
switch {
case r.URL.Query().Has("location"):
code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
case r.Method == http.MethodDelete:
code, body = http.StatusNoContent, ""
case r.Method == http.MethodHead && r.URL.Query().Get("versionId") != "":
code, body = http.StatusMethodNotAllowed, ""
h.Set("X-Amz-Delete-Marker", "true")
}
return &http.Response{StatusCode: code, Header: h, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
}))
t.Cleanup(func() { remoteInstanceTransport = oldTransport })
job := BatchJobRequest{
ID: "marker-success-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
Replicate: &BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
},
}
job.Replicate.Flags.Retry.Attempts = 2
job.Replicate.Flags.Retry.Delay = time.Millisecond
mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
ri := f.jobInfo(t, &job)
if !ri.Failed || ri.RetryAttempts != 2 {
t.Fatalf("control: data failure must cause two attempts: failed=%v attempts=%d", ri.Failed, ri.RetryAttempts)
}
if ri.DeleteMarkers != 1 {
t.Errorf("one successful marker must be counted once in the final attempt, got %d", ri.DeleteMarkers)
}
}
cmd/batch-authz-review4_test.go
+157
View File
@@ -0,0 +1,157 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"fmt"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/policy"
)
// A tag removal in a newer copy must govern both the authorization and the
// metadata sent by a push. The submitter can read data but cannot read tags.
func TestBatchReviewPushSendsOnlyAuthorizedMetadata(t *testing.T) {
z, bucket := consistencyPools(t)
f := newBatchAuthzFixture(t)
const name = "push/removed-tags"
first := map[string]string{
xhttp.AmzObjectTagging: "classification=private",
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
}
second := map[string]string{
xhttp.AmzObjectTagging: "",
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
}
before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if logical.UserTags != "" {
t.Fatal("control: the authoritative version must have no tags")
}
batchAuthzPolicy(f.ctx, t, f.sys, "data-without-tags", fmt.Sprintf(`
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"]},
{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket, bucket))
cred := iamAdminUser(f.ctx, t, f.sys, "data-without-tags-user", "data-without-tags")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: name, versionID: before.VersionID}); !isBatchJobAuthzError(err) {
t.Fatal("control: reading tags must be denied")
}
writes := 0
var sentTags string
target, err := minio.NewCore("target.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.Copy(io.Discard, r.Body)
_ = r.Body.Close()
}
if r.Method == http.MethodPut {
writes++
sentTags = r.Header.Get(xhttp.AmzObjectTagging)
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}),
})
mustIAM(t, err)
r := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
}
err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
if err != nil && !isBatchJobAuthzError(err) {
t.Fatalf("unexpected replication failure: %v", err)
}
if writes > 0 && sentTags != logical.UserTags {
t.Errorf("push must send only metadata covered by authorization: writes=%d sentTags=%q authorizedTags=%q err=%v", writes, sentTags, logical.UserTags, err)
}
}
// LDAP's short username and its parent DN are distinct identities. The
// background merge must not turn a deny for the verified short name into
// a non-matching statement by replacing it with the parent DN.
func TestBatchReviewLDAPUsernameDeny(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "ldap-short-name-deny", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"ldap:username":"alice"}}}`)
const dn = "uid=alice,ou=people,dc=example,dc=org"
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
"exp": UTCNow().Add(time.Hour).Unix(), ldapUser: dn, ldapActualUser: dn, ldapUserN: "alice",
}, globalActiveCred.SecretKey)
mustIAM(t, err)
cred.ParentUser = dn
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "ldap-short-name-deny")
mustIAM(t, err)
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
p, err := sub.principal(f.ctx)
mustIAM(t, err)
cred.Claims = p.claims
req, err := http.NewRequest(http.MethodDelete, "https://localhost/batch-authz/data", nil)
mustIAM(t, err)
args := policy.Args{
AccountName: cred.AccessKey, Claims: p.claims, Action: policy.DeleteObjectVersionAction,
BucketName: f.bucket, ObjectName: "data", ConditionValues: getConditionValues(req, "", cred),
}
if args.ConditionValues["username"][0] != "alice" || f.sys.isAllowedLocally(args) {
t.Fatal("control: the request evaluator must deny the verified LDAP short username")
}
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: mustGetUUID()}); !isBatchJobAuthzError(err) {
t.Errorf("batch authorization must preserve the LDAP username deny or fail closed, got %v", err)
}
}
// Failure to read authoritative metadata is incomplete work, even if one
// pool's copy can still be opened. In particular, skipErrs=false must not
// turn an unreadable entry into a successful archive.
func TestBatchReviewSnowballAuthoritativeReadFailure(t *testing.T) {
z, bucket := consistencyPools(t)
const name = "archive/unreadable-copy"
before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true})
set := z.serverPools[1].sets[0]
getDisks := set.getDisks
disks := append([]StorageAPI(nil), getDisks()...)
for i := range disks {
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: name}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = getDisks }()
gr, err := z.GetObjectNInfo(t.Context(), bucket, name, nil, nil, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
_, logicalErr := batchAuthoritativeRead(t.Context(), z, bucket, name, gr.ObjInfo)
gr.Close()
if logicalErr == nil {
t.Fatal("control: the authoritative read must fail")
}
remote, err := minio.New("archive.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.Copy(io.Discard, r.Body)
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}),
})
mustIAM(t, err)
r := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
}
r.Source.Snowball.InMemory = ptr(true)
r.Source.Snowball.Compress = ptr(false)
r.Source.Snowball.SkipErrs = ptr(false)
if err := r.writeAsArchive(t.Context(), z, remote, []ObjectInfo{before}, "", batchRootSubmitter(t)); err == nil {
t.Error("archive with an unreadable authoritative version must not report success")
}
}
cmd/batch-authz-review5_test.go
+142
View File
@@ -0,0 +1,142 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"fmt"
"io"
"maps"
"net/http"
"strings"
"testing"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
)
// A suspended destination does not keep a source version ID. Authorization
// must therefore describe the null destination version that the write really
// replaces, and must not let a source ID escape that model.
func TestBatchReviewPullSuspendedVersionBinding(t *testing.T) {
f := newBatchAuthzFixture(t)
_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
mustIAM(t, err)
sourceVersion := mustGetUUID()
batchAuthzPolicy(f.ctx, t, f.sys, "pull-source-version-deny", batchAuthzFullAccess+fmt.Sprintf(`,
{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, sourceVersion))
cred := iamAdminUser(f.ctx, t, f.sys, "pull-source-version-user", "pull-source-version-deny")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
job := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
}
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
Name: "pull/suspended-version", VersionID: sourceVersion, ModTime: UTCNow(), ETag: "batchetag",
UserDefined: map[string]string{},
}, false, sub)
if !isBatchJobAuthzError(err) {
t.Fatalf("a source version ID must not evade the destination deny: %v", err)
}
if _, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/suspended-version", ObjectOptions{VersionID: sourceVersion}); gerr == nil {
t.Fatal("a denied suspended-destination write created the source version")
}
}
// A failed source open in a non-skipping snowball must be reported to the
// batch rather than silently omitted from a successful archive.
type batchReviewOpenFailure struct {
ObjectLayer
name string
}
func (a batchReviewOpenFailure) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
if object == a.name {
return nil, fmt.Errorf("test: source open failed")
}
return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
}
func TestBatchReviewSnowballSourceOpenFailure(t *testing.T) {
f := newBatchAuthzFixture(t)
name := "archive/open-failure"
entry := batchPutVersion(t, f.obj, f.bucket, name, nil)
remote, err := minio.New("archive.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.Copy(io.Discard, r.Body)
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}),
})
mustIAM(t, err)
r := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
}
r.Source.Snowball.InMemory = ptr(true)
r.Source.Snowball.Compress = ptr(false)
r.Source.Snowball.SkipErrs = ptr(false)
api := batchReviewOpenFailure{ObjectLayer: f.obj, name: name}
if err := r.writeAsArchive(t.Context(), api, remote, []ObjectInfo{entry}, "", batchRootSubmitter(t)); err == nil {
t.Fatal("a source open failure must not be reported as a successful archive")
}
}
// Rotation must apply tag conditions to the logical version, including
// a more recent tag update stored in a pool other than the selected copy.
func TestBatchReviewRotationAuthoritativeTags(t *testing.T) {
z, bucket := consistencyPools(t)
f := newBatchAuthzFixture(t)
previousKMS := GlobalKMS
GlobalKMS = kms.NewStub("batch-round5-rotation-key")
t.Cleanup(func() { GlobalKMS = previousKMS })
batchAuthzPolicy(f.ctx, t, f.sys, "round5-rotation-tags", fmt.Sprintf(`
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
cred := iamAdminUser(f.ctx, t, f.sys, "round5-rotation-user", "round5-rotation-tags")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
for _, classification := range []string{"private", "public"} {
t.Run(classification, func(t *testing.T) {
name := "rotation/authoritative-" + classification
first := make(map[string]string)
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
mustIAM(t, err)
first[xhttp.AmzObjectTagging] = "classification=public"
first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
second := maps.Clone(first)
second[xhttp.AmzObjectTagging] = "classification=" + classification
second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("test encrypted payload"), key))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if logical.UserTags != "classification="+classification {
t.Fatal("control: the second pool must govern the logical tags")
}
job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
err = job.KeyRotate(f.ctx, z, before, sub)
after, readErr := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, readErr)
changed := after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3]
if classification == "private" {
if !isBatchJobAuthzError(err) || changed {
t.Fatalf("authoritative tag deny must prevent rewrapping: err=%v changed=%v", err, changed)
}
} else if err != nil || !changed {
t.Fatalf("authorized unlocked version must rotate: err=%v changed=%v", err, changed)
}
})
}
}
cmd/batch-authz-review6_test.go
+520
View File
@@ -0,0 +1,520 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"errors"
"fmt"
"io"
"maps"
"net/http"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
)
// Rotation and listing cases found in review.
// Rotation rewrites the copy GetObjectInfo returns, which can carry another
// tag revision than the merged version of all pools: both are authorized.
func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) {
z, bucket := consistencyPools(t)
f := newBatchAuthzFixture(t)
previousKMS := GlobalKMS
GlobalKMS = kms.NewStub("review6-rotation-copy-key")
t.Cleanup(func() { GlobalKMS = previousKMS })
batchAuthzPolicy(f.ctx, t, f.sys, "review6-rotation-copy-deny", fmt.Sprintf(`
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
cred := iamAdminUser(f.ctx, t, f.sys, "review6-rotation-copy-user", "review6-rotation-copy-deny")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
const name = "rotation/written-copy"
first := make(map[string]string)
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
mustIAM(t, err)
first[xhttp.AmzObjectTagging] = "classification=private"
first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
second := maps.Clone(first)
second[xhttp.AmzObjectTagging] = "classification=public"
second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("review6 rotation payload"), key))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if logical.UserTags != "classification=public" {
t.Fatalf("control: merged tags=%q, want public", logical.UserTags)
}
job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
if err = job.KeyRotate(f.ctx, z, before, sub); !isBatchJobAuthzError(err) {
t.Fatalf("rotation of a copy the submitter may not read returned %v", err)
}
after, err := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
t.Fatal("a refused rotation rewrote the copy")
}
}
// batchWalkError fails every listing before it returns a result.
type batchWalkError struct {
ObjectLayer
err error
}
func (w batchWalkError) Walk(context.Context, string, string, chan<- itemOrErr[ObjectInfo], WalkOptions) error {
return w.err
}
// A listing that fails before it returns anything fails the run, as a
// listing error in the results does, and the job is left to a later run.
func TestBatchReviewExpireWalkErrorFailsRun(t *testing.T) {
f := newBatchAuthzFixture(t)
rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
listErr := errors.New("review6: listing failed")
if err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job); !errors.Is(err, listErr) {
t.Fatalf("start returned %v, want %v", err, listErr)
}
if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
t.Fatalf("failed listing: complete=%v failed=%v", ri.Complete, ri.Failed)
}
}
// Two pools hold different writes of the null version, and a newer version
// of the key is in the second one. Rotation rewrites the copy it read and
// authorized, in the first pool, not the copy the latest version of the key
// would route a write to; the other write keeps its key and stays readable.
func TestBatchReviewRotationRewritesTheCopyItRead(t *testing.T) {
for _, otherLatest := range []bool{false, true} {
name := "addressed-version-is-latest"
if otherLatest {
name = "another-version-is-latest"
}
t.Run(name, func(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
z, _ := consistencyPools(t)
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
const bucket = "review7-rotation-copies"
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
_, versionErr := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, versionErr)
oldKMS := GlobalKMS
GlobalKMS = kms.NewStub("review7-rotation-key")
t.Cleanup(func() { GlobalKMS = oldKMS })
const object = "rotation/null-copy"
const payload = "rotation must preserve this data"
readCopy := func(pool int) ([]byte, error) {
gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: nullVersionID})
if err != nil {
return nil, err
}
defer gr.Close()
return io.ReadAll(gr)
}
var before [2]ObjectInfo
var keys [2]crypto.ObjectKey
kinds := []crypto.Type{crypto.S3, crypto.S3KMS}
for pool, kind := range kinds {
meta := make(map[string]string)
key, err := newEncryptMetadata(t.Context(), kind, "review7-rotation-key", nil, bucket, object, meta, nil)
mustIAM(t, err)
keys[pool] = key
meta[xhttp.AmzObjectTagging] = "classification=public"
if pool == 1 {
meta[xhttp.AmzObjectTagging] = "classification=private"
}
meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Duration(pool+2) * time.Hour).Format(time.RFC3339Nano)
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
mustIAM(t, err)
putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{
Versioned: true, VersionID: nullVersionID, UserDefined: meta,
MTime: UTCNow().Add(-time.Duration(pool+2) * time.Hour),
})
// As a read returns it, without the write's internal fields.
before[pool], err = z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
mustIAM(t, err)
data, err := readCopy(pool)
if err != nil || string(data) != payload {
t.Fatalf("control: pool %d was not readable before rotation: %v", pool, err)
}
}
if otherLatest {
putConsistencyObject(t, z, bucket, object, 1, "later version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
}
selected, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID, Versioned: true})
mustIAM(t, err)
if selected.VersionID != nullVersionID || !selected.ModTime.Equal(before[0].ModTime) || !crypto.S3.IsEncrypted(selected.UserDefined) {
t.Fatal("control: the addressed null version must select the SSE-S3 write in pool zero")
}
logical, err := batchAuthoritativeVersion(t.Context(), z, bucket, object, selected)
mustIAM(t, err)
if logical.UserTags != "classification=public" || selected.UserTags != logical.UserTags {
t.Fatal("control: both current authorization inputs must have public tags")
}
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review7-rotation-key"}}
sub := batchRootSubmitter(t)
if otherLatest {
batchAuthzPolicy(t.Context(), t, globalIAMSys, "review7-rotate-public", fmt.Sprintf(`
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
cred := iamAdminUser(t.Context(), t, globalIAMSys, "review7-rotate-user", "review7-rotate-public")
sub, err = newBatchJobSubmitter(cred)
mustIAM(t, err)
if err := sub.authorize(t.Context(), r.rotateAuthzRequests(before[1])...); !isBatchJobAuthzError(err) {
t.Fatal("control: the older write in pool one must be denied")
}
}
err = r.KeyRotate(t.Context(), z, selected, sub)
if err != nil {
t.Logf("rotation refused: %v", err)
}
for pool, want := range keys {
after, readErr := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
mustIAM(t, readErr)
if otherLatest && pool == 1 && (!maps.Equal(after.UserDefined, before[pool].UserDefined) || !after.ModTime.Equal(before[pool].ModTime) || after.UserTags != before[pool].UserTags) {
t.Error("rotation rewrote a copy denied by the submitter's tag-conditioned policy")
}
kind, _ := crypto.IsEncrypted(after.UserDefined)
var got crypto.ObjectKey
switch kind {
case crypto.S3:
got, readErr = crypto.S3.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
case crypto.S3KMS:
got, readErr = crypto.S3KMS.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
default:
t.Fatalf("pool %d: rotation lost encryption metadata", pool)
}
mustIAM(t, readErr)
if got != want {
t.Errorf("pool %d: rotation changed the data key without reencrypting its ciphertext (rotation error: %v)", pool, err)
}
data, readErr := readCopy(pool)
if readErr != nil || string(data) != payload {
t.Errorf("pool %d: rotation left unreadable ciphertext: %v", pool, readErr)
}
}
})
}
}
// A version in a pool that a decommission or a rebalance is moving is not
// rewritten there: only the source holds it, so the rotation is left to a
// later run.
func TestBatchReviewRotationLeavesMovingSource(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
z, _ := consistencyPools(t)
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
const bucket = "review8-rotation-draining"
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
oldKMS := GlobalKMS
GlobalKMS = kms.NewStub("review8-rotation-key")
t.Cleanup(func() { GlobalKMS = oldKMS })
const object = "rotation/draining"
meta := make(map[string]string)
key, err := newEncryptMetadata(t.Context(), crypto.S3, "review8-rotation-key", nil, bucket, object, meta, nil)
mustIAM(t, err)
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("draining rotation"), key))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
z.poolMetaMutex.Lock()
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
z.poolMetaMutex.Unlock()
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
err = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
if !errors.Is(err, errBatchRotateMoving) {
t.Errorf("rotation of a version only a draining pool holds returned %v", err)
}
after, readErr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, readErr)
if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
t.Fatal("rotation rewrote a draining pool")
}
}
// Once a move copied a version, the rotation leaves it to a later run, so the
// source stays as the move's cleanup expects it.
func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) {
for _, tc := range []struct {
kind string
rotate bool
}{{"decommission", false}, {"decommission", true}, {"rebalance", false}, {"rebalance", true}} {
name := tc.kind + "/without-rotation"
if tc.rotate {
name = tc.kind + "/with-rotation"
}
t.Run(name, func(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
z, _ := migrationTestPools(t)
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
const bucket = "review8-rotation-migration"
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
oldKMS := GlobalKMS
GlobalKMS = kms.NewStub("review8-rotation-key")
t.Cleanup(func() { GlobalKMS = oldKMS })
const object = "rotation/migration"
meta := make(map[string]string)
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
mustIAM(t, err)
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation and migration"), key))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
move := migrationTestMover(t, z, 0, tc.kind)
mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, beforeInfo)))
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); err != nil {
t.Fatalf("control: migration did not copy the version: %v", err)
}
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
var rotationErr error
if tc.rotate {
moved, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
rotationErr = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
if !errors.Is(rotationErr, errBatchRotateMoving) {
t.Errorf("rotation during a move returned %v", rotationErr)
}
after, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if !maps.Equal(after.UserDefined, moved.UserDefined) || after.UserTags != moved.UserTags {
t.Error("rotation during a move rewrote the moved copy")
}
}
after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if !after.ModTime.Equal(beforeInfo.ModTime) || !maps.Equal(after.UserDefined, beforeInfo.UserDefined) {
t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
}
snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
t.Fatalf("rotation prevented migration cleanup: %v", err)
}
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
t.Fatalf("migration left a source version behind: %v", err)
}
})
}
}
// The rewrite keeps the tags and the expiry of the version, which a read
// returns apart from its metadata.
func TestBatchReviewRotationKeepsTagsAndExpiry(t *testing.T) {
for _, mode := range []string{"single-pool", "multiple-pools"} {
t.Run(mode, func(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
var z *erasureServerPools
if mode == "single-pool" {
z = purgeSinglePool(t, 4)
} else {
z, _ = consistencyPools(t)
}
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
const bucket = "review8-rotation-metadata"
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
oldKMS := GlobalKMS
GlobalKMS = kms.NewStub("review8-rotation-metadata-key")
t.Cleanup(func() { GlobalKMS = oldKMS })
const object = "rotation/tagged"
meta := make(map[string]string)
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
mustIAM(t, err)
meta[xhttp.AmzObjectTagging] = "classification=private"
meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
meta["expires"] = UTCNow().Add(time.Hour).Format(http.TimeFormat)
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("metadata preservation"), key))
mustIAM(t, err)
written := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta})
before, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: written.VersionID})
mustIAM(t, err)
if before.UserTags == "" || before.Expires.IsZero() {
t.Fatal("control: stored version is missing the metadata to preserve")
}
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
mustIAM(t, r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)))
after, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if after.UserDefined[crypto.MetaSealedKeyS3] == before.UserDefined[crypto.MetaSealedKeyS3] {
t.Error("control: rotation did not rewrap the key")
}
if after.UserTags != before.UserTags {
t.Errorf("rotation changed stored tags: got %q, want %q", after.UserTags, before.UserTags)
}
if !after.Expires.Equal(before.Expires) {
t.Errorf("rotation changed Expires: got %v, want %v", after.Expires, before.Expires)
}
})
}
}
// Two pools hold the same write with different tags, with or without
// tagging timestamps. A rotation, also during a move, leaves the tags that
// the merged version has.
func TestBatchReviewRotationKeepsMergedTags(t *testing.T) {
for _, tc := range []struct {
ordered, moving bool
}{{false, false}, {false, true}, {true, false}, {true, true}} {
t.Run(fmt.Sprintf("timestamp=%t/moving=%t", tc.ordered, tc.moving), func(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
z, _ := consistencyPools(t)
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
const bucket = "review9-rotation-tag-tie"
const object = "rotation/tag-tie"
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
previousKMS := GlobalKMS
GlobalKMS = kms.NewStub("review9-tag-tie-key")
t.Cleanup(func() { GlobalKMS = previousKMS })
meta := make(map[string]string)
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
mustIAM(t, err)
meta[xhttp.AmzObjectTagging] = "classification=private"
if tc.ordered {
meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
}
other := maps.Clone(meta)
other[xhttp.AmzObjectTagging] = "classification=public"
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("same write"), key))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
putConsistencyObject(t, z, bucket, object, 1, string(body), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: other})
prior, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if prior.UserTags != "classification=private" {
t.Fatalf("control: logical tags=%q", prior.UserTags)
}
if tc.moving {
migrationTestMover(t, z, 0, "decommission")
}
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
rotationErr := r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
after, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if after.UserTags != prior.UserTags {
t.Fatalf("rotation changed logical tags from %q to %q (rotation error: %v)", prior.UserTags, after.UserTags, rotationErr)
}
})
}
}
// batchRotateFlipDisk starts a decommission of pool 0 at the flipAt-th
// version read, after the rotation's first read of the version.
type batchRotateFlipDisk struct {
StorageAPI
z *erasureServerPools
reads *atomic.Int32
flipAt int32
flipped *atomic.Bool
}
func (d batchRotateFlipDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
n := d.reads.Add(1)
if n == d.flipAt && d.flipped.CompareAndSwap(false, true) {
d.z.poolMetaMutex.Lock()
d.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
d.z.poolMetaMutex.Unlock()
}
return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
}
// A move that starts while the rotation authorizes and rewraps the version
// defers it: the state is checked again right before the write, where the
// pools layer checks it.
func TestBatchReviewRotationRechecksPoolBeforeWrite(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
z, _ := consistencyPools(t)
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
const bucket = "review11-rotation-window"
const object = "rotation/window"
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
oldKMS := GlobalKMS
GlobalKMS = kms.NewStub("review11-rotation-key")
t.Cleanup(func() { GlobalKMS = oldKMS })
meta := make(map[string]string)
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
mustIAM(t, err)
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation window"), key))
mustIAM(t, err)
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{
Versioned: true,
UserDefined: meta,
MTime: UTCNow().Add(-time.Hour),
})
beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
var reads atomic.Int32
var flipped atomic.Bool
firstPass := int32(0)
for _, pool := range z.serverPools {
firstPass += int32(pool.SetDriveCount())
}
var restore []func()
for _, pool := range z.serverPools {
restore = append(restore, conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI {
return batchRotateFlipDisk{
StorageAPI: d,
z: z,
reads: &reads,
flipAt: firstPass + 1,
flipped: &flipped,
}
}))
}
t.Cleanup(func() {
for _, fn := range restore {
fn()
}
})
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
gotErr := r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
if !flipped.Load() {
t.Fatal("fault injection did not occur after the initial rotation read")
}
if !errors.Is(gotErr, errBatchRotateMoving) {
t.Errorf("rotation did not defer after the pool became moving during pre-work: %v", gotErr)
}
after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
mustIAM(t, err)
if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
t.Error("rotation rewrote the pool after it became moving during pre-work")
}
}
cmd/batch-authz-review_test.go
+287
View File
@@ -0,0 +1,287 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"testing"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
polplugin "github.com/minio/minio/internal/config/policy/plugin"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/policy"
)
func batchPullJob(bucket string) BatchJobReplicateV1 {
return BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
}
}
// A pull does not read the destination version it may replace, so a policy
// that conditions the write on that version's tags refuses it; a version
// condition sees the version the pull writes.
func TestBatchPullDestinationConditions(t *testing.T) {
for _, tc := range []struct {
name string
action policy.Action
marker bool
condition func(versionID string) string
}{
{"tags-put", policy.ReplicateObjectAction, false, func(string) string {
return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
}},
{"tags-marker", policy.ReplicateDeleteAction, true, func(string) string {
return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
}},
{"version-put", policy.PutObjectAction, false, func(v string) string {
return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v)
}},
} {
t.Run(tc.name, func(t *testing.T) {
f := newBatchAuthzFixture(t)
before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"})
batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`,
{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID)))
cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
job := batchPullJob(f.bucket)
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag",
DeleteMarker: tc.marker, UserDefined: map[string]string{},
}, false, sub)
if !isBatchJobAuthzError(err) {
t.Errorf("pull returned %v, want a refusal", err)
}
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID})
if err != nil || after.ETag != before.ETag {
t.Errorf("refused pull changed the destination version: %v", err)
}
})
}
}
// A pull does not encrypt what it writes; encryption headers reported by the
// source do not satisfy a destination policy that requires encryption.
func TestBatchPullEncryptionCondition(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", `
{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]},
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
job := batchPullJob(f.bucket)
version := mustGetUUID()
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{
Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag",
UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"},
}, false, sub)
if err == nil {
oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version})
mustIAM(t, gerr)
if !crypto.S3KMS.IsEncrypted(oi.UserDefined) {
t.Fatal("a pull was allowed by an encryption condition it did not meet")
}
}
}
// A conditional Deny on an S3 Tables action covers the S3 data actions it
// implies, under the tables resource form; the job refuses rather than skip it.
func TestBatchJobTableConditionalDeny(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context")
if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) {
t.Fatal("control: with the request context the Deny applies")
}
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) {
t.Fatalf("authorize returned %v, want a refusal", err)
}
}
type batchArchiveRevocation struct {
ObjectLayer
revoke func()
}
func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
if object == "archive/second" {
a.revoke()
}
return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
}
// Each snowball entry is authorized for the submitter as it is then.
func TestBatchPushSnowballEntryRevalidation(t *testing.T) {
f := newBatchAuthzFixture(t)
cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/")
mustIAM(t, err)
previous := newGlobalAuthZPluginFn()
setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() },
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil
}),
}))
t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
entries := []ObjectInfo{
batchPutVersion(t, f.obj, f.bucket, "archive/first", nil),
batchPutVersion(t, f.obj, f.bucket, "archive/second", nil),
}
remote, err := minio.New("archive.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.Copy(io.Discard, r.Body)
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}),
})
mustIAM(t, err)
job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}}
job.Source.Snowball.InMemory = ptr(true)
job.Source.Snowball.Compress = ptr(false)
job.Source.Snowball.SkipErrs = ptr(false)
api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }}
if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) {
t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err)
}
}
// A definition holds one job type; one with several is refused, so that what
// is authorized is what runs.
func TestBatchJobSeveralSections(t *testing.T) {
f := newBatchAuthzFixture(t)
older := f.putVersions(t, "data/a")
job := f.expireYAML("") + `keyrotate:
apiVersion: v1
bucket: other
encryption:
type: sse-s3
`
rec, queued := f.submit(t, globalActiveCred, job)
if queued != nil {
f.run(t, queued)
}
if rec.Code != http.StatusBadRequest {
t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String())
}
if !f.versionExists(t, "data/a", older) {
t.Fatal("a definition with two job types ran")
}
}
// Prefixes are walked once each in one order: a cursor in a later prefix is
// after every key of the earlier ones, and an overlapping prefix is not
// walked twice.
func TestBatchExpirePrefixes(t *testing.T) {
if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" {
t.Fatalf("canonical prefixes %q", got)
}
if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" {
t.Fatalf("canonical prefixes %q", got)
}
f := newBatchAuthzFixture(t)
done := f.putVersions(t, "a/done")
todo := f.putVersions(t, "z/todo")
batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD"))
batchPutVersion(t, f.obj, f.bucket, "x/data", nil)
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
apiVersion: v1
bucket: %s
prefix: ["z/", "x/", "a/", "x/"]
rules:
- type: object
purge:
retainVersions: 1
`, f.bucket))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
// A run that completed a/ and stopped.
ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"}
mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
f.run(t, job)
if !f.versionExists(t, "a/done", done) {
t.Error("resume did again a key before the cursor")
}
if f.versionExists(t, "z/todo", todo) {
t.Error("resume skipped a later prefix")
}
if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 {
t.Errorf("a locked version under two equal prefixes was counted %d times", got)
}
}
type batchPartialWalk struct {
ObjectLayer
version ObjectInfo
cancel context.CancelFunc
read chan struct{}
}
func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
go func() {
defer close(results)
results <- itemOrErr[ObjectInfo]{Item: a.version}
if a.cancel != nil {
<-a.read
a.cancel()
return
}
results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")}
}()
return nil
}
func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
if a.read != nil {
close(a.read)
}
return a.version, nil
}
// A rotation whose walk fails or is canceled within a key keeps that key
// after its cursor and does not report completion.
func TestBatchKeyRotationIncompleteWalk(t *testing.T) {
for _, canceled := range []bool{false, true} {
t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) {
f := newBatchAuthzFixture(t)
ctx, cancel := context.WithCancel(f.ctx)
defer cancel()
meta := batchLockMeta("LEGALHOLD")
meta[crypto.MetaSealedKeyS3] = "test-sealed-key"
api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}}
if canceled {
api.cancel, api.read = cancel, make(chan struct{})
}
job := BatchJobRequest{
ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}},
}
_ = job.KeyRotate.Start(ctx, api, job)
ri := f.jobInfo(t, &job)
if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 {
t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked)
}
})
}
}
cmd/batch-authz.go
+742
View File
@@ -0,0 +1,742 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/auth"
"github.com/pgsty/silo-pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/policy/condition"
)
// A batch job acts on objects long after the request that submitted it has
// gone. It therefore records who submitted it and acts only as that
// principal:
//
// - StartBatchJob binds the authenticated credential to the job
// (batchJobSubmitter). Nothing in the submitted definition names it.
// - Every start of the job, and every object it touches, resolves the
// binding against IAM again. A credential that is gone, disabled,
// expired, revoked or edited since submission fails the job; a reused
// access key does not match the binding.
// - Each action is evaluated with the submitter's policies and session
// policy but without a request. A policy statement that would need
// request context the job does not have (source address, transport,
// headers and so on) refuses the action instead of being skipped.
//
// A job without a binding was queued by an earlier release. It is never
// authorized, whoever submitted it, and has to be submitted again.
//go:generate msgp -file $GOFILE -unexported
// batchJobSubmitter is the server-assigned identity of the credential that
// submitted a batch job.
type batchJobSubmitter struct {
Version int `msg:"v"`
Type string `msg:"t"`
AccessKey string `msg:"ak"`
// ParentUser is the parent of a derived credential.
ParentUser string `msg:"pu"`
// UpdatedAt and RevokedBefore are the revision of the identity record;
// any edit of the identity changes them. They are unset for root.
UpdatedAt time.Time `msg:"ua"`
RevokedBefore time.Time `msg:"rb"`
// Binding is a keyed fingerprint of the credential itself, see
// batchSubmitterBinding.
Binding []byte `msg:"bd"`
}
const batchJobSubmitterVersion = 1
// Submitter types. A credential is classified in the order IsAllowed
// evaluates it.
const (
batchSubmitterRoot = "root"
batchSubmitterUser = "user"
batchSubmitterTemporary = "temporary"
batchSubmitterService = "service-account"
)
//msgp:ignore batchJobAuthzError
// batchJobAuthzError refuses a batch job, or one of its actions, for its
// submitter. It ends the job: it is persisted as the job's failure and is
// never retried.
type batchJobAuthzError struct {
reason string
}
func (e batchJobAuthzError) Error() string {
return e.reason
}
func isBatchJobAuthzError(err error) bool {
var e batchJobAuthzError
return errors.As(err, &e)
}
var (
errBatchJobResubmit = batchJobAuthzError{reason: "the batch job does not record who submitted it; submit it again"}
errBatchJobChanged = batchJobAuthzError{reason: "the credential that submitted the batch job has changed since submission; submit it again"}
errBatchJobInvalid = batchJobAuthzError{reason: "the credential that submitted the batch job is no longer valid"}
)
func batchSubmitterType(cred auth.Credentials) string {
switch {
case cred.IsTemp():
return batchSubmitterTemporary
case cred.IsServiceAccount():
return batchSubmitterService
default:
return batchSubmitterUser
}
}
// batchSubmitterBinding is a fingerprint of one credential keyed with its own
// secret. It changes when the secret, the session token or the parent
// changes, and a new credential that reuses the access key does not have it.
func batchSubmitterBinding(typ string, cred auth.Credentials) []byte {
mac := hmac.New(sha256.New, []byte(cred.SecretKey))
for _, s := range []string{"silo batch-job submitter v1", typ, cred.AccessKey, cred.ParentUser, cred.SessionToken} {
var n [8]byte
binary.BigEndian.PutUint64(n[:], uint64(len(s)))
mac.Write(n[:])
mac.Write([]byte(s))
}
return mac.Sum(nil)
}
// identityRecord returns the stored identity of a non-root access key.
func (sys *IAMSys) identityRecord(accessKey string) (UserIdentity, bool, error) {
if !sys.Initialized() {
return UserIdentity{}, false, errServerNotInitialized
}
u, ok := sys.store.GetUser(accessKey)
return u, ok, nil
}
// newBatchJobSubmitter binds the credential that authenticated a
// StartBatchJob request.
func newBatchJobSubmitter(cred auth.Credentials) (*batchJobSubmitter, error) {
root := globalActiveCred
if cred.AccessKey == "" {
return nil, errBatchJobInvalid
}
if cred.AccessKey == root.AccessKey {
if subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(root.SecretKey)) != 1 {
return nil, errBatchJobChanged
}
return &batchJobSubmitter{
Version: batchJobSubmitterVersion,
Type: batchSubmitterRoot,
AccessKey: root.AccessKey,
Binding: batchSubmitterBinding(batchSubmitterRoot, root),
}, nil
}
u, ok, err := globalIAMSys.identityRecord(cred.AccessKey)
if err != nil {
return nil, err
}
// The record must be the one the request was authenticated with.
if !ok || u.Credentials.AccessKey != cred.AccessKey || u.Credentials.ParentUser != cred.ParentUser ||
subtle.ConstantTimeCompare([]byte(u.Credentials.SecretKey), []byte(cred.SecretKey)) != 1 ||
subtle.ConstantTimeCompare([]byte(u.Credentials.SessionToken), []byte(cred.SessionToken)) != 1 {
return nil, errBatchJobChanged
}
typ := batchSubmitterType(u.Credentials)
return &batchJobSubmitter{
Version: batchJobSubmitterVersion,
Type: typ,
AccessKey: u.Credentials.AccessKey,
ParentUser: u.Credentials.ParentUser,
UpdatedAt: u.UpdatedAt,
RevokedBefore: u.RevokedBefore,
Binding: batchSubmitterBinding(typ, u.Credentials),
}, nil
}
//msgp:ignore batchJobPrincipal batchAuthzRequest
// batchJobPrincipal is a submitter resolved against current IAM state.
type batchJobPrincipal struct {
typ string
accessKey string
parentUser string
groups []string
claims map[string]any
owner bool
temp bool // evaluated as a temporary credential
service bool // evaluated as a service account
}
// principal resolves the binding against current IAM state. It fails if the
// credential no longer exists, is no longer valid, or is no longer the one
// that was bound.
func (s *batchJobSubmitter) principal(ctx context.Context) (batchJobPrincipal, error) {
if s == nil || s.AccessKey == "" {
return batchJobPrincipal{}, errBatchJobResubmit
}
if s.Version != batchJobSubmitterVersion {
return batchJobPrincipal{}, batchJobAuthzError{reason: fmt.Sprintf("unsupported batch job submitter version %d; submit the job again", s.Version)}
}
root := globalActiveCred
if s.Type == batchSubmitterRoot {
if s.AccessKey != root.AccessKey || !hmac.Equal(s.Binding, batchSubmitterBinding(batchSubmitterRoot, root)) {
return batchJobPrincipal{}, errBatchJobChanged
}
if !globalAPIConfig.permitRootAccess() {
return batchJobPrincipal{}, errBatchJobInvalid
}
return batchJobPrincipal{typ: batchSubmitterRoot, accessKey: root.AccessKey, owner: true}, nil
}
switch s.Type {
case batchSubmitterUser, batchSubmitterTemporary, batchSubmitterService:
default:
return batchJobPrincipal{}, errBatchJobResubmit
}
if s.AccessKey == root.AccessKey {
return batchJobPrincipal{}, errBatchJobChanged
}
u, ok, err := globalIAMSys.CheckKey(ctx, s.AccessKey)
if err != nil {
return batchJobPrincipal{}, err
}
if !ok {
// Deleted, disabled, expired or revoked.
return batchJobPrincipal{}, errBatchJobInvalid
}
cred := u.Credentials
typ := batchSubmitterType(cred)
if typ != s.Type || cred.AccessKey != s.AccessKey || cred.ParentUser != s.ParentUser ||
!u.UpdatedAt.Equal(s.UpdatedAt) || !u.RevokedBefore.Equal(s.RevokedBefore) ||
!hmac.Equal(s.Binding, batchSubmitterBinding(typ, cred)) {
return batchJobPrincipal{}, errBatchJobChanged
}
if cred.IsExpired() || cred.Status == auth.AccountOff {
return batchJobPrincipal{}, errBatchJobInvalid
}
p := batchJobPrincipal{
typ: typ,
accessKey: cred.AccessKey,
parentUser: cred.ParentUser,
groups: cred.Groups,
claims: map[string]any{},
temp: cred.IsTemp(),
service: !cred.IsTemp() && cred.IsServiceAccount(),
}
if typ != batchSubmitterUser {
// Verify the token as a request would, with the current signing keys.
jwtClaims, err := extractJWTClaims(u)
if err != nil {
return batchJobPrincipal{}, errBatchJobInvalid
}
p.claims = jwtClaims.Map()
if err := decodeBatchSessionPolicy(p.claims); err != nil {
return batchJobPrincipal{}, errBatchJobInvalid
}
}
p.owner = cred.ParentUser == root.AccessKey && cred.AccessKey != siteReplicatorSvcAcc
if p.owner && !globalAPIConfig.permitRootAccess() {
return batchJobPrincipal{}, errBatchJobInvalid
}
if _, ok := p.claims[policy.SessionPolicyName]; ok {
p.owner = false
}
return p, nil
}
// decodeBatchSessionPolicy decodes a session policy claim for evaluation.
// A request decodes it while verifying its token, except when an
// authorization plugin is configured; a batch job decodes it always, so the
// session policy also limits what the plugin allows.
func decodeBatchSessionPolicy(claims map[string]any) error {
sp, ok := claims[policy.SessionPolicyName]
if !ok {
return nil
}
s, ok := sp.(string)
if !ok {
return errAuthentication
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return errAuthentication
}
claims[sessionPolicyNameExtracted] = string(b)
return nil
}
// batchAuthzRequest is one action a batch job is about to take.
type batchAuthzRequest struct {
action policy.Action
bucket string
object string // empty for a bucket action
// versionID is the version an object version action addresses.
versionID string
// tags are the current tags of the object version acted on.
tags string
// tagsUnknown is set when the job does not read the version it acts
// on, so s3:ExistingObjectTag is context it does not have.
tagsUnknown bool
// values are request values the job supplies itself, such as the
// listing prefix or the encryption and retention it writes, keyed by
// condition key name.
values map[string][]string
}
// batchAuthzMarker marks the condition values of a batch job for an
// authorization plugin. Requests cannot set s3:authType.
const batchAuthzMarker = "batch-job"
// Condition keys a batch job always provides, as a request would: identity,
// time, the addressed version and the stored tags. Keys a request takes from
// itself are provided only when the job sets them (see batchAuthzRequest
// values); every other key is request context the job does not have.
var batchServerConditionKeys = map[string]bool{
condition.AWSCurrentTime.ToKey().String(): true,
condition.AWSEpochTime.ToKey().String(): true,
condition.AWSPrincipalType.ToKey().String(): true,
condition.AWSUserID.ToKey().String(): true,
condition.AWSUsername.ToKey().String(): true,
condition.AWSGroups.ToKey().String(): true,
condition.S3VersionID.ToKey().String(): true,
condition.ExistingObjectTag.ToKey().String(): true,
}
// Request keys a job may supply, by the name they have in condition values.
var batchRequestConditionKeys = map[string]condition.KeyName{
condition.S3Prefix.Name(): condition.S3Prefix,
condition.S3XAmzServerSideEncryption.Name(): condition.S3XAmzServerSideEncryption,
condition.S3XAmzServerSideEncryptionAwsKmsKeyID.Name(): condition.S3XAmzServerSideEncryptionAwsKmsKeyID,
condition.S3ObjectLockMode.Name(): condition.S3ObjectLockMode,
condition.S3ObjectLockRetainUntilDate.Name(): condition.S3ObjectLockRetainUntilDate,
condition.S3ObjectLockLegalHold.Name(): condition.S3ObjectLockLegalHold,
condition.RequestObjectTagKeys.Name(): condition.RequestObjectTagKeys,
}
// suppliedConditionKeys returns the condition keys whose values req provides.
func (req batchAuthzRequest) suppliedConditionKeys() map[string]bool {
supplied := make(map[string]bool, len(batchServerConditionKeys)+len(req.values))
for k := range batchServerConditionKeys {
supplied[k] = true
}
if req.tagsUnknown {
delete(supplied, condition.ExistingObjectTag.ToKey().String())
}
for name := range req.values {
if key, ok := batchRequestConditionKeys[name]; ok {
supplied[key.ToKey().String()] = true
}
if strings.HasPrefix(name, condition.RequestObjectTag.Name()+"/") {
supplied[condition.RequestObjectTag.ToKey().String()] = true
}
}
return supplied
}
func batchKeySupplied(supplied map[string]bool, name string) bool {
if i := strings.IndexByte(name, '/'); i >= 0 {
name = name[:i]
}
if strings.HasPrefix(name, "jwt:") || strings.HasPrefix(name, "ldap:") {
// Claims of the verified token, as a request has them.
return true
}
return supplied[name]
}
// conditionValues are the values a request with the same identity would
// carry, without any value taken from the request itself.
func (p batchJobPrincipal) conditionValues(req batchAuthzRequest) map[string][]string {
now := UTCNow()
username := p.accessKey
if p.temp || p.service {
username = p.parentUser
}
principalType := "User"
if len(p.claims) > 0 {
principalType = "AssumedRole"
}
if username == globalActiveCred.AccessKey {
principalType = "Account"
}
// Identity and time, then the claims over them, as a request has them
// (an LDAP session's short username replaces its parent DN).
values := map[string][]string{
"CurrentTime": {now.Format(time.RFC3339)},
"EpochTime": {strconv.FormatInt(now.Unix(), 10)},
"principaltype": {principalType},
"userid": {username},
"username": {username},
}
for k, v := range p.claims {
if s, ok := v.(string); ok {
values[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{s}
}
}
// No claim stands for a value of the object or of the job, which
// follow and always win.
for k := range values {
if k == "versionid" || k == "prefix" || k == "authtype" ||
strings.HasPrefix(k, "existingobjecttag/") || strings.HasPrefix(k, "requestobjecttag") ||
strings.HasPrefix(k, "x-amz-") || strings.HasPrefix(k, "object-lock-") {
delete(values, k)
}
}
// For an authorization plugin: this is a batch job, not a request.
values["authType"] = []string{batchAuthzMarker}
if req.versionID != "" {
values["versionid"] = []string{req.versionID}
}
if req.tags != "" {
if t, err := tags.ParseObjectTags(req.tags); err == nil {
for k, v := range t.ToMap() {
values[pathJoin("ExistingObjectTag", k)] = []string{v}
}
}
}
for k, v := range req.values {
values[k] = v
}
if grpsVal, ok := p.claims["groups"]; ok {
if grpsIs, ok := grpsVal.([]any); ok {
grps := []string{}
for _, gI := range grpsIs {
if g, ok := gI.(string); ok {
grps = append(grps, g)
}
}
if len(grps) > 0 {
values["groups"] = grps
}
}
}
if _, ok := values["groups"]; !ok && len(p.groups) > 0 {
values["groups"] = p.groups
}
return values
}
// authorize evaluates one action of the job as its submitter.
func (p batchJobPrincipal) authorize(req batchAuthzRequest) error {
args := policy.Args{
AccountName: p.accessKey,
Groups: p.groups,
Action: req.action,
BucketName: req.bucket,
ObjectName: req.object,
ConditionValues: p.conditionValues(req),
IsOwner: p.owner,
Claims: p.claims,
}
denied := batchJobAuthzError{reason: batchDeniedReason(req)}
plugin := newGlobalAuthZPluginFn()
policies, err := globalIAMSys.batchLocalPolicies(p, args, plugin == nil)
if err != nil {
return denied
}
supplied := req.suppliedConditionKeys()
for _, pol := range policies {
for _, st := range pol.Statements {
if batchStatementNeedsRequest(st, args, supplied) {
return batchJobAuthzError{reason: denied.reason + ": a policy statement depends on request context that a batch job does not have"}
}
}
}
if plugin != nil {
ok, err := plugin.IsAllowed(args)
if err != nil {
authZLogIf(GlobalContext, err)
}
if !ok {
return denied
}
// The plugin does not see a session policy; it only limits further.
if has, allowed := p.sessionPolicyDecision(args); has && !allowed {
return denied
}
return nil
}
if !globalIAMSys.isAllowedLocally(args) {
return denied
}
return nil
}
func batchDeniedReason(req batchAuthzRequest) string {
resource := req.bucket
if req.object != "" {
resource = pathJoin(req.bucket, req.object)
}
if req.versionID != "" {
resource += " (versionId " + req.versionID + ")"
}
return fmt.Sprintf("the batch job submitter is not allowed %s on %s", req.action, resource)
}
// sessionPolicyDecision evaluates the session policy the way IsAllowed would
// for this credential.
func (p batchJobPrincipal) sessionPolicyDecision(args policy.Args) (bool, bool) {
switch {
case p.temp:
return isAllowedBySessionPolicy(args)
case p.service:
if p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
return false, false
}
return isAllowedBySessionPolicyForServiceAccount(args)
}
return false, false
}
// batchLocalPolicies returns every local policy that can take part in the
// decision on args: the mapped policies that IsAllowed would merge when
// withMapped is set (no authorization plugin), and the session policy.
func (sys *IAMSys) batchLocalPolicies(p batchJobPrincipal, args policy.Args, withMapped bool) ([]policy.Policy, error) {
var out []policy.Policy
if withMapped && args.IsOwner {
// IsAllowed does not evaluate any policy for the owner.
return nil, nil
}
if withMapped {
var names []string
ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey
switch {
case p.typ == batchSubmitterRoot || ownerDerived:
case p.temp || p.service:
if roleArn := args.GetRoleArn(); roleArn != "" {
a, err := arn.Parse(roleArn)
if err != nil {
return nil, err
}
names = append(names, newMappedPolicy(sys.rolesMap[a]).toSlice()...)
} else {
mapped, err := sys.PolicyDBGet(p.parentUser, args.Groups...)
if err != nil {
return nil, err
}
names = append(names, mapped...)
// The claimed policies apply only when nothing is mapped.
if len(mapped) == 0 {
if claimed, ok := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID()); ok {
names = append(names, claimed.ToSlice()...)
}
}
}
default:
mapped, err := sys.PolicyDBGet(p.accessKey, args.Groups...)
if err != nil {
return nil, err
}
names = append(names, mapped...)
}
if len(names) > 0 {
out = append(out, sys.GetCombinedPolicy(names...))
}
}
if p.temp || p.service {
if p.service && p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
return out, nil
}
if sp, ok := p.claims[sessionPolicyNameExtracted]; ok {
s, ok := sp.(string)
if !ok {
return nil, errAuthentication
}
parsed, err := policy.ParseConfig(strings.NewReader(s))
if err != nil {
return nil, err
}
out = append(out, *parsed)
}
}
return out, nil
}
// batchStatementNeedsRequest reports whether a statement that applies to
// args depends on a condition key that the job does not supply. Applying
// such a statement without its value could grant (Allow) or drop (Deny)
// what a request with that context would not, so the job refuses the action.
// Resources are matched without conditions; a statement is taken to apply
// whenever it might.
func batchStatementNeedsRequest(st policy.Statement, args policy.Args, supplied map[string]bool) bool {
if (!st.Actions.Match(args.Action) && !st.Actions.IsEmpty()) || st.NotActions.Match(args.Action) {
return false
}
// Policy variables in resources or condition values.
if data, err := json.Marshal(st); err == nil {
s := string(data)
for {
i := strings.Index(s, "${")
if i < 0 {
break
}
s = s[i+2:]
j := strings.IndexByte(s, '}')
if j < 0 {
break
}
if !batchKeySupplied(supplied, s[:j]) {
return true
}
s = s[j+1:]
}
}
if !batchStatementMayApply(st, args) {
return false
}
for key := range st.Conditions.Keys() {
if !batchKeySupplied(supplied, key.String()) {
return true
}
}
return false
}
// batchStatementMayApply matches the resources of a statement, in both forms
// a bucket request can take. A statement whose resources the evaluator
// translates (S3 Tables and vectors actions cover S3 data actions) is taken
// to apply.
func batchStatementMayApply(st policy.Statement, args policy.Args) bool {
for _, actions := range []policy.ActionSet{st.Actions, st.NotActions} {
for action := range actions {
if policy.TableAction(action).IsValid() || policy.VectorsAction(action).IsValid() {
return true
}
}
}
for _, resources := range []policy.ResourceSet{st.Resources, st.NotResources} {
for r := range resources {
if r.Type != policy.ResourceARNS3 && r.Type != policy.ResourceARNAll {
return true
}
}
}
resources := []string{args.BucketName + "/" + strings.TrimPrefix(args.ObjectName, "/")}
if args.ObjectName == "" {
resources = []string{args.BucketName, args.BucketName + "/"}
}
for _, resource := range resources {
if len(st.Resources) > 0 && !st.Resources.Match(resource, args.ConditionValues) {
continue
}
if len(st.NotResources) > 0 && st.NotResources.Match(resource, args.ConditionValues) {
continue
}
return true
}
return false
}
// authorize resolves the submitter and evaluates each request.
func (s *batchJobSubmitter) authorize(ctx context.Context, reqs ...batchAuthzRequest) error {
p, err := s.principal(ctx)
if err != nil {
return err
}
for _, req := range reqs {
if err := p.authorize(req); err != nil {
return err
}
}
return nil
}
// startAuthzRequests are the bucket actions a job needs before it acts on any
// object: the listings it walks. Object actions are authorized per object.
func (j BatchJobRequest) startAuthzRequests() []batchAuthzRequest {
list := func(bucket string, prefixes []string) []batchAuthzRequest {
if len(prefixes) == 0 {
prefixes = []string{""}
}
var reqs []batchAuthzRequest
for _, prefix := range prefixes {
values := map[string][]string{condition.S3Prefix.Name(): {prefix}}
reqs = append(reqs,
batchAuthzRequest{action: policy.ListBucketAction, bucket: bucket, values: values},
batchAuthzRequest{action: policy.ListBucketVersionsAction, bucket: bucket, values: values},
)
}
return reqs
}
// The same order as Type() and the job pool.
switch {
case j.Replicate != nil:
if !j.Replicate.RemoteToLocal() {
return list(j.Replicate.Source.Bucket, j.Replicate.Source.Prefix.F())
}
case j.KeyRotate != nil:
return list(j.KeyRotate.Bucket, []string{j.KeyRotate.Prefix})
case j.Expire != nil:
return list(j.Expire.Bucket, batchCanonicalPrefixes(j.Expire.Prefix.F()))
}
return nil
}
// jobSections counts the job types a definition holds. A job is one of them.
func (j BatchJobRequest) jobSections() int {
n := 0
for _, set := range []bool{j.Replicate != nil, j.KeyRotate != nil, j.Expire != nil} {
if set {
n++
}
}
return n
}
var errBatchJobSections = batchJobAuthzError{reason: "a batch job must define exactly one job type"}
// refuse ends the job for its submitter.
func (ri *batchJobInfo) refuse(err error) {
ri.mu.Lock()
defer ri.mu.Unlock()
if ri.FailureReason == "" {
ri.FailureReason = err.Error()
}
ri.Complete = false
ri.Failed = true
}
func (ri *batchJobInfo) refused() bool {
ri.mu.RLock()
defer ri.mu.RUnlock()
return ri.FailureReason != ""
}
// startAuthorized checks the submitter of a job that is about to run. It
// returns done for a job that must not run now: one refused before, which
// stays refused, or one refused now, which is persisted as failed first.
// An error that is not a decision leaves the job to a later start.
func (ri *batchJobInfo) startAuthorized(ctx context.Context, api ObjectLayer, job BatchJobRequest) (done bool, err error) {
if !ri.refused() {
err := job.Submitter.authorize(ctx, job.startAuthzRequests()...)
if err == nil && job.jobSections() != 1 {
err = errBatchJobSections
}
if err == nil {
return false, nil
}
if !isBatchJobAuthzError(err) {
return true, err
}
ri.refuse(err)
batchLogIf(ctx, fmt.Errorf("batch job %s refused: %w", job.ID, err))
}
globalBatchJobsMetrics.save(job.ID, ri)
return true, ri.updateAfter(ctx, api, 0, job)
}
cmd/batch-authz_gen.go
+260
View File
@@ -0,0 +1,260 @@
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
package cmd
import (
"github.com/tinylib/msgp/msgp"
)
// DecodeMsg implements msgp.Decodable
func (z *batchJobSubmitter) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "v":
z.Version, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "Version")
return
}
case "t":
z.Type, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
case "ak":
z.AccessKey, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "AccessKey")
return
}
case "pu":
z.ParentUser, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "ParentUser")
return
}
case "ua":
z.UpdatedAt, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "UpdatedAt")
return
}
case "rb":
z.RevokedBefore, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "RevokedBefore")
return
}
case "bd":
z.Binding, err = dc.ReadBytes(z.Binding)
if err != nil {
err = msgp.WrapError(err, "Binding")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *batchJobSubmitter) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 7
// write "v"
err = en.Append(0x87, 0xa1, 0x76)
if err != nil {
return
}
err = en.WriteInt(z.Version)
if err != nil {
err = msgp.WrapError(err, "Version")
return
}
// write "t"
err = en.Append(0xa1, 0x74)
if err != nil {
return
}
err = en.WriteString(z.Type)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
// write "ak"
err = en.Append(0xa2, 0x61, 0x6b)
if err != nil {
return
}
err = en.WriteString(z.AccessKey)
if err != nil {
err = msgp.WrapError(err, "AccessKey")
return
}
// write "pu"
err = en.Append(0xa2, 0x70, 0x75)
if err != nil {
return
}
err = en.WriteString(z.ParentUser)
if err != nil {
err = msgp.WrapError(err, "ParentUser")
return
}
// write "ua"
err = en.Append(0xa2, 0x75, 0x61)
if err != nil {
return
}
err = en.WriteTime(z.UpdatedAt)
if err != nil {
err = msgp.WrapError(err, "UpdatedAt")
return
}
// write "rb"
err = en.Append(0xa2, 0x72, 0x62)
if err != nil {
return
}
err = en.WriteTime(z.RevokedBefore)
if err != nil {
err = msgp.WrapError(err, "RevokedBefore")
return
}
// write "bd"
err = en.Append(0xa2, 0x62, 0x64)
if err != nil {
return
}
err = en.WriteBytes(z.Binding)
if err != nil {
err = msgp.WrapError(err, "Binding")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *batchJobSubmitter) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 7
// string "v"
o = append(o, 0x87, 0xa1, 0x76)
o = msgp.AppendInt(o, z.Version)
// string "t"
o = append(o, 0xa1, 0x74)
o = msgp.AppendString(o, z.Type)
// string "ak"
o = append(o, 0xa2, 0x61, 0x6b)
o = msgp.AppendString(o, z.AccessKey)
// string "pu"
o = append(o, 0xa2, 0x70, 0x75)
o = msgp.AppendString(o, z.ParentUser)
// string "ua"
o = append(o, 0xa2, 0x75, 0x61)
o = msgp.AppendTime(o, z.UpdatedAt)
// string "rb"
o = append(o, 0xa2, 0x72, 0x62)
o = msgp.AppendTime(o, z.RevokedBefore)
// string "bd"
o = append(o, 0xa2, 0x62, 0x64)
o = msgp.AppendBytes(o, z.Binding)
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *batchJobSubmitter) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "v":
z.Version, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Version")
return
}
case "t":
z.Type, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
case "ak":
z.AccessKey, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "AccessKey")
return
}
case "pu":
z.ParentUser, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ParentUser")
return
}
case "ua":
z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "UpdatedAt")
return
}
case "rb":
z.RevokedBefore, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "RevokedBefore")
return
}
case "bd":
z.Binding, bts, err = msgp.ReadBytesBytes(bts, z.Binding)
if err != nil {
err = msgp.WrapError(err, "Binding")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *batchJobSubmitter) Msgsize() (s int) {
s = 1 + 2 + msgp.IntSize + 2 + msgp.StringPrefixSize + len(z.Type) + 3 + msgp.StringPrefixSize + len(z.AccessKey) + 3 + msgp.StringPrefixSize + len(z.ParentUser) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.BytesPrefixSize + len(z.Binding)
return
}
cmd/batch-authz_gen_test.go
+123
View File
@@ -0,0 +1,123 @@
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
package cmd
import (
"bytes"
"testing"
"github.com/tinylib/msgp/msgp"
)
func TestMarshalUnmarshalbatchJobSubmitter(t *testing.T) {
v := batchJobSubmitter{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgbatchJobSubmitter(b *testing.B) {
v := batchJobSubmitter{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgbatchJobSubmitter(b *testing.B) {
v := batchJobSubmitter{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalbatchJobSubmitter(b *testing.B) {
v := batchJobSubmitter{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodebatchJobSubmitter(t *testing.T) {
v := batchJobSubmitter{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodebatchJobSubmitter Msgsize() is inaccurate")
}
vn := batchJobSubmitter{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodebatchJobSubmitter(b *testing.B) {
v := batchJobSubmitter{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodebatchJobSubmitter(b *testing.B) {
v := batchJobSubmitter{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
cmd/batch-expire-purge-authz_test.go
+524
View File
@@ -0,0 +1,524 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config/storageclass"
xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/policy"
)
// A batch expiration that keeps no versions purges each key it matched under
// the purge's lock. What it authorizes for the submitter, and checks for
// Object Lock, are the versions the purge deletes then, not the versions the
// walk saw before.
// batchPurgeHook runs change once, after the walk of a job and before its
// first purge of all versions.
type batchPurgeHook struct {
ObjectLayer
once sync.Once
change func()
}
func (h *batchPurgeHook) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
if opts.DeleteAllVersions {
h.once.Do(h.change)
}
return h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
}
func (f *batchAuthzFixture) purgeYAML(prefix string) string {
return fmt.Sprintf(`expire:
apiVersion: v1
bucket: %s
prefix: %q
rules:
- type: object
purge:
retainVersions: 0
`, f.bucket, prefix)
}
// runChanged submits a purge of prefix as cred and runs it, with change made
// between its walk and its purge, and returns the job's final state.
func (f *batchAuthzFixture) runChanged(t *testing.T, cred auth.Credentials, prefix string, change func()) *batchJobInfo {
t.Helper()
rec, job := f.submit(t, cred, f.purgeYAML(prefix))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
hook := &batchPurgeHook{ObjectLayer: f.obj, change: change}
if err := job.Expire.Start(f.ctx, hook, *job); err != nil {
t.Logf("start: %v", err)
}
return f.jobInfo(t, job)
}
// putOlder writes a version of key older than every version it has, as a
// replicated or pulled version can be, with metadata meta.
func (f *batchAuthzFixture) putOlder(t *testing.T, key, versionID string, meta map[string]string) {
t.Helper()
data := []byte("older " + key)
_, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
ObjectOptions{Versioned: true, VersionID: versionID, MTime: UTCNow().Add(-time.Hour), UserDefined: meta})
mustIAM(t, err)
}
func batchPurgeVersionIDs(t *testing.T, f *batchAuthzFixture, key string) []string {
t.Helper()
var ids []string
marker, vmarker := "", ""
for {
res, err := f.obj.ListObjectVersions(f.ctx, f.bucket, key, marker, vmarker, "", 1000)
mustIAM(t, err)
for _, o := range res.Objects {
if o.Name == key {
ids = append(ids, o.VersionID)
}
}
if !res.IsTruncated {
return ids
}
marker, vmarker = res.NextMarker, res.NextVersionIDMarker
}
}
// A version that the submitter may not expire, because of its tags or its
// version ID, is refused although the walk did not see it so: the job fails
// and the key keeps all its versions.
func TestBatchExpirePurgeAuthorizesCurrentVersions(t *testing.T) {
for _, tc := range []struct {
name string
action string // the action a conditional Deny denies
deny string // its condition
// change returns the version IDs that change made.
change func(t *testing.T, f *batchAuthzFixture, key, older string) []string
}{
{
name: "tagged-after-walk",
action: "s3:GetObjectVersion",
deny: `{"StringEquals":{"s3:ExistingObjectTag/keep":["yes"]}}`,
change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older})
mustIAM(t, err)
return nil
},
},
{
name: "version-after-walk",
action: "s3:DeleteObjectVersion",
deny: `{"StringEquals":{"s3:versionid":["%s"]}}`,
change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
f.putOlder(t, key, batchPurgeDeniedVersion, nil)
return []string{batchPurgeDeniedVersion}
},
},
} {
t.Run(tc.name, func(t *testing.T) {
f := newBatchAuthzFixture(t)
deny := tc.deny
if strings.Contains(deny, "%s") {
deny = fmt.Sprintf(deny, batchPurgeDeniedVersion)
}
batchAuthzPolicy(f.ctx, t, f.sys, "batch-purge", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["`+tc.action+`"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":`+deny+`}`)
cred := iamAdminUser(f.ctx, t, f.sys, "batch-purge-user", "batch-purge")
const key = "purge/key"
older := f.putVersions(t, key)
want := batchPurgeVersionIDs(t, f, key)
ri := f.runChanged(t, cred, "purge/", func() {
want = append(want, tc.change(t, f, key, older)...)
})
for _, id := range want {
if !f.versionExists(t, key, id) {
t.Errorf("version %s was purged although the submitter may not delete it", id)
}
}
if !ri.Failed || ri.Complete || ri.FailureReason == "" {
t.Errorf("job complete=%v failed=%v reason=%q, want a refused job", ri.Complete, ri.Failed, ri.FailureReason)
}
if ri.Objects != 0 || ri.KeysRefused != 0 {
t.Errorf("job counts objects=%d refused=%d", ri.Objects, ri.KeysRefused)
}
})
}
}
const batchPurgeDeniedVersion = "6c4b2f8e-1d3a-4e5b-9c7d-0a1b2c3d4e5f"
// A version that Object Lock protects refuses the whole key, also when it
// was added after the walk. The job completes with the key counted as
// refused, not failed.
func TestBatchExpirePurgeRefusesLockedAfterWalk(t *testing.T) {
for _, kind := range batchLockKinds {
t.Run(kind, func(t *testing.T) {
f := newBatchAuthzFixture(t)
enableBucketObjectLock(t, f.bucket)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
const key = "purge/locked"
f.putVersions(t, key)
want := batchPurgeVersionIDs(t, f, key)
locked := mustGetUUID()
ri := f.runChanged(t, cred, "purge/", func() {
f.putOlder(t, key, locked, batchLockMeta(kind))
want = append(want, locked)
})
for _, id := range want {
if !f.versionExists(t, key, id) {
t.Errorf("version %s of a key with a locked version was purged", id)
}
}
if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
}
})
}
}
// Controls: a purge by root and by an authorized user removes every version,
// including one added after the walk.
func TestBatchExpirePurgeAuthorizedControls(t *testing.T) {
for _, root := range []bool{true, false} {
t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
f := newBatchAuthzFixture(t)
cred := globalActiveCred
if !root {
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
}
const key = "purge/key"
f.putVersions(t, key)
added := mustGetUUID()
ri := f.runChanged(t, cred, "purge/", func() {
f.putOlder(t, key, added, map[string]string{xhttp.AmzObjectTagging: "keep=yes"})
})
if ids := batchPurgeVersionIDs(t, f, key); len(ids) != 0 {
t.Errorf("an authorized purge kept %v", ids)
}
if !ri.Complete || ri.Failed || ri.Objects != 3 || ri.KeysRefused != 0 {
t.Errorf("job complete=%v failed=%v objects=%d refused=%d, want complete with 3 objects",
ri.Complete, ri.Failed, ri.Objects, ri.KeysRefused)
}
})
}
}
// Cases found in the sixth review round.
// batchPurgeBudgetChange runs change once the first purge stopped at its
// budget after removing versions.
type batchPurgeBudgetChange struct {
ObjectLayer
change func()
calls int
}
func (h *batchPurgeBudgetChange) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
oi, err := h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
if opts.DeleteAllVersions {
h.calls++
if h.calls == 1 && errors.Is(err, errDeleteAllBudget) && opts.DeleteAllResult.Removed > 0 {
h.change()
}
}
return oi, err
}
// A purge that continues after its budget authorizes the versions again, as
// they are then; the refusal and the versions already removed persist
// across another start.
func TestBatchExpirePurgeBudgetReauthorizes(t *testing.T) {
f := newBatchAuthzFixture(t)
const key = "purge/budget"
oldest := batchPutVersion(t, f.obj, f.bucket, key, nil)
middle := batchPutVersion(t, f.obj, f.bucket, key, nil)
latest := batchPutVersion(t, f.obj, f.bucket, key, nil)
batchAuthzPolicy(f.ctx, t, f.sys, "review6-budget-deny", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`)
cred := iamAdminUser(f.ctx, t, f.sys, "review6-budget-user", "review6-budget-deny")
rec, job := f.submit(t, cred, f.purgeYAML("purge/"))
if job == nil {
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
}
oldBudget := deleteAllBudget
deleteAllBudget.versions = 1
t.Cleanup(func() { deleteAllBudget = oldBudget })
hook := &batchPurgeBudgetChange{ObjectLayer: f.obj, change: func() {
_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: middle.VersionID})
mustIAM(t, err)
}}
mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
ri := f.jobInfo(t, job)
if hook.calls != 2 || !ri.Failed || ri.Complete || ri.FailureReason == "" || ri.Objects != 1 || ri.KeysRefused != 0 {
t.Fatalf("continuation: calls=%d complete=%v failed=%v reason=%q objects=%d refused=%d", hook.calls, ri.Complete, ri.Failed, ri.FailureReason, ri.Objects, ri.KeysRefused)
}
if f.versionExists(t, key, oldest.VersionID) || !f.versionExists(t, key, middle.VersionID) || !f.versionExists(t, key, latest.VersionID) {
t.Fatal("only the first authorized version may be removed")
}
_, err := f.obj.DeleteObjectTags(f.ctx, f.bucket, key, ObjectOptions{VersionID: middle.VersionID})
mustIAM(t, err)
mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
after := f.jobInfo(t, job)
if hook.calls != 2 || after.FailureReason != ri.FailureReason || after.Objects != 1 || after.Complete {
t.Fatal("persisted refusal must prevent a fresh purge after the tags change")
}
}
// The purge's lock excludes tag writers from its authorization through its
// last delete, for a directory key in a lock domain of its own on one pool
// and for a key in a second pool.
func TestBatchExpirePurgeExcludesTagWriter(t *testing.T) {
for _, mode := range []string{ldDsync, "two-pools"} {
t.Run(mode, func(t *testing.T) {
var z *erasureServerPools
var bucket string
pool, key := 0, "purge/tags/"
if mode == "two-pools" {
z, bucket = consistencyPools(t)
pool = 1
} else {
z, bucket = ldPool(t, mode)
for i := 0; ; i++ {
key = fmt.Sprintf("purge/tags-%d/", i)
if z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) == 1 {
break
}
}
}
before := putConsistencyObject(t, z, bucket, encodeDirObject(key), pool, "", ObjectOptions{Versioned: true})
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
entered, release := make(chan struct{}), make(chan struct{})
var released atomic.Bool
unblock := func() {
if released.CompareAndSwap(false, true) {
close(release)
}
}
defer unblock()
purged := make(chan error, 1)
go func() {
_, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: before.VersionID, ModTime: before.ModTime},
deleteAllAuthorize: func(versions []ObjectInfo) error {
close(entered)
select {
case <-release:
return nil
case <-ctx.Done():
return ctx.Err()
}
},
})
purged <- err
}()
select {
case <-entered:
case <-ctx.Done():
t.Fatal("purge never reached authorization")
}
written := make(chan error, 1)
go func() {
_, err := z.PutObjectTags(ctx, bucket, key, "keep=yes", ObjectOptions{VersionID: before.VersionID})
written <- err
}()
select {
case err := <-written:
t.Errorf("tag writer crossed the purge's authorization lock: %v", err)
case <-time.After(ldWait):
unblock()
err := <-written
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
t.Errorf("writer must observe the completed purge: %v", err)
}
}
unblock()
mustIAM(t, <-purged)
})
}
}
// batchTagWriteFault fails metadata updates of one key, so that a tag write
// reaches only the drives it does not wrap.
type batchTagWriteFault struct {
StorageAPI
bucket, object string
}
func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object string, fi FileInfo, opts UpdateMetadataOpts) error {
if bucket == d.bucket && object == d.object {
return errDiskNotFound
}
return d.StorageAPI.UpdateMetadata(ctx, bucket, object, fi, opts)
}
// A failed tag write can leave a revision that the submitter may not expire
// on a minority of drives. The purge removes it with the version, so it is
// authorized too.
func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) {
resetTestGlobals()
t.Cleanup(resetTestGlobals)
// One drive of sixteen holds the failed tag write, so the read quorum of
// the metadata must stay below sixteen. Earlier tests can leave a
// storage class of parity 0 behind (newTestConfig on a single drive).
previous := globalStorageClass
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
t.Cleanup(func() { globalStorageClass.Update(previous) })
z := purgeSinglePool(t, 16)
globalIAMSys.initStore(z, nil)
mustIAM(t, globalIAMSys.Load(t.Context(), true))
f := &batchAuthzFixture{ctx: t.Context(), sys: globalIAMSys, obj: z, bucket: "review6-batch-authz"}
mustIAM(t, z.MakeBucket(f.ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
batchTestJobPool(f.ctx, t, z)
const key = "purge/minority-tags"
older := f.putVersions(t, key)
batchAuthzPolicy(f.ctx, t, f.sys, "review6-minority-deny", strings.ReplaceAll(batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`, "batch-authz", f.bucket))
cred := iamAdminUser(f.ctx, t, f.sys, "review6-minority-user", "review6-minority-deny")
sub, err := newBatchJobSubmitter(cred)
mustIAM(t, err)
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectVersionAction, bucket: f.bucket, object: key, versionID: older, tags: "keep=yes"}); !isBatchJobAuthzError(err) {
t.Fatal("control: this tag revision must deny the read needed for expiration")
}
set := z.serverPools[0].getHashedSet(key)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
wrapped := append([]StorageAPI(nil), disks...)
for i := 1; i < len(wrapped); i++ {
wrapped[i] = batchTagWriteFault{StorageAPI: wrapped[i], bucket: f.bucket, object: key}
}
set.getDisks = func() []StorageAPI { return wrapped }
_, err = f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older, UserDefined: map[string]string{
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
}})
set.getDisks = original
if err == nil {
t.Fatal("control: the tag write must fail its quorum")
}
fi, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{})
mustIAM(t, err)
if fi.Metadata[xhttp.AmzObjectTagging] != "keep=yes" {
t.Fatal("control: the successful drive must hold the denied tag revision")
}
history, err := z.deleteAllHistory(f.ctx, f.bucket, key)
mustIAM(t, err)
for _, v := range history {
for _, c := range v.copies {
if c.fi.Metadata[xhttp.AmzObjectTagging] != "" {
t.Fatal("control: the quorum history must still have the original tags")
}
}
}
ri := f.runChanged(t, cred, "purge/", func() {})
if !ri.Failed || ri.Complete || !strings.Contains(ri.FailureReason, "is not allowed") {
t.Errorf("denied physical revision must refuse the job: complete=%v failed=%v reason=%q", ri.Complete, ri.Failed, ri.FailureReason)
}
if _, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}); err != nil {
t.Errorf("denied physical revision was removed: %v", err)
}
if got := batchPurgeVersionIDs(t, f, key); len(got) != 2 {
t.Errorf("denial must keep the whole key; remaining versions=%v", got)
}
}
// Cases found in the seventh review round.
// The time the authorization takes counts against the purge's budget: a
// late allow and a deny both delete nothing.
func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) {
z := purgeSinglePool(t, 16)
p := newPurgeTest(t, z, "review7-purge-auth-time", MakeBucketOptions{VersioningEnabled: true}, "")
const key = "purge/key"
p.version(0, key, "older", 2)
p.version(0, key, "latest", 1)
exp := p.matched(key)
oldBudget := deleteAllBudget
deleteAllBudget.duration = 10 * time.Millisecond
t.Cleanup(func() { deleteAllBudget = oldBudget })
denied := errors.New("review7: revision denied")
for _, decision := range []error{nil, denied} {
called := false
var res deleteAllResult
_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
deleteAllAuthorize: func(versions []ObjectInfo) error {
called = true
if len(versions) < 2 {
t.Error("authorization did not receive both approved versions")
}
time.Sleep(20 * time.Millisecond)
return decision
},
})
want := decision
if want == nil {
want = errDeleteAllBudget
}
if !called || !errors.Is(err, want) || res.Started || res.Complete || res.Removed != 0 {
t.Errorf("late authorization: called=%v err=%v result=%+v", called, err, res)
}
if got := p.held(0, key); len(got) != 2 {
t.Fatalf("purge removed versions after late authorization: %v", got)
}
}
}
// A listing that failed before any result fails the run without moving the
// cursor or refusing the job; a later run completes it, in both modes.
func TestBatchExpireWalkFailureResumes(t *testing.T) {
for _, purgeAll := range []bool{false, true} {
name := "retain-one"
if purgeAll {
name = "retain-zero"
}
t.Run(name, func(t *testing.T) {
f := newBatchAuthzFixture(t)
const key = "data/key"
older := f.putVersions(t, key)
definition := f.expireYAML("data/")
if purgeAll {
definition = f.purgeYAML("data/")
}
rec, job := f.submit(t, globalActiveCred, definition)
if job == nil {
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
}
listErr := errors.New("review7: transient listing failure")
err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job)
if !errors.Is(err, listErr) {
t.Fatalf("listing error: %v", err)
}
ri := f.jobInfo(t, job)
if !ri.Failed || ri.Complete || ri.FailureReason != "" || ri.Object != "" || !f.versionExists(t, key, older) {
t.Fatalf("listing failure changed progress or became terminal: failed=%v complete=%v reason=%q cursor=%q", ri.Failed, ri.Complete, ri.FailureReason, ri.Object)
}
mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
ri = f.jobInfo(t, job)
wantRemoved := int64(1)
if purgeAll {
wantRemoved = 2
}
if ri.Failed || !ri.Complete || ri.FailureReason != "" || ri.Objects != wantRemoved || f.versionExists(t, key, older) {
t.Fatalf("resume did not complete: failed=%v complete=%v reason=%q removed=%d", ri.Failed, ri.Complete, ri.FailureReason, ri.Objects)
}
})
}
}
cmd/batch-expire.go
+329 -207
View File
@@ -27,6 +27,7 @@ import (
"net/http"
"runtime"
"strconv"
"sync/atomic"
"time"
"github.com/minio/minio-go/v7/pkg/tags"
@@ -34,6 +35,7 @@ import (
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/pgsty/silo-pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/wildcard"
"github.com/pgsty/silo-pkg/v3/workers"
"github.com/pgsty/silo-pkg/v3/xtime"
@@ -289,16 +291,6 @@ type BatchJobExpire struct {
var _ yaml.Unmarshaler = &BatchJobExpire{}
// RedactSensitive will redact any sensitive information in b.
func (r *BatchJobExpire) RedactSensitive() {
if r == nil {
return
}
if r.NotificationCfg.Token != "" {
r.NotificationCfg.Token = redactedText
}
}
// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
type expireJob BatchJobExpire
@@ -345,32 +337,6 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
return nil
}
// Expire expires object versions which have already matched supplied filter conditions
func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
opts := ObjectOptions{
PrefixEnabledFn: vc.PrefixEnabled,
VersionSuspended: vc.Suspended(),
}
allErrs := make([]error, 0, len(objsToDel))
for {
count := len(objsToDel)
if count == 0 {
break
}
if count > maxDeleteList {
count = maxDeleteList
}
_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
allErrs = append(allErrs, errs...)
// Next batch of deletion
objsToDel = objsToDel[count:]
}
return allErrs
}
const (
batchExpireName = "batch-expire.bin"
batchExpireFormat = 1
@@ -381,22 +347,233 @@ const (
batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
)
type objInfoCache map[string]*ObjectInfo
func newObjInfoCache() objInfoCache {
return objInfoCache(make(map[string]*ObjectInfo))
type expireObjInfo struct {
ObjectInfo
ExpireAll bool
DeleteMarkerCount int64
}
func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
// isErrObjectLocked reports a version that Object Lock protects.
func isErrObjectLocked(err error) bool {
var locked ObjectLocked
return errors.As(err, &locked)
}
func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
return oi, ok
func (r *BatchJobExpire) tagsFiltered() bool {
for _, rule := range r.Rules {
if len(rule.Tags) > 0 {
return true
}
}
return false
}
func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
// expireAuthzRequests are the actions the job takes on one version: it reads
// the version, and its tags when a rule filters by them, and deletes it.
func (r *BatchJobExpire) expireAuthzRequests(name, versionID, userTags string) []batchAuthzRequest {
if versionID == "" {
versionID = nullVersionID
}
reqs := []batchAuthzRequest{
{action: policy.GetObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
{action: policy.DeleteObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
{action: policy.DeleteObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
}
if r.tagsFiltered() {
reqs = append(reqs,
batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
)
}
return reqs
}
// expireVersion deletes one version that is older than the versions the rule
// retains. Under the object's write lock, with the version as it is then, it
// leaves a version that Object Lock protects (ObjectLocked) and refuses one
// that the submitter may not delete (batchJobAuthzError). An unreadable
// version is not deleted.
func (r *BatchJobExpire) expireVersion(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, exp expireObjInfo) error {
p, err := job.Submitter.principal(ctx)
if err != nil {
return err
}
versionID := exp.VersionID
if versionID == "" {
// An empty version ID would add a delete marker instead.
versionID = nullVersionID
}
opts := ObjectOptions{
VersionID: versionID,
Versioned: vc.PrefixEnabled(exp.Name),
VersionSuspended: vc.Suspended(),
}
opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) error {
// A delete marker reads as MethodNotAllowed; any other error,
// including a missing version, stops the delete.
marker := isErrMethodNotAllowed(gerr) && (goi.DeleteMarker || !goi.VersionPurgeStatus.Empty())
if gerr != nil && !marker {
return gerr
}
if enforceRetentionForDeletion(ctx, goi) {
return ObjectLocked{Bucket: r.Bucket, Object: exp.Name, VersionID: versionID}
}
for _, req := range r.expireAuthzRequests(exp.Name, versionID, goi.UserTags) {
if err := p.authorize(req); err != nil {
return err
}
}
return nil
})
_, err = api.DeleteObject(ctx, r.Bucket, exp.Name, opts)
return err
}
// expireVersions deletes the versions of one batch entry and records the
// outcome in res. It returns an error that ends the job.
func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, delay time.Duration, res *batchCounts) error {
for attempts := 1; attempts <= retryAttempts; attempts++ {
if err := ctx.Err(); err != nil {
return err
}
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
err := r.expireVersion(ctx, api, vc, job, exp)
stopFn(exp, err)
switch {
case err == nil:
if exp.DeleteMarker {
ri.countRemoved(r.Bucket, 0, 1)
} else {
ri.countRemoved(r.Bucket, 1, 0)
}
return nil
case isBatchJobAuthzError(err):
return err
case isErrObjectLocked(err):
// Left in place, not retried and not a failure.
res.versionsSkipped++
return nil
case isErrVersionNotFound(err) || isErrObjectNotFound(err):
// Removed concurrently.
return nil
}
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", r.Bucket, exp.Name, exp.VersionID, err, attempts))
if attempts < retryAttempts {
time.Sleep(delay)
}
}
if exp.DeleteMarker {
res.deleteMarkersFailed++
} else {
res.objectsFailed++
}
return nil
}
// expireAllVersions purges every version of a key whose rule retains none,
// with the purge of exactly that key (erasureServerPools.deleteAllVersions),
// while the version the walk matched is still its latest. Under its lock,
// before it deletes anything, the purge refuses the key when Object Lock
// protects a version and has every write it would delete, each copy and
// each revision a drive holds, authorized for the submitter, with the
// version ID and tags it has then.
//
// What the purge removed is counted at once; a refused key is counted in res
// as refused, and the versions a failed purge left in res as failures. A
// failed purge is retried up to retryAttempts times, each time with the
// version the walk matched; a purge that stopped at its budget after
// removing versions continues without spending an attempt. A key whose
// latest version changed since the walk is not expired. It returns an error
// that ends the job: an authorization failure or the end of ctx.
func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error {
var objects, markers int64
matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
for attempts := 1; attempts <= retryAttempts; attempts++ {
if err := ctx.Err(); err != nil {
return err
}
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
latest := matched
var purge deleteAllResult
p, err := job.Submitter.principal(ctx)
if err == nil {
_, err = api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &latest,
DeleteAllResult: &purge,
deleteAllAuthorize: r.expireAllAuthorize(p, exp.Name),
})
}
// Removed versions are gone for any later run: count them now.
if purge.Removed > 0 {
ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers))
objects += int64(purge.Removed - purge.Markers)
markers += int64(purge.Markers)
}
stopFn(exp, err)
switch {
case err == nil:
return nil
case isBatchJobAuthzError(err):
return err
case errors.Is(err, errDeleteAllChanged):
// Changed since the walk matched it: not expired.
return nil
case isErrObjectLocked(err):
// Refused as a whole: nothing removed, not retried and not a
// failure.
res.keysRefused++
return nil
case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
attempts-- // progress
continue
}
if err := ctx.Err(); err != nil {
return err
}
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
if errors.Is(err, errDeleteAllTiered) {
break // refused; a later run of the job checks again
}
}
res.expireAllFailed(exp, objects, markers)
return nil
}
// expireAllAuthorize returns the authorization of a purge of all versions of
// name for p: each version the purge would delete, as it holds it, needs the
// actions of expiring it (expireAuthzRequests).
func (r *BatchJobExpire) expireAllAuthorize(p batchJobPrincipal, name string) func([]ObjectInfo) error {
return func(versions []ObjectInfo) error {
for _, v := range versions {
for _, req := range r.expireAuthzRequests(name, v.VersionID, v.UserTags) {
if err := p.authorize(req); err != nil {
return err
}
}
}
return nil
}
}
// expireAllFailed counts the versions of info, as the walk saw them, that a
// failed purge of all its versions did not remove: those beyond the objects
// and markers it removed, at least one.
func (c *batchCounts) expireAllFailed(info expireObjInfo, objects, markers int64) {
failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
failedMarkers := max(info.DeleteMarkerCount-markers, 0)
if failedObjects+failedMarkers == 0 {
failedObjects = 1
}
c.objectsFailed += failedObjects
c.deleteMarkersFailed += failedMarkers
}
// batchObjsForDelete expires the batches of expireCh. walkDone reports, once
// expireCh is closed, whether the walk reached its end: only then is its last
// key known to be complete.
func batchObjsForDelete(ctx context.Context, refuse func(error), walkDone func() bool, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
vc, _ := globalBucketVersioningSys.Get(r.Bucket)
retryAttempts := job.Expire.Retry.Attempts
if retryAttempts <= 0 {
@@ -407,162 +584,53 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
delay = batchExpireJobDefaultRetryDelay
}
var i int
progress := newBatchProgress(ri, r.Bucket)
defer func() {
wk.Wait()
if walkDone() {
progress.finish()
}
}()
for toExpire := range expireCh {
select {
case <-ctx.Done():
return
default:
}
if i > 0 {
seq := progress.begin()
if seq > 0 {
if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
time.Sleep(wait)
}
}
i++
wk.Take()
go func(toExpire []expireObjInfo) {
go func(seq uint64, toExpire []expireObjInfo) {
defer wk.Give()
toExpireAll := make([]expireObjInfo, 0, len(toExpire))
toDel := make([]ObjectToDelete, 0, len(toExpire))
oiCache := newObjInfoCache()
var res batchCounts
for _, exp := range toExpire {
var err error
if exp.ExpireAll {
toExpireAll = append(toExpireAll, exp)
continue
err = r.expireAllVersions(ctx, api, job, ri, exp, retryAttempts, &res)
} else {
err = r.expireVersions(ctx, api, vc, job, ri, exp, retryAttempts, delay, &res)
}
// Cache ObjectInfo value via pointers for
// subsequent use to track objects which
// couldn't be deleted.
od := ObjectToDelete{
ObjectV: ObjectV{
ObjectName: exp.Name,
VersionID: exp.VersionID,
},
}
toDel = append(toDel, od)
oiCache.Add(od, &exp.ObjectInfo)
}
// Purge all versions of exactly each key while the version the
// walk matched is still its latest; see
// erasureServerPools.deleteAllVersions.
for _, exp := range toExpireAll {
if !expireAllVersions(ctx, ri, api, exp, retryAttempts) {
return
}
}
// DeleteMultiple objects
toDelCopy := make([]ObjectToDelete, len(toDel))
for attempts := 1; attempts <= retryAttempts; attempts++ {
select {
case <-ctx.Done():
return
default:
}
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
// Copying toDel to select from objects whose
// deletion failed
copy(toDelCopy, toDel)
var failed int
errs := r.Expire(ctx, api, vc, toDel)
// reslice toDel in preparation for next retry attempt
toDel = toDel[:0]
for i, err := range errs {
if err != nil {
stopFn(toDelCopy[i], err)
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
err, attempts))
failed++
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)
}
if attempts != retryAttempts {
// retry
toDel = append(toDel, toDelCopy[i])
}
} else {
stopFn(toDelCopy[i], nil)
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts)
}
if err != nil {
// The batch is not done; a later run does it again.
if isBatchJobAuthzError(err) {
refuse(err)
}
}
globalBatchJobsMetrics.save(ri.JobID, ri)
if failed == 0 {
break
}
// Add a delay between retry attempts
if attempts < retryAttempts {
time.Sleep(delay)
globalBatchJobsMetrics.save(ri.JobID, ri)
return
}
}
}(toExpire)
if n := len(toExpire); n > 0 {
progress.done(seq, toExpire[n-1].Name, res)
}
globalBatchJobsMetrics.save(ri.JobID, ri)
}(seq, toExpire)
}
}
// expireAllVersions purges all versions of the key exp, which the walk
// matched by its latest version, and counts what the purge removed. It
// retries a failed purge up to attempts times, each time with the version
// the walk matched; a purge that stopped at its budget continues without
// spending an attempt. It does not retry once the key no longer has that
// latest version, which is not an expiry, or once the purge refused the
// key. It returns false once ctx ends.
func expireAllVersions(ctx context.Context, ri *batchJobInfo, api ObjectLayer, exp expireObjInfo, attempts int) bool {
var objects, markers int64
matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
for attempt := 1; attempt <= attempts; attempt++ {
select {
case <-ctx.Done():
ri.trackExpireAll(exp, objects, markers, true)
return false
default:
}
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempt)
latest := matched
var purge deleteAllResult
_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &latest,
DeleteAllResult: &purge,
})
objects += int64(purge.Removed - purge.Markers)
markers += int64(purge.Markers)
stopFn(exp, err)
switch {
case err == nil:
ri.trackExpireAll(exp, objects, markers, false)
return true
case errors.Is(err, errDeleteAllChanged):
// Changed since the walk matched it: not expired.
ri.trackExpireAll(exp, objects, markers, false)
return true
case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
attempt-- // progress
continue
}
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempt))
var locked ObjectLocked
if errors.Is(err, errDeleteAllTiered) || errors.As(err, &locked) {
break // refused; a later run of the job checks again
}
}
ri.trackExpireAll(exp, objects, markers, true)
return true
}
type expireObjInfo struct {
ObjectInfo
ExpireAll bool
DeleteMarkerCount int64
}
// Start the batch expiration job, resumes if there was a pending job via "job.ID"
func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
ri := &batchJobInfo{
@@ -573,6 +641,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
if err := ri.loadOrInit(ctx, api, job); err != nil {
return err
}
if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
return err
}
globalBatchJobsMetrics.save(job.ID, ri)
lastObject := ri.Object
@@ -592,27 +663,46 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
ctx, cancelCause := context.WithCancelCause(ctx)
defer cancelCause(nil)
// refuse ends the job for its submitter: record why before stopping.
refuse := func(err error) {
ri.refuse(err)
cancelCause(err)
}
// The walk stops at a listing failure: work after it is left to a later
// run, so the cursor cannot move past what was not listed.
walkCtx, stopWalk := context.WithCancel(ctx)
defer stopWalk()
// walkFailed: a listing failed before it returned any result.
var walkFailed atomic.Bool
results := make(chan itemOrErr[ObjectInfo], workerSize)
go func() {
prefixes := r.Prefix.F()
if len(prefixes) == 0 {
prefixes = []string{""}
}
for _, prefix := range prefixes {
// One ascending walk over disjoint prefixes, which the cursor
// describes.
for _, prefix := range batchCanonicalPrefixes(r.Prefix.F()) {
if walkCtx.Err() != nil {
break
}
prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
err := api.Walk(walkCtx, r.Bucket, prefix, prefixResultCh, WalkOptions{
Marker: lastObject,
LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions
VersionsSort: WalkVersionsSortDesc,
})
if err != nil {
cancelCause(err)
if walkCtx.Err() == nil {
walkFailed.Store(true)
cancelCause(err)
}
xioutil.SafeClose(results)
return
}
for result := range prefixResultCh {
results <- result
select {
case results <- result:
case <-walkCtx.Done():
// Stopped: let the walk end without a reader.
}
}
}
xioutil.SafeClose(results)
@@ -620,7 +710,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
// Goroutine to periodically save batch-expire job's in-memory state
saverQuitCh := make(chan struct{})
saverDoneCh := make(chan struct{})
go func() {
defer close(saverDoneCh)
saveTicker := time.NewTicker(10 * time.Second)
defer saveTicker.Stop()
quit := false
@@ -647,9 +739,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
expireCh := make(chan []expireObjInfo, workerSize)
expireDoneCh := make(chan struct{})
var walkDone atomic.Bool
go func() {
defer close(expireDoneCh)
batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
batchObjsForDelete(ctx, refuse, walkDone.Load, r, ri, job, api, wk, expireCh)
}()
var (
@@ -661,6 +754,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
done bool
)
deleteMarkerCountMap := map[string]int64{}
// pushToExpire is called between keys only, so that a batch holds all
// the versions of its keys and its last key is done with it.
pushToExpire := func() {
// set preObject deleteMarkerCount
if len(toDel) > 0 {
@@ -695,6 +790,12 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
if result.Err != nil {
failed = true
batchLogIf(ctx, result.Err)
stopWalk()
done = true
break
}
if !batchResumeAfter(lastObject, result.Item.Name) {
// Completed by the run that set the cursor.
continue
}
if result.Item.DeleteMarker {
@@ -752,7 +853,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
toDel = append(toDel, expireObjInfo{
ObjectInfo: result.Item,
})
pushToExpire()
case <-ctx.Done():
done = true
}
@@ -761,16 +861,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
}
}
if context.Cause(ctx) != nil {
xioutil.SafeClose(expireCh)
return context.Cause(ctx)
}
pushToExpire()
// Send any remaining objects downstream
if len(toDel) > 0 {
select {
case <-ctx.Done():
case expireCh <- toDel:
if context.Cause(ctx) == nil {
walkDone.Store(!failed)
pushToExpire()
// Send any remaining objects downstream
if len(toDel) > 0 {
select {
case <-ctx.Done():
case expireCh <- toDel:
}
}
}
xioutil.SafeClose(expireCh)
@@ -778,13 +877,36 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
<-expireDoneCh // waits for the expire goroutine to complete
wk.Wait() // waits for all expire workers to retire
ri.Complete = !failed && !ri.expireFailures()
ri.Failed = failed || ri.expireFailures()
globalBatchJobsMetrics.save(job.ID, ri)
// Close the saverQuitCh - this also triggers saving in-memory state
// immediately one last time before we exit this method.
// Persist the final state once the saver has stopped, so that its last
// save cannot overwrite it.
xioutil.SafeClose(saverQuitCh)
<-saverDoneCh
refused := ri.refused()
cause := context.Cause(ctx)
if cause != nil && !refused {
if walkFailed.Load() {
// This run failed; a later run lists again from the cursor.
ri.mu.Lock()
ri.Complete = false
ri.Failed = true
ri.mu.Unlock()
}
saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
cancel()
globalBatchJobsMetrics.save(job.ID, ri)
return cause
}
ri.mu.Lock()
itemsFailed := ri.expireFailures()
ri.Complete = !refused && !failed && !itemsFailed
ri.Failed = refused || failed || itemsFailed
ri.mu.Unlock()
globalBatchJobsMetrics.save(job.ID, ri)
saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
cancel()
// Notify expire jobs final status to the configured endpoint
buf, _ := json.Marshal(ri)
cmd/batch-handlers.go
+450 -113
View File
@@ -43,6 +43,8 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/amztime"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/config/batch"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/hash"
@@ -72,32 +74,47 @@ type BatchJobRequest struct {
Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
Expire *BatchJobExpire `yaml:"expire" json:"expire"`
ctx context.Context `msg:"-"`
// Submitter is the credential the job acts as. The server assigns it
// at submission; a job definition cannot name it.
Submitter *batchJobSubmitter `yaml:"-" json:"-"`
ctx context.Context `msg:"-"`
}
// RedactSensitive will redact any sensitive information in b.
func (j *BatchJobRequest) RedactSensitive() {
j.Replicate.RedactSensitive()
j.Expire.RedactSensitive()
j.KeyRotate.RedactSensitive()
// redacted returns a copy of the job for output outside the server: the
// describe response, logs and errors. The copy has no secret keys, session
// tokens or notification tokens, and no submitter binding; the job itself,
// which a run needs as it is, is not changed.
func (j BatchJobRequest) redacted() (BatchJobRequest, error) {
buf, err := j.MarshalMsg(nil)
if err != nil {
return BatchJobRequest{}, err
}
var c BatchJobRequest
if _, err = c.UnmarshalMsg(buf); err != nil {
return BatchJobRequest{}, err
}
redact := func(s *string) {
if *s != "" {
*s = redactedText
}
}
c.Submitter = nil
if r := c.Replicate; r != nil {
redact(&r.Source.Creds.SecretKey)
redact(&r.Source.Creds.SessionToken)
redact(&r.Target.Creds.SecretKey)
redact(&r.Target.Creds.SessionToken)
redact(&r.Flags.Notify.Token)
}
if r := c.KeyRotate; r != nil {
redact(&r.Flags.Notify.Token)
}
if r := c.Expire; r != nil {
redact(&r.NotificationCfg.Token)
}
return c, nil
}
// RedactSensitive will redact any sensitive information in b.
func (r *BatchJobReplicateV1) RedactSensitive() {
if r == nil {
return
}
if r.Target.Creds.SecretKey != "" {
r.Target.Creds.SecretKey = redactedText
}
if r.Target.Creds.SessionToken != "" {
r.Target.Creds.SessionToken = redactedText
}
}
// RedactSensitive will redact any sensitive information in b.
func (r *BatchJobKeyRotateV1) RedactSensitive() {}
func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error {
if endpoint == "" {
return nil
@@ -140,8 +157,114 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error
return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
}
// ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
// batchPullObjectLock prepares the Object Lock metadata that a pull writes
// with a version, as a PUT would on the destination: Object Lock metadata
// copied from the source must be valid, a version without retention takes the
// destination bucket's default retention, and both are dated with the
// destination's clock. Outside a lock-enabled bucket the metadata has no
// meaning and is not written. It reports the retention and legal hold that
// the pull copies from the source.
func batchPullObjectLock(bucket string, meta map[string]string) (retention, legalHold bool, err error) {
cfg, err := globalBucketObjectLockSys.Get(bucket)
if err != nil {
return false, false, err
}
h := make(http.Header)
for k, v := range meta {
for _, name := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate, xhttp.AmzObjectLockLegalHold} {
if strings.EqualFold(k, name) {
h.Set(name, v)
delete(meta, k)
}
}
}
if !cfg.LockEnabled {
return false, false, nil
}
if h.Get(xhttp.AmzObjectLockMode) != "" || h.Get(xhttp.AmzObjectLockRetainUntilDate) != "" {
// A copied retention may have expired already.
mode, until, err := objectlock.ParseObjectLockRetentionHeaders(h, true)
if err != nil {
return false, false, err
}
meta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(mode)
meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(until.UTC())
retention = true
} else if cfg.Validity > 0 {
now, err := objectlock.UTCNowNTP()
if err != nil {
return false, false, err
}
meta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(cfg.Mode)
meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(now.Add(cfg.Validity).UTC())
}
if _, ok := h[xhttp.AmzObjectLockLegalHold]; ok {
hold, err := objectlock.ParseObjectLockLegalHoldHeaders(h)
if err != nil {
return false, false, err
}
meta[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(hold.Status)
legalHold = true
}
return retention, legalHold, nil
}
// pullAuthzRequests are the writes of one version that a pull makes to the
// destination key: the version, with the tags, retention and legal hold it
// copies, and the replication permission for a version ID it keeps.
//
// The pull does not encrypt what it writes, so it supplies no encryption
// values, whatever the source reports; and it does not read the destination
// version it may replace, so the tags of that version are unknown.
func pullAuthzRequests(bucket, object, versionID string, keepsVersion bool, opts ObjectOptions, retention, legalHold bool) []batchAuthzRequest {
values := map[string][]string{}
meta := caseInsensitiveMap(opts.UserDefined)
if retention {
mode, _ := meta.Lookup(xhttp.AmzObjectLockMode)
until, _ := meta.Lookup(xhttp.AmzObjectLockRetainUntilDate)
values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockMode), "x-amz-")] = []string{mode}
values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), "x-amz-")] = []string{until}
}
if legalHold {
hold, _ := meta.Lookup(xhttp.AmzObjectLockLegalHold)
values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockLegalHold), "x-amz-")] = []string{hold}
}
tagStr, hasTags := meta.Lookup(xhttp.AmzObjectTagging)
if hasTags && tagStr != "" {
if t, err := tags.ParseObjectTags(tagStr); err == nil {
m := t.ToMap()
keys := make([]string, 0, len(m))
for k, v := range m {
values[pathJoin("RequestObjectTag", k)] = []string{v}
keys = append(keys, k)
}
values["RequestObjectTagKeys"] = keys
}
}
actions := []policy.Action{policy.PutObjectAction}
if keepsVersion {
actions = append(actions, policy.ReplicateObjectAction)
}
if hasTags && tagStr != "" {
actions = append(actions, policy.PutObjectTaggingAction)
}
if retention {
actions = append(actions, policy.PutObjectRetentionAction)
}
if legalHold {
actions = append(actions, policy.PutObjectLegalHoldAction)
}
reqs := make([]batchAuthzRequest, 0, len(actions))
for _, action := range actions {
reqs = append(reqs, batchAuthzRequest{action: action, bucket: bucket, object: object, versionID: versionID, tagsUnknown: true, values: values})
}
return reqs
}
// ReplicateFromSource copies one version from the remote source to the local
// target. A write that would replace a version protected by Object Lock
// returns ObjectLocked and leaves it.
func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool, sub *batchJobSubmitter) error {
srcBucket := r.Source.Bucket
tgtBucket := r.Target.Bucket
srcObject := srcObjInfo.Name
@@ -150,11 +273,28 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
}
p, err := sub.principal(ctx)
if err != nil {
return err
}
versionID := srcObjInfo.VersionID
if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
versionID = ""
}
if srcObjInfo.DeleteMarker {
markerVersion := versionID
if markerVersion == "" {
markerVersion = nullVersionID
}
for _, req := range []batchAuthzRequest{
{action: policy.DeleteObjectAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true},
{action: policy.DeleteObjectVersionAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true},
{action: policy.ReplicateDeleteAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true},
} {
if err := p.authorize(req); err != nil {
return err
}
}
_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
VersionID: versionID,
// Since we are preserving a delete marker, we have to make sure this is always true.
@@ -164,22 +304,43 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
MTime: srcObjInfo.ModTime,
DeleteMarker: srcObjInfo.DeleteMarker,
ReplicationRequest: true,
// A new marker may hide a locked version; one that would replace
// it is refused.
RefuseLockedOverwrite: true,
})
return err
}
opts := ObjectOptions{
VersionID: srcObjInfo.VersionID,
MTime: srcObjInfo.ModTime,
PreserveETag: srcObjInfo.ETag,
UserDefined: srcObjInfo.UserDefined,
userDefined := cloneMSS(srcObjInfo.UserDefined)
retention, legalHold, err := batchPullObjectLock(tgtBucket, userDefined)
if err != nil {
return err
}
if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
opts.VersionID = ""
opts := ObjectOptions{
VersionID: versionID,
Versioned: globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject),
VersionSuspended: globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject),
MTime: srcObjInfo.ModTime,
PreserveETag: srcObjInfo.ETag,
UserDefined: userDefined,
// Checked with the commit, under the destination's write lock.
RefuseLockedOverwrite: true,
}
if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) {
opts.ServerSideEncryption = encrypt.NewSSE()
}
// The version the write replaces or creates: the source's version ID it
// keeps, or the null version of a destination that does not version
// the key. A versioned destination gives a write without an ID a new one.
writeVersion := versionID
if writeVersion == "" && !opts.Versioned {
writeVersion = nullVersionID
}
for _, req := range pullAuthzRequests(tgtBucket, tgtObject, writeVersion, versionID != "", opts, retention, legalHold) {
if err := p.authorize(req); err != nil {
return err
}
}
slc := strings.Split(srcObjInfo.ETag, "-")
if len(slc) == 2 {
partsCount, err := strconv.Atoi(slc[1])
@@ -298,6 +459,9 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
if ri.Complete {
return nil
}
if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
return err
}
globalBatchJobsMetrics.save(job.ID, ri)
retryAttempts := job.Replicate.Flags.Retry.Attempts
@@ -411,7 +575,17 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
// one of source/target is s3, skip delete marker and all versions under the same object name.
s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
ctx, cancel := context.WithCancel(ctx)
ctx, cancelCause := context.WithCancelCause(ctx)
// refuse ends the job for its submitter: record why before stopping.
refuse := func(err error) {
ri.refuse(err)
cancelCause(err)
}
// Each run lists the source from the start and counts every locked
// collision again.
ri.mu.Lock()
ri.VersionsSkippedLocked = 0
ri.mu.Unlock()
objInfoCh := make(chan minio.ObjectInfo, 1)
go func() {
@@ -437,6 +611,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
skipReplicate := false
for obj := range objInfoCh {
if context.Cause(ctx) != nil {
// Refused or canceled: schedule nothing more.
continue
}
oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
if !minioSrc {
// Check if metadata filter was requested and it is expected to have
@@ -484,7 +662,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
defer wk.Give()
stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
success := true
if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil {
if err := r.ReplicateFromSource(ctx, api, core, oi, retry, job.Submitter); err != nil {
if isBatchJobAuthzError(err) {
refuse(err)
return
}
// A locked destination version is left as it is: the
// job completes with it skipped, not replicated.
if isErrObjectLocked(err) {
stopFn(oi, err)
ri.countSkipped()
globalBatchJobsMetrics.save(job.ID, ri)
return
}
// object must be deleted concurrently, allow these failures but do not count them
if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
return
@@ -495,7 +685,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
} else {
stopFn(oi, nil)
}
ri.trackCurrentBucketObject(r.Target.Bucket, oi, success, attempts)
ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk after every 10secs.
batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
@@ -507,9 +697,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
}
wk.Wait()
if ri.refused() {
cancelCause(nil)
globalBatchJobsMetrics.save(job.ID, ri)
batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
if err := r.Notify(GlobalContext, ri); err != nil {
batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
}
return nil
}
ri.RetryAttempts = attempts
ri.Complete = ri.ObjectsFailed == 0
ri.Failed = ri.ObjectsFailed > 0
ri.Complete = ri.ObjectsFailed == 0 && ri.DeleteMarkersFailed == 0
ri.Failed = !ri.Complete
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk.
@@ -519,12 +719,14 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
}
cancel()
cancelCause(nil)
if ri.Failed {
ri.ObjectsFailed = 0
ri.DeleteMarkersFailed = 0
ri.Bucket = ""
ri.Object = ""
ri.Objects = 0
ri.DeleteMarkers = 0
ri.BytesFailed = 0
ri.BytesTransferred = 0
retry = true // indicate we are retrying..
@@ -580,7 +782,10 @@ func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
return oi
}
func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string, sub *batchJobSubmitter) error {
// An entry the submitter may not read ends the archive and the job; an
// entry whose metadata cannot be read ends the archive.
var refused, readErr error
input := make(chan minio.SnowballObject, 1)
opts := minio.SnowballOptions{
Opts: minio.PutObjectOptions{
@@ -593,17 +798,41 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
SkipErrs: *r.Source.Snowball.SkipErrs,
}
uploadCtx, cancelUpload := context.WithCancel(ctx)
defer cancelUpload()
producerDone := make(chan struct{})
go func() {
defer close(producerDone)
defer xioutil.SafeClose(input)
for _, entry := range entries {
gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket,
gr, err := objAPI.GetObjectNInfo(uploadCtx, r.Source.Bucket,
entry.Name, nil, nil, ObjectOptions{
VersionID: entry.VersionID,
})
if err != nil {
batchLogIf(ctx, err)
continue
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || *r.Source.Snowball.SkipErrs {
continue
}
// Not sent and not skippable: the batch goes to the
// per-object path.
readErr = err
return
}
// Each entry is a decision of its own, for the submitter as it is,
// on the metadata of all copies of the version.
logical, lerr := batchAuthoritativeRead(uploadCtx, objAPI, r.Source.Bucket, entry.Name, gr.ObjInfo)
if lerr != nil {
// Not sent: the batch goes to the per-object path.
gr.Close()
readErr = lerr
return
}
refused = sub.authorize(uploadCtx, append(r.pushAuthzRequests(gr.ObjInfo), r.pushAuthzRequests(logical)...)...)
if refused != nil {
gr.Close()
return
}
if prefix != "" {
@@ -625,6 +854,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
opts, _, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
if err != nil {
gr.Close()
batchLogIf(ctx, err)
continue
}
@@ -635,23 +865,75 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
}
}
input <- snowballObj
select {
case input <- snowballObj:
case <-uploadCtx.Done():
gr.Close()
return
}
}
}()
// Collect and upload all entries.
return remoteClnt.PutObjectsSnowball(ctx, r.Target.Bucket, opts, input)
err := remoteClnt.PutObjectsSnowball(uploadCtx, r.Target.Bucket, opts, input)
cancelUpload()
<-producerDone
if refused != nil {
return refused
}
if readErr != nil {
return readErr
}
return err
}
// pushAuthzRequests are the reads of one source version that a push copies to
// the target: the version itself, and its tags, retention and legal hold.
func (r *BatchJobReplicateV1) pushAuthzRequests(oi ObjectInfo) []batchAuthzRequest {
bucket, object, versionID := r.Source.Bucket, oi.Name, oi.VersionID
if versionID == "" {
versionID = nullVersionID
}
reqs := []batchAuthzRequest{
{action: policy.GetObjectAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
{action: policy.GetObjectVersionAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
}
if oi.UserTags != "" {
reqs = append(reqs,
batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
)
}
meta := caseInsensitiveMap(oi.UserDefined)
_, mode := meta.Lookup(xhttp.AmzObjectLockMode)
_, until := meta.Lookup(xhttp.AmzObjectLockRetainUntilDate)
if mode || until {
reqs = append(reqs, batchAuthzRequest{action: policy.GetObjectRetentionAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags})
}
if _, hold := meta.Lookup(xhttp.AmzObjectLockLegalHold); hold {
reqs = append(reqs, batchAuthzRequest{action: policy.GetObjectLegalHoldAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags})
}
return reqs
}
// ReplicateToTarget read from source and replicate to configured target
func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool, sub *batchJobSubmitter) error {
srcBucket := r.Source.Bucket
tgtBucket := r.Target.Bucket
tgtPrefix := r.Target.Prefix
srcObject := srcObjInfo.Name
s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
p, err := sub.principal(ctx)
if err != nil {
return err
}
if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() {
for _, req := range r.pushAuthzRequests(srcObjInfo) {
if err := p.authorize(req); err != nil {
return err
}
}
if retry && !s3Type {
if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.StatObjectOptions{
VersionID: srcObjInfo.VersionID,
@@ -707,6 +989,19 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
}
defer rd.Close()
objInfo := rd.ObjInfo
// Authorize the version as it is read, with the metadata of all its
// copies.
logical, err := batchAuthoritativeRead(ctx, api, srcBucket, srcObject, objInfo)
if err != nil {
return err
}
// What is sent comes from the opened copy; both it and the version as
// all copies describe it must be authorized.
for _, req := range append(r.pushAuthzRequests(objInfo), r.pushAuthzRequests(logical)...) {
if err := p.authorize(req); err != nil {
return err
}
}
size, err := objInfo.GetActualSize()
if err != nil {
@@ -760,6 +1055,16 @@ type batchJobInfo struct {
DeleteMarkersFailed int64 `json:"deleteMarkersFailed" msg:"dmf"`
BytesTransferred int64 `json:"bytesTransferred" msg:"bt"`
BytesFailed int64 `json:"bytesFailed" msg:"bf"`
// Versions the job left in place because Object Lock protects them,
// and keys it refused as a whole for the same reason. A job that
// completes with them has not acted on those versions.
VersionsSkippedLocked int64 `json:"versionsSkippedLocked" msg:"vsl"`
KeysRefused int64 `json:"keysRefused" msg:"kr"`
// FailureReason is set when the job was refused for its submitter. It
// ends the job: a job that has one does not run again.
FailureReason string `json:"failureReason,omitempty" msg:"fr"`
}
const (
@@ -901,40 +1206,31 @@ func (ri *batchJobInfo) clone() *batchJobInfo {
BytesTransferred: ri.BytesTransferred,
BytesFailed: ri.BytesFailed,
Attempts: ri.Attempts,
VersionsSkippedLocked: ri.VersionsSkippedLocked,
KeysRefused: ri.KeysRefused,
FailureReason: ri.FailureReason,
}
}
func (ri *batchJobInfo) countItem(size int64, dmarker, success bool, attempt int) {
// countItem counts the outcome of one item in the current attempt of a
// replicate job, which resets its counters before it retries.
func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) {
if ri == nil {
return
}
ri.Attempts++
if success {
if dmarker {
ri.DeleteMarkers++
} else {
ri.Objects++
ri.BytesTransferred += size
}
if attempt > 1 {
if dmarker {
ri.DeleteMarkersFailed--
} else {
ri.ObjectsFailed--
ri.BytesFailed += size
}
}
} else {
if attempt > 1 {
// Only count first attempt
return
}
if dmarker {
ri.DeleteMarkersFailed++
} else {
ri.ObjectsFailed++
ri.BytesFailed += size
}
switch {
case success && dmarker:
ri.DeleteMarkers++
case success:
ri.Objects++
ri.BytesTransferred += size
case dmarker:
ri.DeleteMarkersFailed++
default:
ri.ObjectsFailed++
ri.BytesFailed += size
}
}
@@ -1002,35 +1298,7 @@ func (ri *batchJobInfo) expireFailures() bool {
return ri.ObjectsFailed > 0 || ri.DeleteMarkersFailed > 0
}
// trackExpireAll counts the objects and delete markers that an expiry of all
// versions of info removed and, when it failed, those the walk saw that it
// did not remove, at least one. Only batch-expire uses it, for a rule that
// expires all versions of an object.
func (ri *batchJobInfo) trackExpireAll(info expireObjInfo, objects, markers int64, failed bool) {
if ri == nil {
return
}
ri.mu.Lock()
defer ri.mu.Unlock()
ri.Objects += objects
ri.DeleteMarkers += markers
if !failed {
ri.Bucket = info.Bucket
ri.Object = info.Name
return
}
failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
failedMarkers := max(info.DeleteMarkerCount-markers, 0)
if failedObjects+failedMarkers == 0 {
failedObjects = 1
}
ri.ObjectsFailed += failedObjects
ri.DeleteMarkersFailed += failedMarkers
}
func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool, attempt int) {
func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool) {
if ri == nil {
return
}
@@ -1042,7 +1310,7 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
ri.Bucket = bucket
ri.Object = info.Name
}
ri.countItem(info.Size, info.DeleteMarker, success, attempt)
ri.countItem(info.Size, info.DeleteMarker, success)
}
func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInfo) {
@@ -1056,7 +1324,7 @@ func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInf
ri.Bucket = bucket
for i := range batch {
ri.Object = batch[i].Name
ri.countItem(batch[i].Size, batch[i].DeleteMarker, true, 1)
ri.countItem(batch[i].Size, batch[i].DeleteMarker, true)
}
}
@@ -1073,6 +1341,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if ri.Complete {
return nil
}
if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
return err
}
globalBatchJobsMetrics.save(job.ID, ri)
lastObject := ri.Object
@@ -1180,6 +1451,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
walkCh = make(chan itemOrErr[ObjectInfo], 100)
slowCh = make(chan itemOrErr[ObjectInfo], 100)
)
ctx, cancelCause := context.WithCancelCause(ctx)
// refuse ends the job for its submitter: record why before stopping.
refuse := func(err error) {
ri.refuse(err)
cancelCause(err)
}
if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
go func() {
@@ -1201,7 +1478,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
writeFn := func(batch []ObjectInfo) {
if len(batch) > 0 {
if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil {
if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix, job.Submitter); err != nil {
if isBatchJobAuthzError(err) {
refuse(err)
return
}
batchLogOnceIf(ctx, err, job.ID+"writeAsArchive")
for _, b := range batch {
slowCh <- itemOrErr[ObjectInfo]{Item: b}
@@ -1250,7 +1531,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if walkQuorum == "" {
walkQuorum = "strict"
}
ctx, cancelCause := context.WithCancelCause(ctx)
// one of source/target is s3, skip delete marker and all versions under the same object name.
s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
@@ -1281,6 +1561,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
skipReplicate := false
for res := range slowCh {
if context.Cause(ctx) != nil {
// Refused or canceled: schedule nothing more.
continue
}
if res.Err != nil {
ri.Failed = true
batchLogOnceIf(ctx, res.Err, job.ID+"res.Err")
@@ -1300,7 +1584,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
success := true
if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
if err := r.ReplicateToTarget(ctx, api, c, result, retry, job.Submitter); err != nil {
if isBatchJobAuthzError(err) {
refuse(err)
return
}
if minio.ToErrorResponse(err).Code == "PreconditionFailed" {
// pre-condition failed means we already have the object copied over.
return
@@ -1315,7 +1603,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
} else {
stopFn(result, nil)
}
ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts)
ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk after every 10secs.
batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2")
@@ -1326,13 +1614,22 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
}()
}
wk.Wait()
if ri.refused() {
cancelCause(nil)
globalBatchJobsMetrics.save(job.ID, ri)
batchLogOnceIf(ctx, ri.updateAfter(GlobalContext, api, 0, job), job.ID+"updateAfter3")
if err := r.Notify(GlobalContext, ri); err != nil {
batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify")
}
return nil
}
// Do not need to retry if we can't list objects on source.
if context.Cause(ctx) != nil {
return context.Cause(ctx)
}
ri.RetryAttempts = attempts
ri.Complete = ri.ObjectsFailed == 0
ri.Failed = ri.ObjectsFailed > 0
ri.Complete = ri.ObjectsFailed == 0 && ri.DeleteMarkersFailed == 0
ri.Failed = !ri.Complete
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk.
@@ -1345,9 +1642,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
cancelCause(nil)
if ri.Failed {
ri.ObjectsFailed = 0
ri.DeleteMarkersFailed = 0
ri.Bucket = ""
ri.Object = ""
ri.Objects = 0
ri.DeleteMarkers = 0
ri.BytesFailed = 0
ri.BytesTransferred = 0
retry = true // indicate we are retrying..
@@ -1549,8 +1848,12 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
return errInvalidArgument
}
func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
deleteConfig(ctx, api, getJobPath(j))
func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) error {
err := deleteConfig(ctx, api, getJobPath(j))
if errors.Is(err, errConfigNotFound) {
return nil
}
return err
}
func (j BatchJobRequest) getJobReportPath() (string, error) {
@@ -1709,7 +2012,7 @@ func (a adminAPIHandlers) BatchJobStatus(w http.ResponseWriter, r *http.Request)
return
}
buf, err := json.Marshal(madmin.BatchJobStatus{LastMetric: ri.metric()})
buf, err := json.Marshal(batchJobStatus{LastMetric: ri.statusMetric()})
if err != nil {
batchLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1747,8 +2050,13 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
}
// Remove sensitive fields.
req.RedactSensitive()
buf, err := yaml.Marshal(req)
out, err := req.redacted()
if err != nil {
batchLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
buf, err := yaml.Marshal(out)
if err != nil {
batchLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1806,6 +2114,27 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
}
}
if job.jobSections() != 1 {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, errBatchJobSections), r.URL)
return
}
// The job acts as the credential that submits it, and only as far as
// that credential may act on the job's buckets.
submitter, err := newBatchJobSubmitter(creds)
if err == nil {
job.Submitter = submitter
err = submitter.authorize(ctx, job.startAuthzRequests()...)
}
if err != nil {
apiErr := toAPIError(ctx, err)
if isBatchJobAuthzError(err) {
apiErr = errorCodes.ToAPIErrWithErr(ErrAccessDenied, err)
}
writeErrorResponseJSON(ctx, w, apiErr, r.URL)
return
}
// Validate the incoming job request
if err := job.Validate(ctx, objectAPI); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1942,6 +2271,14 @@ func (j *BatchJobPool) cleanupReports(randomWait func() time.Duration) {
continue
}
if (ri.Complete || ri.Failed) && time.Since(ri.LastUpdate) > oldJobsExpiration {
// Remove the job before its report, so that a finished
// job is never resumed without its outcome.
if ri.JobID != "" {
if err := (BatchJobRequest{ID: ri.JobID}).delete(ctx, j.objLayer); err != nil {
batchLogIf(ctx, err)
continue
}
}
deleteConfig(ctx, j.objLayer, result.Item.Name)
}
}
@@ -2082,7 +2419,7 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
})
case j.jobCh <- req:
default:
return fmt.Errorf("batch job queue is currently full please try again later %#v", req)
return fmt.Errorf("batch job queue is currently full please try again later (job %s)", req.ID)
}
return nil
}
cmd/batch-handlers_gen.go
+153 -9
View File
@@ -179,6 +179,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
}
case "Submitter":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "Submitter")
return
}
z.Submitter = nil
} else {
if z.Submitter == nil {
z.Submitter = new(batchJobSubmitter)
}
err = z.Submitter.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "Submitter")
return
}
}
default:
err = dc.Skip()
if err != nil {
@@ -192,9 +210,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 6
// map header, size 7
// write "ID"
err = en.Append(0x86, 0xa2, 0x49, 0x44)
err = en.Append(0x87, 0xa2, 0x49, 0x44)
if err != nil {
return
}
@@ -274,15 +292,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
return
}
}
// write "Submitter"
err = en.Append(0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
if err != nil {
return
}
if z.Submitter == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
err = z.Submitter.EncodeMsg(en)
if err != nil {
err = msgp.WrapError(err, "Submitter")
return
}
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 6
// map header, size 7
// string "ID"
o = append(o, 0x86, 0xa2, 0x49, 0x44)
o = append(o, 0x87, 0xa2, 0x49, 0x44)
o = msgp.AppendString(o, z.ID)
// string "User"
o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -323,6 +358,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
return
}
}
// string "Submitter"
o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
if z.Submitter == nil {
o = msgp.AppendNil(o)
} else {
o, err = z.Submitter.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "Submitter")
return
}
}
return
}
@@ -413,6 +459,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
}
case "Submitter":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
if err != nil {
return
}
z.Submitter = nil
} else {
if z.Submitter == nil {
z.Submitter = new(batchJobSubmitter)
}
bts, err = z.Submitter.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "Submitter")
return
}
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -445,6 +508,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
} else {
s += z.Expire.Msgsize()
}
s += 10
if z.Submitter == nil {
s += msgp.NilSize
} else {
s += z.Submitter.Msgsize()
}
return
}
@@ -568,6 +637,24 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "BytesFailed")
return
}
case "vsl":
z.VersionsSkippedLocked, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "VersionsSkippedLocked")
return
}
case "kr":
z.KeysRefused, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "KeysRefused")
return
}
case "fr":
z.FailureReason, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "FailureReason")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -581,9 +668,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 17
// map header, size 20
// write "v"
err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76)
err = en.Append(0xde, 0x0, 0x14, 0xa1, 0x76)
if err != nil {
return
}
@@ -752,15 +839,45 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "BytesFailed")
return
}
// write "vsl"
err = en.Append(0xa3, 0x76, 0x73, 0x6c)
if err != nil {
return
}
err = en.WriteInt64(z.VersionsSkippedLocked)
if err != nil {
err = msgp.WrapError(err, "VersionsSkippedLocked")
return
}
// write "kr"
err = en.Append(0xa2, 0x6b, 0x72)
if err != nil {
return
}
err = en.WriteInt64(z.KeysRefused)
if err != nil {
err = msgp.WrapError(err, "KeysRefused")
return
}
// write "fr"
err = en.Append(0xa2, 0x66, 0x72)
if err != nil {
return
}
err = en.WriteString(z.FailureReason)
if err != nil {
err = msgp.WrapError(err, "FailureReason")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 17
// map header, size 20
// string "v"
o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76)
o = append(o, 0xde, 0x0, 0x14, 0xa1, 0x76)
o = msgp.AppendInt(o, z.Version)
// string "jid"
o = append(o, 0xa3, 0x6a, 0x69, 0x64)
@@ -810,6 +927,15 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
// string "bf"
o = append(o, 0xa2, 0x62, 0x66)
o = msgp.AppendInt64(o, z.BytesFailed)
// string "vsl"
o = append(o, 0xa3, 0x76, 0x73, 0x6c)
o = msgp.AppendInt64(o, z.VersionsSkippedLocked)
// string "kr"
o = append(o, 0xa2, 0x6b, 0x72)
o = msgp.AppendInt64(o, z.KeysRefused)
// string "fr"
o = append(o, 0xa2, 0x66, 0x72)
o = msgp.AppendString(o, z.FailureReason)
return
}
@@ -933,6 +1059,24 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "BytesFailed")
return
}
case "vsl":
z.VersionsSkippedLocked, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "VersionsSkippedLocked")
return
}
case "kr":
z.KeysRefused, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "KeysRefused")
return
}
case "fr":
z.FailureReason, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "FailureReason")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -947,6 +1091,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *batchJobInfo) Msgsize() (s int) {
s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.StringPrefixSize + len(z.FailureReason)
return
}
cmd/batch-object-lock_test.go
+479
View File
@@ -0,0 +1,479 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/auth"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/pgsty/silo-pkg/v3/policy"
)
var batchLockKinds = []string{"COMPLIANCE", "GOVERNANCE", "LEGALHOLD"}
func batchLockMeta(kind string) map[string]string {
m := map[string]string{}
switch kind {
case "COMPLIANCE", "GOVERNANCE":
m[strings.ToLower(xhttp.AmzObjectLockMode)] = kind
m[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(24 * time.Hour))
case "LEGALHOLD":
m[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
}
return m
}
func batchPutVersion(t *testing.T, obj ObjectLayer, bucket, key string, meta map[string]string) ObjectInfo {
t.Helper()
data := []byte("batch lock " + key)
oi, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
ObjectOptions{Versioned: true, UserDefined: meta})
if err != nil {
t.Fatal(err)
}
return oi
}
func batchVersionExists(t *testing.T, obj ObjectLayer, bucket, key, versionID string) bool {
t.Helper()
_, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: versionID})
switch {
case err == nil:
return true
case isErrVersionNotFound(err) || isErrObjectNotFound(err):
return false
}
t.Fatal(err)
return false
}
// batchJobStatusJSON returns the status response of a job as decoded JSON.
func batchJobStatusJSON(t *testing.T, jobID string) map[string]any {
t.Helper()
cred := globalActiveCred
req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/status-job?jobId="+jobID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
if err = req.ParseForm(); err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
req = req.WithContext(newContext(req, rec, "BatchJobStatus"))
adminAPIHandlers{}.BatchJobStatus(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
var out struct{ LastMetric map[string]any }
if err = json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
return out.LastMetric
}
// Batch expiration keeps versions under retention or legal hold, on both
// purge paths, and reports them; unlocked versions are still expired.
func TestBatchExpireObjectLock(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t, endpoints: []string{"PutObject"},
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
defer r5Capacity(obj.(*erasureServerPools))()
enableBucketObjectLock(t, bucket)
f := &batchAuthzFixture{ctx: t.Context(), obj: obj, bucket: bucket}
batchTestJobPool(t.Context(), t, obj)
type version struct{ key, id string }
var locked, unlocked []version
for _, retain := range []int{0, 1} {
for _, kind := range batchLockKinds {
key := fmt.Sprintf("r%d/%s", retain, kind)
older := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
batchPutVersion(t, obj, bucket, key, nil)
locked = append(locked, version{key, older.VersionID})
}
key := fmt.Sprintf("r%d/plain", retain)
older := batchPutVersion(t, obj, bucket, key, nil)
batchPutVersion(t, obj, bucket, key, nil)
unlocked = append(unlocked, version{key, older.VersionID})
}
var jobs []*BatchJobRequest
for _, retain := range []int{0, 1} {
job := fmt.Sprintf(`expire:
apiVersion: v1
bucket: %s
prefix: r%d/
rules:
- type: object
purge:
retainVersions: %d
`, bucket, retain, retain)
rec, queued := f.submit(t, globalActiveCred, job)
if queued == nil {
t.Fatalf("%s: submission status %d: %s", instance, rec.Code, rec.Body.String())
}
// Running a job again, as a resumed job does, must not count
// the versions it left a second time.
f.run(t, queued)
f.run(t, queued)
jobs = append(jobs, queued)
}
for _, v := range locked {
if !batchVersionExists(t, obj, bucket, v.key, v.id) {
t.Errorf("%s: %s: locked version %s was expired", instance, v.key, v.id)
}
}
for _, v := range unlocked {
if batchVersionExists(t, obj, bucket, v.key, v.id) {
t.Errorf("%s: %s: control: unlocked version %s was not expired", instance, v.key, v.id)
}
}
for i, field := range []string{"keysRefused", "versionsSkippedLocked"} {
m := batchJobStatusJSON(t, jobs[i].ID)
if got, _ := m[field].(float64); got != float64(len(batchLockKinds)) {
t.Errorf("%s: retain %d: %s = %v, want %d", instance, i, field, m[field], len(batchLockKinds))
}
if m["complete"] != true || m["failed"] != false {
t.Errorf("%s: retain %d: complete=%v failed=%v", instance, i, m["complete"], m["failed"])
}
}
},
})
}
type batchLockTransport func(*http.Request) (*http.Response, error)
func (f batchLockTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
// batchLockRemote is a source that serves replacement for every read.
func batchLockRemote(t *testing.T, replacement string) *minio.Core {
t.Helper()
core, err := minio.NewCore("source.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{
"Content-Length": []string{strconv.Itoa(len(replacement))},
"Last-Modified": []string{UTCNow().Format(http.TimeFormat)},
"Etag": []string{"\"batchetag\""},
}, Body: io.NopCloser(strings.NewReader(replacement)), ContentLength: int64(len(replacement)), Request: r}, nil
}),
})
if err != nil {
t.Fatal(err)
}
return core
}
func batchRootSubmitter(t *testing.T) *batchJobSubmitter {
t.Helper()
sub, err := newBatchJobSubmitter(globalActiveCred)
if err != nil {
t.Fatal(err)
}
return sub
}
// A pull leaves a destination version under Object Lock as it is, whether a
// single PUT, a multipart completion or a delete marker would replace it.
func TestBatchPullObjectLock(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t, endpoints: []string{"PutObject"},
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
defer r5Capacity(obj.(*erasureServerPools))()
enableBucketObjectLock(t, bucket)
const replacement = "replacement from remote"
core := batchLockRemote(t, replacement)
sub := batchRootSubmitter(t)
job := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
}
for _, kind := range batchLockKinds {
for _, path := range []string{"put", "multipart", "marker", "directory"} {
key := "pull/" + kind + "/" + path
if path == "directory" {
key += "/"
}
before := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
src := ObjectInfo{Name: key, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}}
switch path {
case "multipart":
src.ETag = "batchetag-1"
case "marker":
src.DeleteMarker = true
}
err := job.ReplicateFromSource(t.Context(), obj, core, src, false, sub)
if !isErrObjectLocked(err) {
t.Errorf("%s: %s: pull returned %v, want ObjectLocked", instance, key, err)
}
after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
if gerr != nil {
t.Errorf("%s: %s: locked version lost: %v", instance, key, gerr)
continue
}
if !enforceRetentionForDeletion(t.Context(), after) || after.ETag != before.ETag {
t.Errorf("%s: %s: locked version was replaced", instance, key)
}
}
}
// Controls: a new version, and a new delete marker that hides a
// locked version without replacing it.
locked := batchPutVersion(t, obj, bucket, "pull/new", batchLockMeta("COMPLIANCE"))
newVersion := mustGetUUID()
if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
Name: "pull/new", VersionID: newVersion, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
}, false, sub); err != nil {
t.Fatalf("%s: new version: %v", instance, err)
}
if got := readCopyChecksumObject(t, obj, bucket, "pull/new", ObjectOptions{VersionID: newVersion}); string(got) != replacement {
t.Errorf("%s: new version content %q", instance, got)
}
marker := mustGetUUID()
if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
Name: "pull/new", VersionID: marker, ModTime: UTCNow(), DeleteMarker: true,
}, false, sub); err != nil {
t.Fatalf("%s: new marker: %v", instance, err)
}
if !batchVersionExists(t, obj, bucket, "pull/new", locked.VersionID) {
t.Errorf("%s: a new marker removed a locked version", instance)
}
},
})
}
// A pull applies retention as a PUT to the destination would: copied
// retention is validated, and a version without one gets the bucket default.
func TestBatchPullRetentionMetadata(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t, endpoints: []string{"PutObject"},
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
defer r5Capacity(obj.(*erasureServerPools))()
meta, err := globalBucketMetadataSys.Get(bucket)
if err != nil {
t.Fatal(err)
}
meta.ObjectLockConfigXML = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>1</Days></DefaultRetention></Rule></ObjectLockConfiguration>`)
meta.VersioningConfigXML = enabledBucketVersioningConfig
if err = meta.parseAllConfigs(t.Context(), obj); err != nil {
t.Fatal(err)
}
globalBucketMetadataSys.Set(bucket, meta)
core := batchLockRemote(t, "retention content")
sub := batchRootSubmitter(t)
job := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
}
pull := func(key string, meta map[string]string) (ObjectInfo, error) {
vid := mustGetUUID()
err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{Name: key, VersionID: vid, ModTime: UTCNow().Add(-time.Hour), ETag: "batchetag", UserDefined: meta}, false, sub)
if err != nil {
return ObjectInfo{}, err
}
return obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: vid})
}
oi, err := pull("retention/default", map[string]string{})
if err != nil {
t.Fatal(err)
}
ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
if ret.Mode != objectlock.RetGovernance || !ret.RetainUntilDate.After(UTCNow().Add(23*time.Hour)) {
t.Errorf("%s: default retention not applied with the destination's time: %+v", instance, ret)
}
until := UTCNow().Add(48 * time.Hour)
oi, err = pull("retention/copied", map[string]string{
xhttp.AmzObjectLockMode: "COMPLIANCE", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until),
})
if err != nil {
t.Fatal(err)
}
if ret = objectlock.GetObjectRetentionMeta(oi.UserDefined); ret.Mode != objectlock.RetCompliance || !ret.RetainUntilDate.Equal(until.Truncate(time.Millisecond)) {
t.Errorf("%s: copied retention: %+v", instance, ret)
}
if _, err = pull("retention/invalid", map[string]string{xhttp.AmzObjectLockMode: "FOREVER", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until)}); err == nil {
t.Errorf("%s: invalid copied retention was accepted", instance)
}
},
})
}
// Key rotation does not rewrite a version under Object Lock.
func TestBatchKeyRotationObjectLock(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t, endpoints: []string{"PutObject", "GetObject"},
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
defer r5Capacity(obj.(*erasureServerPools))()
enableBucketObjectLock(t, bucket)
oldKMS := GlobalKMS
GlobalKMS = kms.NewStub("batch-lock-key")
defer func() { GlobalKMS = oldKMS }()
sub := batchRootSubmitter(t)
job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
for _, kind := range append([]string{"none"}, batchLockKinds...) {
key := "rotate/" + kind
headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
switch kind {
case "COMPLIANCE", "GOVERNANCE":
headers[xhttp.AmzObjectLockMode] = kind
headers[xhttp.AmzObjectLockRetainUntilDate] = UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
case "LEGALHOLD":
headers[xhttp.AmzObjectLockLegalHold] = "ON"
}
putCopyChecksumSource(t, router, cred, bucket, key, []byte("rotate "+kind), headers)
before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
if err != nil || !crypto.S3.IsEncrypted(before.UserDefined) {
t.Fatalf("%s: %s: encrypted control: %v", instance, key, err)
}
err = job.KeyRotate(t.Context(), obj, before, sub)
after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
if gerr != nil {
t.Fatal(gerr)
}
if kind == "none" {
if err != nil || after.ModTime.Equal(before.ModTime) {
t.Errorf("%s: control: unlocked version not rotated: %v", instance, err)
}
continue
}
var locked ObjectLocked
if !errors.As(err, &locked) || !after.ModTime.Equal(before.ModTime) {
t.Errorf("%s: %s: rotation of a locked version returned %v; rewritten=%v", instance, key, err, !after.ModTime.Equal(before.ModTime))
}
}
},
})
}
// A push reads locked versions as any authorized read does, and a submitter
// that may not read a version does not send it.
func TestBatchPushReads(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t, endpoints: []string{"PutObject"},
objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
defer r5Capacity(obj.(*erasureServerPools))()
enableBucketObjectLock(t, bucket)
var puts []string
target, err := minio.NewCore("target.invalid", &minio.Options{
Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
io.Copy(io.Discard, r.Body)
r.Body.Close()
}
if r.Method == http.MethodPut {
puts = append(puts, r.URL.Path)
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": []string{"\"pushetag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
}),
})
if err != nil {
t.Fatal(err)
}
job := BatchJobReplicateV1{
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
}
src := batchPutVersion(t, obj, bucket, "push/locked", batchLockMeta("COMPLIANCE"))
if err := job.ReplicateToTarget(t.Context(), obj, target, src, false, batchRootSubmitter(t)); err != nil {
t.Fatalf("%s: authorized push of a locked version: %v", instance, err)
}
if len(puts) != 1 {
t.Fatalf("%s: authorized push sent %d writes", instance, len(puts))
}
// A user that may list but not read.
p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:StartBatchJob"]},{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::` + bucket + `"]}]}`))
if err != nil {
t.Fatal(err)
}
if _, err = globalIAMSys.SetPolicy(t.Context(), "batch-list-only", *p); err != nil {
t.Fatal(err)
}
user := auth.Credentials{AccessKey: "batch-list-user", SecretKey: "batch-list-user-secret"}
if _, err = globalIAMSys.CreateUser(t.Context(), user.AccessKey, madmin.AddOrUpdateUserReq{SecretKey: user.SecretKey, Status: madmin.AccountEnabled}); err != nil {
t.Fatal(err)
}
if _, err = globalIAMSys.PolicyDBSet(t.Context(), user.AccessKey, "batch-list-only", regUser, false); err != nil {
t.Fatal(err)
}
sub, err := newBatchJobSubmitter(globalIAMSysUser(t, user.AccessKey))
if err != nil {
t.Fatal(err)
}
err = job.ReplicateToTarget(t.Context(), obj, target, src, false, sub)
if !isBatchJobAuthzError(err) || len(puts) != 1 {
t.Fatalf("%s: unauthorized push returned %v and sent %d writes", instance, err, len(puts)-1)
}
},
})
}
// globalIAMSysUser returns the stored credential of accessKey, as a request
// authenticated with it has.
func globalIAMSysUser(t *testing.T, accessKey string) auth.Credentials {
t.Helper()
u, ok := globalIAMSys.store.GetUser(accessKey)
if !ok {
t.Fatalf("no user %s", accessKey)
}
return u.Credentials
}
// The cursor of a walk done out of order moves only past keys whose items
// are all done, and counts each item once.
func TestBatchProgressCursor(t *testing.T) {
ri := &batchJobInfo{}
p := newBatchProgress(ri, "bucket")
items := []string{"a", "a", "b", "c", "c"}
seqs := make([]uint64, len(items))
for i := range items {
seqs[i] = p.begin()
}
skip := batchCounts{versionsSkipped: 1}
p.done(seqs[1], "a", skip)
p.done(seqs[3], "c", skip)
if ri.Object != "" || ri.VersionsSkippedLocked != 0 {
t.Fatalf("cursor moved before the first item: %q %d", ri.Object, ri.VersionsSkippedLocked)
}
p.done(seqs[0], "a", skip)
p.done(seqs[2], "b", skip)
// a is done; b is done but c may still have items.
if ri.Object != "b" && ri.Object != "a" {
t.Fatalf("cursor %q", ri.Object)
}
if ri.Object == "a" && ri.VersionsSkippedLocked != 2 {
t.Fatalf("counts %d at cursor a", ri.VersionsSkippedLocked)
}
// c's last item never finishes, as when a run stops: the cursor stays
// before c and c is not counted.
p.finish()
if ri.Object != "b" || ri.VersionsSkippedLocked != 3 {
t.Fatalf("interrupted: cursor %q counts %d, want b 3", ri.Object, ri.VersionsSkippedLocked)
}
p.done(seqs[4], "c", skip)
p.finish()
if ri.Object != "c" || ri.VersionsSkippedLocked != 5 {
t.Fatalf("finished: cursor %q counts %d, want c 5", ri.Object, ri.VersionsSkippedLocked)
}
}
cmd/batch-progress.go
+187
View File
@@ -0,0 +1,187 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"slices"
"strings"
"sync"
"github.com/minio/madmin-go/v3"
)
// batchCounts are outcomes of a batch job that a later run of the job would
// see again, and so must be persisted together with the cursor of the walk.
type batchCounts struct {
objects int64
objectsFailed int64
deleteMarkersFailed int64
versionsSkipped int64
keysRefused int64
}
func (c *batchCounts) add(o batchCounts) {
c.objects += o.objects
c.objectsFailed += o.objectsFailed
c.deleteMarkersFailed += o.deleteMarkersFailed
c.versionsSkipped += o.versionsSkipped
c.keysRefused += o.keysRefused
}
//msgp:ignore batchProgress batchProgressItem batchCounts
// batchProgress is the cursor of a walk whose items are done concurrently.
//
// The items of a walk are numbered in walk order. A key is done when every
// item of it, and every item before it, is done; only then does the cursor
// move past it and its counts become part of the job's counts. The cursor and
// the counts are persisted together, so a job resumed after the cursor
// neither skips an item that was not done nor counts an item twice.
type batchProgress struct {
mu sync.Mutex
ri *batchJobInfo
bucket string
issued uint64
next uint64
pending map[uint64]batchProgressItem
// key is the key of the item last committed, and held are the counts of
// its items so far.
key string
held batchCounts
}
type batchProgressItem struct {
key string
counts batchCounts
}
func newBatchProgress(ri *batchJobInfo, bucket string) *batchProgress {
return &batchProgress{ri: ri, bucket: bucket, pending: make(map[uint64]batchProgressItem)}
}
// begin numbers the next item of the walk.
func (p *batchProgress) begin() uint64 {
p.mu.Lock()
defer p.mu.Unlock()
seq := p.issued
p.issued++
return seq
}
// done records item seq, of key, as done with counts.
func (p *batchProgress) done(seq uint64, key string, counts batchCounts) {
p.mu.Lock()
defer p.mu.Unlock()
p.pending[seq] = batchProgressItem{key: key, counts: counts}
for {
item, ok := p.pending[p.next]
if !ok {
return
}
delete(p.pending, p.next)
p.next++
if item.key != p.key {
p.commitLocked()
p.key = item.key
}
p.held.add(item.counts)
}
}
// finish commits the last key if every item of the walk is done. Call it
// once no item is in progress.
func (p *batchProgress) finish() {
p.mu.Lock()
defer p.mu.Unlock()
if p.next == p.issued {
p.commitLocked()
}
}
func (p *batchProgress) commitLocked() {
if p.key == "" {
return
}
ri := p.ri
ri.mu.Lock()
ri.Bucket = p.bucket
ri.Object = p.key
ri.Objects += p.held.objects
ri.ObjectsFailed += p.held.objectsFailed
ri.DeleteMarkersFailed += p.held.deleteMarkersFailed
ri.VersionsSkippedLocked += p.held.versionsSkipped
ri.KeysRefused += p.held.keysRefused
ri.mu.Unlock()
p.held = batchCounts{}
}
// countRemoved counts versions a job removed. A removed version is gone for
// any later run, so it is counted at once, not at the cursor.
func (ri *batchJobInfo) countRemoved(bucket string, objects, markers int64) {
ri.mu.Lock()
defer ri.mu.Unlock()
ri.Bucket = bucket
ri.Objects += objects
ri.DeleteMarkers += markers
}
// countSkipped counts a version a job left in place because Object Lock
// protects it, for a job that is counted per run rather than at a cursor.
func (ri *batchJobInfo) countSkipped() {
ri.mu.Lock()
defer ri.mu.Unlock()
ri.VersionsSkippedLocked++
}
// batchCanonicalPrefixes orders the prefixes of a walk and drops a prefix
// that another one covers. The keys of the walk are then visited once each,
// in one ascending order, which a single cursor can describe.
func batchCanonicalPrefixes(prefixes []string) []string {
if len(prefixes) == 0 {
return []string{""}
}
sorted := slices.Clone(prefixes)
slices.Sort(sorted)
out := sorted[:0]
for _, p := range sorted {
if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) {
continue
}
out = append(out, p)
}
return out
}
// batchResumeAfter reports whether a resumed walk returned a key after the
// cursor. The cursor key is done; the walk forwards conservatively and
// returns it again. Keys compare as the walk orders them, by name, so a
// directory object comes before the keys under it.
func batchResumeAfter(cursor, name string) bool {
return cursor == "" || name > cursor
}
// batchJobStatus is the status response: madmin.BatchJobStatus with the
// fields this server adds. The added fields are additive; a client that does
// not know them ignores them.
type batchJobStatus struct {
LastMetric batchJobStatusMetric
}
type batchJobStatusMetric struct {
madmin.JobMetric
VersionsSkippedLocked int64 `json:"versionsSkippedLocked,omitempty"`
KeysRefused int64 `json:"keysRefused,omitempty"`
FailureReason string `json:"failureReason,omitempty"`
}
func (ri *batchJobInfo) statusMetric() batchJobStatusMetric {
ri.mu.RLock()
defer ri.mu.RUnlock()
return batchJobStatusMetric{
JobMetric: ri.metric(),
VersionsSkippedLocked: ri.VersionsSkippedLocked,
KeysRefused: ri.KeysRefused,
FailureReason: ri.FailureReason,
}
}
cmd/batch-rotate.go
+221 -50
View File
@@ -20,6 +20,7 @@ package cmd
import (
"context"
"encoding/base64"
"errors"
"fmt"
"maps"
"math/rand"
@@ -35,6 +36,7 @@ import (
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/pgsty/silo-pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/workers"
)
@@ -161,8 +163,68 @@ func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error
return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
}
// KeyRotate rotates encryption key of an object
func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
// batchAuthoritativeVersion returns the version obj as all its copies
// describe it. With several pools a version can have copies in more than one
// pool, and a read returns one of them; the merged copy carries the newest
// Object Lock and tag state. The caller holds the object's write lock.
func batchAuthoritativeVersion(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
z, ok := api.(*erasureServerPools)
if !ok || z.SinglePool() {
return obj, nil
}
versionID := obj.VersionID
if versionID == "" {
versionID = nullVersionID
}
return z.replicaObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{VersionID: versionID, NoLock: true, NoAuditLog: true})
}
// batchAuthoritativeRead is batchAuthoritativeVersion for a version that has
// been opened for reading. It takes the object's read lock itself, since an
// opened reader may hold none.
func batchAuthoritativeRead(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
z, ok := api.(*erasureServerPools)
if !ok || z.SinglePool() {
return obj, nil
}
lk := z.NewNSLock(bucket, encodeDirObject(object))
lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
if err != nil {
return ObjectInfo{}, err
}
defer lk.RUnlock(lkctx)
return batchAuthoritativeVersion(lkctx.Context(), api, bucket, object, obj)
}
// rotateAuthzRequests are the actions of rotating one version: it is read and
// written again in place with the new key.
func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzRequest {
versionID := oi.VersionID
if versionID == "" {
versionID = nullVersionID
}
// The encryption the rewrite leaves: rotation keeps the kind of the
// version (rotateKey), and SSE-KMS takes the requested key.
values := map[string][]string{}
switch {
case crypto.S3KMS.IsEncrypted(oi.UserDefined):
values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
}
case crypto.S3.IsEncrypted(oi.UserDefined):
values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionAES}
}
return []batchAuthzRequest{
{action: policy.GetObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
{action: policy.PutObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags, values: values},
}
}
// KeyRotate rotates encryption key of an object. A version that Object Lock
// protects is not rewritten: it returns ObjectLocked.
func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo, sub *batchJobSubmitter) error {
srcBucket := r.Bucket
srcObject := objInfo.Name
@@ -180,7 +242,8 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
lock := api.NewObjectNSLock(r.Bucket, objInfo.Name)
// Lock the name writers lock: the stored name of a directory object.
lock := api.NewObjectNSLock(r.Bucket, encodeDirObject(objInfo.Name))
lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
@@ -194,10 +257,32 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
VersionSuspended: versionSuspended,
NoLock: true,
}
obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
obj, pool, err := batchRotateRead(ctx, api, r.Bucket, objInfo.Name, opts)
if err != nil {
return err
}
// Decide on the version as it is under the lock that the rewrite holds,
// before any key is requested, with the metadata of all its copies.
logical, err := batchAuthoritativeVersion(ctx, api, r.Bucket, objInfo.Name, obj)
if err != nil {
return err
}
if enforceRetentionForDeletion(ctx, logical) {
return ObjectLocked{Bucket: r.Bucket, Object: obj.Name, VersionID: obj.VersionID}
}
p, err := sub.principal(ctx)
if err != nil {
return err
}
// Authorize the copy that is read and rewritten, and the version as all
// its copies describe it.
for _, v := range []ObjectInfo{obj, logical} {
for _, req := range r.rotateAuthzRequests(v) {
if err := p.authorize(req); err != nil {
return err
}
}
}
oi := obj.Clone()
var (
newKeyID string
@@ -225,16 +310,61 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
oi.metadataOnly = true
oi.keyRotation = true
maps.Copy(oi.UserDefined, encMetadata)
if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
VersionID: oi.VersionID,
}, ObjectOptions{
VersionID: oi.VersionID,
NoLock: true,
}); err != nil {
return err
// The rewrite replaces the stored metadata with UserDefined, which a
// read returns without the tags and the expiry: keep them.
if oi.UserTags != "" {
oi.UserDefined[xhttp.AmzObjectTagging] = oi.UserTags
}
if !oi.Expires.IsZero() {
oi.UserDefined["expires"] = oi.ExpiresStr()
}
srcOpts := ObjectOptions{VersionID: oi.VersionID}
dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true}
if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
// Rewrite the copy that was read and authorized. The pools layer
// would route the rewrite by the latest version of the key, which
// can be in a pool that holds another write of this version ID.
//
// Check the pool's state again where the pools layer checks it,
// right before the write: a move can have started since the read.
if z.IsSuspended(pool) || z.IsPoolRebalancing(pool) {
return errBatchRotateMoving
}
dstOpts.replicaObjectInfo = z.replicaObjectInfo
name := encodeDirObject(oi.Name)
_, err = z.serverPools[pool].CopyObject(ctx, r.Bucket, name, r.Bucket, name, oi, srcOpts, dstOpts)
} else {
_, err = api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, srcOpts, dstOpts)
}
return err
}
return nil
// errBatchRotateMoving: the version is read from a pool that a decommission
// or a rebalance is moving.
var errBatchRotateMoving = errors.New("the version is being moved to another pool; rotate it once the move is done")
// batchRotateRead reads the version that a rotation rewrites, as
// GetObjectInfo does. With several pools it also returns the pool of the
// copy it read, otherwise -1. The caller holds the object lock.
//
// A copy in a pool that a decommission or a rebalance is moving is the
// source of that move, which must not change under it, and rewriting a copy
// in another pool would change which copy the version's metadata comes
// from. Such a version is left to a later run (errBatchRotateMoving).
func batchRotateRead(ctx context.Context, api ObjectLayer, bucket, object string, opts ObjectOptions) (ObjectInfo, int, error) {
z, ok := api.(*erasureServerPools)
if !ok || z.SinglePool() {
oi, err := api.GetObjectInfo(ctx, bucket, object, opts)
return oi, -1, err
}
if err := checkGetObjArgs(ctx, bucket, object); err != nil {
return ObjectInfo{}, -1, err
}
oi, idx, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
if err == nil && (z.IsSuspended(idx) || z.IsPoolRebalancing(idx)) {
return ObjectInfo{}, -1, errBatchRotateMoving
}
return oi, idx, err
}
const (
@@ -260,6 +390,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if ri.Complete {
return nil
}
if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
return err
}
globalBatchJobsMetrics.save(job.ID, ri)
lastObject := ri.Object
@@ -273,8 +406,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
delay = batchKeyRotateJobDefaultRetryDelay
}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
selectObj := func(info FileInfo) (ok bool) {
if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
// skip all objects that are newer than specified older duration
@@ -355,65 +486,92 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
return err
}
ctx, cancel := context.WithCancel(ctx)
ctx, cancelCause := context.WithCancelCause(ctx)
defer cancelCause(nil)
// refuse ends the job for its submitter: record why before stopping.
refuse := func(err error) {
ri.refuse(err)
cancelCause(err)
}
progress := newBatchProgress(ri, r.Bucket)
results := make(chan itemOrErr[ObjectInfo], 100)
if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
Marker: lastObject,
Filter: selectObj,
}); err != nil {
cancel()
// Do not need to retry if we can't list objects on source.
return err
}
failed := false
for res := range results {
if context.Cause(ctx) != nil {
// Refused or canceled: schedule nothing more.
continue
}
if res.Err != nil {
failed = true
batchLogIf(ctx, res.Err)
break
}
result := res.Item
if !batchResumeAfter(lastObject, result.Name) {
// Completed by the run that set the cursor.
continue
}
sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
continue
}
seq := progress.begin()
wk.Take()
go func() {
defer wk.Give()
var counts batchCounts
for attempts := 1; attempts <= retryAttempts; attempts++ {
stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
success := true
if err := r.KeyRotate(ctx, api, result); err != nil {
stopFn(result, err)
batchLogIf(ctx, err)
success = false
if attempts >= retryAttempts {
auditOptions := AuditLogOptions{
Event: "KeyRotate",
APIName: "StartBatchJob",
Bucket: result.Bucket,
Object: result.Name,
VersionID: result.VersionID,
Error: err.Error(),
}
auditLogInternal(ctx, auditOptions)
}
} else {
stopFn(result, nil)
err := r.KeyRotate(ctx, api, result, job.Submitter)
stopFn(result, err)
if err == nil {
counts.objects = 1
break
}
ri.trackCurrentBucketObject(r.Bucket, result, success, attempts)
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk after every 10secs.
batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
if success {
if isBatchJobAuthzError(err) {
refuse(err)
return
}
if isErrObjectLocked(err) {
// Left as it is, not retried and not a failure.
counts.versionsSkipped = 1
break
}
if ctx.Err() != nil {
// Not done; a later run rotates it.
return
}
batchLogIf(ctx, err)
if attempts >= retryAttempts {
auditOptions := AuditLogOptions{
Event: "KeyRotate",
APIName: "StartBatchJob",
Bucket: result.Bucket,
Object: result.Name,
VersionID: result.VersionID,
Error: err.Error(),
}
auditLogInternal(ctx, auditOptions)
counts.objectsFailed = 1
break
}
if delay > 0 {
time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
time.Sleep(delay + time.Duration(rand.Float64()*float64(delay)))
}
}
progress.done(seq, result.Name, counts)
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk after every 10secs.
batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
time.Sleep(wait)
@@ -421,18 +579,31 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
}()
}
wk.Wait()
ri.Complete = !failed && ri.ObjectsFailed == 0
ri.Failed = failed || ri.ObjectsFailed > 0
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk.
batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
if err := r.Notify(ctx, ri); err != nil {
batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
cause := context.Cause(ctx)
if !failed && cause == nil {
// The walk ended: its last key is done.
progress.finish()
}
if cause != nil && !ri.refused() {
// Stopped before the end: keep the work after the cursor pending.
globalBatchJobsMetrics.save(job.ID, ri)
batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
return cause
}
cancel()
if !ri.refused() {
ri.mu.Lock()
ri.Complete = !failed && ri.ObjectsFailed == 0
ri.Failed = failed || ri.ObjectsFailed > 0
ri.mu.Unlock()
}
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk.
batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
if err := r.Notify(GlobalContext, ri); err != nil {
batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
}
return nil
}
cmd/batch-submitter-authz_test.go
+456
View File
@@ -0,0 +1,456 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/pgsty/silo-pkg/v3/policy"
"go.yaml.in/yaml/v3"
)
// batchAuthzFixture runs batch jobs submitted through the admin API, one job
// at a time and in the test's goroutine.
type batchAuthzFixture struct {
ctx context.Context
sys *IAMSys
obj ObjectLayer
bucket string
}
func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture {
t.Helper()
ctx, sys := iamAdminFixture(t)
obj := newObjectLayerFn()
batchTestJobPool(ctx, t, obj)
f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
_, err := globalBucketMetadataSys.Update(ctx, f.bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
return f
}
// batchTestJobPool queues submitted jobs without running them.
func batchTestJobPool(ctx context.Context, t *testing.T, obj ObjectLayer) {
t.Helper()
prevPool := globalBatchJobPool
globalBatchJobPool = &BatchJobPool{
ctx: ctx,
objLayer: obj,
jobCh: make(chan *BatchJobRequest, 10),
jobCancelers: make(map[string]context.CancelFunc),
}
globalBatchJobsMetrics.Lock()
globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo)
globalBatchJobsMetrics.Unlock()
t.Cleanup(func() { globalBatchJobPool = prevPool })
}
// putVersions writes two versions of key and returns the older one.
func (f *batchAuthzFixture) putVersions(t *testing.T, key string) string {
t.Helper()
var older string
for i := range 2 {
data := []byte(fmt.Sprintf("version %d", i))
oi, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
mustIAM(t, err)
if i == 0 {
older = oi.VersionID
}
}
return older
}
func (f *batchAuthzFixture) versionExists(t *testing.T, key, versionID string) bool {
t.Helper()
_, err := f.obj.GetObjectInfo(f.ctx, f.bucket, key, ObjectOptions{VersionID: versionID})
if err == nil {
return true
}
if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
return false
}
t.Fatal(err)
return false
}
func (f *batchAuthzFixture) expireYAML(prefix string) string {
return fmt.Sprintf(`expire:
apiVersion: v1
bucket: %s
prefix: %q
rules:
- type: object
purge:
retainVersions: 1
`, f.bucket, prefix)
}
// submit sends a job definition to StartBatchJob as cred. An accepted job is
// returned from the queue, not run.
func (f *batchAuthzFixture) submit(t *testing.T, cred auth.Credentials, job string) (*httptest.ResponseRecorder, *BatchJobRequest) {
t.Helper()
headers := map[string]string{}
if cred.IsTemp() && !cred.IsServiceAccount() {
headers["X-Amz-Security-Token"] = cred.SessionToken
}
body := []byte(job)
req, err := newTestSignedRequestV4(http.MethodPost, "http://localhost/minio/admin/v3/start-job", int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
mustIAM(t, err)
mustIAM(t, req.ParseForm())
rec := httptest.NewRecorder()
req = req.WithContext(newContext(req, rec, "StartBatchJob"))
adminAPIHandlers{}.StartBatchJob(rec, req)
if rec.Code != http.StatusOK {
return rec, nil
}
select {
case queued := <-globalBatchJobPool.jobCh:
return rec, queued
default:
t.Fatal("accepted job was not queued")
return rec, nil
}
}
func (f *batchAuthzFixture) run(t *testing.T, job *BatchJobRequest) {
t.Helper()
ctx := job.ctx
if ctx == nil {
ctx = f.ctx
}
switch {
case job.Expire != nil:
if err := job.Expire.Start(ctx, f.obj, *job); err != nil {
t.Logf("start: %v", err)
}
default:
t.Fatalf("unexpected job type %s", job.Type())
}
ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
if err := ri.load(f.ctx, f.obj, *job); err == nil {
t.Logf("job %s: %+v", job.ID, ri.clone())
}
}
func (f *batchAuthzFixture) jobInfo(t *testing.T, job *BatchJobRequest) *batchJobInfo {
t.Helper()
ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
mustIAM(t, ri.load(f.ctx, f.obj, *job))
return ri
}
func batchAuthzPolicy(ctx context.Context, t *testing.T, sys *IAMSys, name, statements string) {
t.Helper()
p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[` + statements + `]}`))
mustIAM(t, err)
_, err = sys.SetPolicy(ctx, name, *p)
mustIAM(t, err)
}
const batchAuthzFullAccess = `{"Effect":"Allow","Action":["admin:StartBatchJob"]},
{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}`
// A credential that may start batch jobs but has no access to the bucket
// cannot delete through a job.
func TestBatchJobSubmitterWithoutDataAccess(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-only", `{"Effect":"Allow","Action":["admin:StartBatchJob","admin:ListBatchJobs"]}`)
cred := iamAdminUser(f.ctx, t, f.sys, "batch-only-user", "batch-only")
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job != nil {
f.run(t, job)
}
if !f.versionExists(t, "data/a", older) {
t.Fatal("a job submitted without data access deleted a version")
}
if rec.Code != http.StatusForbidden {
t.Fatalf("submission status %d, want 403: %s", rec.Code, rec.Body.String())
}
}
// A job acts only while the credential that submitted it is the same valid
// credential.
func TestBatchJobSubmitterChangedBeforeStart(t *testing.T) {
for _, tc := range []struct {
name string
change func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials)
}{
{"unchanged", nil},
{"deleted", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
}},
{"disabled", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
_, err := f.sys.SetUserStatus(f.ctx, cred.AccessKey, madmin.AccountDisabled)
mustIAM(t, err)
}},
{"secret-changed", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
mustIAM(t, f.sys.SetUserSecretKey(f.ctx, cred.AccessKey, "batch-authz-changed-secret"))
}},
{"access-key-reused", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
time.Sleep(10 * time.Millisecond)
iamAdminUser(f.ctx, t, f.sys, cred.AccessKey, "batch-full")
}},
} {
t.Run(tc.name, func(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
if tc.change != nil {
tc.change(t, f, cred)
}
f.run(t, job)
exists := f.versionExists(t, "data/a", older)
if tc.change == nil && exists {
t.Fatal("control: an authorized job did not expire the older version")
}
if tc.change != nil && !exists {
t.Fatal("a job ran as a credential that changed after submission")
}
})
}
}
// An expired temporary credential does not keep its job running.
func TestBatchJobSubmitterExpired(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(2 * time.Second).Unix()}, globalActiveCred.SecretKey)
mustIAM(t, err)
cred.ParentUser = parent.AccessKey
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
time.Sleep(time.Until(cred.Expiration) + time.Second)
f.run(t, job)
if !f.versionExists(t, "data/a", older) {
t.Fatal("a job ran as an expired credential")
}
}
// A session policy limits the job as it limits requests, also when an
// authorization plugin decides the rest.
func TestBatchJobSessionPolicyDeny(t *testing.T) {
session := `{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":["admin:StartBatchJob"]},
{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]},
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`
for _, tc := range []struct {
name string
plugin bool
}{{"local", false}, {"plugin-allows", true}} {
t.Run(tc.name, func(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
cred := iamAdminServiceAccount(f.ctx, t, f.sys, parent.AccessKey, "batch-restricted", session)
if tc.plugin {
iamAdminPlugin(t, true)
}
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job != nil {
f.run(t, job)
} else {
t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
}
if !f.versionExists(t, "data/a", older) {
t.Fatal("a job deleted a version its session policy denies")
}
})
}
}
// A temporary credential's session policy limits its job too.
func TestBatchJobTemporarySessionPolicyDeny(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
session := `{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":["admin:StartBatchJob"]},
{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions","s3:GetObject","s3:GetObjectVersion","s3:DeleteObject"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}]}`
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(session)),
}, globalActiveCred.SecretKey)
mustIAM(t, err)
cred.ParentUser = parent.AccessKey
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job != nil {
f.run(t, job)
} else {
t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
}
if !f.versionExists(t, "data/a", older) {
t.Fatal("a job deleted a version its session policy does not allow")
}
}
// A Deny that depends on request context the job does not have refuses the
// action instead of being skipped.
func TestBatchJobAbsentContextDeny(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-context", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-context")
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job != nil {
f.run(t, job)
} else {
t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
}
if !f.versionExists(t, "data/a", older) {
t.Fatal("a job skipped a Deny that depends on request context")
}
}
// Each object is authorized: a Deny under an allowed prefix keeps its objects.
func TestBatchJobPerObjectDeny(t *testing.T) {
f := newBatchAuthzFixture(t)
batchAuthzPolicy(f.ctx, t, f.sys, "batch-keep", batchAuthzFullAccess+`,
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/data/keep*"]}`)
cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-keep")
kept := f.putVersions(t, "data/keep")
rec, job := f.submit(t, cred, f.expireYAML("data/"))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
f.run(t, job)
if !f.versionExists(t, "data/keep", kept) {
t.Fatal("a job deleted a version that a per-object Deny protects")
}
if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
t.Fatalf("refused job: complete=%v failed=%v", ri.Complete, ri.Failed)
}
}
// A job queued without its submitter, as an earlier release stored it, fails
// and stays failed across another start.
func TestBatchJobWithoutSubmitter(t *testing.T) {
f := newBatchAuthzFixture(t)
older := f.putVersions(t, "data/a")
job := &BatchJobRequest{}
mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
job.ID = "expire-legacy" + getKeySeparator() + "0"
job.User = globalActiveCred.AccessKey
job.Started = time.Now()
mustIAM(t, job.save(f.ctx, f.obj))
for range 2 {
loaded := &BatchJobRequest{}
mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
f.run(t, loaded)
if !f.versionExists(t, "data/a", older) {
t.Fatal("a job without a recorded submitter deleted a version")
}
if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
t.Fatalf("job without submitter: complete=%v failed=%v", ri.Complete, ri.Failed)
}
}
}
// Root and an authorized user can still run jobs.
func TestBatchJobAuthorizedControls(t *testing.T) {
for _, root := range []bool{true, false} {
t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
f := newBatchAuthzFixture(t)
cred := globalActiveCred
if !root {
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
}
older := f.putVersions(t, "data/a")
rec, job := f.submit(t, cred, f.expireYAML(""))
if job == nil {
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
}
f.run(t, job)
if f.versionExists(t, "data/a", older) {
t.Fatal("an authorized job did not expire the older version")
}
if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed {
t.Fatalf("authorized job: complete=%v failed=%v", ri.Complete, ri.Failed)
}
})
}
}
// The describe output has no secrets of the job.
func TestBatchJobDescribeRedactsSecrets(t *testing.T) {
f := newBatchAuthzFixture(t)
secrets := []string{"describe-source-secret", "describe-source-token", "describe-target-secret", "describe-target-token", "describe-notify-token"}
for _, job := range []BatchJobRequest{
{ID: "replicate-describe" + getKeySeparator() + "0", Replicate: &BatchJobReplicateV1{
APIVersion: batchReplJobAPIVersion,
Source: BatchJobReplicateSource{Bucket: "remote", Endpoint: "https://remote.invalid", Creds: BatchJobReplicateCredentials{
AccessKey: "describe-source-access", SecretKey: secrets[0], SessionToken: secrets[1],
}},
Target: BatchJobReplicateTarget{Bucket: f.bucket, Creds: BatchJobReplicateCredentials{
AccessKey: "describe-target-access", SecretKey: secrets[2], SessionToken: secrets[3],
}},
Flags: BatchJobReplicateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
}},
{ID: "expire-describe" + getKeySeparator() + "0", Expire: &BatchJobExpire{
APIVersion: batchExpireAPIVersion, Bucket: f.bucket,
NotificationCfg: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]},
}},
{ID: "keyrotate-describe" + getKeySeparator() + "0", KeyRotate: &BatchJobKeyRotateV1{
APIVersion: batchKeyRotateAPIVersion, Bucket: f.bucket,
Flags: BatchJobKeyRotateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
}},
} {
data, err := job.MarshalMsg(nil)
mustIAM(t, err)
mustIAM(t, saveConfig(f.ctx, f.obj, getJobPath(job), data))
cred := globalActiveCred
req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/describe-job?jobId="+job.ID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
mustIAM(t, err)
mustIAM(t, req.ParseForm())
rec := httptest.NewRecorder()
req = req.WithContext(newContext(req, rec, "DescribeBatchJob"))
adminAPIHandlers{}.DescribeBatchJob(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: describe status %d: %s", job.ID, rec.Code, rec.Body.String())
}
for _, s := range secrets {
if strings.Contains(rec.Body.String(), s) {
t.Errorf("%s: describe output contains %s", job.ID, s)
}
}
if job.Replicate != nil && !strings.Contains(rec.Body.String(), "describe-source-access") {
t.Errorf("%s: describe output lost the access key", job.ID)
}
// The stored job keeps what a run needs.
stored := &BatchJobRequest{}
mustIAM(t, stored.load(f.ctx, f.obj, getJobPath(job)))
if stored.Replicate != nil && stored.Replicate.Source.Creds.SecretKey != secrets[0] {
t.Errorf("%s: describe changed the stored job", job.ID)
}
}
}
cmd/bucket-object-lock.go
+47
View File
@@ -291,6 +291,53 @@ func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, g
return nil
}
// checkVersionLockedForOverwrite re-checks under the object write lock
// whether a write would replace the stored version versionID (the null
// version if empty) while Object Lock protects it. It reads the version's own
// metadata, whatever the bucket configuration, and a failed read is an error.
func checkVersionLockedForOverwrite(ctx context.Context, bucket, object, versionID string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error {
if isMinioMetaBucketName(bucket) {
return nil
}
if versionID == "" {
versionID = nullVersionID
}
objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
VersionID: versionID,
Transition: transition,
NoLock: true,
NoAuditLog: true,
})
if err != nil {
// A delete marker is not protected.
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
return nil
}
return err
}
if enforceRetentionForDeletion(ctx, objInfo) {
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
}
return nil
}
// refuseLockedMarkerOverwrite refuses a delete marker write that would replace
// the stored version it names while Object Lock protects it. goi and gerr are
// that version as read under the write lock. A new marker, or one replacing a
// marker, is allowed.
func refuseLockedMarkerOverwrite(ctx context.Context, bucket, object, versionID string, goi ObjectInfo, gerr error) error {
if gerr != nil {
if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) || errors.As(gerr, &MethodNotAllowed{}) {
return nil
}
return gerr
}
if enforceRetentionForDeletion(ctx, goi) {
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
}
return nil
}
// checkPutObjectLockAllowed enforces object retention policy and legal hold policy
// for requests with WORM headers
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec.
cmd/erasure-multipart.go
+9
View File
@@ -1687,6 +1687,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
defer lk.Unlock(lkctx)
}
if opts.RefuseLockedOverwrite {
getObjectInfo := er.getObjectInfo
if opts.replicaObjectInfo != nil {
getObjectInfo = opts.replicaObjectInfo
}
if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil {
return oi, toObjectErr(err, bucket, object)
}
}
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement {
getObjectInfo := er.getObjectInfo
if opts.replicaObjectInfo != nil {
cmd/erasure-object.go
+16
View File
@@ -1676,6 +1676,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
}
if opts.RefuseLockedOverwrite {
getObjectInfo := er.getObjectInfo
if opts.replicaObjectInfo != nil {
getObjectInfo = opts.replicaObjectInfo
}
if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil {
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
}
// Rename the successfully written temporary object to final location.
onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
@@ -2199,6 +2208,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
// A delete marker that names a stored version replaces it.
if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" {
if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil {
return ObjectInfo{}, err
}
}
if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools {
if gerr == nil {
evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
cmd/erasure-server-pool-delete-all-options_test.go
+88 -15
View File
@@ -25,6 +25,8 @@ import (
"sync/atomic"
"testing"
"time"
xhttp "github.com/minio/minio/internal/http"
)
// The parts of the purge of all versions that need its own options.
@@ -65,16 +67,22 @@ func TestPurgeAllVersionsOptions(t *testing.T) {
p.version(0, "ea/key", "v1", 5)
match := &deleteAllMatch{}
expire := ExpirationOptions{Expire: true}
allow := func([]ObjectInfo) error { return nil }
for name, opts := range map[string]ObjectOptions{
"no approval": {DeleteAllVersions: true},
"two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match},
"prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true},
"prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true},
"version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()},
"movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true},
"marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true},
"versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true},
"suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true},
"two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match, deleteAllAuthorize: allow},
// A batch purge deletes as the submitter of its job, an ILM purge
// as the server, and no other delete takes an authorization.
"batch unauthorized": {DeleteAllVersions: true, DeleteAllLatest: match},
"ilm authorized": {DeleteAllVersions: true, Expiration: expire, deleteAllAuthorize: allow},
"authorized delete": {deleteAllAuthorize: allow},
"prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true},
"prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true},
"version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()},
"movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true},
"marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true},
"versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true},
"suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true},
} {
if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(InvalidArgument)) {
t.Errorf("%s: %v", name, err)
@@ -134,9 +142,10 @@ func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) {
t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() })
var res deleteAllResult
_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
deleteAllAuthorize: func([]ObjectInfo) error { return nil },
})
if !errors.Is(err, errDeleteAllBudget) || res.Started || res.Removed != 0 {
t.Errorf("purge after its budget ran out on reads: err=%v %+v", err, res)
@@ -149,7 +158,11 @@ func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) {
// A marker that an expiration job failed to remove fails the job.
func TestPurgeAllVersionsMarkerFailureFailsJob(t *testing.T) {
ri := &batchJobInfo{}
ri.trackExpireAll(expireObjInfo{ObjectInfo: ObjectInfo{NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0, true)
var res batchCounts
res.expireAllFailed(expireObjInfo{ObjectInfo: ObjectInfo{Name: "key", NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0)
progress := newBatchProgress(ri, "bucket")
progress.done(progress.begin(), "key", res)
progress.finish()
if ri.ObjectsFailed != 0 || ri.DeleteMarkersFailed != 1 || !ri.expireFailures() {
t.Errorf("failed=%d/%d failures=%t", ri.ObjectsFailed, ri.DeleteMarkersFailed, ri.expireFailures())
}
@@ -209,9 +222,10 @@ func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) {
})
var res deleteAllResult
if _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
deleteAllAuthorize: func([]ObjectInfo) error { return nil },
}); err != nil || !res.Complete {
t.Fatalf("purge: %v %+v", err, res)
}
@@ -219,3 +233,62 @@ func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) {
t.Errorf("reads=%d deletes on the replacement handle=%d", reads.Load(), deletes.Load())
}
}
// A batch purge has every copy of every version it approved authorized, as
// the copy is now, before it deletes any; an authorization error deletes
// nothing and is returned as it is.
func TestPurgeAllVersionsAuthorizesEveryCopy(t *testing.T) {
z, _ := consistencyPools(t)
p := newPurgeTest(t, z, "purge-authorize", MakeBucketOptions{VersioningEnabled: true}, "")
const key = "bx/authorize"
p.put(0, key, "old", daysAgo(6), ObjectOptions{})
p.put(1, key, "new", daysAgo(5), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: "copy=new"}})
tagged := p.put(0, key, "v1", daysAgo(4), ObjectOptions{Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "keep=yes"}})
exp := p.matched(key)
denied := errors.New("denied")
var seen []string // the distinct versions and tags authorized
purge := func(authorize func([]ObjectInfo) error) (deleteAllResult, error) {
var res deleteAllResult
_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
deleteAllAuthorize: authorize,
})
return res, err
}
res, err := purge(func(versions []ObjectInfo) error {
for _, v := range versions {
if got := v.VersionID + " " + v.UserTags; !slices.Contains(seen, got) {
seen = append(seen, got)
}
}
return denied
})
if !errors.Is(err, denied) || res.Started || res.Removed != 0 {
t.Errorf("denied purge: err=%v %+v", err, res)
}
want := []string{"null ", "null copy=new", tagged.VersionID + " keep=yes"}
slices.Sort(seen)
slices.Sort(want)
if !slices.Equal(seen, want) {
t.Errorf("authorized %q, want every copy %q", seen, want)
}
for pool := range z.serverPools {
if got := p.held(pool, key); len(got) == 0 {
t.Errorf("pool %d lost its copies to a denied purge", pool)
}
}
if got := p.get(key); got != "v1" {
t.Errorf("reads %s after a denied purge, want v1", got)
}
// Control: an allowed purge deletes them all.
if res, err = purge(func([]ObjectInfo) error { return nil }); err != nil || !res.Complete {
t.Fatalf("allowed purge: %v %+v", err, res)
}
for pool := range z.serverPools {
if got := p.held(pool, key); got != nil {
t.Errorf("pool %d keeps %v", pool, got)
}
}
}
cmd/erasure-server-pool-delete-all.go
+56 -6
View File
@@ -26,6 +26,7 @@ import (
"sync"
"time"
xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
)
@@ -49,6 +50,9 @@ import (
// when any revision of an approved version there is transitioned or,
// when the bucket has Object Lock, protected by retention or a legal
// hold. It deletes on exactly the drives it read;
// - a batch expiry then authorizes every copy of every approved version,
// and every revision of one that a drive holds, for the submitter of the
// job, and the purge deletes nothing unless all of them are allowed;
// - it deletes the oldest version first and the latest version last and
// alone, once every older version is gone from every pool. A purge that
// stops early keeps the latest version, so it never exposes an older
@@ -118,11 +122,14 @@ func deleteAllVersionID(versionID string) string {
}
// deleteAllInvalid returns why opts cannot ask for a purge of all versions.
// The purge is approved either as an ILM expiry or by a batch match, and it
// only deletes: it never creates a marker, addresses a single version,
// moves data or deletes a prefix.
// The purge is approved either as an ILM expiry or by a batch match, which
// also authorizes the versions it deletes, and it only deletes: it never
// creates a marker, addresses a single version, moves data or deletes a
// prefix.
func (opts ObjectOptions) deleteAllInvalid() error {
switch {
case !opts.DeleteAllVersions:
return errors.New("only a purge of all versions authorizes the versions it deletes")
case opts.DeletePrefix || opts.DeletePrefixObject:
return errors.New("a purge of all versions is not a prefix delete")
case opts.VersionID != "":
@@ -137,6 +144,8 @@ func (opts ObjectOptions) deleteAllInvalid() error {
return errors.New("a purge of all versions takes no options of other deletes")
case opts.Expiration.Expire == (opts.DeleteAllLatest != nil):
return errors.New("a purge of all versions needs exactly one approval")
case (opts.DeleteAllLatest != nil) != (opts.deleteAllAuthorize != nil):
return errors.New("a batch purge of all versions, and only it, authorizes the versions it deletes")
}
return nil
}
@@ -297,12 +306,29 @@ func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, obje
}
}
}
// A batch expiry deletes as its submitter: every write it is about to
// delete, the accepted copies and every revision a drive holds, must be
// allowed with the version ID and tags it has now.
var writes *deleteAllWrites
if opts.deleteAllAuthorize != nil {
writes = &deleteAllWrites{bucket: bucket, name: name, seen: make(map[[2]string]bool, len(history))}
for _, v := range history {
for _, c := range v.copies {
writes.add(c.fi)
}
}
}
drives := make([][]StorageAPI, len(z.serverPools))
for pool := range z.serverPools {
if !held[pool] {
continue
}
if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled); err != nil {
if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled, writes); err != nil {
return fail(err)
}
}
if opts.deleteAllAuthorize != nil {
if err := opts.deleteAllAuthorize(writes.versions); err != nil {
return fail(err)
}
}
@@ -329,6 +355,28 @@ func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, obje
return res.Latest, nil
}
// deleteAllWrites collects, for the authorization of a batch purge, the
// writes it deletes: one of each distinct version ID and tags.
type deleteAllWrites struct {
bucket, name string
seen map[[2]string]bool
versions []ObjectInfo
}
// add adds the write fi unless w is nil or holds one with its version ID and
// tags.
func (w *deleteAllWrites) add(fi FileInfo) {
if w == nil {
return
}
key := [2]string{deleteAllVersionID(fi.VersionID), fi.Metadata[xhttp.AmzObjectTagging]}
if w.seen[key] {
return
}
w.seen[key] = true
w.versions = append(w.versions, fi.ToObjectInfo(w.bucket, w.name, true))
}
// deleteAllRefuses returns why the purge must keep the key that holds the
// revision fi of an approved version: it is transitioned, whose remote
// object a later release hands to the tier cleanup, or, with Object Lock,
@@ -344,7 +392,8 @@ func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, l
}
// deleteAllDrives reads object on every drive of the set and checks each
// revision of the versions ids that a drive holds with deleteAllRefuses.
// revision of the versions ids that a drive holds with deleteAllRefuses, and
// adds it to writes.
// A minority of drives can hold a revision other than the accepted one, and
// heal would later remove what the purge leaves on a drive, so a drive that
// cannot be read or decoded refuses the key. A drive without the object
@@ -353,7 +402,7 @@ func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, l
// handle names a drive, not a backend: a backend that replaces the drive
// behind it starts empty and receives this object only from a write or a
// heal of it, which take the object lock the purge holds.
func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool) ([]StorageAPI, error) {
func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool, writes *deleteAllWrites) ([]StorageAPI, error) {
disks := slices.Clone(er.getDisks())
raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
if err := ctx.Err(); err != nil {
@@ -381,6 +430,7 @@ func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object str
if err := deleteAllRefuses(ctx, bucket, object, fi, lockEnabled); err != nil {
return nil, err
}
writes.add(fi)
}
}
return disks, nil
cmd/erasure-server-pool-delete-all_test.go
+12 -6
View File
@@ -332,8 +332,9 @@ func (p *purgeTest) matched(name string) expireObjInfo {
return expireObjInfo{ObjectInfo: oi, ExpireAll: true, DeleteMarkerCount: markers}
}
// batch runs the deletes of a batch expiration job for walked, the keys its
// walk matched, and returns the counts of the job.
// batch runs the deletes of a batch expiration job of the root user for
// walked, the keys its walk matched, as one batch of a walk that reached its
// end, and returns the counts of the job.
func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo {
p.t.Helper()
globalBatchJobsMetrics.Lock()
@@ -341,7 +342,11 @@ func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo {
globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo)
}
globalBatchJobsMetrics.Unlock()
job := BatchJobRequest{ID: "purge-" + mustGetUUID(), Started: time.Now(), Expire: &BatchJobExpire{Bucket: p.bucket}}
sub, err := newBatchJobSubmitter(globalActiveCred)
if err != nil {
p.t.Fatal(err)
}
job := BatchJobRequest{ID: "purge-" + mustGetUUID(), Started: time.Now(), Submitter: sub, Expire: &BatchJobExpire{Bucket: p.bucket}}
ri := &batchJobInfo{JobID: job.ID}
wk, err := workers.New(1)
if err != nil {
@@ -350,7 +355,8 @@ func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo {
ch := make(chan []expireObjInfo, 1)
ch <- walked
close(ch)
batchObjsForDelete(p.t.Context(), job.Expire, ri, job, p.z, wk, ch)
refuse := func(err error) { p.t.Errorf("batch job refused: %v", err) }
batchObjsForDelete(p.t.Context(), refuse, func() bool { return true }, job.Expire, ri, job, p.z, wk, ch)
wk.Wait()
return ri
}
@@ -837,8 +843,8 @@ func TestPurgeAllVersionsKeepsProtectedWrites(t *testing.T) {
t.Errorf("%s holds %v, want both versions", name, got)
}
}
if ri.Objects != 2 || ri.ObjectsFailed == 0 {
t.Errorf("job counts objects=%d failed=%d, want 2 and the refusals", ri.Objects, ri.ObjectsFailed)
if ri.Objects != 2 || ri.KeysRefused != 3 || ri.ObjectsFailed != 0 {
t.Errorf("job counts objects=%d refused=%d failed=%d, want 2, 3 and 0", ri.Objects, ri.KeysRefused, ri.ObjectsFailed)
}
}
cmd/erasure-server-pool-lock-domain_test.go
+5 -1
View File
@@ -1013,9 +1013,13 @@ func TestLockDomainKeyRotationWaitsForWriter(t *testing.T) {
// The rotation's listing found an SSE-S3 object; it reads it again
// under the lock.
listed := ObjectInfo{Bucket: bucket, Name: key, UserDefined: map[string]string{crypto.MetaSealedKeyS3: "listed"}}
sub, err := newBatchJobSubmitter(globalActiveCred)
if err != nil {
t.Fatal(err)
}
waited, _, _ := ldOverlapRun(t,
func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") },
func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed) })
func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed, sub) })
if !waited {
t.Errorf("key rotation ran while a PUT held its commit")
}
cmd/erasure-server-pool.go
+10 -1
View File
@@ -1234,7 +1234,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
if err = checkDelObjArgs(ctx, bucket, object); err != nil {
return objInfo, err
}
if opts.DeleteAllVersions {
if opts.DeleteAllVersions || opts.deleteAllAuthorize != nil {
if err := opts.deleteAllInvalid(); err != nil {
return objInfo, InvalidArgument{Bucket: bucket, Object: object, Err: err}
}
@@ -1277,6 +1277,15 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
return z.deleteAllVersions(ctx, bucket, object, opts)
}
// A delete marker that names a version replaces that version in
// whichever pool holds it.
if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" && !z.SinglePool() {
goi, gerr := z.replicaObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: opts.VersionID, NoLock: true, NoAuditLog: true})
if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil {
return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err
}
}
// An ILM expiry is decided on the versions in one pool. Under the lock,
// check it with the versions in all pools; the sets then take it as
// decided, without a recheck by the versions each one holds. They only
cmd/iam.go
+5
View File
@@ -2549,7 +2549,12 @@ func (sys *IAMSys) IsAllowed(args policy.Args) bool {
}
return ok
}
return sys.isAllowedLocally(args)
}
// isAllowedLocally evaluates args with the local policies only, as IsAllowed
// does when no authorization plugin is configured.
func (sys *IAMSys) isAllowedLocally(args policy.Args) bool {
// Policies don't apply to the owner.
if args.IsOwner {
return true
cmd/object-api-interface.go
+12
View File
@@ -113,10 +113,22 @@ type ObjectOptions struct {
// DeleteAllResult, when set, receives what a purge of all versions
// confirmed, also when it fails part way.
DeleteAllResult *deleteAllResult
// deleteAllAuthorize authorizes a batch purge of all versions, which
// needs it, on every copy of every version the purge approved and every
// revision of one that a drive holds, one of each distinct version ID
// and tags, before it deletes any; its error refuses the key and is
// returned. Set by batch expiration only, never from a request.
deleteAllAuthorize func(versions []ObjectInfo) error
// Pools-layer version resolver; the caller holds the shared object lock.
replicaObjectInfo GetObjectInfoFn
// RefuseLockedOverwrite refuses, under the destination's write lock, a
// write that would replace a stored version Object Lock protects: the
// version it writes, or the version a delete marker it writes names.
// It returns ObjectLocked. Set by batch replication pulls.
RefuseLockedOverwrite bool
Speedtest bool // object call specifically meant for SpeedTest code, set to 'true' when invoked by SpeedtestHandler.
// Use the maximum parity (N/2), used when saving server configuration files