diff --git a/cmd/batch-authz-review2_test.go b/cmd/batch-authz-review2_test.go
new file mode 100644
index 000000000..5692d1d52
--- /dev/null
+++ b/cmd/batch-authz-review2_test.go
@@ -0,0 +1,356 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/auth"
+ "github.com/minio/minio/internal/crypto"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/kms"
+ "github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A pull into a suspended bucket replaces the null version. Its policy
+// decision must name that version even when the source is unversioned S3.
+func TestBatchReviewPullImplicitNullVersionAuthorization(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ _, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`Suspended`))
+ mustIAM(t, err)
+ data := []byte("original null version")
+ before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: true})
+ mustIAM(t, err)
+ batchAuthzPolicy(f.ctx, t, f.sys, "null-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"null"}}}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "null-deny-user", "null-deny")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ job := batchPullJob(f.bucket)
+ job.Source.Type = BatchJobReplicateResourceS3
+ err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+ Name: before.Name, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
+ }, false, sub)
+ if !isBatchJobAuthzError(err) {
+ t.Errorf("null version policy must refuse the write, got %v", err)
+ }
+ after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: nullVersionID})
+ if gerr != nil || before.ETag != after.ETag {
+ t.Errorf("denied null version must remain unchanged: read=%v unchanged=%v", gerr, before.ETag == after.ETag)
+ }
+}
+
+type batchReviewPrefixWalkFailure struct{ ObjectLayer }
+
+func (a batchReviewPrefixWalkFailure) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
+ if prefix != "a/" {
+ return a.ObjectLayer.Walk(ctx, bucket, prefix, results, opts)
+ }
+ go func() {
+ defer close(results)
+ results <- itemOrErr[ObjectInfo]{Err: errors.New("test: incomplete first prefix")}
+ }()
+ return nil
+}
+
+type batchReviewWalkCutoff struct {
+ ObjectLayer
+ cutoff string
+}
+
+func (a batchReviewWalkCutoff) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
+ inner := make(chan itemOrErr[ObjectInfo], 1)
+ if err := a.ObjectLayer.Walk(ctx, bucket, prefix, inner, opts); err != nil {
+ close(results)
+ return err
+ }
+ go func() {
+ defer close(results)
+ stopped := false
+ for result := range inner {
+ if stopped {
+ continue
+ }
+ if result.Err == nil && result.Item.Name >= a.cutoff {
+ stopped = true
+ result = itemOrErr[ObjectInfo]{Err: errors.New("test: listing stopped before remaining child keys")}
+ }
+ select {
+ case results <- result:
+ case <-ctx.Done():
+ stopped = true
+ }
+ }
+ }()
+ return nil
+}
+
+// Listing a directory prefix returns the directory object before its child
+// keys. A durable cursor for that directory must preserve unfinished children.
+func TestBatchReviewDirectoryCursorKeepsChildrenPending(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ for range 12 {
+ batchPutVersion(t, f.obj, f.bucket, "a/", nil)
+ }
+ var pending string
+ for i := 0; i < 13; i++ {
+ older := f.putVersions(t, fmt.Sprintf("a/key-%02d", i))
+ if i == 11 {
+ pending = older
+ }
+ }
+ rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+ apiVersion: v1
+ bucket: %s
+ prefix: ["a/"]
+ rules:
+ - type: object
+ purge:
+ retainVersions: 1
+`, f.bucket))
+ if job == nil {
+ t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+ }
+ _ = job.Expire.Start(f.ctx, batchReviewWalkCutoff{ObjectLayer: f.obj, cutoff: "a/key-11"}, *job)
+ ri := f.jobInfo(t, job)
+ if ri.Object != "a/" {
+ t.Fatalf("control: the completed directory batch must be the cursor, got %q", ri.Object)
+ }
+ mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+ if f.versionExists(t, "a/key-11", pending) {
+ t.Error("a directory cursor must not suppress the unvisited child key on resume")
+ }
+}
+
+// Later prefixes must not advance the completion cursor past an earlier
+// prefix whose listing failed. This state is produced by Start itself.
+func TestBatchReviewExpireFailedPrefixKeepsWorkPending(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ older := f.putVersions(t, "a/todo")
+ for i := 0; i < 13; i++ {
+ f.putVersions(t, fmt.Sprintf("z/key-%02d", i))
+ }
+ rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+ apiVersion: v1
+ bucket: %s
+ prefix: ["z/", "a/"]
+ rules:
+ - type: object
+ purge:
+ retainVersions: 1
+`, f.bucket))
+ if job == nil {
+ t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+ }
+ _ = job.Expire.Start(f.ctx, batchReviewPrefixWalkFailure{f.obj}, *job)
+ ri := f.jobInfo(t, job)
+ if ri.Object >= "a/todo" {
+ t.Errorf("cursor must not cross an incomplete prefix, got %q", ri.Object)
+ }
+ mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+ if f.versionExists(t, "a/todo", older) {
+ t.Error("a fresh Start must revisit the key omitted by the failed listing")
+ }
+}
+
+// A repeated marker replication error must remain a failure on the second
+// attempt, including after the retry loop resets its counters.
+func TestBatchReviewMarkerRetryRemainsFailed(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchPutVersion(t, f.obj, f.bucket, "marker", nil)
+ _, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
+ mustIAM(t, err)
+ oldTransport := globalRemoteTargetTransport
+ globalRemoteTargetTransport = batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ code, body := http.StatusForbidden, `AccessDeniedtest refusal`
+ if r.URL.Query().Has("location") {
+ code, body = http.StatusOK, `us-east-1`
+ }
+ return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
+ })
+ t.Cleanup(func() { globalRemoteTargetTransport = oldTransport })
+ job := BatchJobRequest{
+ ID: "replicate-marker-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+ Replicate: &BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
+ },
+ }
+ job.Replicate.Flags.Retry.Attempts = 2
+ job.Replicate.Flags.Retry.Delay = time.Millisecond
+ mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
+ ri := f.jobInfo(t, &job)
+ if ri.Complete || !ri.Failed || ri.DeleteMarkersFailed != 1 {
+ t.Errorf("both marker attempts failed: complete=%v failed=%v markerFailures=%d attempts=%d", ri.Complete, ri.Failed, ri.DeleteMarkersFailed, ri.RetryAttempts)
+ }
+}
+
+// A current parent mapping suppresses the OpenID policy-name fallback.
+// Background-condition validation must inspect the same effective policies.
+func TestBatchReviewMappedPolicySuppressesClaimFallback(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
+ batchAuthzPolicy(f.ctx, t, f.sys, "unused-claim-policy", `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+ parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "mapped-full")
+ cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+ parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), iamPolicyClaimNameOpenID(): "unused-claim-policy",
+ }, globalActiveCred.SecretKey)
+ mustIAM(t, err)
+ cred.ParentUser = parent.AccessKey
+ _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+ mustIAM(t, err)
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ p, err := sub.principal(f.ctx)
+ mustIAM(t, err)
+ req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "mapped/key", versionID: mustGetUUID()}
+ if !f.sys.isAllowedLocally(policy.Args{AccountName: p.accessKey, Claims: p.claims, Action: req.action, BucketName: req.bucket, ObjectName: req.object, ConditionValues: p.conditionValues(req)}) {
+ t.Fatal("control: the current mapped policy must allow this read")
+ }
+ if err := sub.authorize(f.ctx, req); err != nil {
+ t.Errorf("an unused fallback policy must not refuse a mapped-policy grant: %v", err)
+ }
+}
+
+func TestBatchReviewBindingAndExpiringServiceControls(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ root := batchRootSubmitter(t)
+ priorRoot := globalActiveCred
+ globalActiveCred.SecretKey = "different-test-root-secret"
+ _, err := root.principal(f.ctx)
+ globalActiveCred = priorRoot
+ if !isBatchJobAuthzError(err) {
+ t.Errorf("root-secret change must invalidate the binding: %v", err)
+ }
+ missing := *root
+ missing.Binding = nil
+ if _, err := missing.principal(f.ctx); !isBatchJobAuthzError(err) {
+ t.Errorf("missing fingerprint must refuse the principal: %v", err)
+ }
+ batchAuthzPolicy(f.ctx, t, f.sys, "service-parent-full", batchAuthzFullAccess)
+ parent := iamAdminUser(f.ctx, t, f.sys, "service-parent", "service-parent-full")
+ sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`))
+ mustIAM(t, err)
+ expires := UTCNow().Add(time.Hour)
+ cred, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp, expiration: &expires})
+ mustIAM(t, err)
+ stored, ok := f.sys.GetUser(f.ctx, cred.AccessKey)
+ if !ok {
+ t.Fatal("created service account must resolve")
+ }
+ cred = stored.Credentials
+ if !cred.IsTemp() || !cred.IsServiceAccount() {
+ t.Fatal("fixture must be an expiring service account")
+ }
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "allowed"}); err != nil {
+ t.Errorf("expiring service account must retain its allowed read: %v", err)
+ }
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectAction, bucket: f.bucket, object: "allowed"}); !isBatchJobAuthzError(err) {
+ t.Errorf("expiring service account must remain session-policy restricted: %v", err)
+ }
+}
+
+// An interrupted cross-pool copy can leave the same version in both pools.
+// A newer legal-hold field in either copy must prevent key rotation.
+func TestBatchReviewRotationUsesAuthoritativePoolHold(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ const name = "rotation/split-hold"
+ previousKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("batch-rotation-key")
+ t.Cleanup(func() { GlobalKMS = previousKMS })
+ dek, err := GlobalKMS.GenerateKey(t.Context(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{bucket: bucket + "/" + name}})
+ mustIAM(t, err)
+ objectKey := crypto.GenerateKey(dek.Plaintext, nil)
+ sealed := objectKey.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), bucket, name)
+ first := crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed)
+ first[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "OFF"
+ first[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ second := maps.Clone(first)
+ second[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
+ second[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
+ ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("protected payload"), objectKey))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+ putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+ logical, err := z.replicaObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if !enforceRetentionForDeletion(t.Context(), logical) {
+ t.Fatal("control: the authoritative logical version must be held")
+ }
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ err = r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
+ if !isErrObjectLocked(err) {
+ t.Errorf("rotation must refuse the authoritative legal hold, got %v", err)
+ }
+ after, gerr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, gerr)
+ if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
+ t.Error("held logical version must not have its key rewritten")
+ }
+}
+
+// An encryption condition must describe what rotation actually commits,
+// even when the requested encryption type differs from the existing type.
+func TestBatchReviewRotationEncryptionConditionMatchesResult(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ previousKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("required-key")
+ t.Cleanup(func() { GlobalKMS = previousKMS })
+ const name = "rotation/encryption-condition"
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, name, meta, nil)
+ mustIAM(t, err)
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("encrypted payload"), key))
+ mustIAM(t, err)
+ before, err := f.obj.PutObject(f.ctx, f.bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, UserDefined: meta})
+ mustIAM(t, err)
+ batchAuthzPolicy(f.ctx, t, f.sys, "kms-write-only", `
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "kms-write-user", "kms-write-only")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ r := BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "required-key"}}
+ err = r.KeyRotate(f.ctx, f.obj, before, sub)
+ after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, gerr)
+ if err == nil && !crypto.S3KMS.IsEncrypted(after.UserDefined) {
+ t.Error("rotation allowed by an aws:kms-only write policy must produce SSE-KMS or refuse the operation")
+ }
+ if err != nil && before.UserDefined[crypto.MetaSealedKeyS3] != after.UserDefined[crypto.MetaSealedKeyS3] {
+ t.Error("a refused rotation must leave the version unchanged")
+ }
+}
+
+// An explicit null marker collision must retain a held null data version.
+func TestBatchReviewPullMarkerNullLock(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ meta := batchLockMeta("LEGALHOLD")
+ before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, strings.NewReader("held null"), 9, "", ""), ObjectOptions{VersionID: nullVersionID, Versioned: true, UserDefined: meta})
+ mustIAM(t, err)
+ job := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+ }
+ err = job.ReplicateFromSource(f.ctx, f.obj, nil, ObjectInfo{
+ Name: before.Name, VersionID: nullVersionID, ModTime: UTCNow(), DeleteMarker: true,
+ }, false, batchRootSubmitter(t))
+ if !isErrObjectLocked(err) {
+ t.Errorf("marker collision with protected null data must return ObjectLocked, got %v", err)
+ }
+ if !batchVersionExists(t, f.obj, f.bucket, before.Name, nullVersionID) {
+ t.Fatal("protected null data must remain present")
+ }
+}
diff --git a/cmd/batch-authz-review3_test.go b/cmd/batch-authz-review3_test.go
new file mode 100644
index 000000000..d6d9d9313
--- /dev/null
+++ b/cmd/batch-authz-review3_test.go
@@ -0,0 +1,140 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/minio/minio/internal/auth"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A read policy must use the newest tag state of the logical version even
+// when an interrupted movement left an older copy in the preferred pool.
+func TestBatchReviewPushUsesAuthoritativePoolTags(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ f := newBatchAuthzFixture(t)
+ const name = "push/tagged"
+ first := map[string]string{
+ xhttp.AmzObjectTagging: "classification=public",
+ ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
+ }
+ second := map[string]string{
+ xhttp.AmzObjectTagging: "classification=private",
+ ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+ }
+ before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
+ putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+ logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if logical.UserTags != "classification=private" {
+ t.Fatal("control: logical version must have the newer private tag")
+ }
+ batchAuthzPolicy(f.ctx, t, f.sys, "pooled-push", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+ cred := iamAdminUser(f.ctx, t, f.sys, "pooled-push-user", "pooled-push")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: bucket, object: name, versionID: before.VersionID, tags: logical.UserTags}); !isBatchJobAuthzError(err) {
+ t.Fatal("control: authoritative tags must deny the read")
+ }
+ writes := 0
+ target, err := minio.NewCore("target.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.Copy(io.Discard, r.Body)
+ _ = r.Body.Close()
+ }
+ if r.Method == http.MethodPut {
+ writes++
+ }
+ return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+ }),
+ })
+ mustIAM(t, err)
+ r := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ }
+ err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
+ if !isBatchJobAuthzError(err) || writes != 0 {
+ t.Errorf("newer authoritative tags must deny push: err=%v writes=%d", err, writes)
+ }
+}
+
+// Verified token claims must not replace the actual version being authorized.
+func TestBatchReviewVersionConditionResistsClaimCollision(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ deniedVersion := mustGetUUID()
+ batchAuthzPolicy(f.ctx, t, f.sys, "version-claims", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, deniedVersion))
+ parent := iamAdminUser(f.ctx, t, f.sys, "version-claims-parent", "version-claims")
+ cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+ parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), "versionid": "an-unrelated-claim-value",
+ }, globalActiveCred.SecretKey)
+ mustIAM(t, err)
+ cred.ParentUser = parent.AccessKey
+ _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+ mustIAM(t, err)
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ req := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: deniedVersion}
+ if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
+ t.Errorf("a token claim must not replace the actual denied version: %v", err)
+ }
+}
+
+// Retry totals describe a pass, including markers already successful before
+// another item caused that pass to retry.
+func TestBatchReviewMarkerSuccessCountResetsOnRetry(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchPutVersion(t, f.obj, f.bucket, "marker", nil)
+ _, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
+ mustIAM(t, err)
+ oldTransport := remoteInstanceTransport
+ remoteInstanceTransport = atomic.Value{}
+ setRemoteInstanceTransport(batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ code, body := http.StatusForbidden, `AccessDeniedtest refusal`
+ h := make(http.Header)
+ switch {
+ case r.URL.Query().Has("location"):
+ code, body = http.StatusOK, `us-east-1`
+ case r.Method == http.MethodDelete:
+ code, body = http.StatusNoContent, ""
+ case r.Method == http.MethodHead && r.URL.Query().Get("versionId") != "":
+ code, body = http.StatusMethodNotAllowed, ""
+ h.Set("X-Amz-Delete-Marker", "true")
+ }
+ return &http.Response{StatusCode: code, Header: h, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
+ }))
+ t.Cleanup(func() { remoteInstanceTransport = oldTransport })
+ job := BatchJobRequest{
+ ID: "marker-success-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+ Replicate: &BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
+ },
+ }
+ job.Replicate.Flags.Retry.Attempts = 2
+ job.Replicate.Flags.Retry.Delay = time.Millisecond
+ mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
+ ri := f.jobInfo(t, &job)
+ if !ri.Failed || ri.RetryAttempts != 2 {
+ t.Fatalf("control: data failure must cause two attempts: failed=%v attempts=%d", ri.Failed, ri.RetryAttempts)
+ }
+ if ri.DeleteMarkers != 1 {
+ t.Errorf("one successful marker must be counted once in the final attempt, got %d", ri.DeleteMarkers)
+ }
+}
diff --git a/cmd/batch-authz-review4_test.go b/cmd/batch-authz-review4_test.go
new file mode 100644
index 000000000..a4b3fb02a
--- /dev/null
+++ b/cmd/batch-authz-review4_test.go
@@ -0,0 +1,157 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/minio/minio/internal/auth"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A tag removal in a newer copy must govern both the authorization and the
+// metadata sent by a push. The submitter can read data but cannot read tags.
+func TestBatchReviewPushSendsOnlyAuthorizedMetadata(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ f := newBatchAuthzFixture(t)
+ const name = "push/removed-tags"
+ first := map[string]string{
+ xhttp.AmzObjectTagging: "classification=private",
+ ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
+ }
+ second := map[string]string{
+ xhttp.AmzObjectTagging: "",
+ ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+ }
+ before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
+ putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+ logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if logical.UserTags != "" {
+ t.Fatal("control: the authoritative version must have no tags")
+ }
+ batchAuthzPolicy(f.ctx, t, f.sys, "data-without-tags", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket, bucket))
+ cred := iamAdminUser(f.ctx, t, f.sys, "data-without-tags-user", "data-without-tags")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: name, versionID: before.VersionID}); !isBatchJobAuthzError(err) {
+ t.Fatal("control: reading tags must be denied")
+ }
+ writes := 0
+ var sentTags string
+ target, err := minio.NewCore("target.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.Copy(io.Discard, r.Body)
+ _ = r.Body.Close()
+ }
+ if r.Method == http.MethodPut {
+ writes++
+ sentTags = r.Header.Get(xhttp.AmzObjectTagging)
+ }
+ return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+ }),
+ })
+ mustIAM(t, err)
+ r := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ }
+ err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
+ if err != nil && !isBatchJobAuthzError(err) {
+ t.Fatalf("unexpected replication failure: %v", err)
+ }
+ if writes > 0 && sentTags != logical.UserTags {
+ t.Errorf("push must send only metadata covered by authorization: writes=%d sentTags=%q authorizedTags=%q err=%v", writes, sentTags, logical.UserTags, err)
+ }
+}
+
+// LDAP's short username and its parent DN are distinct identities. The
+// background merge must not turn a deny for the verified short name into
+// a non-matching statement by replacing it with the parent DN.
+func TestBatchReviewLDAPUsernameDeny(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "ldap-short-name-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"ldap:username":"alice"}}}`)
+ const dn = "uid=alice,ou=people,dc=example,dc=org"
+ cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+ "exp": UTCNow().Add(time.Hour).Unix(), ldapUser: dn, ldapActualUser: dn, ldapUserN: "alice",
+ }, globalActiveCred.SecretKey)
+ mustIAM(t, err)
+ cred.ParentUser = dn
+ _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "ldap-short-name-deny")
+ mustIAM(t, err)
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ p, err := sub.principal(f.ctx)
+ mustIAM(t, err)
+ cred.Claims = p.claims
+ req, err := http.NewRequest(http.MethodDelete, "https://localhost/batch-authz/data", nil)
+ mustIAM(t, err)
+ args := policy.Args{
+ AccountName: cred.AccessKey, Claims: p.claims, Action: policy.DeleteObjectVersionAction,
+ BucketName: f.bucket, ObjectName: "data", ConditionValues: getConditionValues(req, "", cred),
+ }
+ if args.ConditionValues["username"][0] != "alice" || f.sys.isAllowedLocally(args) {
+ t.Fatal("control: the request evaluator must deny the verified LDAP short username")
+ }
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: mustGetUUID()}); !isBatchJobAuthzError(err) {
+ t.Errorf("batch authorization must preserve the LDAP username deny or fail closed, got %v", err)
+ }
+}
+
+// Failure to read authoritative metadata is incomplete work, even if one
+// pool's copy can still be opened. In particular, skipErrs=false must not
+// turn an unreadable entry into a successful archive.
+func TestBatchReviewSnowballAuthoritativeReadFailure(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ const name = "archive/unreadable-copy"
+ before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true})
+ set := z.serverPools[1].sets[0]
+ getDisks := set.getDisks
+ disks := append([]StorageAPI(nil), getDisks()...)
+ for i := range disks {
+ disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: name}
+ }
+ set.getDisks = func() []StorageAPI { return disks }
+ defer func() { set.getDisks = getDisks }()
+ gr, err := z.GetObjectNInfo(t.Context(), bucket, name, nil, nil, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ _, logicalErr := batchAuthoritativeRead(t.Context(), z, bucket, name, gr.ObjInfo)
+ gr.Close()
+ if logicalErr == nil {
+ t.Fatal("control: the authoritative read must fail")
+ }
+ remote, err := minio.New("archive.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.Copy(io.Discard, r.Body)
+ }
+ return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+ }),
+ })
+ mustIAM(t, err)
+ r := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ }
+ r.Source.Snowball.InMemory = ptr(true)
+ r.Source.Snowball.Compress = ptr(false)
+ r.Source.Snowball.SkipErrs = ptr(false)
+ if err := r.writeAsArchive(t.Context(), z, remote, []ObjectInfo{before}, "", batchRootSubmitter(t)); err == nil {
+ t.Error("archive with an unreadable authoritative version must not report success")
+ }
+}
diff --git a/cmd/batch-authz-review5_test.go b/cmd/batch-authz-review5_test.go
new file mode 100644
index 000000000..ccc2520e9
--- /dev/null
+++ b/cmd/batch-authz-review5_test.go
@@ -0,0 +1,142 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "maps"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/minio/minio/internal/crypto"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/kms"
+)
+
+// A suspended destination does not keep a source version ID. Authorization
+// must therefore describe the null destination version that the write really
+// replaces, and must not let a source ID escape that model.
+func TestBatchReviewPullSuspendedVersionBinding(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ _, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`Suspended`))
+ mustIAM(t, err)
+
+ sourceVersion := mustGetUUID()
+ batchAuthzPolicy(f.ctx, t, f.sys, "pull-source-version-deny", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, sourceVersion))
+ cred := iamAdminUser(f.ctx, t, f.sys, "pull-source-version-user", "pull-source-version-deny")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ job := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+ }
+ err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+ Name: "pull/suspended-version", VersionID: sourceVersion, ModTime: UTCNow(), ETag: "batchetag",
+ UserDefined: map[string]string{},
+ }, false, sub)
+ if !isBatchJobAuthzError(err) {
+ t.Fatalf("a source version ID must not evade the destination deny: %v", err)
+ }
+ if _, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/suspended-version", ObjectOptions{VersionID: sourceVersion}); gerr == nil {
+ t.Fatal("a denied suspended-destination write created the source version")
+ }
+}
+
+// A failed source open in a non-skipping snowball must be reported to the
+// batch rather than silently omitted from a successful archive.
+type batchReviewOpenFailure struct {
+ ObjectLayer
+ name string
+}
+
+func (a batchReviewOpenFailure) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+ if object == a.name {
+ return nil, fmt.Errorf("test: source open failed")
+ }
+ return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
+}
+
+func TestBatchReviewSnowballSourceOpenFailure(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ name := "archive/open-failure"
+ entry := batchPutVersion(t, f.obj, f.bucket, name, nil)
+ remote, err := minio.New("archive.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.Copy(io.Discard, r.Body)
+ }
+ return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+ }),
+ })
+ mustIAM(t, err)
+ r := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ }
+ r.Source.Snowball.InMemory = ptr(true)
+ r.Source.Snowball.Compress = ptr(false)
+ r.Source.Snowball.SkipErrs = ptr(false)
+ api := batchReviewOpenFailure{ObjectLayer: f.obj, name: name}
+ if err := r.writeAsArchive(t.Context(), api, remote, []ObjectInfo{entry}, "", batchRootSubmitter(t)); err == nil {
+ t.Fatal("a source open failure must not be reported as a successful archive")
+ }
+}
+
+// Rotation must apply tag conditions to the logical version, including
+// a more recent tag update stored in a pool other than the selected copy.
+func TestBatchReviewRotationAuthoritativeTags(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ f := newBatchAuthzFixture(t)
+ previousKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("batch-round5-rotation-key")
+ t.Cleanup(func() { GlobalKMS = previousKMS })
+ batchAuthzPolicy(f.ctx, t, f.sys, "round5-rotation-tags", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+ cred := iamAdminUser(f.ctx, t, f.sys, "round5-rotation-user", "round5-rotation-tags")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ for _, classification := range []string{"private", "public"} {
+ t.Run(classification, func(t *testing.T) {
+ name := "rotation/authoritative-" + classification
+ first := make(map[string]string)
+ key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
+ mustIAM(t, err)
+ first[xhttp.AmzObjectTagging] = "classification=public"
+ first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ second := maps.Clone(first)
+ second[xhttp.AmzObjectTagging] = "classification=" + classification
+ second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+ ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("test encrypted payload"), key))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+ putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+ logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if logical.UserTags != "classification="+classification {
+ t.Fatal("control: the second pool must govern the logical tags")
+ }
+ job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ err = job.KeyRotate(f.ctx, z, before, sub)
+ after, readErr := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, readErr)
+ changed := after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3]
+ if classification == "private" {
+ if !isBatchJobAuthzError(err) || changed {
+ t.Fatalf("authoritative tag deny must prevent rewrapping: err=%v changed=%v", err, changed)
+ }
+ } else if err != nil || !changed {
+ t.Fatalf("authorized unlocked version must rotate: err=%v changed=%v", err, changed)
+ }
+ })
+ }
+}
diff --git a/cmd/batch-authz-review6_test.go b/cmd/batch-authz-review6_test.go
new file mode 100644
index 000000000..63303b733
--- /dev/null
+++ b/cmd/batch-authz-review6_test.go
@@ -0,0 +1,520 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "net/http"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/crypto"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/kms"
+)
+
+// Rotation and listing cases found in review.
+
+// Rotation rewrites the copy GetObjectInfo returns, which can carry another
+// tag revision than the merged version of all pools: both are authorized.
+func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ f := newBatchAuthzFixture(t)
+ previousKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review6-rotation-copy-key")
+ t.Cleanup(func() { GlobalKMS = previousKMS })
+ batchAuthzPolicy(f.ctx, t, f.sys, "review6-rotation-copy-deny", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+ cred := iamAdminUser(f.ctx, t, f.sys, "review6-rotation-copy-user", "review6-rotation-copy-deny")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ const name = "rotation/written-copy"
+ first := make(map[string]string)
+ key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
+ mustIAM(t, err)
+ first[xhttp.AmzObjectTagging] = "classification=private"
+ first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ second := maps.Clone(first)
+ second[xhttp.AmzObjectTagging] = "classification=public"
+ second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
+ ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("review6 rotation payload"), key))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
+ putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
+ logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if logical.UserTags != "classification=public" {
+ t.Fatalf("control: merged tags=%q, want public", logical.UserTags)
+ }
+ job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ if err = job.KeyRotate(f.ctx, z, before, sub); !isBatchJobAuthzError(err) {
+ t.Fatalf("rotation of a copy the submitter may not read returned %v", err)
+ }
+ after, err := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
+ t.Fatal("a refused rotation rewrote the copy")
+ }
+}
+
+// batchWalkError fails every listing before it returns a result.
+type batchWalkError struct {
+ ObjectLayer
+ err error
+}
+
+func (w batchWalkError) Walk(context.Context, string, string, chan<- itemOrErr[ObjectInfo], WalkOptions) error {
+ return w.err
+}
+
+// A listing that fails before it returns anything fails the run, as a
+// listing error in the results does, and the job is left to a later run.
+func TestBatchReviewExpireWalkErrorFailsRun(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ listErr := errors.New("review6: listing failed")
+ if err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job); !errors.Is(err, listErr) {
+ t.Fatalf("start returned %v, want %v", err, listErr)
+ }
+ if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+ t.Fatalf("failed listing: complete=%v failed=%v", ri.Complete, ri.Failed)
+ }
+}
+
+// Two pools hold different writes of the null version, and a newer version
+// of the key is in the second one. Rotation rewrites the copy it read and
+// authorized, in the first pool, not the copy the latest version of the key
+// would route a write to; the other write keeps its key and stays readable.
+func TestBatchReviewRotationRewritesTheCopyItRead(t *testing.T) {
+ for _, otherLatest := range []bool{false, true} {
+ name := "addressed-version-is-latest"
+ if otherLatest {
+ name = "another-version-is-latest"
+ }
+ t.Run(name, func(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ z, _ := consistencyPools(t)
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ const bucket = "review7-rotation-copies"
+ mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, versionErr := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, versionErr)
+ oldKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review7-rotation-key")
+ t.Cleanup(func() { GlobalKMS = oldKMS })
+ const object = "rotation/null-copy"
+ const payload = "rotation must preserve this data"
+ readCopy := func(pool int) ([]byte, error) {
+ gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: nullVersionID})
+ if err != nil {
+ return nil, err
+ }
+ defer gr.Close()
+ return io.ReadAll(gr)
+ }
+ var before [2]ObjectInfo
+ var keys [2]crypto.ObjectKey
+ kinds := []crypto.Type{crypto.S3, crypto.S3KMS}
+ for pool, kind := range kinds {
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(t.Context(), kind, "review7-rotation-key", nil, bucket, object, meta, nil)
+ mustIAM(t, err)
+ keys[pool] = key
+ meta[xhttp.AmzObjectTagging] = "classification=public"
+ if pool == 1 {
+ meta[xhttp.AmzObjectTagging] = "classification=private"
+ }
+ meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Duration(pool+2) * time.Hour).Format(time.RFC3339Nano)
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
+ mustIAM(t, err)
+ putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{
+ Versioned: true, VersionID: nullVersionID, UserDefined: meta,
+ MTime: UTCNow().Add(-time.Duration(pool+2) * time.Hour),
+ })
+ // As a read returns it, without the write's internal fields.
+ before[pool], err = z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
+ mustIAM(t, err)
+ data, err := readCopy(pool)
+ if err != nil || string(data) != payload {
+ t.Fatalf("control: pool %d was not readable before rotation: %v", pool, err)
+ }
+ }
+ if otherLatest {
+ putConsistencyObject(t, z, bucket, object, 1, "later version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
+ }
+ selected, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID, Versioned: true})
+ mustIAM(t, err)
+ if selected.VersionID != nullVersionID || !selected.ModTime.Equal(before[0].ModTime) || !crypto.S3.IsEncrypted(selected.UserDefined) {
+ t.Fatal("control: the addressed null version must select the SSE-S3 write in pool zero")
+ }
+ logical, err := batchAuthoritativeVersion(t.Context(), z, bucket, object, selected)
+ mustIAM(t, err)
+ if logical.UserTags != "classification=public" || selected.UserTags != logical.UserTags {
+ t.Fatal("control: both current authorization inputs must have public tags")
+ }
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review7-rotation-key"}}
+ sub := batchRootSubmitter(t)
+ if otherLatest {
+ batchAuthzPolicy(t.Context(), t, globalIAMSys, "review7-rotate-public", fmt.Sprintf(`
+{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
+{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
+ cred := iamAdminUser(t.Context(), t, globalIAMSys, "review7-rotate-user", "review7-rotate-public")
+ sub, err = newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ if err := sub.authorize(t.Context(), r.rotateAuthzRequests(before[1])...); !isBatchJobAuthzError(err) {
+ t.Fatal("control: the older write in pool one must be denied")
+ }
+ }
+ err = r.KeyRotate(t.Context(), z, selected, sub)
+ if err != nil {
+ t.Logf("rotation refused: %v", err)
+ }
+ for pool, want := range keys {
+ after, readErr := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
+ mustIAM(t, readErr)
+ if otherLatest && pool == 1 && (!maps.Equal(after.UserDefined, before[pool].UserDefined) || !after.ModTime.Equal(before[pool].ModTime) || after.UserTags != before[pool].UserTags) {
+ t.Error("rotation rewrote a copy denied by the submitter's tag-conditioned policy")
+ }
+ kind, _ := crypto.IsEncrypted(after.UserDefined)
+ var got crypto.ObjectKey
+ switch kind {
+ case crypto.S3:
+ got, readErr = crypto.S3.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
+ case crypto.S3KMS:
+ got, readErr = crypto.S3KMS.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
+ default:
+ t.Fatalf("pool %d: rotation lost encryption metadata", pool)
+ }
+ mustIAM(t, readErr)
+ if got != want {
+ t.Errorf("pool %d: rotation changed the data key without reencrypting its ciphertext (rotation error: %v)", pool, err)
+ }
+ data, readErr := readCopy(pool)
+ if readErr != nil || string(data) != payload {
+ t.Errorf("pool %d: rotation left unreadable ciphertext: %v", pool, readErr)
+ }
+ }
+ })
+ }
+}
+
+// A version in a pool that a decommission or a rebalance is moving is not
+// rewritten there: only the source holds it, so the rotation is left to a
+// later run.
+func TestBatchReviewRotationLeavesMovingSource(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ z, _ := consistencyPools(t)
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ const bucket = "review8-rotation-draining"
+ mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, err)
+ oldKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review8-rotation-key")
+ t.Cleanup(func() { GlobalKMS = oldKMS })
+ const object = "rotation/draining"
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(t.Context(), crypto.S3, "review8-rotation-key", nil, bucket, object, meta, nil)
+ mustIAM(t, err)
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("draining rotation"), key))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+ beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+
+ z.poolMetaMutex.Lock()
+ z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+ z.poolMetaMutex.Unlock()
+
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
+ err = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+ if !errors.Is(err, errBatchRotateMoving) {
+ t.Errorf("rotation of a version only a draining pool holds returned %v", err)
+ }
+ after, readErr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, readErr)
+ if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
+ t.Fatal("rotation rewrote a draining pool")
+ }
+}
+
+// Once a move copied a version, the rotation leaves it to a later run, so the
+// source stays as the move's cleanup expects it.
+func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) {
+ for _, tc := range []struct {
+ kind string
+ rotate bool
+ }{{"decommission", false}, {"decommission", true}, {"rebalance", false}, {"rebalance", true}} {
+ name := tc.kind + "/without-rotation"
+ if tc.rotate {
+ name = tc.kind + "/with-rotation"
+ }
+ t.Run(name, func(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ z, _ := migrationTestPools(t)
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ const bucket = "review8-rotation-migration"
+ mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, err)
+ oldKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review8-rotation-key")
+ t.Cleanup(func() { GlobalKMS = oldKMS })
+ const object = "rotation/migration"
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+ mustIAM(t, err)
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation and migration"), key))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+ beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ move := migrationTestMover(t, z, 0, tc.kind)
+ mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, beforeInfo)))
+ if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); err != nil {
+ t.Fatalf("control: migration did not copy the version: %v", err)
+ }
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
+ var rotationErr error
+ if tc.rotate {
+ moved, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ rotationErr = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+ if !errors.Is(rotationErr, errBatchRotateMoving) {
+ t.Errorf("rotation during a move returned %v", rotationErr)
+ }
+ after, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if !maps.Equal(after.UserDefined, moved.UserDefined) || after.UserTags != moved.UserTags {
+ t.Error("rotation during a move rewrote the moved copy")
+ }
+ }
+ after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if !after.ModTime.Equal(beforeInfo.ModTime) || !maps.Equal(after.UserDefined, beforeInfo.UserDefined) {
+ t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
+ }
+ snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
+ if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
+ t.Fatalf("rotation prevented migration cleanup: %v", err)
+ }
+ if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+ t.Fatalf("migration left a source version behind: %v", err)
+ }
+ })
+ }
+}
+
+// The rewrite keeps the tags and the expiry of the version, which a read
+// returns apart from its metadata.
+func TestBatchReviewRotationKeepsTagsAndExpiry(t *testing.T) {
+ for _, mode := range []string{"single-pool", "multiple-pools"} {
+ t.Run(mode, func(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ var z *erasureServerPools
+ if mode == "single-pool" {
+ z = purgeSinglePool(t, 4)
+ } else {
+ z, _ = consistencyPools(t)
+ }
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ const bucket = "review8-rotation-metadata"
+ mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, err)
+ oldKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review8-rotation-metadata-key")
+ t.Cleanup(func() { GlobalKMS = oldKMS })
+ const object = "rotation/tagged"
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+ mustIAM(t, err)
+ meta[xhttp.AmzObjectTagging] = "classification=private"
+ meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
+ meta["expires"] = UTCNow().Add(time.Hour).Format(http.TimeFormat)
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("metadata preservation"), key))
+ mustIAM(t, err)
+ written := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta})
+ before, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: written.VersionID})
+ mustIAM(t, err)
+ if before.UserTags == "" || before.Expires.IsZero() {
+ t.Fatal("control: stored version is missing the metadata to preserve")
+ }
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ mustIAM(t, r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)))
+ after, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if after.UserDefined[crypto.MetaSealedKeyS3] == before.UserDefined[crypto.MetaSealedKeyS3] {
+ t.Error("control: rotation did not rewrap the key")
+ }
+ if after.UserTags != before.UserTags {
+ t.Errorf("rotation changed stored tags: got %q, want %q", after.UserTags, before.UserTags)
+ }
+ if !after.Expires.Equal(before.Expires) {
+ t.Errorf("rotation changed Expires: got %v, want %v", after.Expires, before.Expires)
+ }
+ })
+ }
+}
+
+// Two pools hold the same write with different tags, with or without
+// tagging timestamps. A rotation, also during a move, leaves the tags that
+// the merged version has.
+func TestBatchReviewRotationKeepsMergedTags(t *testing.T) {
+ for _, tc := range []struct {
+ ordered, moving bool
+ }{{false, false}, {false, true}, {true, false}, {true, true}} {
+ t.Run(fmt.Sprintf("timestamp=%t/moving=%t", tc.ordered, tc.moving), func(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ z, _ := consistencyPools(t)
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ const bucket = "review9-rotation-tag-tie"
+ const object = "rotation/tag-tie"
+ mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, err)
+ previousKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review9-tag-tie-key")
+ t.Cleanup(func() { GlobalKMS = previousKMS })
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+ mustIAM(t, err)
+ meta[xhttp.AmzObjectTagging] = "classification=private"
+ if tc.ordered {
+ meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
+ }
+ other := maps.Clone(meta)
+ other[xhttp.AmzObjectTagging] = "classification=public"
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("same write"), key))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
+ putConsistencyObject(t, z, bucket, object, 1, string(body), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: other})
+ prior, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if prior.UserTags != "classification=private" {
+ t.Fatalf("control: logical tags=%q", prior.UserTags)
+ }
+ if tc.moving {
+ migrationTestMover(t, z, 0, "decommission")
+ }
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ rotationErr := r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
+ after, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if after.UserTags != prior.UserTags {
+ t.Fatalf("rotation changed logical tags from %q to %q (rotation error: %v)", prior.UserTags, after.UserTags, rotationErr)
+ }
+ })
+ }
+}
+
+// batchRotateFlipDisk starts a decommission of pool 0 at the flipAt-th
+// version read, after the rotation's first read of the version.
+type batchRotateFlipDisk struct {
+ StorageAPI
+ z *erasureServerPools
+ reads *atomic.Int32
+ flipAt int32
+ flipped *atomic.Bool
+}
+
+func (d batchRotateFlipDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
+ n := d.reads.Add(1)
+ if n == d.flipAt && d.flipped.CompareAndSwap(false, true) {
+ d.z.poolMetaMutex.Lock()
+ d.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+ d.z.poolMetaMutex.Unlock()
+ }
+ return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
+}
+
+// A move that starts while the rotation authorizes and rewraps the version
+// defers it: the state is checked again right before the write, where the
+// pools layer checks it.
+func TestBatchReviewRotationRechecksPoolBeforeWrite(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ z, _ := consistencyPools(t)
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ const bucket = "review11-rotation-window"
+ const object = "rotation/window"
+ mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, err)
+ oldKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("review11-rotation-key")
+ t.Cleanup(func() { GlobalKMS = oldKMS })
+
+ meta := make(map[string]string)
+ key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
+ mustIAM(t, err)
+ body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation window"), key))
+ mustIAM(t, err)
+ before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{
+ Versioned: true,
+ UserDefined: meta,
+ MTime: UTCNow().Add(-time.Hour),
+ })
+ beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+
+ var reads atomic.Int32
+ var flipped atomic.Bool
+ firstPass := int32(0)
+ for _, pool := range z.serverPools {
+ firstPass += int32(pool.SetDriveCount())
+ }
+ var restore []func()
+ for _, pool := range z.serverPools {
+ restore = append(restore, conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI {
+ return batchRotateFlipDisk{
+ StorageAPI: d,
+ z: z,
+ reads: &reads,
+ flipAt: firstPass + 1,
+ flipped: &flipped,
+ }
+ }))
+ }
+ t.Cleanup(func() {
+ for _, fn := range restore {
+ fn()
+ }
+ })
+
+ r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ gotErr := r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
+ if !flipped.Load() {
+ t.Fatal("fault injection did not occur after the initial rotation read")
+ }
+ if !errors.Is(gotErr, errBatchRotateMoving) {
+ t.Errorf("rotation did not defer after the pool became moving during pre-work: %v", gotErr)
+ }
+ after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
+ mustIAM(t, err)
+ if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
+ t.Error("rotation rewrote the pool after it became moving during pre-work")
+ }
+}
diff --git a/cmd/batch-authz-review_test.go b/cmd/batch-authz-review_test.go
new file mode 100644
index 000000000..c69ee43e4
--- /dev/null
+++ b/cmd/batch-authz-review_test.go
@@ -0,0 +1,287 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ polplugin "github.com/minio/minio/internal/config/policy/plugin"
+ "github.com/minio/minio/internal/crypto"
+ xhttp "github.com/minio/minio/internal/http"
+ xnet "github.com/pgsty/silo-pkg/v3/net"
+ "github.com/pgsty/silo-pkg/v3/policy"
+)
+
+func batchPullJob(bucket string) BatchJobReplicateV1 {
+ return BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ }
+}
+
+// A pull does not read the destination version it may replace, so a policy
+// that conditions the write on that version's tags refuses it; a version
+// condition sees the version the pull writes.
+func TestBatchPullDestinationConditions(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ action policy.Action
+ marker bool
+ condition func(versionID string) string
+ }{
+ {"tags-put", policy.ReplicateObjectAction, false, func(string) string {
+ return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
+ }},
+ {"tags-marker", policy.ReplicateDeleteAction, true, func(string) string {
+ return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
+ }},
+ {"version-put", policy.PutObjectAction, false, func(v string) string {
+ return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v)
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"})
+ batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`,
+{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID)))
+ cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ job := batchPullJob(f.bucket)
+ err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
+ Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag",
+ DeleteMarker: tc.marker, UserDefined: map[string]string{},
+ }, false, sub)
+ if !isBatchJobAuthzError(err) {
+ t.Errorf("pull returned %v, want a refusal", err)
+ }
+ after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID})
+ if err != nil || after.ETag != before.ETag {
+ t.Errorf("refused pull changed the destination version: %v", err)
+ }
+ })
+ }
+}
+
+// A pull does not encrypt what it writes; encryption headers reported by the
+// source do not satisfy a destination policy that requires encryption.
+func TestBatchPullEncryptionCondition(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", `
+{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ job := batchPullJob(f.bucket)
+ version := mustGetUUID()
+ err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{
+ Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag",
+ UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"},
+ }, false, sub)
+ if err == nil {
+ oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version})
+ mustIAM(t, gerr)
+ if !crypto.S3KMS.IsEncrypted(oi.UserDefined) {
+ t.Fatal("a pull was allowed by an encryption condition it did not meet")
+ }
+ }
+}
+
+// A conditional Deny on an S3 Tables action covers the S3 data actions it
+// implies, under the tables resource form; the job refuses rather than skip it.
+func TestBatchJobTableConditionalDeny(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context")
+ if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) {
+ t.Fatal("control: with the request context the Deny applies")
+ }
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) {
+ t.Fatalf("authorize returned %v, want a refusal", err)
+ }
+}
+
+type batchArchiveRevocation struct {
+ ObjectLayer
+ revoke func()
+}
+
+func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
+ if object == "archive/second" {
+ a.revoke()
+ }
+ return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
+}
+
+// Each snowball entry is authorized for the submitter as it is then.
+func TestBatchPushSnowballEntryRevalidation(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/")
+ mustIAM(t, err)
+ previous := newGlobalAuthZPluginFn()
+ setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
+ URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() },
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil
+ }),
+ }))
+ t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
+ entries := []ObjectInfo{
+ batchPutVersion(t, f.obj, f.bucket, "archive/first", nil),
+ batchPutVersion(t, f.obj, f.bucket, "archive/second", nil),
+ }
+ remote, err := minio.New("archive.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ _, _ = io.Copy(io.Discard, r.Body)
+ }
+ return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+ }),
+ })
+ mustIAM(t, err)
+ job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}}
+ job.Source.Snowball.InMemory = ptr(true)
+ job.Source.Snowball.Compress = ptr(false)
+ job.Source.Snowball.SkipErrs = ptr(false)
+ api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }}
+ if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) {
+ t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err)
+ }
+}
+
+// A definition holds one job type; one with several is refused, so that what
+// is authorized is what runs.
+func TestBatchJobSeveralSections(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ older := f.putVersions(t, "data/a")
+ job := f.expireYAML("") + `keyrotate:
+ apiVersion: v1
+ bucket: other
+ encryption:
+ type: sse-s3
+`
+ rec, queued := f.submit(t, globalActiveCred, job)
+ if queued != nil {
+ f.run(t, queued)
+ }
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String())
+ }
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a definition with two job types ran")
+ }
+}
+
+// Prefixes are walked once each in one order: a cursor in a later prefix is
+// after every key of the earlier ones, and an overlapping prefix is not
+// walked twice.
+func TestBatchExpirePrefixes(t *testing.T) {
+ if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" {
+ t.Fatalf("canonical prefixes %q", got)
+ }
+ if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" {
+ t.Fatalf("canonical prefixes %q", got)
+ }
+ f := newBatchAuthzFixture(t)
+ done := f.putVersions(t, "a/done")
+ todo := f.putVersions(t, "z/todo")
+ batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD"))
+ batchPutVersion(t, f.obj, f.bucket, "x/data", nil)
+ rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
+ apiVersion: v1
+ bucket: %s
+ prefix: ["z/", "x/", "a/", "x/"]
+ rules:
+ - type: object
+ purge:
+ retainVersions: 1
+`, f.bucket))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ // A run that completed a/ and stopped.
+ ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"}
+ mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
+ f.run(t, job)
+ if !f.versionExists(t, "a/done", done) {
+ t.Error("resume did again a key before the cursor")
+ }
+ if f.versionExists(t, "z/todo", todo) {
+ t.Error("resume skipped a later prefix")
+ }
+ if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 {
+ t.Errorf("a locked version under two equal prefixes was counted %d times", got)
+ }
+}
+
+type batchPartialWalk struct {
+ ObjectLayer
+ version ObjectInfo
+ cancel context.CancelFunc
+ read chan struct{}
+}
+
+func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
+ go func() {
+ defer close(results)
+ results <- itemOrErr[ObjectInfo]{Item: a.version}
+ if a.cancel != nil {
+ <-a.read
+ a.cancel()
+ return
+ }
+ results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")}
+ }()
+ return nil
+}
+
+func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
+ if a.read != nil {
+ close(a.read)
+ }
+ return a.version, nil
+}
+
+// A rotation whose walk fails or is canceled within a key keeps that key
+// after its cursor and does not report completion.
+func TestBatchKeyRotationIncompleteWalk(t *testing.T) {
+ for _, canceled := range []bool{false, true} {
+ t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ ctx, cancel := context.WithCancel(f.ctx)
+ defer cancel()
+ meta := batchLockMeta("LEGALHOLD")
+ meta[crypto.MetaSealedKeyS3] = "test-sealed-key"
+ api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}}
+ if canceled {
+ api.cancel, api.read = cancel, make(chan struct{})
+ }
+ job := BatchJobRequest{
+ ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
+ KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}},
+ }
+ _ = job.KeyRotate.Start(ctx, api, job)
+ ri := f.jobInfo(t, &job)
+ if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 {
+ t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked)
+ }
+ })
+ }
+}
diff --git a/cmd/batch-authz.go b/cmd/batch-authz.go
new file mode 100644
index 000000000..192fbb473
--- /dev/null
+++ b/cmd/batch-authz.go
@@ -0,0 +1,742 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "crypto/subtle"
+ "encoding/base64"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/minio/minio-go/v7/pkg/tags"
+ "github.com/minio/minio/internal/arn"
+ "github.com/minio/minio/internal/auth"
+ "github.com/pgsty/silo-pkg/v3/policy"
+ "github.com/pgsty/silo-pkg/v3/policy/condition"
+)
+
+// A batch job acts on objects long after the request that submitted it has
+// gone. It therefore records who submitted it and acts only as that
+// principal:
+//
+// - StartBatchJob binds the authenticated credential to the job
+// (batchJobSubmitter). Nothing in the submitted definition names it.
+// - Every start of the job, and every object it touches, resolves the
+// binding against IAM again. A credential that is gone, disabled,
+// expired, revoked or edited since submission fails the job; a reused
+// access key does not match the binding.
+// - Each action is evaluated with the submitter's policies and session
+// policy but without a request. A policy statement that would need
+// request context the job does not have (source address, transport,
+// headers and so on) refuses the action instead of being skipped.
+//
+// A job without a binding was queued by an earlier release. It is never
+// authorized, whoever submitted it, and has to be submitted again.
+
+//go:generate msgp -file $GOFILE -unexported
+
+// batchJobSubmitter is the server-assigned identity of the credential that
+// submitted a batch job.
+type batchJobSubmitter struct {
+ Version int `msg:"v"`
+ Type string `msg:"t"`
+ AccessKey string `msg:"ak"`
+ // ParentUser is the parent of a derived credential.
+ ParentUser string `msg:"pu"`
+ // UpdatedAt and RevokedBefore are the revision of the identity record;
+ // any edit of the identity changes them. They are unset for root.
+ UpdatedAt time.Time `msg:"ua"`
+ RevokedBefore time.Time `msg:"rb"`
+ // Binding is a keyed fingerprint of the credential itself, see
+ // batchSubmitterBinding.
+ Binding []byte `msg:"bd"`
+}
+
+const batchJobSubmitterVersion = 1
+
+// Submitter types. A credential is classified in the order IsAllowed
+// evaluates it.
+const (
+ batchSubmitterRoot = "root"
+ batchSubmitterUser = "user"
+ batchSubmitterTemporary = "temporary"
+ batchSubmitterService = "service-account"
+)
+
+//msgp:ignore batchJobAuthzError
+
+// batchJobAuthzError refuses a batch job, or one of its actions, for its
+// submitter. It ends the job: it is persisted as the job's failure and is
+// never retried.
+type batchJobAuthzError struct {
+ reason string
+}
+
+func (e batchJobAuthzError) Error() string {
+ return e.reason
+}
+
+func isBatchJobAuthzError(err error) bool {
+ var e batchJobAuthzError
+ return errors.As(err, &e)
+}
+
+var (
+ errBatchJobResubmit = batchJobAuthzError{reason: "the batch job does not record who submitted it; submit it again"}
+ errBatchJobChanged = batchJobAuthzError{reason: "the credential that submitted the batch job has changed since submission; submit it again"}
+ errBatchJobInvalid = batchJobAuthzError{reason: "the credential that submitted the batch job is no longer valid"}
+)
+
+func batchSubmitterType(cred auth.Credentials) string {
+ switch {
+ case cred.IsTemp():
+ return batchSubmitterTemporary
+ case cred.IsServiceAccount():
+ return batchSubmitterService
+ default:
+ return batchSubmitterUser
+ }
+}
+
+// batchSubmitterBinding is a fingerprint of one credential keyed with its own
+// secret. It changes when the secret, the session token or the parent
+// changes, and a new credential that reuses the access key does not have it.
+func batchSubmitterBinding(typ string, cred auth.Credentials) []byte {
+ mac := hmac.New(sha256.New, []byte(cred.SecretKey))
+ for _, s := range []string{"silo batch-job submitter v1", typ, cred.AccessKey, cred.ParentUser, cred.SessionToken} {
+ var n [8]byte
+ binary.BigEndian.PutUint64(n[:], uint64(len(s)))
+ mac.Write(n[:])
+ mac.Write([]byte(s))
+ }
+ return mac.Sum(nil)
+}
+
+// identityRecord returns the stored identity of a non-root access key.
+func (sys *IAMSys) identityRecord(accessKey string) (UserIdentity, bool, error) {
+ if !sys.Initialized() {
+ return UserIdentity{}, false, errServerNotInitialized
+ }
+ u, ok := sys.store.GetUser(accessKey)
+ return u, ok, nil
+}
+
+// newBatchJobSubmitter binds the credential that authenticated a
+// StartBatchJob request.
+func newBatchJobSubmitter(cred auth.Credentials) (*batchJobSubmitter, error) {
+ root := globalActiveCred
+ if cred.AccessKey == "" {
+ return nil, errBatchJobInvalid
+ }
+ if cred.AccessKey == root.AccessKey {
+ if subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(root.SecretKey)) != 1 {
+ return nil, errBatchJobChanged
+ }
+ return &batchJobSubmitter{
+ Version: batchJobSubmitterVersion,
+ Type: batchSubmitterRoot,
+ AccessKey: root.AccessKey,
+ Binding: batchSubmitterBinding(batchSubmitterRoot, root),
+ }, nil
+ }
+ u, ok, err := globalIAMSys.identityRecord(cred.AccessKey)
+ if err != nil {
+ return nil, err
+ }
+ // The record must be the one the request was authenticated with.
+ if !ok || u.Credentials.AccessKey != cred.AccessKey || u.Credentials.ParentUser != cred.ParentUser ||
+ subtle.ConstantTimeCompare([]byte(u.Credentials.SecretKey), []byte(cred.SecretKey)) != 1 ||
+ subtle.ConstantTimeCompare([]byte(u.Credentials.SessionToken), []byte(cred.SessionToken)) != 1 {
+ return nil, errBatchJobChanged
+ }
+ typ := batchSubmitterType(u.Credentials)
+ return &batchJobSubmitter{
+ Version: batchJobSubmitterVersion,
+ Type: typ,
+ AccessKey: u.Credentials.AccessKey,
+ ParentUser: u.Credentials.ParentUser,
+ UpdatedAt: u.UpdatedAt,
+ RevokedBefore: u.RevokedBefore,
+ Binding: batchSubmitterBinding(typ, u.Credentials),
+ }, nil
+}
+
+//msgp:ignore batchJobPrincipal batchAuthzRequest
+
+// batchJobPrincipal is a submitter resolved against current IAM state.
+type batchJobPrincipal struct {
+ typ string
+ accessKey string
+ parentUser string
+ groups []string
+ claims map[string]any
+ owner bool
+ temp bool // evaluated as a temporary credential
+ service bool // evaluated as a service account
+}
+
+// principal resolves the binding against current IAM state. It fails if the
+// credential no longer exists, is no longer valid, or is no longer the one
+// that was bound.
+func (s *batchJobSubmitter) principal(ctx context.Context) (batchJobPrincipal, error) {
+ if s == nil || s.AccessKey == "" {
+ return batchJobPrincipal{}, errBatchJobResubmit
+ }
+ if s.Version != batchJobSubmitterVersion {
+ return batchJobPrincipal{}, batchJobAuthzError{reason: fmt.Sprintf("unsupported batch job submitter version %d; submit the job again", s.Version)}
+ }
+ root := globalActiveCred
+ if s.Type == batchSubmitterRoot {
+ if s.AccessKey != root.AccessKey || !hmac.Equal(s.Binding, batchSubmitterBinding(batchSubmitterRoot, root)) {
+ return batchJobPrincipal{}, errBatchJobChanged
+ }
+ if !globalAPIConfig.permitRootAccess() {
+ return batchJobPrincipal{}, errBatchJobInvalid
+ }
+ return batchJobPrincipal{typ: batchSubmitterRoot, accessKey: root.AccessKey, owner: true}, nil
+ }
+ switch s.Type {
+ case batchSubmitterUser, batchSubmitterTemporary, batchSubmitterService:
+ default:
+ return batchJobPrincipal{}, errBatchJobResubmit
+ }
+ if s.AccessKey == root.AccessKey {
+ return batchJobPrincipal{}, errBatchJobChanged
+ }
+ u, ok, err := globalIAMSys.CheckKey(ctx, s.AccessKey)
+ if err != nil {
+ return batchJobPrincipal{}, err
+ }
+ if !ok {
+ // Deleted, disabled, expired or revoked.
+ return batchJobPrincipal{}, errBatchJobInvalid
+ }
+ cred := u.Credentials
+ typ := batchSubmitterType(cred)
+ if typ != s.Type || cred.AccessKey != s.AccessKey || cred.ParentUser != s.ParentUser ||
+ !u.UpdatedAt.Equal(s.UpdatedAt) || !u.RevokedBefore.Equal(s.RevokedBefore) ||
+ !hmac.Equal(s.Binding, batchSubmitterBinding(typ, cred)) {
+ return batchJobPrincipal{}, errBatchJobChanged
+ }
+ if cred.IsExpired() || cred.Status == auth.AccountOff {
+ return batchJobPrincipal{}, errBatchJobInvalid
+ }
+ p := batchJobPrincipal{
+ typ: typ,
+ accessKey: cred.AccessKey,
+ parentUser: cred.ParentUser,
+ groups: cred.Groups,
+ claims: map[string]any{},
+ temp: cred.IsTemp(),
+ service: !cred.IsTemp() && cred.IsServiceAccount(),
+ }
+ if typ != batchSubmitterUser {
+ // Verify the token as a request would, with the current signing keys.
+ jwtClaims, err := extractJWTClaims(u)
+ if err != nil {
+ return batchJobPrincipal{}, errBatchJobInvalid
+ }
+ p.claims = jwtClaims.Map()
+ if err := decodeBatchSessionPolicy(p.claims); err != nil {
+ return batchJobPrincipal{}, errBatchJobInvalid
+ }
+ }
+ p.owner = cred.ParentUser == root.AccessKey && cred.AccessKey != siteReplicatorSvcAcc
+ if p.owner && !globalAPIConfig.permitRootAccess() {
+ return batchJobPrincipal{}, errBatchJobInvalid
+ }
+ if _, ok := p.claims[policy.SessionPolicyName]; ok {
+ p.owner = false
+ }
+ return p, nil
+}
+
+// decodeBatchSessionPolicy decodes a session policy claim for evaluation.
+// A request decodes it while verifying its token, except when an
+// authorization plugin is configured; a batch job decodes it always, so the
+// session policy also limits what the plugin allows.
+func decodeBatchSessionPolicy(claims map[string]any) error {
+ sp, ok := claims[policy.SessionPolicyName]
+ if !ok {
+ return nil
+ }
+ s, ok := sp.(string)
+ if !ok {
+ return errAuthentication
+ }
+ b, err := base64.StdEncoding.DecodeString(s)
+ if err != nil {
+ return errAuthentication
+ }
+ claims[sessionPolicyNameExtracted] = string(b)
+ return nil
+}
+
+// batchAuthzRequest is one action a batch job is about to take.
+type batchAuthzRequest struct {
+ action policy.Action
+ bucket string
+ object string // empty for a bucket action
+ // versionID is the version an object version action addresses.
+ versionID string
+ // tags are the current tags of the object version acted on.
+ tags string
+ // tagsUnknown is set when the job does not read the version it acts
+ // on, so s3:ExistingObjectTag is context it does not have.
+ tagsUnknown bool
+ // values are request values the job supplies itself, such as the
+ // listing prefix or the encryption and retention it writes, keyed by
+ // condition key name.
+ values map[string][]string
+}
+
+// batchAuthzMarker marks the condition values of a batch job for an
+// authorization plugin. Requests cannot set s3:authType.
+const batchAuthzMarker = "batch-job"
+
+// Condition keys a batch job always provides, as a request would: identity,
+// time, the addressed version and the stored tags. Keys a request takes from
+// itself are provided only when the job sets them (see batchAuthzRequest
+// values); every other key is request context the job does not have.
+var batchServerConditionKeys = map[string]bool{
+ condition.AWSCurrentTime.ToKey().String(): true,
+ condition.AWSEpochTime.ToKey().String(): true,
+ condition.AWSPrincipalType.ToKey().String(): true,
+ condition.AWSUserID.ToKey().String(): true,
+ condition.AWSUsername.ToKey().String(): true,
+ condition.AWSGroups.ToKey().String(): true,
+ condition.S3VersionID.ToKey().String(): true,
+ condition.ExistingObjectTag.ToKey().String(): true,
+}
+
+// Request keys a job may supply, by the name they have in condition values.
+var batchRequestConditionKeys = map[string]condition.KeyName{
+ condition.S3Prefix.Name(): condition.S3Prefix,
+ condition.S3XAmzServerSideEncryption.Name(): condition.S3XAmzServerSideEncryption,
+ condition.S3XAmzServerSideEncryptionAwsKmsKeyID.Name(): condition.S3XAmzServerSideEncryptionAwsKmsKeyID,
+ condition.S3ObjectLockMode.Name(): condition.S3ObjectLockMode,
+ condition.S3ObjectLockRetainUntilDate.Name(): condition.S3ObjectLockRetainUntilDate,
+ condition.S3ObjectLockLegalHold.Name(): condition.S3ObjectLockLegalHold,
+ condition.RequestObjectTagKeys.Name(): condition.RequestObjectTagKeys,
+}
+
+// suppliedConditionKeys returns the condition keys whose values req provides.
+func (req batchAuthzRequest) suppliedConditionKeys() map[string]bool {
+ supplied := make(map[string]bool, len(batchServerConditionKeys)+len(req.values))
+ for k := range batchServerConditionKeys {
+ supplied[k] = true
+ }
+ if req.tagsUnknown {
+ delete(supplied, condition.ExistingObjectTag.ToKey().String())
+ }
+ for name := range req.values {
+ if key, ok := batchRequestConditionKeys[name]; ok {
+ supplied[key.ToKey().String()] = true
+ }
+ if strings.HasPrefix(name, condition.RequestObjectTag.Name()+"/") {
+ supplied[condition.RequestObjectTag.ToKey().String()] = true
+ }
+ }
+ return supplied
+}
+
+func batchKeySupplied(supplied map[string]bool, name string) bool {
+ if i := strings.IndexByte(name, '/'); i >= 0 {
+ name = name[:i]
+ }
+ if strings.HasPrefix(name, "jwt:") || strings.HasPrefix(name, "ldap:") {
+ // Claims of the verified token, as a request has them.
+ return true
+ }
+ return supplied[name]
+}
+
+// conditionValues are the values a request with the same identity would
+// carry, without any value taken from the request itself.
+func (p batchJobPrincipal) conditionValues(req batchAuthzRequest) map[string][]string {
+ now := UTCNow()
+ username := p.accessKey
+ if p.temp || p.service {
+ username = p.parentUser
+ }
+ principalType := "User"
+ if len(p.claims) > 0 {
+ principalType = "AssumedRole"
+ }
+ if username == globalActiveCred.AccessKey {
+ principalType = "Account"
+ }
+ // Identity and time, then the claims over them, as a request has them
+ // (an LDAP session's short username replaces its parent DN).
+ values := map[string][]string{
+ "CurrentTime": {now.Format(time.RFC3339)},
+ "EpochTime": {strconv.FormatInt(now.Unix(), 10)},
+ "principaltype": {principalType},
+ "userid": {username},
+ "username": {username},
+ }
+ for k, v := range p.claims {
+ if s, ok := v.(string); ok {
+ values[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{s}
+ }
+ }
+ // No claim stands for a value of the object or of the job, which
+ // follow and always win.
+ for k := range values {
+ if k == "versionid" || k == "prefix" || k == "authtype" ||
+ strings.HasPrefix(k, "existingobjecttag/") || strings.HasPrefix(k, "requestobjecttag") ||
+ strings.HasPrefix(k, "x-amz-") || strings.HasPrefix(k, "object-lock-") {
+ delete(values, k)
+ }
+ }
+ // For an authorization plugin: this is a batch job, not a request.
+ values["authType"] = []string{batchAuthzMarker}
+ if req.versionID != "" {
+ values["versionid"] = []string{req.versionID}
+ }
+ if req.tags != "" {
+ if t, err := tags.ParseObjectTags(req.tags); err == nil {
+ for k, v := range t.ToMap() {
+ values[pathJoin("ExistingObjectTag", k)] = []string{v}
+ }
+ }
+ }
+ for k, v := range req.values {
+ values[k] = v
+ }
+ if grpsVal, ok := p.claims["groups"]; ok {
+ if grpsIs, ok := grpsVal.([]any); ok {
+ grps := []string{}
+ for _, gI := range grpsIs {
+ if g, ok := gI.(string); ok {
+ grps = append(grps, g)
+ }
+ }
+ if len(grps) > 0 {
+ values["groups"] = grps
+ }
+ }
+ }
+ if _, ok := values["groups"]; !ok && len(p.groups) > 0 {
+ values["groups"] = p.groups
+ }
+ return values
+}
+
+// authorize evaluates one action of the job as its submitter.
+func (p batchJobPrincipal) authorize(req batchAuthzRequest) error {
+ args := policy.Args{
+ AccountName: p.accessKey,
+ Groups: p.groups,
+ Action: req.action,
+ BucketName: req.bucket,
+ ObjectName: req.object,
+ ConditionValues: p.conditionValues(req),
+ IsOwner: p.owner,
+ Claims: p.claims,
+ }
+ denied := batchJobAuthzError{reason: batchDeniedReason(req)}
+ plugin := newGlobalAuthZPluginFn()
+ policies, err := globalIAMSys.batchLocalPolicies(p, args, plugin == nil)
+ if err != nil {
+ return denied
+ }
+ supplied := req.suppliedConditionKeys()
+ for _, pol := range policies {
+ for _, st := range pol.Statements {
+ if batchStatementNeedsRequest(st, args, supplied) {
+ return batchJobAuthzError{reason: denied.reason + ": a policy statement depends on request context that a batch job does not have"}
+ }
+ }
+ }
+ if plugin != nil {
+ ok, err := plugin.IsAllowed(args)
+ if err != nil {
+ authZLogIf(GlobalContext, err)
+ }
+ if !ok {
+ return denied
+ }
+ // The plugin does not see a session policy; it only limits further.
+ if has, allowed := p.sessionPolicyDecision(args); has && !allowed {
+ return denied
+ }
+ return nil
+ }
+ if !globalIAMSys.isAllowedLocally(args) {
+ return denied
+ }
+ return nil
+}
+
+func batchDeniedReason(req batchAuthzRequest) string {
+ resource := req.bucket
+ if req.object != "" {
+ resource = pathJoin(req.bucket, req.object)
+ }
+ if req.versionID != "" {
+ resource += " (versionId " + req.versionID + ")"
+ }
+ return fmt.Sprintf("the batch job submitter is not allowed %s on %s", req.action, resource)
+}
+
+// sessionPolicyDecision evaluates the session policy the way IsAllowed would
+// for this credential.
+func (p batchJobPrincipal) sessionPolicyDecision(args policy.Args) (bool, bool) {
+ switch {
+ case p.temp:
+ return isAllowedBySessionPolicy(args)
+ case p.service:
+ if p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
+ return false, false
+ }
+ return isAllowedBySessionPolicyForServiceAccount(args)
+ }
+ return false, false
+}
+
+// batchLocalPolicies returns every local policy that can take part in the
+// decision on args: the mapped policies that IsAllowed would merge when
+// withMapped is set (no authorization plugin), and the session policy.
+func (sys *IAMSys) batchLocalPolicies(p batchJobPrincipal, args policy.Args, withMapped bool) ([]policy.Policy, error) {
+ var out []policy.Policy
+ if withMapped && args.IsOwner {
+ // IsAllowed does not evaluate any policy for the owner.
+ return nil, nil
+ }
+ if withMapped {
+ var names []string
+ ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey
+ switch {
+ case p.typ == batchSubmitterRoot || ownerDerived:
+ case p.temp || p.service:
+ if roleArn := args.GetRoleArn(); roleArn != "" {
+ a, err := arn.Parse(roleArn)
+ if err != nil {
+ return nil, err
+ }
+ names = append(names, newMappedPolicy(sys.rolesMap[a]).toSlice()...)
+ } else {
+ mapped, err := sys.PolicyDBGet(p.parentUser, args.Groups...)
+ if err != nil {
+ return nil, err
+ }
+ names = append(names, mapped...)
+ // The claimed policies apply only when nothing is mapped.
+ if len(mapped) == 0 {
+ if claimed, ok := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID()); ok {
+ names = append(names, claimed.ToSlice()...)
+ }
+ }
+ }
+ default:
+ mapped, err := sys.PolicyDBGet(p.accessKey, args.Groups...)
+ if err != nil {
+ return nil, err
+ }
+ names = append(names, mapped...)
+ }
+ if len(names) > 0 {
+ out = append(out, sys.GetCombinedPolicy(names...))
+ }
+ }
+ if p.temp || p.service {
+ if p.service && p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
+ return out, nil
+ }
+ if sp, ok := p.claims[sessionPolicyNameExtracted]; ok {
+ s, ok := sp.(string)
+ if !ok {
+ return nil, errAuthentication
+ }
+ parsed, err := policy.ParseConfig(strings.NewReader(s))
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, *parsed)
+ }
+ }
+ return out, nil
+}
+
+// batchStatementNeedsRequest reports whether a statement that applies to
+// args depends on a condition key that the job does not supply. Applying
+// such a statement without its value could grant (Allow) or drop (Deny)
+// what a request with that context would not, so the job refuses the action.
+// Resources are matched without conditions; a statement is taken to apply
+// whenever it might.
+func batchStatementNeedsRequest(st policy.Statement, args policy.Args, supplied map[string]bool) bool {
+ if (!st.Actions.Match(args.Action) && !st.Actions.IsEmpty()) || st.NotActions.Match(args.Action) {
+ return false
+ }
+ // Policy variables in resources or condition values.
+ if data, err := json.Marshal(st); err == nil {
+ s := string(data)
+ for {
+ i := strings.Index(s, "${")
+ if i < 0 {
+ break
+ }
+ s = s[i+2:]
+ j := strings.IndexByte(s, '}')
+ if j < 0 {
+ break
+ }
+ if !batchKeySupplied(supplied, s[:j]) {
+ return true
+ }
+ s = s[j+1:]
+ }
+ }
+ if !batchStatementMayApply(st, args) {
+ return false
+ }
+ for key := range st.Conditions.Keys() {
+ if !batchKeySupplied(supplied, key.String()) {
+ return true
+ }
+ }
+ return false
+}
+
+// batchStatementMayApply matches the resources of a statement, in both forms
+// a bucket request can take. A statement whose resources the evaluator
+// translates (S3 Tables and vectors actions cover S3 data actions) is taken
+// to apply.
+func batchStatementMayApply(st policy.Statement, args policy.Args) bool {
+ for _, actions := range []policy.ActionSet{st.Actions, st.NotActions} {
+ for action := range actions {
+ if policy.TableAction(action).IsValid() || policy.VectorsAction(action).IsValid() {
+ return true
+ }
+ }
+ }
+ for _, resources := range []policy.ResourceSet{st.Resources, st.NotResources} {
+ for r := range resources {
+ if r.Type != policy.ResourceARNS3 && r.Type != policy.ResourceARNAll {
+ return true
+ }
+ }
+ }
+ resources := []string{args.BucketName + "/" + strings.TrimPrefix(args.ObjectName, "/")}
+ if args.ObjectName == "" {
+ resources = []string{args.BucketName, args.BucketName + "/"}
+ }
+ for _, resource := range resources {
+ if len(st.Resources) > 0 && !st.Resources.Match(resource, args.ConditionValues) {
+ continue
+ }
+ if len(st.NotResources) > 0 && st.NotResources.Match(resource, args.ConditionValues) {
+ continue
+ }
+ return true
+ }
+ return false
+}
+
+// authorize resolves the submitter and evaluates each request.
+func (s *batchJobSubmitter) authorize(ctx context.Context, reqs ...batchAuthzRequest) error {
+ p, err := s.principal(ctx)
+ if err != nil {
+ return err
+ }
+ for _, req := range reqs {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// startAuthzRequests are the bucket actions a job needs before it acts on any
+// object: the listings it walks. Object actions are authorized per object.
+func (j BatchJobRequest) startAuthzRequests() []batchAuthzRequest {
+ list := func(bucket string, prefixes []string) []batchAuthzRequest {
+ if len(prefixes) == 0 {
+ prefixes = []string{""}
+ }
+ var reqs []batchAuthzRequest
+ for _, prefix := range prefixes {
+ values := map[string][]string{condition.S3Prefix.Name(): {prefix}}
+ reqs = append(reqs,
+ batchAuthzRequest{action: policy.ListBucketAction, bucket: bucket, values: values},
+ batchAuthzRequest{action: policy.ListBucketVersionsAction, bucket: bucket, values: values},
+ )
+ }
+ return reqs
+ }
+ // The same order as Type() and the job pool.
+ switch {
+ case j.Replicate != nil:
+ if !j.Replicate.RemoteToLocal() {
+ return list(j.Replicate.Source.Bucket, j.Replicate.Source.Prefix.F())
+ }
+ case j.KeyRotate != nil:
+ return list(j.KeyRotate.Bucket, []string{j.KeyRotate.Prefix})
+ case j.Expire != nil:
+ return list(j.Expire.Bucket, batchCanonicalPrefixes(j.Expire.Prefix.F()))
+ }
+ return nil
+}
+
+// jobSections counts the job types a definition holds. A job is one of them.
+func (j BatchJobRequest) jobSections() int {
+ n := 0
+ for _, set := range []bool{j.Replicate != nil, j.KeyRotate != nil, j.Expire != nil} {
+ if set {
+ n++
+ }
+ }
+ return n
+}
+
+var errBatchJobSections = batchJobAuthzError{reason: "a batch job must define exactly one job type"}
+
+// refuse ends the job for its submitter.
+func (ri *batchJobInfo) refuse(err error) {
+ ri.mu.Lock()
+ defer ri.mu.Unlock()
+ if ri.FailureReason == "" {
+ ri.FailureReason = err.Error()
+ }
+ ri.Complete = false
+ ri.Failed = true
+}
+
+func (ri *batchJobInfo) refused() bool {
+ ri.mu.RLock()
+ defer ri.mu.RUnlock()
+ return ri.FailureReason != ""
+}
+
+// startAuthorized checks the submitter of a job that is about to run. It
+// returns done for a job that must not run now: one refused before, which
+// stays refused, or one refused now, which is persisted as failed first.
+// An error that is not a decision leaves the job to a later start.
+func (ri *batchJobInfo) startAuthorized(ctx context.Context, api ObjectLayer, job BatchJobRequest) (done bool, err error) {
+ if !ri.refused() {
+ err := job.Submitter.authorize(ctx, job.startAuthzRequests()...)
+ if err == nil && job.jobSections() != 1 {
+ err = errBatchJobSections
+ }
+ if err == nil {
+ return false, nil
+ }
+ if !isBatchJobAuthzError(err) {
+ return true, err
+ }
+ ri.refuse(err)
+ batchLogIf(ctx, fmt.Errorf("batch job %s refused: %w", job.ID, err))
+ }
+ globalBatchJobsMetrics.save(job.ID, ri)
+ return true, ri.updateAfter(ctx, api, 0, job)
+}
diff --git a/cmd/batch-authz_gen.go b/cmd/batch-authz_gen.go
new file mode 100644
index 000000000..b5ebe3f64
--- /dev/null
+++ b/cmd/batch-authz_gen.go
@@ -0,0 +1,260 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package cmd
+
+import (
+ "github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *batchJobSubmitter) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zb0001 uint32
+ zb0001, err = dc.ReadMapHeader()
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ for zb0001 > 0 {
+ zb0001--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "v":
+ z.Version, err = dc.ReadInt()
+ if err != nil {
+ err = msgp.WrapError(err, "Version")
+ return
+ }
+ case "t":
+ z.Type, err = dc.ReadString()
+ if err != nil {
+ err = msgp.WrapError(err, "Type")
+ return
+ }
+ case "ak":
+ z.AccessKey, err = dc.ReadString()
+ if err != nil {
+ err = msgp.WrapError(err, "AccessKey")
+ return
+ }
+ case "pu":
+ z.ParentUser, err = dc.ReadString()
+ if err != nil {
+ err = msgp.WrapError(err, "ParentUser")
+ return
+ }
+ case "ua":
+ z.UpdatedAt, err = dc.ReadTime()
+ if err != nil {
+ err = msgp.WrapError(err, "UpdatedAt")
+ return
+ }
+ case "rb":
+ z.RevokedBefore, err = dc.ReadTime()
+ if err != nil {
+ err = msgp.WrapError(err, "RevokedBefore")
+ return
+ }
+ case "bd":
+ z.Binding, err = dc.ReadBytes(z.Binding)
+ if err != nil {
+ err = msgp.WrapError(err, "Binding")
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ }
+ }
+ return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *batchJobSubmitter) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 7
+ // write "v"
+ err = en.Append(0x87, 0xa1, 0x76)
+ if err != nil {
+ return
+ }
+ err = en.WriteInt(z.Version)
+ if err != nil {
+ err = msgp.WrapError(err, "Version")
+ return
+ }
+ // write "t"
+ err = en.Append(0xa1, 0x74)
+ if err != nil {
+ return
+ }
+ err = en.WriteString(z.Type)
+ if err != nil {
+ err = msgp.WrapError(err, "Type")
+ return
+ }
+ // write "ak"
+ err = en.Append(0xa2, 0x61, 0x6b)
+ if err != nil {
+ return
+ }
+ err = en.WriteString(z.AccessKey)
+ if err != nil {
+ err = msgp.WrapError(err, "AccessKey")
+ return
+ }
+ // write "pu"
+ err = en.Append(0xa2, 0x70, 0x75)
+ if err != nil {
+ return
+ }
+ err = en.WriteString(z.ParentUser)
+ if err != nil {
+ err = msgp.WrapError(err, "ParentUser")
+ return
+ }
+ // write "ua"
+ err = en.Append(0xa2, 0x75, 0x61)
+ if err != nil {
+ return
+ }
+ err = en.WriteTime(z.UpdatedAt)
+ if err != nil {
+ err = msgp.WrapError(err, "UpdatedAt")
+ return
+ }
+ // write "rb"
+ err = en.Append(0xa2, 0x72, 0x62)
+ if err != nil {
+ return
+ }
+ err = en.WriteTime(z.RevokedBefore)
+ if err != nil {
+ err = msgp.WrapError(err, "RevokedBefore")
+ return
+ }
+ // write "bd"
+ err = en.Append(0xa2, 0x62, 0x64)
+ if err != nil {
+ return
+ }
+ err = en.WriteBytes(z.Binding)
+ if err != nil {
+ err = msgp.WrapError(err, "Binding")
+ return
+ }
+ return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *batchJobSubmitter) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 7
+ // string "v"
+ o = append(o, 0x87, 0xa1, 0x76)
+ o = msgp.AppendInt(o, z.Version)
+ // string "t"
+ o = append(o, 0xa1, 0x74)
+ o = msgp.AppendString(o, z.Type)
+ // string "ak"
+ o = append(o, 0xa2, 0x61, 0x6b)
+ o = msgp.AppendString(o, z.AccessKey)
+ // string "pu"
+ o = append(o, 0xa2, 0x70, 0x75)
+ o = msgp.AppendString(o, z.ParentUser)
+ // string "ua"
+ o = append(o, 0xa2, 0x75, 0x61)
+ o = msgp.AppendTime(o, z.UpdatedAt)
+ // string "rb"
+ o = append(o, 0xa2, 0x72, 0x62)
+ o = msgp.AppendTime(o, z.RevokedBefore)
+ // string "bd"
+ o = append(o, 0xa2, 0x62, 0x64)
+ o = msgp.AppendBytes(o, z.Binding)
+ return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *batchJobSubmitter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zb0001 uint32
+ zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ for zb0001 > 0 {
+ zb0001--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "v":
+ z.Version, bts, err = msgp.ReadIntBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "Version")
+ return
+ }
+ case "t":
+ z.Type, bts, err = msgp.ReadStringBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "Type")
+ return
+ }
+ case "ak":
+ z.AccessKey, bts, err = msgp.ReadStringBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "AccessKey")
+ return
+ }
+ case "pu":
+ z.ParentUser, bts, err = msgp.ReadStringBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "ParentUser")
+ return
+ }
+ case "ua":
+ z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "UpdatedAt")
+ return
+ }
+ case "rb":
+ z.RevokedBefore, bts, err = msgp.ReadTimeBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "RevokedBefore")
+ return
+ }
+ case "bd":
+ z.Binding, bts, err = msgp.ReadBytesBytes(bts, z.Binding)
+ if err != nil {
+ err = msgp.WrapError(err, "Binding")
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *batchJobSubmitter) Msgsize() (s int) {
+ s = 1 + 2 + msgp.IntSize + 2 + msgp.StringPrefixSize + len(z.Type) + 3 + msgp.StringPrefixSize + len(z.AccessKey) + 3 + msgp.StringPrefixSize + len(z.ParentUser) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.BytesPrefixSize + len(z.Binding)
+ return
+}
diff --git a/cmd/batch-authz_gen_test.go b/cmd/batch-authz_gen_test.go
new file mode 100644
index 000000000..15f9cdfa1
--- /dev/null
+++ b/cmd/batch-authz_gen_test.go
@@ -0,0 +1,123 @@
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+package cmd
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalbatchJobSubmitter(t *testing.T) {
+ v := batchJobSubmitter{}
+ bts, err := v.MarshalMsg(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ left, err := v.UnmarshalMsg(bts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) > 0 {
+ t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+ }
+
+ left, err = msgp.Skip(bts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) > 0 {
+ t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+ }
+}
+
+func BenchmarkMarshalMsgbatchJobSubmitter(b *testing.B) {
+ v := batchJobSubmitter{}
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ v.MarshalMsg(nil)
+ }
+}
+
+func BenchmarkAppendMsgbatchJobSubmitter(b *testing.B) {
+ v := batchJobSubmitter{}
+ bts := make([]byte, 0, v.Msgsize())
+ bts, _ = v.MarshalMsg(bts[0:0])
+ b.SetBytes(int64(len(bts)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ bts, _ = v.MarshalMsg(bts[0:0])
+ }
+}
+
+func BenchmarkUnmarshalbatchJobSubmitter(b *testing.B) {
+ v := batchJobSubmitter{}
+ bts, _ := v.MarshalMsg(nil)
+ b.ReportAllocs()
+ b.SetBytes(int64(len(bts)))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _, err := v.UnmarshalMsg(bts)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
+
+func TestEncodeDecodebatchJobSubmitter(t *testing.T) {
+ v := batchJobSubmitter{}
+ var buf bytes.Buffer
+ msgp.Encode(&buf, &v)
+
+ m := v.Msgsize()
+ if buf.Len() > m {
+ t.Log("WARNING: TestEncodeDecodebatchJobSubmitter Msgsize() is inaccurate")
+ }
+
+ vn := batchJobSubmitter{}
+ err := msgp.Decode(&buf, &vn)
+ if err != nil {
+ t.Error(err)
+ }
+
+ buf.Reset()
+ msgp.Encode(&buf, &v)
+ err = msgp.NewReader(&buf).Skip()
+ if err != nil {
+ t.Error(err)
+ }
+}
+
+func BenchmarkEncodebatchJobSubmitter(b *testing.B) {
+ v := batchJobSubmitter{}
+ var buf bytes.Buffer
+ msgp.Encode(&buf, &v)
+ b.SetBytes(int64(buf.Len()))
+ en := msgp.NewWriter(msgp.Nowhere)
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ v.EncodeMsg(en)
+ }
+ en.Flush()
+}
+
+func BenchmarkDecodebatchJobSubmitter(b *testing.B) {
+ v := batchJobSubmitter{}
+ var buf bytes.Buffer
+ msgp.Encode(&buf, &v)
+ b.SetBytes(int64(buf.Len()))
+ rd := msgp.NewEndlessReader(buf.Bytes(), b)
+ dc := msgp.NewReader(rd)
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ err := v.DecodeMsg(dc)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
diff --git a/cmd/batch-expire-purge-authz_test.go b/cmd/batch-expire-purge-authz_test.go
new file mode 100644
index 000000000..6c8b14dfd
--- /dev/null
+++ b/cmd/batch-expire-purge-authz_test.go
@@ -0,0 +1,524 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/auth"
+ "github.com/minio/minio/internal/config/storageclass"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/pgsty/silo-pkg/v3/policy"
+)
+
+// A batch expiration that keeps no versions purges each key it matched under
+// the purge's lock. What it authorizes for the submitter, and checks for
+// Object Lock, are the versions the purge deletes then, not the versions the
+// walk saw before.
+
+// batchPurgeHook runs change once, after the walk of a job and before its
+// first purge of all versions.
+type batchPurgeHook struct {
+ ObjectLayer
+ once sync.Once
+ change func()
+}
+
+func (h *batchPurgeHook) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ if opts.DeleteAllVersions {
+ h.once.Do(h.change)
+ }
+ return h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+}
+
+func (f *batchAuthzFixture) purgeYAML(prefix string) string {
+ return fmt.Sprintf(`expire:
+ apiVersion: v1
+ bucket: %s
+ prefix: %q
+ rules:
+ - type: object
+ purge:
+ retainVersions: 0
+`, f.bucket, prefix)
+}
+
+// runChanged submits a purge of prefix as cred and runs it, with change made
+// between its walk and its purge, and returns the job's final state.
+func (f *batchAuthzFixture) runChanged(t *testing.T, cred auth.Credentials, prefix string, change func()) *batchJobInfo {
+ t.Helper()
+ rec, job := f.submit(t, cred, f.purgeYAML(prefix))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ hook := &batchPurgeHook{ObjectLayer: f.obj, change: change}
+ if err := job.Expire.Start(f.ctx, hook, *job); err != nil {
+ t.Logf("start: %v", err)
+ }
+ return f.jobInfo(t, job)
+}
+
+// putOlder writes a version of key older than every version it has, as a
+// replicated or pulled version can be, with metadata meta.
+func (f *batchAuthzFixture) putOlder(t *testing.T, key, versionID string, meta map[string]string) {
+ t.Helper()
+ data := []byte("older " + key)
+ _, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
+ ObjectOptions{Versioned: true, VersionID: versionID, MTime: UTCNow().Add(-time.Hour), UserDefined: meta})
+ mustIAM(t, err)
+}
+
+func batchPurgeVersionIDs(t *testing.T, f *batchAuthzFixture, key string) []string {
+ t.Helper()
+ var ids []string
+ marker, vmarker := "", ""
+ for {
+ res, err := f.obj.ListObjectVersions(f.ctx, f.bucket, key, marker, vmarker, "", 1000)
+ mustIAM(t, err)
+ for _, o := range res.Objects {
+ if o.Name == key {
+ ids = append(ids, o.VersionID)
+ }
+ }
+ if !res.IsTruncated {
+ return ids
+ }
+ marker, vmarker = res.NextMarker, res.NextVersionIDMarker
+ }
+}
+
+// A version that the submitter may not expire, because of its tags or its
+// version ID, is refused although the walk did not see it so: the job fails
+// and the key keeps all its versions.
+func TestBatchExpirePurgeAuthorizesCurrentVersions(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ action string // the action a conditional Deny denies
+ deny string // its condition
+ // change returns the version IDs that change made.
+ change func(t *testing.T, f *batchAuthzFixture, key, older string) []string
+ }{
+ {
+ name: "tagged-after-walk",
+ action: "s3:GetObjectVersion",
+ deny: `{"StringEquals":{"s3:ExistingObjectTag/keep":["yes"]}}`,
+ change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
+ _, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older})
+ mustIAM(t, err)
+ return nil
+ },
+ },
+ {
+ name: "version-after-walk",
+ action: "s3:DeleteObjectVersion",
+ deny: `{"StringEquals":{"s3:versionid":["%s"]}}`,
+ change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
+ f.putOlder(t, key, batchPurgeDeniedVersion, nil)
+ return []string{batchPurgeDeniedVersion}
+ },
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ deny := tc.deny
+ if strings.Contains(deny, "%s") {
+ deny = fmt.Sprintf(deny, batchPurgeDeniedVersion)
+ }
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-purge", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["`+tc.action+`"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":`+deny+`}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "batch-purge-user", "batch-purge")
+ const key = "purge/key"
+ older := f.putVersions(t, key)
+ want := batchPurgeVersionIDs(t, f, key)
+ ri := f.runChanged(t, cred, "purge/", func() {
+ want = append(want, tc.change(t, f, key, older)...)
+ })
+ for _, id := range want {
+ if !f.versionExists(t, key, id) {
+ t.Errorf("version %s was purged although the submitter may not delete it", id)
+ }
+ }
+ if !ri.Failed || ri.Complete || ri.FailureReason == "" {
+ t.Errorf("job complete=%v failed=%v reason=%q, want a refused job", ri.Complete, ri.Failed, ri.FailureReason)
+ }
+ if ri.Objects != 0 || ri.KeysRefused != 0 {
+ t.Errorf("job counts objects=%d refused=%d", ri.Objects, ri.KeysRefused)
+ }
+ })
+ }
+}
+
+const batchPurgeDeniedVersion = "6c4b2f8e-1d3a-4e5b-9c7d-0a1b2c3d4e5f"
+
+// A version that Object Lock protects refuses the whole key, also when it
+// was added after the walk. The job completes with the key counted as
+// refused, not failed.
+func TestBatchExpirePurgeRefusesLockedAfterWalk(t *testing.T) {
+ for _, kind := range batchLockKinds {
+ t.Run(kind, func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ enableBucketObjectLock(t, f.bucket)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+ const key = "purge/locked"
+ f.putVersions(t, key)
+ want := batchPurgeVersionIDs(t, f, key)
+ locked := mustGetUUID()
+ ri := f.runChanged(t, cred, "purge/", func() {
+ f.putOlder(t, key, locked, batchLockMeta(kind))
+ want = append(want, locked)
+ })
+ for _, id := range want {
+ if !f.versionExists(t, key, id) {
+ t.Errorf("version %s of a key with a locked version was purged", id)
+ }
+ }
+ if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
+ t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
+ ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
+ }
+ })
+ }
+}
+
+// Controls: a purge by root and by an authorized user removes every version,
+// including one added after the walk.
+func TestBatchExpirePurgeAuthorizedControls(t *testing.T) {
+ for _, root := range []bool{true, false} {
+ t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ cred := globalActiveCred
+ if !root {
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+ }
+ const key = "purge/key"
+ f.putVersions(t, key)
+ added := mustGetUUID()
+ ri := f.runChanged(t, cred, "purge/", func() {
+ f.putOlder(t, key, added, map[string]string{xhttp.AmzObjectTagging: "keep=yes"})
+ })
+ if ids := batchPurgeVersionIDs(t, f, key); len(ids) != 0 {
+ t.Errorf("an authorized purge kept %v", ids)
+ }
+ if !ri.Complete || ri.Failed || ri.Objects != 3 || ri.KeysRefused != 0 {
+ t.Errorf("job complete=%v failed=%v objects=%d refused=%d, want complete with 3 objects",
+ ri.Complete, ri.Failed, ri.Objects, ri.KeysRefused)
+ }
+ })
+ }
+}
+
+// Cases found in the sixth review round.
+
+// batchPurgeBudgetChange runs change once the first purge stopped at its
+// budget after removing versions.
+type batchPurgeBudgetChange struct {
+ ObjectLayer
+ change func()
+ calls int
+}
+
+func (h *batchPurgeBudgetChange) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ oi, err := h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
+ if opts.DeleteAllVersions {
+ h.calls++
+ if h.calls == 1 && errors.Is(err, errDeleteAllBudget) && opts.DeleteAllResult.Removed > 0 {
+ h.change()
+ }
+ }
+ return oi, err
+}
+
+// A purge that continues after its budget authorizes the versions again, as
+// they are then; the refusal and the versions already removed persist
+// across another start.
+func TestBatchExpirePurgeBudgetReauthorizes(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ const key = "purge/budget"
+ oldest := batchPutVersion(t, f.obj, f.bucket, key, nil)
+ middle := batchPutVersion(t, f.obj, f.bucket, key, nil)
+ latest := batchPutVersion(t, f.obj, f.bucket, key, nil)
+ batchAuthzPolicy(f.ctx, t, f.sys, "review6-budget-deny", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "review6-budget-user", "review6-budget-deny")
+ rec, job := f.submit(t, cred, f.purgeYAML("purge/"))
+ if job == nil {
+ t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+ }
+ oldBudget := deleteAllBudget
+ deleteAllBudget.versions = 1
+ t.Cleanup(func() { deleteAllBudget = oldBudget })
+ hook := &batchPurgeBudgetChange{ObjectLayer: f.obj, change: func() {
+ _, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: middle.VersionID})
+ mustIAM(t, err)
+ }}
+ mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
+ ri := f.jobInfo(t, job)
+ if hook.calls != 2 || !ri.Failed || ri.Complete || ri.FailureReason == "" || ri.Objects != 1 || ri.KeysRefused != 0 {
+ t.Fatalf("continuation: calls=%d complete=%v failed=%v reason=%q objects=%d refused=%d", hook.calls, ri.Complete, ri.Failed, ri.FailureReason, ri.Objects, ri.KeysRefused)
+ }
+ if f.versionExists(t, key, oldest.VersionID) || !f.versionExists(t, key, middle.VersionID) || !f.versionExists(t, key, latest.VersionID) {
+ t.Fatal("only the first authorized version may be removed")
+ }
+ _, err := f.obj.DeleteObjectTags(f.ctx, f.bucket, key, ObjectOptions{VersionID: middle.VersionID})
+ mustIAM(t, err)
+ mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
+ after := f.jobInfo(t, job)
+ if hook.calls != 2 || after.FailureReason != ri.FailureReason || after.Objects != 1 || after.Complete {
+ t.Fatal("persisted refusal must prevent a fresh purge after the tags change")
+ }
+}
+
+// The purge's lock excludes tag writers from its authorization through its
+// last delete, for a directory key in a lock domain of its own on one pool
+// and for a key in a second pool.
+func TestBatchExpirePurgeExcludesTagWriter(t *testing.T) {
+ for _, mode := range []string{ldDsync, "two-pools"} {
+ t.Run(mode, func(t *testing.T) {
+ var z *erasureServerPools
+ var bucket string
+ pool, key := 0, "purge/tags/"
+ if mode == "two-pools" {
+ z, bucket = consistencyPools(t)
+ pool = 1
+ } else {
+ z, bucket = ldPool(t, mode)
+ for i := 0; ; i++ {
+ key = fmt.Sprintf("purge/tags-%d/", i)
+ if z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) == 1 {
+ break
+ }
+ }
+ }
+ before := putConsistencyObject(t, z, bucket, encodeDirObject(key), pool, "", ObjectOptions{Versioned: true})
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancel()
+ entered, release := make(chan struct{}), make(chan struct{})
+ var released atomic.Bool
+ unblock := func() {
+ if released.CompareAndSwap(false, true) {
+ close(release)
+ }
+ }
+ defer unblock()
+ purged := make(chan error, 1)
+ go func() {
+ _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{
+ DeleteAllVersions: true,
+ DeleteAllLatest: &deleteAllMatch{VersionID: before.VersionID, ModTime: before.ModTime},
+ deleteAllAuthorize: func(versions []ObjectInfo) error {
+ close(entered)
+ select {
+ case <-release:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ },
+ })
+ purged <- err
+ }()
+ select {
+ case <-entered:
+ case <-ctx.Done():
+ t.Fatal("purge never reached authorization")
+ }
+ written := make(chan error, 1)
+ go func() {
+ _, err := z.PutObjectTags(ctx, bucket, key, "keep=yes", ObjectOptions{VersionID: before.VersionID})
+ written <- err
+ }()
+ select {
+ case err := <-written:
+ t.Errorf("tag writer crossed the purge's authorization lock: %v", err)
+ case <-time.After(ldWait):
+ unblock()
+ err := <-written
+ if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+ t.Errorf("writer must observe the completed purge: %v", err)
+ }
+ }
+ unblock()
+ mustIAM(t, <-purged)
+ })
+ }
+}
+
+// batchTagWriteFault fails metadata updates of one key, so that a tag write
+// reaches only the drives it does not wrap.
+type batchTagWriteFault struct {
+ StorageAPI
+ bucket, object string
+}
+
+func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object string, fi FileInfo, opts UpdateMetadataOpts) error {
+ if bucket == d.bucket && object == d.object {
+ return errDiskNotFound
+ }
+ return d.StorageAPI.UpdateMetadata(ctx, bucket, object, fi, opts)
+}
+
+// A failed tag write can leave a revision that the submitter may not expire
+// on a minority of drives. The purge removes it with the version, so it is
+// authorized too.
+func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) {
+ resetTestGlobals()
+ t.Cleanup(resetTestGlobals)
+ // One drive of sixteen holds the failed tag write, so the read quorum of
+ // the metadata must stay below sixteen. Earlier tests can leave a
+ // storage class of parity 0 behind (newTestConfig on a single drive).
+ previous := globalStorageClass
+ globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
+ t.Cleanup(func() { globalStorageClass.Update(previous) })
+ z := purgeSinglePool(t, 16)
+ globalIAMSys.initStore(z, nil)
+ mustIAM(t, globalIAMSys.Load(t.Context(), true))
+ f := &batchAuthzFixture{ctx: t.Context(), sys: globalIAMSys, obj: z, bucket: "review6-batch-authz"}
+ mustIAM(t, z.MakeBucket(f.ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+ batchTestJobPool(f.ctx, t, z)
+ const key = "purge/minority-tags"
+ older := f.putVersions(t, key)
+ batchAuthzPolicy(f.ctx, t, f.sys, "review6-minority-deny", strings.ReplaceAll(batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`, "batch-authz", f.bucket))
+ cred := iamAdminUser(f.ctx, t, f.sys, "review6-minority-user", "review6-minority-deny")
+ sub, err := newBatchJobSubmitter(cred)
+ mustIAM(t, err)
+ if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectVersionAction, bucket: f.bucket, object: key, versionID: older, tags: "keep=yes"}); !isBatchJobAuthzError(err) {
+ t.Fatal("control: this tag revision must deny the read needed for expiration")
+ }
+ set := z.serverPools[0].getHashedSet(key)
+ original := set.getDisks
+ disks := append([]StorageAPI(nil), original()...)
+ wrapped := append([]StorageAPI(nil), disks...)
+ for i := 1; i < len(wrapped); i++ {
+ wrapped[i] = batchTagWriteFault{StorageAPI: wrapped[i], bucket: f.bucket, object: key}
+ }
+ set.getDisks = func() []StorageAPI { return wrapped }
+ _, err = f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older, UserDefined: map[string]string{
+ ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
+ }})
+ set.getDisks = original
+ if err == nil {
+ t.Fatal("control: the tag write must fail its quorum")
+ }
+ fi, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{})
+ mustIAM(t, err)
+ if fi.Metadata[xhttp.AmzObjectTagging] != "keep=yes" {
+ t.Fatal("control: the successful drive must hold the denied tag revision")
+ }
+ history, err := z.deleteAllHistory(f.ctx, f.bucket, key)
+ mustIAM(t, err)
+ for _, v := range history {
+ for _, c := range v.copies {
+ if c.fi.Metadata[xhttp.AmzObjectTagging] != "" {
+ t.Fatal("control: the quorum history must still have the original tags")
+ }
+ }
+ }
+ ri := f.runChanged(t, cred, "purge/", func() {})
+ if !ri.Failed || ri.Complete || !strings.Contains(ri.FailureReason, "is not allowed") {
+ t.Errorf("denied physical revision must refuse the job: complete=%v failed=%v reason=%q", ri.Complete, ri.Failed, ri.FailureReason)
+ }
+ if _, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}); err != nil {
+ t.Errorf("denied physical revision was removed: %v", err)
+ }
+ if got := batchPurgeVersionIDs(t, f, key); len(got) != 2 {
+ t.Errorf("denial must keep the whole key; remaining versions=%v", got)
+ }
+}
+
+// Cases found in the seventh review round.
+
+// The time the authorization takes counts against the purge's budget: a
+// late allow and a deny both delete nothing.
+func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) {
+ z := purgeSinglePool(t, 16)
+ p := newPurgeTest(t, z, "review7-purge-auth-time", MakeBucketOptions{VersioningEnabled: true}, "")
+ const key = "purge/key"
+ p.version(0, key, "older", 2)
+ p.version(0, key, "latest", 1)
+ exp := p.matched(key)
+ oldBudget := deleteAllBudget
+ deleteAllBudget.duration = 10 * time.Millisecond
+ t.Cleanup(func() { deleteAllBudget = oldBudget })
+ denied := errors.New("review7: revision denied")
+ for _, decision := range []error{nil, denied} {
+ called := false
+ var res deleteAllResult
+ _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
+ DeleteAllVersions: true,
+ DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+ DeleteAllResult: &res,
+ deleteAllAuthorize: func(versions []ObjectInfo) error {
+ called = true
+ if len(versions) < 2 {
+ t.Error("authorization did not receive both approved versions")
+ }
+ time.Sleep(20 * time.Millisecond)
+ return decision
+ },
+ })
+ want := decision
+ if want == nil {
+ want = errDeleteAllBudget
+ }
+ if !called || !errors.Is(err, want) || res.Started || res.Complete || res.Removed != 0 {
+ t.Errorf("late authorization: called=%v err=%v result=%+v", called, err, res)
+ }
+ if got := p.held(0, key); len(got) != 2 {
+ t.Fatalf("purge removed versions after late authorization: %v", got)
+ }
+ }
+}
+
+// A listing that failed before any result fails the run without moving the
+// cursor or refusing the job; a later run completes it, in both modes.
+func TestBatchExpireWalkFailureResumes(t *testing.T) {
+ for _, purgeAll := range []bool{false, true} {
+ name := "retain-one"
+ if purgeAll {
+ name = "retain-zero"
+ }
+ t.Run(name, func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ const key = "data/key"
+ older := f.putVersions(t, key)
+ definition := f.expireYAML("data/")
+ if purgeAll {
+ definition = f.purgeYAML("data/")
+ }
+ rec, job := f.submit(t, globalActiveCred, definition)
+ if job == nil {
+ t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
+ }
+ listErr := errors.New("review7: transient listing failure")
+ err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job)
+ if !errors.Is(err, listErr) {
+ t.Fatalf("listing error: %v", err)
+ }
+ ri := f.jobInfo(t, job)
+ if !ri.Failed || ri.Complete || ri.FailureReason != "" || ri.Object != "" || !f.versionExists(t, key, older) {
+ t.Fatalf("listing failure changed progress or became terminal: failed=%v complete=%v reason=%q cursor=%q", ri.Failed, ri.Complete, ri.FailureReason, ri.Object)
+ }
+ mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
+ ri = f.jobInfo(t, job)
+ wantRemoved := int64(1)
+ if purgeAll {
+ wantRemoved = 2
+ }
+ if ri.Failed || !ri.Complete || ri.FailureReason != "" || ri.Objects != wantRemoved || f.versionExists(t, key, older) {
+ t.Fatalf("resume did not complete: failed=%v complete=%v reason=%q removed=%d", ri.Failed, ri.Complete, ri.FailureReason, ri.Objects)
+ }
+ })
+ }
+}
diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go
index 17dcf556f..77c444dfc 100644
--- a/cmd/batch-expire.go
+++ b/cmd/batch-expire.go
@@ -27,6 +27,7 @@ import (
"net/http"
"runtime"
"strconv"
+ "sync/atomic"
"time"
"github.com/minio/minio-go/v7/pkg/tags"
@@ -34,6 +35,7 @@ import (
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/pgsty/silo-pkg/v3/env"
+ "github.com/pgsty/silo-pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/wildcard"
"github.com/pgsty/silo-pkg/v3/workers"
"github.com/pgsty/silo-pkg/v3/xtime"
@@ -289,16 +291,6 @@ type BatchJobExpire struct {
var _ yaml.Unmarshaler = &BatchJobExpire{}
-// RedactSensitive will redact any sensitive information in b.
-func (r *BatchJobExpire) RedactSensitive() {
- if r == nil {
- return
- }
- if r.NotificationCfg.Token != "" {
- r.NotificationCfg.Token = redactedText
- }
-}
-
// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
type expireJob BatchJobExpire
@@ -345,32 +337,6 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
return nil
}
-// Expire expires object versions which have already matched supplied filter conditions
-func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
- opts := ObjectOptions{
- PrefixEnabledFn: vc.PrefixEnabled,
- VersionSuspended: vc.Suspended(),
- }
-
- allErrs := make([]error, 0, len(objsToDel))
-
- for {
- count := len(objsToDel)
- if count == 0 {
- break
- }
- if count > maxDeleteList {
- count = maxDeleteList
- }
- _, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
- allErrs = append(allErrs, errs...)
- // Next batch of deletion
- objsToDel = objsToDel[count:]
- }
-
- return allErrs
-}
-
const (
batchExpireName = "batch-expire.bin"
batchExpireFormat = 1
@@ -381,22 +347,233 @@ const (
batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
)
-type objInfoCache map[string]*ObjectInfo
-
-func newObjInfoCache() objInfoCache {
- return objInfoCache(make(map[string]*ObjectInfo))
+type expireObjInfo struct {
+ ObjectInfo
+ ExpireAll bool
+ DeleteMarkerCount int64
}
-func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
- oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
+// isErrObjectLocked reports a version that Object Lock protects.
+func isErrObjectLocked(err error) bool {
+ var locked ObjectLocked
+ return errors.As(err, &locked)
}
-func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
- oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
- return oi, ok
+func (r *BatchJobExpire) tagsFiltered() bool {
+ for _, rule := range r.Rules {
+ if len(rule.Tags) > 0 {
+ return true
+ }
+ }
+ return false
}
-func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
+// expireAuthzRequests are the actions the job takes on one version: it reads
+// the version, and its tags when a rule filters by them, and deletes it.
+func (r *BatchJobExpire) expireAuthzRequests(name, versionID, userTags string) []batchAuthzRequest {
+ if versionID == "" {
+ versionID = nullVersionID
+ }
+ reqs := []batchAuthzRequest{
+ {action: policy.GetObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+ {action: policy.GetObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+ {action: policy.DeleteObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+ {action: policy.DeleteObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+ }
+ if r.tagsFiltered() {
+ reqs = append(reqs,
+ batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+ batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
+ )
+ }
+ return reqs
+}
+
+// expireVersion deletes one version that is older than the versions the rule
+// retains. Under the object's write lock, with the version as it is then, it
+// leaves a version that Object Lock protects (ObjectLocked) and refuses one
+// that the submitter may not delete (batchJobAuthzError). An unreadable
+// version is not deleted.
+func (r *BatchJobExpire) expireVersion(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, exp expireObjInfo) error {
+ p, err := job.Submitter.principal(ctx)
+ if err != nil {
+ return err
+ }
+ versionID := exp.VersionID
+ if versionID == "" {
+ // An empty version ID would add a delete marker instead.
+ versionID = nullVersionID
+ }
+ opts := ObjectOptions{
+ VersionID: versionID,
+ Versioned: vc.PrefixEnabled(exp.Name),
+ VersionSuspended: vc.Suspended(),
+ }
+ opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) error {
+ // A delete marker reads as MethodNotAllowed; any other error,
+ // including a missing version, stops the delete.
+ marker := isErrMethodNotAllowed(gerr) && (goi.DeleteMarker || !goi.VersionPurgeStatus.Empty())
+ if gerr != nil && !marker {
+ return gerr
+ }
+ if enforceRetentionForDeletion(ctx, goi) {
+ return ObjectLocked{Bucket: r.Bucket, Object: exp.Name, VersionID: versionID}
+ }
+ for _, req := range r.expireAuthzRequests(exp.Name, versionID, goi.UserTags) {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ _, err = api.DeleteObject(ctx, r.Bucket, exp.Name, opts)
+ return err
+}
+
+// expireVersions deletes the versions of one batch entry and records the
+// outcome in res. It returns an error that ends the job.
+func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, delay time.Duration, res *batchCounts) error {
+ for attempts := 1; attempts <= retryAttempts; attempts++ {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+ err := r.expireVersion(ctx, api, vc, job, exp)
+ stopFn(exp, err)
+ switch {
+ case err == nil:
+ if exp.DeleteMarker {
+ ri.countRemoved(r.Bucket, 0, 1)
+ } else {
+ ri.countRemoved(r.Bucket, 1, 0)
+ }
+ return nil
+ case isBatchJobAuthzError(err):
+ return err
+ case isErrObjectLocked(err):
+ // Left in place, not retried and not a failure.
+ res.versionsSkipped++
+ return nil
+ case isErrVersionNotFound(err) || isErrObjectNotFound(err):
+ // Removed concurrently.
+ return nil
+ }
+ batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", r.Bucket, exp.Name, exp.VersionID, err, attempts))
+ if attempts < retryAttempts {
+ time.Sleep(delay)
+ }
+ }
+ if exp.DeleteMarker {
+ res.deleteMarkersFailed++
+ } else {
+ res.objectsFailed++
+ }
+ return nil
+}
+
+// expireAllVersions purges every version of a key whose rule retains none,
+// with the purge of exactly that key (erasureServerPools.deleteAllVersions),
+// while the version the walk matched is still its latest. Under its lock,
+// before it deletes anything, the purge refuses the key when Object Lock
+// protects a version and has every write it would delete, each copy and
+// each revision a drive holds, authorized for the submitter, with the
+// version ID and tags it has then.
+//
+// What the purge removed is counted at once; a refused key is counted in res
+// as refused, and the versions a failed purge left in res as failures. A
+// failed purge is retried up to retryAttempts times, each time with the
+// version the walk matched; a purge that stopped at its budget after
+// removing versions continues without spending an attempt. A key whose
+// latest version changed since the walk is not expired. It returns an error
+// that ends the job: an authorization failure or the end of ctx.
+func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error {
+ var objects, markers int64
+ matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
+ for attempts := 1; attempts <= retryAttempts; attempts++ {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+ latest := matched
+ var purge deleteAllResult
+ p, err := job.Submitter.principal(ctx)
+ if err == nil {
+ _, err = api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
+ DeleteAllVersions: true,
+ DeleteAllLatest: &latest,
+ DeleteAllResult: &purge,
+ deleteAllAuthorize: r.expireAllAuthorize(p, exp.Name),
+ })
+ }
+ // Removed versions are gone for any later run: count them now.
+ if purge.Removed > 0 {
+ ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers))
+ objects += int64(purge.Removed - purge.Markers)
+ markers += int64(purge.Markers)
+ }
+ stopFn(exp, err)
+ switch {
+ case err == nil:
+ return nil
+ case isBatchJobAuthzError(err):
+ return err
+ case errors.Is(err, errDeleteAllChanged):
+ // Changed since the walk matched it: not expired.
+ return nil
+ case isErrObjectLocked(err):
+ // Refused as a whole: nothing removed, not retried and not a
+ // failure.
+ res.keysRefused++
+ return nil
+ case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
+ attempts-- // progress
+ continue
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
+ if errors.Is(err, errDeleteAllTiered) {
+ break // refused; a later run of the job checks again
+ }
+ }
+ res.expireAllFailed(exp, objects, markers)
+ return nil
+}
+
+// expireAllAuthorize returns the authorization of a purge of all versions of
+// name for p: each version the purge would delete, as it holds it, needs the
+// actions of expiring it (expireAuthzRequests).
+func (r *BatchJobExpire) expireAllAuthorize(p batchJobPrincipal, name string) func([]ObjectInfo) error {
+ return func(versions []ObjectInfo) error {
+ for _, v := range versions {
+ for _, req := range r.expireAuthzRequests(name, v.VersionID, v.UserTags) {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+ }
+}
+
+// expireAllFailed counts the versions of info, as the walk saw them, that a
+// failed purge of all its versions did not remove: those beyond the objects
+// and markers it removed, at least one.
+func (c *batchCounts) expireAllFailed(info expireObjInfo, objects, markers int64) {
+ failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
+ failedMarkers := max(info.DeleteMarkerCount-markers, 0)
+ if failedObjects+failedMarkers == 0 {
+ failedObjects = 1
+ }
+ c.objectsFailed += failedObjects
+ c.deleteMarkersFailed += failedMarkers
+}
+
+// batchObjsForDelete expires the batches of expireCh. walkDone reports, once
+// expireCh is closed, whether the walk reached its end: only then is its last
+// key known to be complete.
+func batchObjsForDelete(ctx context.Context, refuse func(error), walkDone func() bool, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
vc, _ := globalBucketVersioningSys.Get(r.Bucket)
retryAttempts := job.Expire.Retry.Attempts
if retryAttempts <= 0 {
@@ -407,162 +584,53 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
delay = batchExpireJobDefaultRetryDelay
}
- var i int
+ progress := newBatchProgress(ri, r.Bucket)
+ defer func() {
+ wk.Wait()
+ if walkDone() {
+ progress.finish()
+ }
+ }()
for toExpire := range expireCh {
select {
case <-ctx.Done():
return
default:
}
- if i > 0 {
+ seq := progress.begin()
+ if seq > 0 {
if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
time.Sleep(wait)
}
}
- i++
wk.Take()
- go func(toExpire []expireObjInfo) {
+ go func(seq uint64, toExpire []expireObjInfo) {
defer wk.Give()
-
- toExpireAll := make([]expireObjInfo, 0, len(toExpire))
- toDel := make([]ObjectToDelete, 0, len(toExpire))
- oiCache := newObjInfoCache()
+ var res batchCounts
for _, exp := range toExpire {
+ var err error
if exp.ExpireAll {
- toExpireAll = append(toExpireAll, exp)
- continue
+ err = r.expireAllVersions(ctx, api, job, ri, exp, retryAttempts, &res)
+ } else {
+ err = r.expireVersions(ctx, api, vc, job, ri, exp, retryAttempts, delay, &res)
}
- // Cache ObjectInfo value via pointers for
- // subsequent use to track objects which
- // couldn't be deleted.
- od := ObjectToDelete{
- ObjectV: ObjectV{
- ObjectName: exp.Name,
- VersionID: exp.VersionID,
- },
- }
- toDel = append(toDel, od)
- oiCache.Add(od, &exp.ObjectInfo)
- }
-
- // Purge all versions of exactly each key while the version the
- // walk matched is still its latest; see
- // erasureServerPools.deleteAllVersions.
- for _, exp := range toExpireAll {
- if !expireAllVersions(ctx, ri, api, exp, retryAttempts) {
- return
- }
- }
-
- // DeleteMultiple objects
- toDelCopy := make([]ObjectToDelete, len(toDel))
- for attempts := 1; attempts <= retryAttempts; attempts++ {
- select {
- case <-ctx.Done():
- return
- default:
- }
-
- stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
- // Copying toDel to select from objects whose
- // deletion failed
- copy(toDelCopy, toDel)
- var failed int
- errs := r.Expire(ctx, api, vc, toDel)
- // reslice toDel in preparation for next retry attempt
- toDel = toDel[:0]
- for i, err := range errs {
- if err != nil {
- stopFn(toDelCopy[i], err)
- batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
- err, attempts))
- failed++
- if oi, ok := oiCache.Get(toDelCopy[i]); ok {
- ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)
- }
- if attempts != retryAttempts {
- // retry
- toDel = append(toDel, toDelCopy[i])
- }
- } else {
- stopFn(toDelCopy[i], nil)
- if oi, ok := oiCache.Get(toDelCopy[i]); ok {
- ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts)
- }
+ if err != nil {
+ // The batch is not done; a later run does it again.
+ if isBatchJobAuthzError(err) {
+ refuse(err)
}
- }
-
- globalBatchJobsMetrics.save(ri.JobID, ri)
-
- if failed == 0 {
- break
- }
-
- // Add a delay between retry attempts
- if attempts < retryAttempts {
- time.Sleep(delay)
+ globalBatchJobsMetrics.save(ri.JobID, ri)
+ return
}
}
- }(toExpire)
+ if n := len(toExpire); n > 0 {
+ progress.done(seq, toExpire[n-1].Name, res)
+ }
+ globalBatchJobsMetrics.save(ri.JobID, ri)
+ }(seq, toExpire)
}
}
-// expireAllVersions purges all versions of the key exp, which the walk
-// matched by its latest version, and counts what the purge removed. It
-// retries a failed purge up to attempts times, each time with the version
-// the walk matched; a purge that stopped at its budget continues without
-// spending an attempt. It does not retry once the key no longer has that
-// latest version, which is not an expiry, or once the purge refused the
-// key. It returns false once ctx ends.
-func expireAllVersions(ctx context.Context, ri *batchJobInfo, api ObjectLayer, exp expireObjInfo, attempts int) bool {
- var objects, markers int64
- matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
- for attempt := 1; attempt <= attempts; attempt++ {
- select {
- case <-ctx.Done():
- ri.trackExpireAll(exp, objects, markers, true)
- return false
- default:
- }
- stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempt)
- latest := matched
- var purge deleteAllResult
- _, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
- DeleteAllVersions: true,
- DeleteAllLatest: &latest,
- DeleteAllResult: &purge,
- })
- objects += int64(purge.Removed - purge.Markers)
- markers += int64(purge.Markers)
- stopFn(exp, err)
- switch {
- case err == nil:
- ri.trackExpireAll(exp, objects, markers, false)
- return true
- case errors.Is(err, errDeleteAllChanged):
- // Changed since the walk matched it: not expired.
- ri.trackExpireAll(exp, objects, markers, false)
- return true
- case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
- attempt-- // progress
- continue
- }
- batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempt))
- var locked ObjectLocked
- if errors.Is(err, errDeleteAllTiered) || errors.As(err, &locked) {
- break // refused; a later run of the job checks again
- }
- }
- ri.trackExpireAll(exp, objects, markers, true)
- return true
-}
-
-type expireObjInfo struct {
- ObjectInfo
- ExpireAll bool
- DeleteMarkerCount int64
-}
-
// Start the batch expiration job, resumes if there was a pending job via "job.ID"
func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
ri := &batchJobInfo{
@@ -573,6 +641,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
if err := ri.loadOrInit(ctx, api, job); err != nil {
return err
}
+ if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
+ return err
+ }
globalBatchJobsMetrics.save(job.ID, ri)
lastObject := ri.Object
@@ -592,27 +663,46 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
ctx, cancelCause := context.WithCancelCause(ctx)
defer cancelCause(nil)
+ // refuse ends the job for its submitter: record why before stopping.
+ refuse := func(err error) {
+ ri.refuse(err)
+ cancelCause(err)
+ }
+ // The walk stops at a listing failure: work after it is left to a later
+ // run, so the cursor cannot move past what was not listed.
+ walkCtx, stopWalk := context.WithCancel(ctx)
+ defer stopWalk()
+ // walkFailed: a listing failed before it returned any result.
+ var walkFailed atomic.Bool
results := make(chan itemOrErr[ObjectInfo], workerSize)
go func() {
- prefixes := r.Prefix.F()
- if len(prefixes) == 0 {
- prefixes = []string{""}
- }
- for _, prefix := range prefixes {
+ // One ascending walk over disjoint prefixes, which the cursor
+ // describes.
+ for _, prefix := range batchCanonicalPrefixes(r.Prefix.F()) {
+ if walkCtx.Err() != nil {
+ break
+ }
prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
- err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
+ err := api.Walk(walkCtx, r.Bucket, prefix, prefixResultCh, WalkOptions{
Marker: lastObject,
LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions
VersionsSort: WalkVersionsSortDesc,
})
if err != nil {
- cancelCause(err)
+ if walkCtx.Err() == nil {
+ walkFailed.Store(true)
+ cancelCause(err)
+ }
xioutil.SafeClose(results)
return
}
for result := range prefixResultCh {
- results <- result
+ select {
+ case results <- result:
+ case <-walkCtx.Done():
+ // Stopped: let the walk end without a reader.
+ }
}
}
xioutil.SafeClose(results)
@@ -620,7 +710,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
// Goroutine to periodically save batch-expire job's in-memory state
saverQuitCh := make(chan struct{})
+ saverDoneCh := make(chan struct{})
go func() {
+ defer close(saverDoneCh)
saveTicker := time.NewTicker(10 * time.Second)
defer saveTicker.Stop()
quit := false
@@ -647,9 +739,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
expireCh := make(chan []expireObjInfo, workerSize)
expireDoneCh := make(chan struct{})
+ var walkDone atomic.Bool
go func() {
defer close(expireDoneCh)
- batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+ batchObjsForDelete(ctx, refuse, walkDone.Load, r, ri, job, api, wk, expireCh)
}()
var (
@@ -661,6 +754,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
done bool
)
deleteMarkerCountMap := map[string]int64{}
+ // pushToExpire is called between keys only, so that a batch holds all
+ // the versions of its keys and its last key is done with it.
pushToExpire := func() {
// set preObject deleteMarkerCount
if len(toDel) > 0 {
@@ -695,6 +790,12 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
if result.Err != nil {
failed = true
batchLogIf(ctx, result.Err)
+ stopWalk()
+ done = true
+ break
+ }
+ if !batchResumeAfter(lastObject, result.Item.Name) {
+ // Completed by the run that set the cursor.
continue
}
if result.Item.DeleteMarker {
@@ -752,7 +853,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
toDel = append(toDel, expireObjInfo{
ObjectInfo: result.Item,
})
- pushToExpire()
case <-ctx.Done():
done = true
}
@@ -761,16 +861,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
}
}
- if context.Cause(ctx) != nil {
- xioutil.SafeClose(expireCh)
- return context.Cause(ctx)
- }
- pushToExpire()
- // Send any remaining objects downstream
- if len(toDel) > 0 {
- select {
- case <-ctx.Done():
- case expireCh <- toDel:
+ if context.Cause(ctx) == nil {
+ walkDone.Store(!failed)
+ pushToExpire()
+ // Send any remaining objects downstream
+ if len(toDel) > 0 {
+ select {
+ case <-ctx.Done():
+ case expireCh <- toDel:
+ }
}
}
xioutil.SafeClose(expireCh)
@@ -778,13 +877,36 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
<-expireDoneCh // waits for the expire goroutine to complete
wk.Wait() // waits for all expire workers to retire
- ri.Complete = !failed && !ri.expireFailures()
- ri.Failed = failed || ri.expireFailures()
- globalBatchJobsMetrics.save(job.ID, ri)
-
- // Close the saverQuitCh - this also triggers saving in-memory state
- // immediately one last time before we exit this method.
+ // Persist the final state once the saver has stopped, so that its last
+ // save cannot overwrite it.
xioutil.SafeClose(saverQuitCh)
+ <-saverDoneCh
+ refused := ri.refused()
+ cause := context.Cause(ctx)
+ if cause != nil && !refused {
+ if walkFailed.Load() {
+ // This run failed; a later run lists again from the cursor.
+ ri.mu.Lock()
+ ri.Complete = false
+ ri.Failed = true
+ ri.mu.Unlock()
+ }
+ saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
+ batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
+ cancel()
+ globalBatchJobsMetrics.save(job.ID, ri)
+ return cause
+ }
+
+ ri.mu.Lock()
+ itemsFailed := ri.expireFailures()
+ ri.Complete = !refused && !failed && !itemsFailed
+ ri.Failed = refused || failed || itemsFailed
+ ri.mu.Unlock()
+ globalBatchJobsMetrics.save(job.ID, ri)
+ saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
+ batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
+ cancel()
// Notify expire jobs final status to the configured endpoint
buf, _ := json.Marshal(ri)
diff --git a/cmd/batch-handlers.go b/cmd/batch-handlers.go
index dba3c6a20..28985f728 100644
--- a/cmd/batch-handlers.go
+++ b/cmd/batch-handlers.go
@@ -43,6 +43,8 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/tags"
+ "github.com/minio/minio/internal/amztime"
+ objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/config/batch"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/hash"
@@ -72,32 +74,47 @@ type BatchJobRequest struct {
Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
Expire *BatchJobExpire `yaml:"expire" json:"expire"`
- ctx context.Context `msg:"-"`
+ // Submitter is the credential the job acts as. The server assigns it
+ // at submission; a job definition cannot name it.
+ Submitter *batchJobSubmitter `yaml:"-" json:"-"`
+ ctx context.Context `msg:"-"`
}
-// RedactSensitive will redact any sensitive information in b.
-func (j *BatchJobRequest) RedactSensitive() {
- j.Replicate.RedactSensitive()
- j.Expire.RedactSensitive()
- j.KeyRotate.RedactSensitive()
+// redacted returns a copy of the job for output outside the server: the
+// describe response, logs and errors. The copy has no secret keys, session
+// tokens or notification tokens, and no submitter binding; the job itself,
+// which a run needs as it is, is not changed.
+func (j BatchJobRequest) redacted() (BatchJobRequest, error) {
+ buf, err := j.MarshalMsg(nil)
+ if err != nil {
+ return BatchJobRequest{}, err
+ }
+ var c BatchJobRequest
+ if _, err = c.UnmarshalMsg(buf); err != nil {
+ return BatchJobRequest{}, err
+ }
+ redact := func(s *string) {
+ if *s != "" {
+ *s = redactedText
+ }
+ }
+ c.Submitter = nil
+ if r := c.Replicate; r != nil {
+ redact(&r.Source.Creds.SecretKey)
+ redact(&r.Source.Creds.SessionToken)
+ redact(&r.Target.Creds.SecretKey)
+ redact(&r.Target.Creds.SessionToken)
+ redact(&r.Flags.Notify.Token)
+ }
+ if r := c.KeyRotate; r != nil {
+ redact(&r.Flags.Notify.Token)
+ }
+ if r := c.Expire; r != nil {
+ redact(&r.NotificationCfg.Token)
+ }
+ return c, nil
}
-// RedactSensitive will redact any sensitive information in b.
-func (r *BatchJobReplicateV1) RedactSensitive() {
- if r == nil {
- return
- }
- if r.Target.Creds.SecretKey != "" {
- r.Target.Creds.SecretKey = redactedText
- }
- if r.Target.Creds.SessionToken != "" {
- r.Target.Creds.SessionToken = redactedText
- }
-}
-
-// RedactSensitive will redact any sensitive information in b.
-func (r *BatchJobKeyRotateV1) RedactSensitive() {}
-
func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error {
if endpoint == "" {
return nil
@@ -140,8 +157,114 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error
return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
}
-// ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
-func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
+// batchPullObjectLock prepares the Object Lock metadata that a pull writes
+// with a version, as a PUT would on the destination: Object Lock metadata
+// copied from the source must be valid, a version without retention takes the
+// destination bucket's default retention, and both are dated with the
+// destination's clock. Outside a lock-enabled bucket the metadata has no
+// meaning and is not written. It reports the retention and legal hold that
+// the pull copies from the source.
+func batchPullObjectLock(bucket string, meta map[string]string) (retention, legalHold bool, err error) {
+ cfg, err := globalBucketObjectLockSys.Get(bucket)
+ if err != nil {
+ return false, false, err
+ }
+ h := make(http.Header)
+ for k, v := range meta {
+ for _, name := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate, xhttp.AmzObjectLockLegalHold} {
+ if strings.EqualFold(k, name) {
+ h.Set(name, v)
+ delete(meta, k)
+ }
+ }
+ }
+ if !cfg.LockEnabled {
+ return false, false, nil
+ }
+ if h.Get(xhttp.AmzObjectLockMode) != "" || h.Get(xhttp.AmzObjectLockRetainUntilDate) != "" {
+ // A copied retention may have expired already.
+ mode, until, err := objectlock.ParseObjectLockRetentionHeaders(h, true)
+ if err != nil {
+ return false, false, err
+ }
+ meta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(mode)
+ meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(until.UTC())
+ retention = true
+ } else if cfg.Validity > 0 {
+ now, err := objectlock.UTCNowNTP()
+ if err != nil {
+ return false, false, err
+ }
+ meta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(cfg.Mode)
+ meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(now.Add(cfg.Validity).UTC())
+ }
+ if _, ok := h[xhttp.AmzObjectLockLegalHold]; ok {
+ hold, err := objectlock.ParseObjectLockLegalHoldHeaders(h)
+ if err != nil {
+ return false, false, err
+ }
+ meta[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(hold.Status)
+ legalHold = true
+ }
+ return retention, legalHold, nil
+}
+
+// pullAuthzRequests are the writes of one version that a pull makes to the
+// destination key: the version, with the tags, retention and legal hold it
+// copies, and the replication permission for a version ID it keeps.
+//
+// The pull does not encrypt what it writes, so it supplies no encryption
+// values, whatever the source reports; and it does not read the destination
+// version it may replace, so the tags of that version are unknown.
+func pullAuthzRequests(bucket, object, versionID string, keepsVersion bool, opts ObjectOptions, retention, legalHold bool) []batchAuthzRequest {
+ values := map[string][]string{}
+ meta := caseInsensitiveMap(opts.UserDefined)
+ if retention {
+ mode, _ := meta.Lookup(xhttp.AmzObjectLockMode)
+ until, _ := meta.Lookup(xhttp.AmzObjectLockRetainUntilDate)
+ values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockMode), "x-amz-")] = []string{mode}
+ values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), "x-amz-")] = []string{until}
+ }
+ if legalHold {
+ hold, _ := meta.Lookup(xhttp.AmzObjectLockLegalHold)
+ values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockLegalHold), "x-amz-")] = []string{hold}
+ }
+ tagStr, hasTags := meta.Lookup(xhttp.AmzObjectTagging)
+ if hasTags && tagStr != "" {
+ if t, err := tags.ParseObjectTags(tagStr); err == nil {
+ m := t.ToMap()
+ keys := make([]string, 0, len(m))
+ for k, v := range m {
+ values[pathJoin("RequestObjectTag", k)] = []string{v}
+ keys = append(keys, k)
+ }
+ values["RequestObjectTagKeys"] = keys
+ }
+ }
+ actions := []policy.Action{policy.PutObjectAction}
+ if keepsVersion {
+ actions = append(actions, policy.ReplicateObjectAction)
+ }
+ if hasTags && tagStr != "" {
+ actions = append(actions, policy.PutObjectTaggingAction)
+ }
+ if retention {
+ actions = append(actions, policy.PutObjectRetentionAction)
+ }
+ if legalHold {
+ actions = append(actions, policy.PutObjectLegalHoldAction)
+ }
+ reqs := make([]batchAuthzRequest, 0, len(actions))
+ for _, action := range actions {
+ reqs = append(reqs, batchAuthzRequest{action: action, bucket: bucket, object: object, versionID: versionID, tagsUnknown: true, values: values})
+ }
+ return reqs
+}
+
+// ReplicateFromSource copies one version from the remote source to the local
+// target. A write that would replace a version protected by Object Lock
+// returns ObjectLocked and leaves it.
+func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool, sub *batchJobSubmitter) error {
srcBucket := r.Source.Bucket
tgtBucket := r.Target.Bucket
srcObject := srcObjInfo.Name
@@ -150,11 +273,28 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
}
+ p, err := sub.principal(ctx)
+ if err != nil {
+ return err
+ }
versionID := srcObjInfo.VersionID
if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
versionID = ""
}
if srcObjInfo.DeleteMarker {
+ markerVersion := versionID
+ if markerVersion == "" {
+ markerVersion = nullVersionID
+ }
+ for _, req := range []batchAuthzRequest{
+ {action: policy.DeleteObjectAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true},
+ {action: policy.DeleteObjectVersionAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true},
+ {action: policy.ReplicateDeleteAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true},
+ } {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
VersionID: versionID,
// Since we are preserving a delete marker, we have to make sure this is always true.
@@ -164,22 +304,43 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
MTime: srcObjInfo.ModTime,
DeleteMarker: srcObjInfo.DeleteMarker,
ReplicationRequest: true,
+ // A new marker may hide a locked version; one that would replace
+ // it is refused.
+ RefuseLockedOverwrite: true,
})
return err
}
- opts := ObjectOptions{
- VersionID: srcObjInfo.VersionID,
- MTime: srcObjInfo.ModTime,
- PreserveETag: srcObjInfo.ETag,
- UserDefined: srcObjInfo.UserDefined,
+ userDefined := cloneMSS(srcObjInfo.UserDefined)
+ retention, legalHold, err := batchPullObjectLock(tgtBucket, userDefined)
+ if err != nil {
+ return err
}
- if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
- opts.VersionID = ""
+ opts := ObjectOptions{
+ VersionID: versionID,
+ Versioned: globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject),
+ VersionSuspended: globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject),
+ MTime: srcObjInfo.ModTime,
+ PreserveETag: srcObjInfo.ETag,
+ UserDefined: userDefined,
+ // Checked with the commit, under the destination's write lock.
+ RefuseLockedOverwrite: true,
}
if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) {
opts.ServerSideEncryption = encrypt.NewSSE()
}
+ // The version the write replaces or creates: the source's version ID it
+ // keeps, or the null version of a destination that does not version
+ // the key. A versioned destination gives a write without an ID a new one.
+ writeVersion := versionID
+ if writeVersion == "" && !opts.Versioned {
+ writeVersion = nullVersionID
+ }
+ for _, req := range pullAuthzRequests(tgtBucket, tgtObject, writeVersion, versionID != "", opts, retention, legalHold) {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
slc := strings.Split(srcObjInfo.ETag, "-")
if len(slc) == 2 {
partsCount, err := strconv.Atoi(slc[1])
@@ -298,6 +459,9 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
if ri.Complete {
return nil
}
+ if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
+ return err
+ }
globalBatchJobsMetrics.save(job.ID, ri)
retryAttempts := job.Replicate.Flags.Retry.Attempts
@@ -411,7 +575,17 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
// one of source/target is s3, skip delete marker and all versions under the same object name.
s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
- ctx, cancel := context.WithCancel(ctx)
+ ctx, cancelCause := context.WithCancelCause(ctx)
+ // refuse ends the job for its submitter: record why before stopping.
+ refuse := func(err error) {
+ ri.refuse(err)
+ cancelCause(err)
+ }
+ // Each run lists the source from the start and counts every locked
+ // collision again.
+ ri.mu.Lock()
+ ri.VersionsSkippedLocked = 0
+ ri.mu.Unlock()
objInfoCh := make(chan minio.ObjectInfo, 1)
go func() {
@@ -437,6 +611,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
skipReplicate := false
for obj := range objInfoCh {
+ if context.Cause(ctx) != nil {
+ // Refused or canceled: schedule nothing more.
+ continue
+ }
oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
if !minioSrc {
// Check if metadata filter was requested and it is expected to have
@@ -484,7 +662,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
defer wk.Give()
stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
success := true
- if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil {
+ if err := r.ReplicateFromSource(ctx, api, core, oi, retry, job.Submitter); err != nil {
+ if isBatchJobAuthzError(err) {
+ refuse(err)
+ return
+ }
+ // A locked destination version is left as it is: the
+ // job completes with it skipped, not replicated.
+ if isErrObjectLocked(err) {
+ stopFn(oi, err)
+ ri.countSkipped()
+ globalBatchJobsMetrics.save(job.ID, ri)
+ return
+ }
// object must be deleted concurrently, allow these failures but do not count them
if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
return
@@ -495,7 +685,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
} else {
stopFn(oi, nil)
}
- ri.trackCurrentBucketObject(r.Target.Bucket, oi, success, attempts)
+ ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk after every 10secs.
batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
@@ -507,9 +697,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
}
wk.Wait()
+ if ri.refused() {
+ cancelCause(nil)
+ globalBatchJobsMetrics.save(job.ID, ri)
+ batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+ if err := r.Notify(GlobalContext, ri); err != nil {
+ batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+ }
+ return nil
+ }
+
ri.RetryAttempts = attempts
- ri.Complete = ri.ObjectsFailed == 0
- ri.Failed = ri.ObjectsFailed > 0
+ ri.Complete = ri.ObjectsFailed == 0 && ri.DeleteMarkersFailed == 0
+ ri.Failed = !ri.Complete
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk.
@@ -519,12 +719,14 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
}
- cancel()
+ cancelCause(nil)
if ri.Failed {
ri.ObjectsFailed = 0
+ ri.DeleteMarkersFailed = 0
ri.Bucket = ""
ri.Object = ""
ri.Objects = 0
+ ri.DeleteMarkers = 0
ri.BytesFailed = 0
ri.BytesTransferred = 0
retry = true // indicate we are retrying..
@@ -580,7 +782,10 @@ func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
return oi
}
-func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
+func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string, sub *batchJobSubmitter) error {
+ // An entry the submitter may not read ends the archive and the job; an
+ // entry whose metadata cannot be read ends the archive.
+ var refused, readErr error
input := make(chan minio.SnowballObject, 1)
opts := minio.SnowballOptions{
Opts: minio.PutObjectOptions{
@@ -593,17 +798,41 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
SkipErrs: *r.Source.Snowball.SkipErrs,
}
+ uploadCtx, cancelUpload := context.WithCancel(ctx)
+ defer cancelUpload()
+ producerDone := make(chan struct{})
go func() {
+ defer close(producerDone)
defer xioutil.SafeClose(input)
for _, entry := range entries {
- gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket,
+ gr, err := objAPI.GetObjectNInfo(uploadCtx, r.Source.Bucket,
entry.Name, nil, nil, ObjectOptions{
VersionID: entry.VersionID,
})
if err != nil {
batchLogIf(ctx, err)
- continue
+ if isErrObjectNotFound(err) || isErrVersionNotFound(err) || *r.Source.Snowball.SkipErrs {
+ continue
+ }
+ // Not sent and not skippable: the batch goes to the
+ // per-object path.
+ readErr = err
+ return
+ }
+ // Each entry is a decision of its own, for the submitter as it is,
+ // on the metadata of all copies of the version.
+ logical, lerr := batchAuthoritativeRead(uploadCtx, objAPI, r.Source.Bucket, entry.Name, gr.ObjInfo)
+ if lerr != nil {
+ // Not sent: the batch goes to the per-object path.
+ gr.Close()
+ readErr = lerr
+ return
+ }
+ refused = sub.authorize(uploadCtx, append(r.pushAuthzRequests(gr.ObjInfo), r.pushAuthzRequests(logical)...)...)
+ if refused != nil {
+ gr.Close()
+ return
}
if prefix != "" {
@@ -625,6 +854,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
opts, _, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
if err != nil {
+ gr.Close()
batchLogIf(ctx, err)
continue
}
@@ -635,23 +865,75 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
}
}
- input <- snowballObj
+ select {
+ case input <- snowballObj:
+ case <-uploadCtx.Done():
+ gr.Close()
+ return
+ }
}
}()
// Collect and upload all entries.
- return remoteClnt.PutObjectsSnowball(ctx, r.Target.Bucket, opts, input)
+ err := remoteClnt.PutObjectsSnowball(uploadCtx, r.Target.Bucket, opts, input)
+ cancelUpload()
+ <-producerDone
+ if refused != nil {
+ return refused
+ }
+ if readErr != nil {
+ return readErr
+ }
+ return err
+}
+
+// pushAuthzRequests are the reads of one source version that a push copies to
+// the target: the version itself, and its tags, retention and legal hold.
+func (r *BatchJobReplicateV1) pushAuthzRequests(oi ObjectInfo) []batchAuthzRequest {
+ bucket, object, versionID := r.Source.Bucket, oi.Name, oi.VersionID
+ if versionID == "" {
+ versionID = nullVersionID
+ }
+ reqs := []batchAuthzRequest{
+ {action: policy.GetObjectAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
+ {action: policy.GetObjectVersionAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
+ }
+ if oi.UserTags != "" {
+ reqs = append(reqs,
+ batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
+ batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags},
+ )
+ }
+ meta := caseInsensitiveMap(oi.UserDefined)
+ _, mode := meta.Lookup(xhttp.AmzObjectLockMode)
+ _, until := meta.Lookup(xhttp.AmzObjectLockRetainUntilDate)
+ if mode || until {
+ reqs = append(reqs, batchAuthzRequest{action: policy.GetObjectRetentionAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags})
+ }
+ if _, hold := meta.Lookup(xhttp.AmzObjectLockLegalHold); hold {
+ reqs = append(reqs, batchAuthzRequest{action: policy.GetObjectLegalHoldAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags})
+ }
+ return reqs
}
// ReplicateToTarget read from source and replicate to configured target
-func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
+func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool, sub *batchJobSubmitter) error {
srcBucket := r.Source.Bucket
tgtBucket := r.Target.Bucket
tgtPrefix := r.Target.Prefix
srcObject := srcObjInfo.Name
s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
+ p, err := sub.principal(ctx)
+ if err != nil {
+ return err
+ }
if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() {
+ for _, req := range r.pushAuthzRequests(srcObjInfo) {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
if retry && !s3Type {
if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.StatObjectOptions{
VersionID: srcObjInfo.VersionID,
@@ -707,6 +989,19 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
}
defer rd.Close()
objInfo := rd.ObjInfo
+ // Authorize the version as it is read, with the metadata of all its
+ // copies.
+ logical, err := batchAuthoritativeRead(ctx, api, srcBucket, srcObject, objInfo)
+ if err != nil {
+ return err
+ }
+ // What is sent comes from the opened copy; both it and the version as
+ // all copies describe it must be authorized.
+ for _, req := range append(r.pushAuthzRequests(objInfo), r.pushAuthzRequests(logical)...) {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
size, err := objInfo.GetActualSize()
if err != nil {
@@ -760,6 +1055,16 @@ type batchJobInfo struct {
DeleteMarkersFailed int64 `json:"deleteMarkersFailed" msg:"dmf"`
BytesTransferred int64 `json:"bytesTransferred" msg:"bt"`
BytesFailed int64 `json:"bytesFailed" msg:"bf"`
+
+ // Versions the job left in place because Object Lock protects them,
+ // and keys it refused as a whole for the same reason. A job that
+ // completes with them has not acted on those versions.
+ VersionsSkippedLocked int64 `json:"versionsSkippedLocked" msg:"vsl"`
+ KeysRefused int64 `json:"keysRefused" msg:"kr"`
+
+ // FailureReason is set when the job was refused for its submitter. It
+ // ends the job: a job that has one does not run again.
+ FailureReason string `json:"failureReason,omitempty" msg:"fr"`
}
const (
@@ -901,40 +1206,31 @@ func (ri *batchJobInfo) clone() *batchJobInfo {
BytesTransferred: ri.BytesTransferred,
BytesFailed: ri.BytesFailed,
Attempts: ri.Attempts,
+
+ VersionsSkippedLocked: ri.VersionsSkippedLocked,
+ KeysRefused: ri.KeysRefused,
+ FailureReason: ri.FailureReason,
}
}
-func (ri *batchJobInfo) countItem(size int64, dmarker, success bool, attempt int) {
+// countItem counts the outcome of one item in the current attempt of a
+// replicate job, which resets its counters before it retries.
+func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) {
if ri == nil {
return
}
ri.Attempts++
- if success {
- if dmarker {
- ri.DeleteMarkers++
- } else {
- ri.Objects++
- ri.BytesTransferred += size
- }
- if attempt > 1 {
- if dmarker {
- ri.DeleteMarkersFailed--
- } else {
- ri.ObjectsFailed--
- ri.BytesFailed += size
- }
- }
- } else {
- if attempt > 1 {
- // Only count first attempt
- return
- }
- if dmarker {
- ri.DeleteMarkersFailed++
- } else {
- ri.ObjectsFailed++
- ri.BytesFailed += size
- }
+ switch {
+ case success && dmarker:
+ ri.DeleteMarkers++
+ case success:
+ ri.Objects++
+ ri.BytesTransferred += size
+ case dmarker:
+ ri.DeleteMarkersFailed++
+ default:
+ ri.ObjectsFailed++
+ ri.BytesFailed += size
}
}
@@ -1002,35 +1298,7 @@ func (ri *batchJobInfo) expireFailures() bool {
return ri.ObjectsFailed > 0 || ri.DeleteMarkersFailed > 0
}
-// trackExpireAll counts the objects and delete markers that an expiry of all
-// versions of info removed and, when it failed, those the walk saw that it
-// did not remove, at least one. Only batch-expire uses it, for a rule that
-// expires all versions of an object.
-func (ri *batchJobInfo) trackExpireAll(info expireObjInfo, objects, markers int64, failed bool) {
- if ri == nil {
- return
- }
-
- ri.mu.Lock()
- defer ri.mu.Unlock()
-
- ri.Objects += objects
- ri.DeleteMarkers += markers
- if !failed {
- ri.Bucket = info.Bucket
- ri.Object = info.Name
- return
- }
- failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
- failedMarkers := max(info.DeleteMarkerCount-markers, 0)
- if failedObjects+failedMarkers == 0 {
- failedObjects = 1
- }
- ri.ObjectsFailed += failedObjects
- ri.DeleteMarkersFailed += failedMarkers
-}
-
-func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool, attempt int) {
+func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool) {
if ri == nil {
return
}
@@ -1042,7 +1310,7 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
ri.Bucket = bucket
ri.Object = info.Name
}
- ri.countItem(info.Size, info.DeleteMarker, success, attempt)
+ ri.countItem(info.Size, info.DeleteMarker, success)
}
func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInfo) {
@@ -1056,7 +1324,7 @@ func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInf
ri.Bucket = bucket
for i := range batch {
ri.Object = batch[i].Name
- ri.countItem(batch[i].Size, batch[i].DeleteMarker, true, 1)
+ ri.countItem(batch[i].Size, batch[i].DeleteMarker, true)
}
}
@@ -1073,6 +1341,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if ri.Complete {
return nil
}
+ if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
+ return err
+ }
globalBatchJobsMetrics.save(job.ID, ri)
lastObject := ri.Object
@@ -1180,6 +1451,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
walkCh = make(chan itemOrErr[ObjectInfo], 100)
slowCh = make(chan itemOrErr[ObjectInfo], 100)
)
+ ctx, cancelCause := context.WithCancelCause(ctx)
+ // refuse ends the job for its submitter: record why before stopping.
+ refuse := func(err error) {
+ ri.refuse(err)
+ cancelCause(err)
+ }
if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
go func() {
@@ -1201,7 +1478,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
writeFn := func(batch []ObjectInfo) {
if len(batch) > 0 {
- if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil {
+ if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix, job.Submitter); err != nil {
+ if isBatchJobAuthzError(err) {
+ refuse(err)
+ return
+ }
batchLogOnceIf(ctx, err, job.ID+"writeAsArchive")
for _, b := range batch {
slowCh <- itemOrErr[ObjectInfo]{Item: b}
@@ -1250,7 +1531,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if walkQuorum == "" {
walkQuorum = "strict"
}
- ctx, cancelCause := context.WithCancelCause(ctx)
// one of source/target is s3, skip delete marker and all versions under the same object name.
s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
@@ -1281,6 +1561,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
skipReplicate := false
for res := range slowCh {
+ if context.Cause(ctx) != nil {
+ // Refused or canceled: schedule nothing more.
+ continue
+ }
if res.Err != nil {
ri.Failed = true
batchLogOnceIf(ctx, res.Err, job.ID+"res.Err")
@@ -1300,7 +1584,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
success := true
- if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
+ if err := r.ReplicateToTarget(ctx, api, c, result, retry, job.Submitter); err != nil {
+ if isBatchJobAuthzError(err) {
+ refuse(err)
+ return
+ }
if minio.ToErrorResponse(err).Code == "PreconditionFailed" {
// pre-condition failed means we already have the object copied over.
return
@@ -1315,7 +1603,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
} else {
stopFn(result, nil)
}
- ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts)
+ ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk after every 10secs.
batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2")
@@ -1326,13 +1614,22 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
}()
}
wk.Wait()
+ if ri.refused() {
+ cancelCause(nil)
+ globalBatchJobsMetrics.save(job.ID, ri)
+ batchLogOnceIf(ctx, ri.updateAfter(GlobalContext, api, 0, job), job.ID+"updateAfter3")
+ if err := r.Notify(GlobalContext, ri); err != nil {
+ batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify")
+ }
+ return nil
+ }
// Do not need to retry if we can't list objects on source.
if context.Cause(ctx) != nil {
return context.Cause(ctx)
}
ri.RetryAttempts = attempts
- ri.Complete = ri.ObjectsFailed == 0
- ri.Failed = ri.ObjectsFailed > 0
+ ri.Complete = ri.ObjectsFailed == 0 && ri.DeleteMarkersFailed == 0
+ ri.Failed = !ri.Complete
globalBatchJobsMetrics.save(job.ID, ri)
// persist in-memory state to disk.
@@ -1345,9 +1642,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
cancelCause(nil)
if ri.Failed {
ri.ObjectsFailed = 0
+ ri.DeleteMarkersFailed = 0
ri.Bucket = ""
ri.Object = ""
ri.Objects = 0
+ ri.DeleteMarkers = 0
ri.BytesFailed = 0
ri.BytesTransferred = 0
retry = true // indicate we are retrying..
@@ -1549,8 +1848,12 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
return errInvalidArgument
}
-func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
- deleteConfig(ctx, api, getJobPath(j))
+func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) error {
+ err := deleteConfig(ctx, api, getJobPath(j))
+ if errors.Is(err, errConfigNotFound) {
+ return nil
+ }
+ return err
}
func (j BatchJobRequest) getJobReportPath() (string, error) {
@@ -1709,7 +2012,7 @@ func (a adminAPIHandlers) BatchJobStatus(w http.ResponseWriter, r *http.Request)
return
}
- buf, err := json.Marshal(madmin.BatchJobStatus{LastMetric: ri.metric()})
+ buf, err := json.Marshal(batchJobStatus{LastMetric: ri.statusMetric()})
if err != nil {
batchLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1747,8 +2050,13 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
}
// Remove sensitive fields.
- req.RedactSensitive()
- buf, err := yaml.Marshal(req)
+ out, err := req.redacted()
+ if err != nil {
+ batchLogIf(ctx, err)
+ writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+ return
+ }
+ buf, err := yaml.Marshal(out)
if err != nil {
batchLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1806,6 +2114,27 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
}
}
+ if job.jobSections() != 1 {
+ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, errBatchJobSections), r.URL)
+ return
+ }
+
+ // The job acts as the credential that submits it, and only as far as
+ // that credential may act on the job's buckets.
+ submitter, err := newBatchJobSubmitter(creds)
+ if err == nil {
+ job.Submitter = submitter
+ err = submitter.authorize(ctx, job.startAuthzRequests()...)
+ }
+ if err != nil {
+ apiErr := toAPIError(ctx, err)
+ if isBatchJobAuthzError(err) {
+ apiErr = errorCodes.ToAPIErrWithErr(ErrAccessDenied, err)
+ }
+ writeErrorResponseJSON(ctx, w, apiErr, r.URL)
+ return
+ }
+
// Validate the incoming job request
if err := job.Validate(ctx, objectAPI); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1942,6 +2271,14 @@ func (j *BatchJobPool) cleanupReports(randomWait func() time.Duration) {
continue
}
if (ri.Complete || ri.Failed) && time.Since(ri.LastUpdate) > oldJobsExpiration {
+ // Remove the job before its report, so that a finished
+ // job is never resumed without its outcome.
+ if ri.JobID != "" {
+ if err := (BatchJobRequest{ID: ri.JobID}).delete(ctx, j.objLayer); err != nil {
+ batchLogIf(ctx, err)
+ continue
+ }
+ }
deleteConfig(ctx, j.objLayer, result.Item.Name)
}
}
@@ -2082,7 +2419,7 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
})
case j.jobCh <- req:
default:
- return fmt.Errorf("batch job queue is currently full please try again later %#v", req)
+ return fmt.Errorf("batch job queue is currently full please try again later (job %s)", req.ID)
}
return nil
}
diff --git a/cmd/batch-handlers_gen.go b/cmd/batch-handlers_gen.go
index ad29932b4..d0e1d7086 100644
--- a/cmd/batch-handlers_gen.go
+++ b/cmd/batch-handlers_gen.go
@@ -179,6 +179,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
}
+ case "Submitter":
+ if dc.IsNil() {
+ err = dc.ReadNil()
+ if err != nil {
+ err = msgp.WrapError(err, "Submitter")
+ return
+ }
+ z.Submitter = nil
+ } else {
+ if z.Submitter == nil {
+ z.Submitter = new(batchJobSubmitter)
+ }
+ err = z.Submitter.DecodeMsg(dc)
+ if err != nil {
+ err = msgp.WrapError(err, "Submitter")
+ return
+ }
+ }
default:
err = dc.Skip()
if err != nil {
@@ -192,9 +210,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 6
+ // map header, size 7
// write "ID"
- err = en.Append(0x86, 0xa2, 0x49, 0x44)
+ err = en.Append(0x87, 0xa2, 0x49, 0x44)
if err != nil {
return
}
@@ -274,15 +292,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
return
}
}
+ // write "Submitter"
+ err = en.Append(0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
+ if err != nil {
+ return
+ }
+ if z.Submitter == nil {
+ err = en.WriteNil()
+ if err != nil {
+ return
+ }
+ } else {
+ err = z.Submitter.EncodeMsg(en)
+ if err != nil {
+ err = msgp.WrapError(err, "Submitter")
+ return
+ }
+ }
return
}
// MarshalMsg implements msgp.Marshaler
func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
- // map header, size 6
+ // map header, size 7
// string "ID"
- o = append(o, 0x86, 0xa2, 0x49, 0x44)
+ o = append(o, 0x87, 0xa2, 0x49, 0x44)
o = msgp.AppendString(o, z.ID)
// string "User"
o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -323,6 +358,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
return
}
}
+ // string "Submitter"
+ o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
+ if z.Submitter == nil {
+ o = msgp.AppendNil(o)
+ } else {
+ o, err = z.Submitter.MarshalMsg(o)
+ if err != nil {
+ err = msgp.WrapError(err, "Submitter")
+ return
+ }
+ }
return
}
@@ -413,6 +459,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
}
+ case "Submitter":
+ if msgp.IsNil(bts) {
+ bts, err = msgp.ReadNilBytes(bts)
+ if err != nil {
+ return
+ }
+ z.Submitter = nil
+ } else {
+ if z.Submitter == nil {
+ z.Submitter = new(batchJobSubmitter)
+ }
+ bts, err = z.Submitter.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "Submitter")
+ return
+ }
+ }
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -445,6 +508,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
} else {
s += z.Expire.Msgsize()
}
+ s += 10
+ if z.Submitter == nil {
+ s += msgp.NilSize
+ } else {
+ s += z.Submitter.Msgsize()
+ }
return
}
@@ -568,6 +637,24 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "BytesFailed")
return
}
+ case "vsl":
+ z.VersionsSkippedLocked, err = dc.ReadInt64()
+ if err != nil {
+ err = msgp.WrapError(err, "VersionsSkippedLocked")
+ return
+ }
+ case "kr":
+ z.KeysRefused, err = dc.ReadInt64()
+ if err != nil {
+ err = msgp.WrapError(err, "KeysRefused")
+ return
+ }
+ case "fr":
+ z.FailureReason, err = dc.ReadString()
+ if err != nil {
+ err = msgp.WrapError(err, "FailureReason")
+ return
+ }
default:
err = dc.Skip()
if err != nil {
@@ -581,9 +668,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 17
+ // map header, size 20
// write "v"
- err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76)
+ err = en.Append(0xde, 0x0, 0x14, 0xa1, 0x76)
if err != nil {
return
}
@@ -752,15 +839,45 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "BytesFailed")
return
}
+ // write "vsl"
+ err = en.Append(0xa3, 0x76, 0x73, 0x6c)
+ if err != nil {
+ return
+ }
+ err = en.WriteInt64(z.VersionsSkippedLocked)
+ if err != nil {
+ err = msgp.WrapError(err, "VersionsSkippedLocked")
+ return
+ }
+ // write "kr"
+ err = en.Append(0xa2, 0x6b, 0x72)
+ if err != nil {
+ return
+ }
+ err = en.WriteInt64(z.KeysRefused)
+ if err != nil {
+ err = msgp.WrapError(err, "KeysRefused")
+ return
+ }
+ // write "fr"
+ err = en.Append(0xa2, 0x66, 0x72)
+ if err != nil {
+ return
+ }
+ err = en.WriteString(z.FailureReason)
+ if err != nil {
+ err = msgp.WrapError(err, "FailureReason")
+ return
+ }
return
}
// MarshalMsg implements msgp.Marshaler
func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
- // map header, size 17
+ // map header, size 20
// string "v"
- o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76)
+ o = append(o, 0xde, 0x0, 0x14, 0xa1, 0x76)
o = msgp.AppendInt(o, z.Version)
// string "jid"
o = append(o, 0xa3, 0x6a, 0x69, 0x64)
@@ -810,6 +927,15 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
// string "bf"
o = append(o, 0xa2, 0x62, 0x66)
o = msgp.AppendInt64(o, z.BytesFailed)
+ // string "vsl"
+ o = append(o, 0xa3, 0x76, 0x73, 0x6c)
+ o = msgp.AppendInt64(o, z.VersionsSkippedLocked)
+ // string "kr"
+ o = append(o, 0xa2, 0x6b, 0x72)
+ o = msgp.AppendInt64(o, z.KeysRefused)
+ // string "fr"
+ o = append(o, 0xa2, 0x66, 0x72)
+ o = msgp.AppendString(o, z.FailureReason)
return
}
@@ -933,6 +1059,24 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "BytesFailed")
return
}
+ case "vsl":
+ z.VersionsSkippedLocked, bts, err = msgp.ReadInt64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "VersionsSkippedLocked")
+ return
+ }
+ case "kr":
+ z.KeysRefused, bts, err = msgp.ReadInt64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "KeysRefused")
+ return
+ }
+ case "fr":
+ z.FailureReason, bts, err = msgp.ReadStringBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "FailureReason")
+ return
+ }
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -947,6 +1091,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *batchJobInfo) Msgsize() (s int) {
- s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
+ s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.StringPrefixSize + len(z.FailureReason)
return
}
diff --git a/cmd/batch-object-lock_test.go b/cmd/batch-object-lock_test.go
new file mode 100644
index 000000000..dc3096c0b
--- /dev/null
+++ b/cmd/batch-object-lock_test.go
@@ -0,0 +1,479 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/minio/minio/internal/amztime"
+ "github.com/minio/minio/internal/auth"
+ objectlock "github.com/minio/minio/internal/bucket/object/lock"
+ "github.com/minio/minio/internal/crypto"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/kms"
+ "github.com/pgsty/silo-pkg/v3/policy"
+)
+
+var batchLockKinds = []string{"COMPLIANCE", "GOVERNANCE", "LEGALHOLD"}
+
+func batchLockMeta(kind string) map[string]string {
+ m := map[string]string{}
+ switch kind {
+ case "COMPLIANCE", "GOVERNANCE":
+ m[strings.ToLower(xhttp.AmzObjectLockMode)] = kind
+ m[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(24 * time.Hour))
+ case "LEGALHOLD":
+ m[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
+ }
+ return m
+}
+
+func batchPutVersion(t *testing.T, obj ObjectLayer, bucket, key string, meta map[string]string) ObjectInfo {
+ t.Helper()
+ data := []byte("batch lock " + key)
+ oi, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
+ ObjectOptions{Versioned: true, UserDefined: meta})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return oi
+}
+
+func batchVersionExists(t *testing.T, obj ObjectLayer, bucket, key, versionID string) bool {
+ t.Helper()
+ _, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: versionID})
+ switch {
+ case err == nil:
+ return true
+ case isErrVersionNotFound(err) || isErrObjectNotFound(err):
+ return false
+ }
+ t.Fatal(err)
+ return false
+}
+
+// batchJobStatusJSON returns the status response of a job as decoded JSON.
+func batchJobStatusJSON(t *testing.T, jobID string) map[string]any {
+ t.Helper()
+ cred := globalActiveCred
+ req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/status-job?jobId="+jobID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ rec := httptest.NewRecorder()
+ req = req.WithContext(newContext(req, rec, "BatchJobStatus"))
+ adminAPIHandlers{}.BatchJobStatus(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+ }
+ var out struct{ LastMetric map[string]any }
+ if err = json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
+ t.Fatal(err)
+ }
+ return out.LastMetric
+}
+
+// Batch expiration keeps versions under retention or legal hold, on both
+// purge paths, and reports them; unlocked versions are still expired.
+func TestBatchExpireObjectLock(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t, endpoints: []string{"PutObject"},
+ objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+ defer r5Capacity(obj.(*erasureServerPools))()
+ enableBucketObjectLock(t, bucket)
+ f := &batchAuthzFixture{ctx: t.Context(), obj: obj, bucket: bucket}
+ batchTestJobPool(t.Context(), t, obj)
+
+ type version struct{ key, id string }
+ var locked, unlocked []version
+ for _, retain := range []int{0, 1} {
+ for _, kind := range batchLockKinds {
+ key := fmt.Sprintf("r%d/%s", retain, kind)
+ older := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
+ batchPutVersion(t, obj, bucket, key, nil)
+ locked = append(locked, version{key, older.VersionID})
+ }
+ key := fmt.Sprintf("r%d/plain", retain)
+ older := batchPutVersion(t, obj, bucket, key, nil)
+ batchPutVersion(t, obj, bucket, key, nil)
+ unlocked = append(unlocked, version{key, older.VersionID})
+ }
+
+ var jobs []*BatchJobRequest
+ for _, retain := range []int{0, 1} {
+ job := fmt.Sprintf(`expire:
+ apiVersion: v1
+ bucket: %s
+ prefix: r%d/
+ rules:
+ - type: object
+ purge:
+ retainVersions: %d
+`, bucket, retain, retain)
+ rec, queued := f.submit(t, globalActiveCred, job)
+ if queued == nil {
+ t.Fatalf("%s: submission status %d: %s", instance, rec.Code, rec.Body.String())
+ }
+ // Running a job again, as a resumed job does, must not count
+ // the versions it left a second time.
+ f.run(t, queued)
+ f.run(t, queued)
+ jobs = append(jobs, queued)
+ }
+
+ for _, v := range locked {
+ if !batchVersionExists(t, obj, bucket, v.key, v.id) {
+ t.Errorf("%s: %s: locked version %s was expired", instance, v.key, v.id)
+ }
+ }
+ for _, v := range unlocked {
+ if batchVersionExists(t, obj, bucket, v.key, v.id) {
+ t.Errorf("%s: %s: control: unlocked version %s was not expired", instance, v.key, v.id)
+ }
+ }
+ for i, field := range []string{"keysRefused", "versionsSkippedLocked"} {
+ m := batchJobStatusJSON(t, jobs[i].ID)
+ if got, _ := m[field].(float64); got != float64(len(batchLockKinds)) {
+ t.Errorf("%s: retain %d: %s = %v, want %d", instance, i, field, m[field], len(batchLockKinds))
+ }
+ if m["complete"] != true || m["failed"] != false {
+ t.Errorf("%s: retain %d: complete=%v failed=%v", instance, i, m["complete"], m["failed"])
+ }
+ }
+ },
+ })
+}
+
+type batchLockTransport func(*http.Request) (*http.Response, error)
+
+func (f batchLockTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// batchLockRemote is a source that serves replacement for every read.
+func batchLockRemote(t *testing.T, replacement string) *minio.Core {
+ t.Helper()
+ core, err := minio.NewCore("source.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: http.StatusOK, Header: http.Header{
+ "Content-Length": []string{strconv.Itoa(len(replacement))},
+ "Last-Modified": []string{UTCNow().Format(http.TimeFormat)},
+ "Etag": []string{"\"batchetag\""},
+ }, Body: io.NopCloser(strings.NewReader(replacement)), ContentLength: int64(len(replacement)), Request: r}, nil
+ }),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return core
+}
+
+func batchRootSubmitter(t *testing.T) *batchJobSubmitter {
+ t.Helper()
+ sub, err := newBatchJobSubmitter(globalActiveCred)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return sub
+}
+
+// A pull leaves a destination version under Object Lock as it is, whether a
+// single PUT, a multipart completion or a delete marker would replace it.
+func TestBatchPullObjectLock(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t, endpoints: []string{"PutObject"},
+ objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+ defer r5Capacity(obj.(*erasureServerPools))()
+ enableBucketObjectLock(t, bucket)
+ const replacement = "replacement from remote"
+ core := batchLockRemote(t, replacement)
+ sub := batchRootSubmitter(t)
+ job := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ }
+ for _, kind := range batchLockKinds {
+ for _, path := range []string{"put", "multipart", "marker", "directory"} {
+ key := "pull/" + kind + "/" + path
+ if path == "directory" {
+ key += "/"
+ }
+ before := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind))
+ src := ObjectInfo{Name: key, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}}
+ switch path {
+ case "multipart":
+ src.ETag = "batchetag-1"
+ case "marker":
+ src.DeleteMarker = true
+ }
+ err := job.ReplicateFromSource(t.Context(), obj, core, src, false, sub)
+ if !isErrObjectLocked(err) {
+ t.Errorf("%s: %s: pull returned %v, want ObjectLocked", instance, key, err)
+ }
+ after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
+ if gerr != nil {
+ t.Errorf("%s: %s: locked version lost: %v", instance, key, gerr)
+ continue
+ }
+ if !enforceRetentionForDeletion(t.Context(), after) || after.ETag != before.ETag {
+ t.Errorf("%s: %s: locked version was replaced", instance, key)
+ }
+ }
+ }
+
+ // Controls: a new version, and a new delete marker that hides a
+ // locked version without replacing it.
+ locked := batchPutVersion(t, obj, bucket, "pull/new", batchLockMeta("COMPLIANCE"))
+ newVersion := mustGetUUID()
+ if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
+ Name: "pull/new", VersionID: newVersion, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
+ }, false, sub); err != nil {
+ t.Fatalf("%s: new version: %v", instance, err)
+ }
+ if got := readCopyChecksumObject(t, obj, bucket, "pull/new", ObjectOptions{VersionID: newVersion}); string(got) != replacement {
+ t.Errorf("%s: new version content %q", instance, got)
+ }
+ marker := mustGetUUID()
+ if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{
+ Name: "pull/new", VersionID: marker, ModTime: UTCNow(), DeleteMarker: true,
+ }, false, sub); err != nil {
+ t.Fatalf("%s: new marker: %v", instance, err)
+ }
+ if !batchVersionExists(t, obj, bucket, "pull/new", locked.VersionID) {
+ t.Errorf("%s: a new marker removed a locked version", instance)
+ }
+ },
+ })
+}
+
+// A pull applies retention as a PUT to the destination would: copied
+// retention is validated, and a version without one gets the bucket default.
+func TestBatchPullRetentionMetadata(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t, endpoints: []string{"PutObject"},
+ objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+ defer r5Capacity(obj.(*erasureServerPools))()
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.ObjectLockConfigXML = []byte(`EnabledGOVERNANCE1`)
+ meta.VersioningConfigXML = enabledBucketVersioningConfig
+ if err = meta.parseAllConfigs(t.Context(), obj); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(bucket, meta)
+ core := batchLockRemote(t, "retention content")
+ sub := batchRootSubmitter(t)
+ job := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ }
+ pull := func(key string, meta map[string]string) (ObjectInfo, error) {
+ vid := mustGetUUID()
+ err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{Name: key, VersionID: vid, ModTime: UTCNow().Add(-time.Hour), ETag: "batchetag", UserDefined: meta}, false, sub)
+ if err != nil {
+ return ObjectInfo{}, err
+ }
+ return obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: vid})
+ }
+ oi, err := pull("retention/default", map[string]string{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
+ if ret.Mode != objectlock.RetGovernance || !ret.RetainUntilDate.After(UTCNow().Add(23*time.Hour)) {
+ t.Errorf("%s: default retention not applied with the destination's time: %+v", instance, ret)
+ }
+ until := UTCNow().Add(48 * time.Hour)
+ oi, err = pull("retention/copied", map[string]string{
+ xhttp.AmzObjectLockMode: "COMPLIANCE", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ret = objectlock.GetObjectRetentionMeta(oi.UserDefined); ret.Mode != objectlock.RetCompliance || !ret.RetainUntilDate.Equal(until.Truncate(time.Millisecond)) {
+ t.Errorf("%s: copied retention: %+v", instance, ret)
+ }
+ if _, err = pull("retention/invalid", map[string]string{xhttp.AmzObjectLockMode: "FOREVER", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until)}); err == nil {
+ t.Errorf("%s: invalid copied retention was accepted", instance)
+ }
+ },
+ })
+}
+
+// Key rotation does not rewrite a version under Object Lock.
+func TestBatchKeyRotationObjectLock(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t, endpoints: []string{"PutObject", "GetObject"},
+ objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+ defer r5Capacity(obj.(*erasureServerPools))()
+ enableBucketObjectLock(t, bucket)
+ oldKMS := GlobalKMS
+ GlobalKMS = kms.NewStub("batch-lock-key")
+ defer func() { GlobalKMS = oldKMS }()
+ sub := batchRootSubmitter(t)
+ job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ for _, kind := range append([]string{"none"}, batchLockKinds...) {
+ key := "rotate/" + kind
+ headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
+ switch kind {
+ case "COMPLIANCE", "GOVERNANCE":
+ headers[xhttp.AmzObjectLockMode] = kind
+ headers[xhttp.AmzObjectLockRetainUntilDate] = UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
+ case "LEGALHOLD":
+ headers[xhttp.AmzObjectLockLegalHold] = "ON"
+ }
+ putCopyChecksumSource(t, router, cred, bucket, key, []byte("rotate "+kind), headers)
+ before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
+ if err != nil || !crypto.S3.IsEncrypted(before.UserDefined) {
+ t.Fatalf("%s: %s: encrypted control: %v", instance, key, err)
+ }
+ err = job.KeyRotate(t.Context(), obj, before, sub)
+ after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID})
+ if gerr != nil {
+ t.Fatal(gerr)
+ }
+ if kind == "none" {
+ if err != nil || after.ModTime.Equal(before.ModTime) {
+ t.Errorf("%s: control: unlocked version not rotated: %v", instance, err)
+ }
+ continue
+ }
+ var locked ObjectLocked
+ if !errors.As(err, &locked) || !after.ModTime.Equal(before.ModTime) {
+ t.Errorf("%s: %s: rotation of a locked version returned %v; rewritten=%v", instance, key, err, !after.ModTime.Equal(before.ModTime))
+ }
+ }
+ },
+ })
+}
+
+// A push reads locked versions as any authorized read does, and a submitter
+// that may not read a version does not send it.
+func TestBatchPushReads(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t, endpoints: []string{"PutObject"},
+ objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
+ defer r5Capacity(obj.(*erasureServerPools))()
+ enableBucketObjectLock(t, bucket)
+ var puts []string
+ target, err := minio.NewCore("target.invalid", &minio.Options{
+ Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
+ Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Body != nil {
+ io.Copy(io.Discard, r.Body)
+ r.Body.Close()
+ }
+ if r.Method == http.MethodPut {
+ puts = append(puts, r.URL.Path)
+ }
+ return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": []string{"\"pushetag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
+ }),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ job := BatchJobReplicateV1{
+ Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
+ Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"},
+ }
+ src := batchPutVersion(t, obj, bucket, "push/locked", batchLockMeta("COMPLIANCE"))
+ if err := job.ReplicateToTarget(t.Context(), obj, target, src, false, batchRootSubmitter(t)); err != nil {
+ t.Fatalf("%s: authorized push of a locked version: %v", instance, err)
+ }
+ if len(puts) != 1 {
+ t.Fatalf("%s: authorized push sent %d writes", instance, len(puts))
+ }
+
+ // A user that may list but not read.
+ p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:StartBatchJob"]},{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::` + bucket + `"]}]}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = globalIAMSys.SetPolicy(t.Context(), "batch-list-only", *p); err != nil {
+ t.Fatal(err)
+ }
+ user := auth.Credentials{AccessKey: "batch-list-user", SecretKey: "batch-list-user-secret"}
+ if _, err = globalIAMSys.CreateUser(t.Context(), user.AccessKey, madmin.AddOrUpdateUserReq{SecretKey: user.SecretKey, Status: madmin.AccountEnabled}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = globalIAMSys.PolicyDBSet(t.Context(), user.AccessKey, "batch-list-only", regUser, false); err != nil {
+ t.Fatal(err)
+ }
+ sub, err := newBatchJobSubmitter(globalIAMSysUser(t, user.AccessKey))
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = job.ReplicateToTarget(t.Context(), obj, target, src, false, sub)
+ if !isBatchJobAuthzError(err) || len(puts) != 1 {
+ t.Fatalf("%s: unauthorized push returned %v and sent %d writes", instance, err, len(puts)-1)
+ }
+ },
+ })
+}
+
+// globalIAMSysUser returns the stored credential of accessKey, as a request
+// authenticated with it has.
+func globalIAMSysUser(t *testing.T, accessKey string) auth.Credentials {
+ t.Helper()
+ u, ok := globalIAMSys.store.GetUser(accessKey)
+ if !ok {
+ t.Fatalf("no user %s", accessKey)
+ }
+ return u.Credentials
+}
+
+// The cursor of a walk done out of order moves only past keys whose items
+// are all done, and counts each item once.
+func TestBatchProgressCursor(t *testing.T) {
+ ri := &batchJobInfo{}
+ p := newBatchProgress(ri, "bucket")
+ items := []string{"a", "a", "b", "c", "c"}
+ seqs := make([]uint64, len(items))
+ for i := range items {
+ seqs[i] = p.begin()
+ }
+ skip := batchCounts{versionsSkipped: 1}
+ p.done(seqs[1], "a", skip)
+ p.done(seqs[3], "c", skip)
+ if ri.Object != "" || ri.VersionsSkippedLocked != 0 {
+ t.Fatalf("cursor moved before the first item: %q %d", ri.Object, ri.VersionsSkippedLocked)
+ }
+ p.done(seqs[0], "a", skip)
+ p.done(seqs[2], "b", skip)
+ // a is done; b is done but c may still have items.
+ if ri.Object != "b" && ri.Object != "a" {
+ t.Fatalf("cursor %q", ri.Object)
+ }
+ if ri.Object == "a" && ri.VersionsSkippedLocked != 2 {
+ t.Fatalf("counts %d at cursor a", ri.VersionsSkippedLocked)
+ }
+ // c's last item never finishes, as when a run stops: the cursor stays
+ // before c and c is not counted.
+ p.finish()
+ if ri.Object != "b" || ri.VersionsSkippedLocked != 3 {
+ t.Fatalf("interrupted: cursor %q counts %d, want b 3", ri.Object, ri.VersionsSkippedLocked)
+ }
+ p.done(seqs[4], "c", skip)
+ p.finish()
+ if ri.Object != "c" || ri.VersionsSkippedLocked != 5 {
+ t.Fatalf("finished: cursor %q counts %d, want c 5", ri.Object, ri.VersionsSkippedLocked)
+ }
+}
diff --git a/cmd/batch-progress.go b/cmd/batch-progress.go
new file mode 100644
index 000000000..968e8e0ef
--- /dev/null
+++ b/cmd/batch-progress.go
@@ -0,0 +1,187 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "slices"
+ "strings"
+ "sync"
+
+ "github.com/minio/madmin-go/v3"
+)
+
+// batchCounts are outcomes of a batch job that a later run of the job would
+// see again, and so must be persisted together with the cursor of the walk.
+type batchCounts struct {
+ objects int64
+ objectsFailed int64
+ deleteMarkersFailed int64
+ versionsSkipped int64
+ keysRefused int64
+}
+
+func (c *batchCounts) add(o batchCounts) {
+ c.objects += o.objects
+ c.objectsFailed += o.objectsFailed
+ c.deleteMarkersFailed += o.deleteMarkersFailed
+ c.versionsSkipped += o.versionsSkipped
+ c.keysRefused += o.keysRefused
+}
+
+//msgp:ignore batchProgress batchProgressItem batchCounts
+
+// batchProgress is the cursor of a walk whose items are done concurrently.
+//
+// The items of a walk are numbered in walk order. A key is done when every
+// item of it, and every item before it, is done; only then does the cursor
+// move past it and its counts become part of the job's counts. The cursor and
+// the counts are persisted together, so a job resumed after the cursor
+// neither skips an item that was not done nor counts an item twice.
+type batchProgress struct {
+ mu sync.Mutex
+ ri *batchJobInfo
+ bucket string
+ issued uint64
+ next uint64
+ pending map[uint64]batchProgressItem
+ // key is the key of the item last committed, and held are the counts of
+ // its items so far.
+ key string
+ held batchCounts
+}
+
+type batchProgressItem struct {
+ key string
+ counts batchCounts
+}
+
+func newBatchProgress(ri *batchJobInfo, bucket string) *batchProgress {
+ return &batchProgress{ri: ri, bucket: bucket, pending: make(map[uint64]batchProgressItem)}
+}
+
+// begin numbers the next item of the walk.
+func (p *batchProgress) begin() uint64 {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ seq := p.issued
+ p.issued++
+ return seq
+}
+
+// done records item seq, of key, as done with counts.
+func (p *batchProgress) done(seq uint64, key string, counts batchCounts) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.pending[seq] = batchProgressItem{key: key, counts: counts}
+ for {
+ item, ok := p.pending[p.next]
+ if !ok {
+ return
+ }
+ delete(p.pending, p.next)
+ p.next++
+ if item.key != p.key {
+ p.commitLocked()
+ p.key = item.key
+ }
+ p.held.add(item.counts)
+ }
+}
+
+// finish commits the last key if every item of the walk is done. Call it
+// once no item is in progress.
+func (p *batchProgress) finish() {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.next == p.issued {
+ p.commitLocked()
+ }
+}
+
+func (p *batchProgress) commitLocked() {
+ if p.key == "" {
+ return
+ }
+ ri := p.ri
+ ri.mu.Lock()
+ ri.Bucket = p.bucket
+ ri.Object = p.key
+ ri.Objects += p.held.objects
+ ri.ObjectsFailed += p.held.objectsFailed
+ ri.DeleteMarkersFailed += p.held.deleteMarkersFailed
+ ri.VersionsSkippedLocked += p.held.versionsSkipped
+ ri.KeysRefused += p.held.keysRefused
+ ri.mu.Unlock()
+ p.held = batchCounts{}
+}
+
+// countRemoved counts versions a job removed. A removed version is gone for
+// any later run, so it is counted at once, not at the cursor.
+func (ri *batchJobInfo) countRemoved(bucket string, objects, markers int64) {
+ ri.mu.Lock()
+ defer ri.mu.Unlock()
+ ri.Bucket = bucket
+ ri.Objects += objects
+ ri.DeleteMarkers += markers
+}
+
+// countSkipped counts a version a job left in place because Object Lock
+// protects it, for a job that is counted per run rather than at a cursor.
+func (ri *batchJobInfo) countSkipped() {
+ ri.mu.Lock()
+ defer ri.mu.Unlock()
+ ri.VersionsSkippedLocked++
+}
+
+// batchCanonicalPrefixes orders the prefixes of a walk and drops a prefix
+// that another one covers. The keys of the walk are then visited once each,
+// in one ascending order, which a single cursor can describe.
+func batchCanonicalPrefixes(prefixes []string) []string {
+ if len(prefixes) == 0 {
+ return []string{""}
+ }
+ sorted := slices.Clone(prefixes)
+ slices.Sort(sorted)
+ out := sorted[:0]
+ for _, p := range sorted {
+ if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) {
+ continue
+ }
+ out = append(out, p)
+ }
+ return out
+}
+
+// batchResumeAfter reports whether a resumed walk returned a key after the
+// cursor. The cursor key is done; the walk forwards conservatively and
+// returns it again. Keys compare as the walk orders them, by name, so a
+// directory object comes before the keys under it.
+func batchResumeAfter(cursor, name string) bool {
+ return cursor == "" || name > cursor
+}
+
+// batchJobStatus is the status response: madmin.BatchJobStatus with the
+// fields this server adds. The added fields are additive; a client that does
+// not know them ignores them.
+type batchJobStatus struct {
+ LastMetric batchJobStatusMetric
+}
+
+type batchJobStatusMetric struct {
+ madmin.JobMetric
+ VersionsSkippedLocked int64 `json:"versionsSkippedLocked,omitempty"`
+ KeysRefused int64 `json:"keysRefused,omitempty"`
+ FailureReason string `json:"failureReason,omitempty"`
+}
+
+func (ri *batchJobInfo) statusMetric() batchJobStatusMetric {
+ ri.mu.RLock()
+ defer ri.mu.RUnlock()
+ return batchJobStatusMetric{
+ JobMetric: ri.metric(),
+ VersionsSkippedLocked: ri.VersionsSkippedLocked,
+ KeysRefused: ri.KeysRefused,
+ FailureReason: ri.FailureReason,
+ }
+}
diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go
index 3ebbc844d..b8599fd1d 100644
--- a/cmd/batch-rotate.go
+++ b/cmd/batch-rotate.go
@@ -20,6 +20,7 @@ package cmd
import (
"context"
"encoding/base64"
+ "errors"
"fmt"
"maps"
"math/rand"
@@ -35,6 +36,7 @@ import (
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/pgsty/silo-pkg/v3/env"
+ "github.com/pgsty/silo-pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/workers"
)
@@ -161,8 +163,68 @@ func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error
return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
}
-// KeyRotate rotates encryption key of an object
-func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
+// batchAuthoritativeVersion returns the version obj as all its copies
+// describe it. With several pools a version can have copies in more than one
+// pool, and a read returns one of them; the merged copy carries the newest
+// Object Lock and tag state. The caller holds the object's write lock.
+func batchAuthoritativeVersion(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
+ z, ok := api.(*erasureServerPools)
+ if !ok || z.SinglePool() {
+ return obj, nil
+ }
+ versionID := obj.VersionID
+ if versionID == "" {
+ versionID = nullVersionID
+ }
+ return z.replicaObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{VersionID: versionID, NoLock: true, NoAuditLog: true})
+}
+
+// batchAuthoritativeRead is batchAuthoritativeVersion for a version that has
+// been opened for reading. It takes the object's read lock itself, since an
+// opened reader may hold none.
+func batchAuthoritativeRead(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) {
+ z, ok := api.(*erasureServerPools)
+ if !ok || z.SinglePool() {
+ return obj, nil
+ }
+ lk := z.NewNSLock(bucket, encodeDirObject(object))
+ lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return ObjectInfo{}, err
+ }
+ defer lk.RUnlock(lkctx)
+ return batchAuthoritativeVersion(lkctx.Context(), api, bucket, object, obj)
+}
+
+// rotateAuthzRequests are the actions of rotating one version: it is read and
+// written again in place with the new key.
+func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzRequest {
+ versionID := oi.VersionID
+ if versionID == "" {
+ versionID = nullVersionID
+ }
+ // The encryption the rewrite leaves: rotation keeps the kind of the
+ // version (rotateKey), and SSE-KMS takes the requested key.
+ values := map[string][]string{}
+ switch {
+ case crypto.S3KMS.IsEncrypted(oi.UserDefined):
+ values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
+ if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
+ values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
+ }
+ case crypto.S3.IsEncrypted(oi.UserDefined):
+ values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionAES}
+ }
+ return []batchAuthzRequest{
+ {action: policy.GetObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
+ {action: policy.GetObjectVersionAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags},
+ {action: policy.PutObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags, values: values},
+ }
+}
+
+// KeyRotate rotates encryption key of an object. A version that Object Lock
+// protects is not rewritten: it returns ObjectLocked.
+func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo, sub *batchJobSubmitter) error {
srcBucket := r.Bucket
srcObject := objInfo.Name
@@ -180,7 +242,8 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
- lock := api.NewObjectNSLock(r.Bucket, objInfo.Name)
+ // Lock the name writers lock: the stored name of a directory object.
+ lock := api.NewObjectNSLock(r.Bucket, encodeDirObject(objInfo.Name))
lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
@@ -194,10 +257,32 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
VersionSuspended: versionSuspended,
NoLock: true,
}
- obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
+ obj, pool, err := batchRotateRead(ctx, api, r.Bucket, objInfo.Name, opts)
if err != nil {
return err
}
+ // Decide on the version as it is under the lock that the rewrite holds,
+ // before any key is requested, with the metadata of all its copies.
+ logical, err := batchAuthoritativeVersion(ctx, api, r.Bucket, objInfo.Name, obj)
+ if err != nil {
+ return err
+ }
+ if enforceRetentionForDeletion(ctx, logical) {
+ return ObjectLocked{Bucket: r.Bucket, Object: obj.Name, VersionID: obj.VersionID}
+ }
+ p, err := sub.principal(ctx)
+ if err != nil {
+ return err
+ }
+ // Authorize the copy that is read and rewritten, and the version as all
+ // its copies describe it.
+ for _, v := range []ObjectInfo{obj, logical} {
+ for _, req := range r.rotateAuthzRequests(v) {
+ if err := p.authorize(req); err != nil {
+ return err
+ }
+ }
+ }
oi := obj.Clone()
var (
newKeyID string
@@ -225,16 +310,61 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
oi.metadataOnly = true
oi.keyRotation = true
maps.Copy(oi.UserDefined, encMetadata)
- if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
- VersionID: oi.VersionID,
- }, ObjectOptions{
- VersionID: oi.VersionID,
- NoLock: true,
- }); err != nil {
- return err
+ // The rewrite replaces the stored metadata with UserDefined, which a
+ // read returns without the tags and the expiry: keep them.
+ if oi.UserTags != "" {
+ oi.UserDefined[xhttp.AmzObjectTagging] = oi.UserTags
}
+ if !oi.Expires.IsZero() {
+ oi.UserDefined["expires"] = oi.ExpiresStr()
+ }
+ srcOpts := ObjectOptions{VersionID: oi.VersionID}
+ dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true}
+ if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
+ // Rewrite the copy that was read and authorized. The pools layer
+ // would route the rewrite by the latest version of the key, which
+ // can be in a pool that holds another write of this version ID.
+ //
+ // Check the pool's state again where the pools layer checks it,
+ // right before the write: a move can have started since the read.
+ if z.IsSuspended(pool) || z.IsPoolRebalancing(pool) {
+ return errBatchRotateMoving
+ }
+ dstOpts.replicaObjectInfo = z.replicaObjectInfo
+ name := encodeDirObject(oi.Name)
+ _, err = z.serverPools[pool].CopyObject(ctx, r.Bucket, name, r.Bucket, name, oi, srcOpts, dstOpts)
+ } else {
+ _, err = api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, srcOpts, dstOpts)
+ }
+ return err
+}
- return nil
+// errBatchRotateMoving: the version is read from a pool that a decommission
+// or a rebalance is moving.
+var errBatchRotateMoving = errors.New("the version is being moved to another pool; rotate it once the move is done")
+
+// batchRotateRead reads the version that a rotation rewrites, as
+// GetObjectInfo does. With several pools it also returns the pool of the
+// copy it read, otherwise -1. The caller holds the object lock.
+//
+// A copy in a pool that a decommission or a rebalance is moving is the
+// source of that move, which must not change under it, and rewriting a copy
+// in another pool would change which copy the version's metadata comes
+// from. Such a version is left to a later run (errBatchRotateMoving).
+func batchRotateRead(ctx context.Context, api ObjectLayer, bucket, object string, opts ObjectOptions) (ObjectInfo, int, error) {
+ z, ok := api.(*erasureServerPools)
+ if !ok || z.SinglePool() {
+ oi, err := api.GetObjectInfo(ctx, bucket, object, opts)
+ return oi, -1, err
+ }
+ if err := checkGetObjArgs(ctx, bucket, object); err != nil {
+ return ObjectInfo{}, -1, err
+ }
+ oi, idx, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
+ if err == nil && (z.IsSuspended(idx) || z.IsPoolRebalancing(idx)) {
+ return ObjectInfo{}, -1, errBatchRotateMoving
+ }
+ return oi, idx, err
}
const (
@@ -260,6 +390,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
if ri.Complete {
return nil
}
+ if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
+ return err
+ }
globalBatchJobsMetrics.save(job.ID, ri)
lastObject := ri.Object
@@ -273,8 +406,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
delay = batchKeyRotateJobDefaultRetryDelay
}
- rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
-
selectObj := func(info FileInfo) (ok bool) {
if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
// skip all objects that are newer than specified older duration
@@ -355,65 +486,92 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
return err
}
- ctx, cancel := context.WithCancel(ctx)
+ ctx, cancelCause := context.WithCancelCause(ctx)
+ defer cancelCause(nil)
+ // refuse ends the job for its submitter: record why before stopping.
+ refuse := func(err error) {
+ ri.refuse(err)
+ cancelCause(err)
+ }
+ progress := newBatchProgress(ri, r.Bucket)
results := make(chan itemOrErr[ObjectInfo], 100)
if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
Marker: lastObject,
Filter: selectObj,
}); err != nil {
- cancel()
// Do not need to retry if we can't list objects on source.
return err
}
failed := false
for res := range results {
+ if context.Cause(ctx) != nil {
+ // Refused or canceled: schedule nothing more.
+ continue
+ }
if res.Err != nil {
failed = true
batchLogIf(ctx, res.Err)
break
}
result := res.Item
+ if !batchResumeAfter(lastObject, result.Name) {
+ // Completed by the run that set the cursor.
+ continue
+ }
sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
continue
}
+ seq := progress.begin()
wk.Take()
go func() {
defer wk.Give()
+ var counts batchCounts
for attempts := 1; attempts <= retryAttempts; attempts++ {
stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
- success := true
- if err := r.KeyRotate(ctx, api, result); err != nil {
- stopFn(result, err)
- batchLogIf(ctx, err)
- success = false
- if attempts >= retryAttempts {
- auditOptions := AuditLogOptions{
- Event: "KeyRotate",
- APIName: "StartBatchJob",
- Bucket: result.Bucket,
- Object: result.Name,
- VersionID: result.VersionID,
- Error: err.Error(),
- }
- auditLogInternal(ctx, auditOptions)
- }
- } else {
- stopFn(result, nil)
+ err := r.KeyRotate(ctx, api, result, job.Submitter)
+ stopFn(result, err)
+ if err == nil {
+ counts.objects = 1
+ break
}
- ri.trackCurrentBucketObject(r.Bucket, result, success, attempts)
- globalBatchJobsMetrics.save(job.ID, ri)
- // persist in-memory state to disk after every 10secs.
- batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
- if success {
+ if isBatchJobAuthzError(err) {
+ refuse(err)
+ return
+ }
+ if isErrObjectLocked(err) {
+ // Left as it is, not retried and not a failure.
+ counts.versionsSkipped = 1
+ break
+ }
+ if ctx.Err() != nil {
+ // Not done; a later run rotates it.
+ return
+ }
+ batchLogIf(ctx, err)
+ if attempts >= retryAttempts {
+ auditOptions := AuditLogOptions{
+ Event: "KeyRotate",
+ APIName: "StartBatchJob",
+ Bucket: result.Bucket,
+ Object: result.Name,
+ VersionID: result.VersionID,
+ Error: err.Error(),
+ }
+ auditLogInternal(ctx, auditOptions)
+ counts.objectsFailed = 1
break
}
if delay > 0 {
- time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+ time.Sleep(delay + time.Duration(rand.Float64()*float64(delay)))
}
}
+ progress.done(seq, result.Name, counts)
+ globalBatchJobsMetrics.save(job.ID, ri)
+ // persist in-memory state to disk after every 10secs.
+ batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
time.Sleep(wait)
@@ -421,18 +579,31 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
}()
}
wk.Wait()
-
- ri.Complete = !failed && ri.ObjectsFailed == 0
- ri.Failed = failed || ri.ObjectsFailed > 0
- globalBatchJobsMetrics.save(job.ID, ri)
- // persist in-memory state to disk.
- batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-
- if err := r.Notify(ctx, ri); err != nil {
- batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+ cause := context.Cause(ctx)
+ if !failed && cause == nil {
+ // The walk ended: its last key is done.
+ progress.finish()
+ }
+ if cause != nil && !ri.refused() {
+ // Stopped before the end: keep the work after the cursor pending.
+ globalBatchJobsMetrics.save(job.ID, ri)
+ batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+ return cause
}
- cancel()
+ if !ri.refused() {
+ ri.mu.Lock()
+ ri.Complete = !failed && ri.ObjectsFailed == 0
+ ri.Failed = failed || ri.ObjectsFailed > 0
+ ri.mu.Unlock()
+ }
+ globalBatchJobsMetrics.save(job.ID, ri)
+ // persist in-memory state to disk.
+ batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job))
+
+ if err := r.Notify(GlobalContext, ri); err != nil {
+ batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+ }
return nil
}
diff --git a/cmd/batch-submitter-authz_test.go b/cmd/batch-submitter-authz_test.go
new file mode 100644
index 000000000..64a16d97c
--- /dev/null
+++ b/cmd/batch-submitter-authz_test.go
@@ -0,0 +1,456 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/auth"
+ "github.com/pgsty/silo-pkg/v3/policy"
+ "go.yaml.in/yaml/v3"
+)
+
+// batchAuthzFixture runs batch jobs submitted through the admin API, one job
+// at a time and in the test's goroutine.
+type batchAuthzFixture struct {
+ ctx context.Context
+ sys *IAMSys
+ obj ObjectLayer
+ bucket string
+}
+
+func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture {
+ t.Helper()
+ ctx, sys := iamAdminFixture(t)
+ obj := newObjectLayerFn()
+ batchTestJobPool(ctx, t, obj)
+ f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
+ mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+ _, err := globalBucketMetadataSys.Update(ctx, f.bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
+ mustIAM(t, err)
+ return f
+}
+
+// batchTestJobPool queues submitted jobs without running them.
+func batchTestJobPool(ctx context.Context, t *testing.T, obj ObjectLayer) {
+ t.Helper()
+ prevPool := globalBatchJobPool
+ globalBatchJobPool = &BatchJobPool{
+ ctx: ctx,
+ objLayer: obj,
+ jobCh: make(chan *BatchJobRequest, 10),
+ jobCancelers: make(map[string]context.CancelFunc),
+ }
+ globalBatchJobsMetrics.Lock()
+ globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo)
+ globalBatchJobsMetrics.Unlock()
+ t.Cleanup(func() { globalBatchJobPool = prevPool })
+}
+
+// putVersions writes two versions of key and returns the older one.
+func (f *batchAuthzFixture) putVersions(t *testing.T, key string) string {
+ t.Helper()
+ var older string
+ for i := range 2 {
+ data := []byte(fmt.Sprintf("version %d", i))
+ oi, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
+ mustIAM(t, err)
+ if i == 0 {
+ older = oi.VersionID
+ }
+ }
+ return older
+}
+
+func (f *batchAuthzFixture) versionExists(t *testing.T, key, versionID string) bool {
+ t.Helper()
+ _, err := f.obj.GetObjectInfo(f.ctx, f.bucket, key, ObjectOptions{VersionID: versionID})
+ if err == nil {
+ return true
+ }
+ if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+ return false
+ }
+ t.Fatal(err)
+ return false
+}
+
+func (f *batchAuthzFixture) expireYAML(prefix string) string {
+ return fmt.Sprintf(`expire:
+ apiVersion: v1
+ bucket: %s
+ prefix: %q
+ rules:
+ - type: object
+ purge:
+ retainVersions: 1
+`, f.bucket, prefix)
+}
+
+// submit sends a job definition to StartBatchJob as cred. An accepted job is
+// returned from the queue, not run.
+func (f *batchAuthzFixture) submit(t *testing.T, cred auth.Credentials, job string) (*httptest.ResponseRecorder, *BatchJobRequest) {
+ t.Helper()
+ headers := map[string]string{}
+ if cred.IsTemp() && !cred.IsServiceAccount() {
+ headers["X-Amz-Security-Token"] = cred.SessionToken
+ }
+ body := []byte(job)
+ req, err := newTestSignedRequestV4(http.MethodPost, "http://localhost/minio/admin/v3/start-job", int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
+ mustIAM(t, err)
+ mustIAM(t, req.ParseForm())
+ rec := httptest.NewRecorder()
+ req = req.WithContext(newContext(req, rec, "StartBatchJob"))
+ adminAPIHandlers{}.StartBatchJob(rec, req)
+ if rec.Code != http.StatusOK {
+ return rec, nil
+ }
+ select {
+ case queued := <-globalBatchJobPool.jobCh:
+ return rec, queued
+ default:
+ t.Fatal("accepted job was not queued")
+ return rec, nil
+ }
+}
+
+func (f *batchAuthzFixture) run(t *testing.T, job *BatchJobRequest) {
+ t.Helper()
+ ctx := job.ctx
+ if ctx == nil {
+ ctx = f.ctx
+ }
+ switch {
+ case job.Expire != nil:
+ if err := job.Expire.Start(ctx, f.obj, *job); err != nil {
+ t.Logf("start: %v", err)
+ }
+ default:
+ t.Fatalf("unexpected job type %s", job.Type())
+ }
+ ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
+ if err := ri.load(f.ctx, f.obj, *job); err == nil {
+ t.Logf("job %s: %+v", job.ID, ri.clone())
+ }
+}
+
+func (f *batchAuthzFixture) jobInfo(t *testing.T, job *BatchJobRequest) *batchJobInfo {
+ t.Helper()
+ ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())}
+ mustIAM(t, ri.load(f.ctx, f.obj, *job))
+ return ri
+}
+
+func batchAuthzPolicy(ctx context.Context, t *testing.T, sys *IAMSys, name, statements string) {
+ t.Helper()
+ p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[` + statements + `]}`))
+ mustIAM(t, err)
+ _, err = sys.SetPolicy(ctx, name, *p)
+ mustIAM(t, err)
+}
+
+const batchAuthzFullAccess = `{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}`
+
+// A credential that may start batch jobs but has no access to the bucket
+// cannot delete through a job.
+func TestBatchJobSubmitterWithoutDataAccess(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-only", `{"Effect":"Allow","Action":["admin:StartBatchJob","admin:ListBatchJobs"]}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "batch-only-user", "batch-only")
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job != nil {
+ f.run(t, job)
+ }
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a job submitted without data access deleted a version")
+ }
+ if rec.Code != http.StatusForbidden {
+ t.Fatalf("submission status %d, want 403: %s", rec.Code, rec.Body.String())
+ }
+}
+
+// A job acts only while the credential that submitted it is the same valid
+// credential.
+func TestBatchJobSubmitterChangedBeforeStart(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ change func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials)
+ }{
+ {"unchanged", nil},
+ {"deleted", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+ mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+ }},
+ {"disabled", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+ _, err := f.sys.SetUserStatus(f.ctx, cred.AccessKey, madmin.AccountDisabled)
+ mustIAM(t, err)
+ }},
+ {"secret-changed", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+ mustIAM(t, f.sys.SetUserSecretKey(f.ctx, cred.AccessKey, "batch-authz-changed-secret"))
+ }},
+ {"access-key-reused", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) {
+ mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false))
+ time.Sleep(10 * time.Millisecond)
+ iamAdminUser(f.ctx, t, f.sys, cred.AccessKey, "batch-full")
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ if tc.change != nil {
+ tc.change(t, f, cred)
+ }
+ f.run(t, job)
+ exists := f.versionExists(t, "data/a", older)
+ if tc.change == nil && exists {
+ t.Fatal("control: an authorized job did not expire the older version")
+ }
+ if tc.change != nil && !exists {
+ t.Fatal("a job ran as a credential that changed after submission")
+ }
+ })
+ }
+}
+
+// An expired temporary credential does not keep its job running.
+func TestBatchJobSubmitterExpired(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+ cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(2 * time.Second).Unix()}, globalActiveCred.SecretKey)
+ mustIAM(t, err)
+ cred.ParentUser = parent.AccessKey
+ _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+ mustIAM(t, err)
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ time.Sleep(time.Until(cred.Expiration) + time.Second)
+ f.run(t, job)
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a job ran as an expired credential")
+ }
+}
+
+// A session policy limits the job as it limits requests, also when an
+// authorization plugin decides the rest.
+func TestBatchJobSessionPolicyDeny(t *testing.T) {
+ session := `{"Version":"2012-10-17","Statement":[
+{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]},
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`
+ for _, tc := range []struct {
+ name string
+ plugin bool
+ }{{"local", false}, {"plugin-allows", true}} {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+ cred := iamAdminServiceAccount(f.ctx, t, f.sys, parent.AccessKey, "batch-restricted", session)
+ if tc.plugin {
+ iamAdminPlugin(t, true)
+ }
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job != nil {
+ f.run(t, job)
+ } else {
+ t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a job deleted a version its session policy denies")
+ }
+ })
+ }
+}
+
+// A temporary credential's session policy limits its job too.
+func TestBatchJobTemporarySessionPolicyDeny(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
+ session := `{"Version":"2012-10-17","Statement":[
+{"Effect":"Allow","Action":["admin:StartBatchJob"]},
+{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions","s3:GetObject","s3:GetObjectVersion","s3:DeleteObject"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}]}`
+ cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
+ parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
+ policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(session)),
+ }, globalActiveCred.SecretKey)
+ mustIAM(t, err)
+ cred.ParentUser = parent.AccessKey
+ _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
+ mustIAM(t, err)
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job != nil {
+ f.run(t, job)
+ } else {
+ t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a job deleted a version its session policy does not allow")
+ }
+}
+
+// A Deny that depends on request context the job does not have refuses the
+// action instead of being skipped.
+func TestBatchJobAbsentContextDeny(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-context", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-context")
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job != nil {
+ f.run(t, job)
+ } else {
+ t.Logf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a job skipped a Deny that depends on request context")
+ }
+}
+
+// Each object is authorized: a Deny under an allowed prefix keeps its objects.
+func TestBatchJobPerObjectDeny(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-keep", batchAuthzFullAccess+`,
+{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/data/keep*"]}`)
+ cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-keep")
+ kept := f.putVersions(t, "data/keep")
+ rec, job := f.submit(t, cred, f.expireYAML("data/"))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ f.run(t, job)
+ if !f.versionExists(t, "data/keep", kept) {
+ t.Fatal("a job deleted a version that a per-object Deny protects")
+ }
+ if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+ t.Fatalf("refused job: complete=%v failed=%v", ri.Complete, ri.Failed)
+ }
+}
+
+// A job queued without its submitter, as an earlier release stored it, fails
+// and stays failed across another start.
+func TestBatchJobWithoutSubmitter(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ older := f.putVersions(t, "data/a")
+ job := &BatchJobRequest{}
+ mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
+ job.ID = "expire-legacy" + getKeySeparator() + "0"
+ job.User = globalActiveCred.AccessKey
+ job.Started = time.Now()
+ mustIAM(t, job.save(f.ctx, f.obj))
+ for range 2 {
+ loaded := &BatchJobRequest{}
+ mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
+ f.run(t, loaded)
+ if !f.versionExists(t, "data/a", older) {
+ t.Fatal("a job without a recorded submitter deleted a version")
+ }
+ if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete {
+ t.Fatalf("job without submitter: complete=%v failed=%v", ri.Complete, ri.Failed)
+ }
+ }
+}
+
+// Root and an authorized user can still run jobs.
+func TestBatchJobAuthorizedControls(t *testing.T) {
+ for _, root := range []bool{true, false} {
+ t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ cred := globalActiveCred
+ if !root {
+ batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
+ cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
+ }
+ older := f.putVersions(t, "data/a")
+ rec, job := f.submit(t, cred, f.expireYAML(""))
+ if job == nil {
+ t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
+ }
+ f.run(t, job)
+ if f.versionExists(t, "data/a", older) {
+ t.Fatal("an authorized job did not expire the older version")
+ }
+ if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed {
+ t.Fatalf("authorized job: complete=%v failed=%v", ri.Complete, ri.Failed)
+ }
+ })
+ }
+}
+
+// The describe output has no secrets of the job.
+func TestBatchJobDescribeRedactsSecrets(t *testing.T) {
+ f := newBatchAuthzFixture(t)
+ secrets := []string{"describe-source-secret", "describe-source-token", "describe-target-secret", "describe-target-token", "describe-notify-token"}
+ for _, job := range []BatchJobRequest{
+ {ID: "replicate-describe" + getKeySeparator() + "0", Replicate: &BatchJobReplicateV1{
+ APIVersion: batchReplJobAPIVersion,
+ Source: BatchJobReplicateSource{Bucket: "remote", Endpoint: "https://remote.invalid", Creds: BatchJobReplicateCredentials{
+ AccessKey: "describe-source-access", SecretKey: secrets[0], SessionToken: secrets[1],
+ }},
+ Target: BatchJobReplicateTarget{Bucket: f.bucket, Creds: BatchJobReplicateCredentials{
+ AccessKey: "describe-target-access", SecretKey: secrets[2], SessionToken: secrets[3],
+ }},
+ Flags: BatchJobReplicateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
+ }},
+ {ID: "expire-describe" + getKeySeparator() + "0", Expire: &BatchJobExpire{
+ APIVersion: batchExpireAPIVersion, Bucket: f.bucket,
+ NotificationCfg: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]},
+ }},
+ {ID: "keyrotate-describe" + getKeySeparator() + "0", KeyRotate: &BatchJobKeyRotateV1{
+ APIVersion: batchKeyRotateAPIVersion, Bucket: f.bucket,
+ Flags: BatchJobKeyRotateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}},
+ }},
+ } {
+ data, err := job.MarshalMsg(nil)
+ mustIAM(t, err)
+ mustIAM(t, saveConfig(f.ctx, f.obj, getJobPath(job), data))
+ cred := globalActiveCred
+ req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/describe-job?jobId="+job.ID, 0, nil, cred.AccessKey, cred.SecretKey, nil)
+ mustIAM(t, err)
+ mustIAM(t, req.ParseForm())
+ rec := httptest.NewRecorder()
+ req = req.WithContext(newContext(req, rec, "DescribeBatchJob"))
+ adminAPIHandlers{}.DescribeBatchJob(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("%s: describe status %d: %s", job.ID, rec.Code, rec.Body.String())
+ }
+ for _, s := range secrets {
+ if strings.Contains(rec.Body.String(), s) {
+ t.Errorf("%s: describe output contains %s", job.ID, s)
+ }
+ }
+ if job.Replicate != nil && !strings.Contains(rec.Body.String(), "describe-source-access") {
+ t.Errorf("%s: describe output lost the access key", job.ID)
+ }
+ // The stored job keeps what a run needs.
+ stored := &BatchJobRequest{}
+ mustIAM(t, stored.load(f.ctx, f.obj, getJobPath(job)))
+ if stored.Replicate != nil && stored.Replicate.Source.Creds.SecretKey != secrets[0] {
+ t.Errorf("%s: describe changed the stored job", job.ID)
+ }
+ }
+}
diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go
index 397b039ef..f791d7a12 100644
--- a/cmd/bucket-object-lock.go
+++ b/cmd/bucket-object-lock.go
@@ -291,6 +291,53 @@ func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, g
return nil
}
+// checkVersionLockedForOverwrite re-checks under the object write lock
+// whether a write would replace the stored version versionID (the null
+// version if empty) while Object Lock protects it. It reads the version's own
+// metadata, whatever the bucket configuration, and a failed read is an error.
+func checkVersionLockedForOverwrite(ctx context.Context, bucket, object, versionID string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error {
+ if isMinioMetaBucketName(bucket) {
+ return nil
+ }
+ if versionID == "" {
+ versionID = nullVersionID
+ }
+ objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{
+ VersionID: versionID,
+ Transition: transition,
+ NoLock: true,
+ NoAuditLog: true,
+ })
+ if err != nil {
+ // A delete marker is not protected.
+ if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) {
+ return nil
+ }
+ return err
+ }
+ if enforceRetentionForDeletion(ctx, objInfo) {
+ return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+ }
+ return nil
+}
+
+// refuseLockedMarkerOverwrite refuses a delete marker write that would replace
+// the stored version it names while Object Lock protects it. goi and gerr are
+// that version as read under the write lock. A new marker, or one replacing a
+// marker, is allowed.
+func refuseLockedMarkerOverwrite(ctx context.Context, bucket, object, versionID string, goi ObjectInfo, gerr error) error {
+ if gerr != nil {
+ if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) || errors.As(gerr, &MethodNotAllowed{}) {
+ return nil
+ }
+ return gerr
+ }
+ if enforceRetentionForDeletion(ctx, goi) {
+ return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+ }
+ return nil
+}
+
// checkPutObjectLockAllowed enforces object retention policy and legal hold policy
// for requests with WORM headers
// See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec.
diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go
index 6d717df26..12b662fe5 100644
--- a/cmd/erasure-multipart.go
+++ b/cmd/erasure-multipart.go
@@ -1687,6 +1687,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
defer lk.Unlock(lkctx)
}
+ if opts.RefuseLockedOverwrite {
+ getObjectInfo := er.getObjectInfo
+ if opts.replicaObjectInfo != nil {
+ getObjectInfo = opts.replicaObjectInfo
+ }
+ if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil {
+ return oi, toObjectErr(err, bucket, object)
+ }
+ }
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement {
getObjectInfo := er.getObjectInfo
if opts.replicaObjectInfo != nil {
diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go
index 15249cf48..92d40d810 100644
--- a/cmd/erasure-object.go
+++ b/cmd/erasure-object.go
@@ -1676,6 +1676,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
return ObjectInfo{}, toObjectErr(err, bucket, object)
}
}
+ if opts.RefuseLockedOverwrite {
+ getObjectInfo := er.getObjectInfo
+ if opts.replicaObjectInfo != nil {
+ getObjectInfo = opts.replicaObjectInfo
+ }
+ if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil {
+ return ObjectInfo{}, toObjectErr(err, bucket, object)
+ }
+ }
// Rename the successfully written temporary object to final location.
onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
@@ -2199,6 +2208,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
+ // A delete marker that names a stored version replaces it.
+ if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" {
+ if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil {
+ return ObjectInfo{}, err
+ }
+ }
+
if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools {
if gerr == nil {
evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
diff --git a/cmd/erasure-server-pool-delete-all-options_test.go b/cmd/erasure-server-pool-delete-all-options_test.go
index 96039ffae..4dfac1b79 100644
--- a/cmd/erasure-server-pool-delete-all-options_test.go
+++ b/cmd/erasure-server-pool-delete-all-options_test.go
@@ -25,6 +25,8 @@ import (
"sync/atomic"
"testing"
"time"
+
+ xhttp "github.com/minio/minio/internal/http"
)
// The parts of the purge of all versions that need its own options.
@@ -65,16 +67,22 @@ func TestPurgeAllVersionsOptions(t *testing.T) {
p.version(0, "ea/key", "v1", 5)
match := &deleteAllMatch{}
expire := ExpirationOptions{Expire: true}
+ allow := func([]ObjectInfo) error { return nil }
for name, opts := range map[string]ObjectOptions{
"no approval": {DeleteAllVersions: true},
- "two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match},
- "prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true},
- "prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true},
- "version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()},
- "movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true},
- "marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true},
- "versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true},
- "suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true},
+ "two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match, deleteAllAuthorize: allow},
+ // A batch purge deletes as the submitter of its job, an ILM purge
+ // as the server, and no other delete takes an authorization.
+ "batch unauthorized": {DeleteAllVersions: true, DeleteAllLatest: match},
+ "ilm authorized": {DeleteAllVersions: true, Expiration: expire, deleteAllAuthorize: allow},
+ "authorized delete": {deleteAllAuthorize: allow},
+ "prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true},
+ "prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true},
+ "version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()},
+ "movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true},
+ "marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true},
+ "versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true},
+ "suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true},
} {
if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(InvalidArgument)) {
t.Errorf("%s: %v", name, err)
@@ -134,9 +142,10 @@ func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) {
t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() })
var res deleteAllResult
_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
- DeleteAllVersions: true,
- DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
- DeleteAllResult: &res,
+ DeleteAllVersions: true,
+ DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+ DeleteAllResult: &res,
+ deleteAllAuthorize: func([]ObjectInfo) error { return nil },
})
if !errors.Is(err, errDeleteAllBudget) || res.Started || res.Removed != 0 {
t.Errorf("purge after its budget ran out on reads: err=%v %+v", err, res)
@@ -149,7 +158,11 @@ func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) {
// A marker that an expiration job failed to remove fails the job.
func TestPurgeAllVersionsMarkerFailureFailsJob(t *testing.T) {
ri := &batchJobInfo{}
- ri.trackExpireAll(expireObjInfo{ObjectInfo: ObjectInfo{NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0, true)
+ var res batchCounts
+ res.expireAllFailed(expireObjInfo{ObjectInfo: ObjectInfo{Name: "key", NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0)
+ progress := newBatchProgress(ri, "bucket")
+ progress.done(progress.begin(), "key", res)
+ progress.finish()
if ri.ObjectsFailed != 0 || ri.DeleteMarkersFailed != 1 || !ri.expireFailures() {
t.Errorf("failed=%d/%d failures=%t", ri.ObjectsFailed, ri.DeleteMarkersFailed, ri.expireFailures())
}
@@ -209,9 +222,10 @@ func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) {
})
var res deleteAllResult
if _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
- DeleteAllVersions: true,
- DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
- DeleteAllResult: &res,
+ DeleteAllVersions: true,
+ DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+ DeleteAllResult: &res,
+ deleteAllAuthorize: func([]ObjectInfo) error { return nil },
}); err != nil || !res.Complete {
t.Fatalf("purge: %v %+v", err, res)
}
@@ -219,3 +233,62 @@ func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) {
t.Errorf("reads=%d deletes on the replacement handle=%d", reads.Load(), deletes.Load())
}
}
+
+// A batch purge has every copy of every version it approved authorized, as
+// the copy is now, before it deletes any; an authorization error deletes
+// nothing and is returned as it is.
+func TestPurgeAllVersionsAuthorizesEveryCopy(t *testing.T) {
+ z, _ := consistencyPools(t)
+ p := newPurgeTest(t, z, "purge-authorize", MakeBucketOptions{VersioningEnabled: true}, "")
+ const key = "bx/authorize"
+ p.put(0, key, "old", daysAgo(6), ObjectOptions{})
+ p.put(1, key, "new", daysAgo(5), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: "copy=new"}})
+ tagged := p.put(0, key, "v1", daysAgo(4), ObjectOptions{Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "keep=yes"}})
+ exp := p.matched(key)
+ denied := errors.New("denied")
+ var seen []string // the distinct versions and tags authorized
+ purge := func(authorize func([]ObjectInfo) error) (deleteAllResult, error) {
+ var res deleteAllResult
+ _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
+ DeleteAllVersions: true,
+ DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+ DeleteAllResult: &res,
+ deleteAllAuthorize: authorize,
+ })
+ return res, err
+ }
+ res, err := purge(func(versions []ObjectInfo) error {
+ for _, v := range versions {
+ if got := v.VersionID + " " + v.UserTags; !slices.Contains(seen, got) {
+ seen = append(seen, got)
+ }
+ }
+ return denied
+ })
+ if !errors.Is(err, denied) || res.Started || res.Removed != 0 {
+ t.Errorf("denied purge: err=%v %+v", err, res)
+ }
+ want := []string{"null ", "null copy=new", tagged.VersionID + " keep=yes"}
+ slices.Sort(seen)
+ slices.Sort(want)
+ if !slices.Equal(seen, want) {
+ t.Errorf("authorized %q, want every copy %q", seen, want)
+ }
+ for pool := range z.serverPools {
+ if got := p.held(pool, key); len(got) == 0 {
+ t.Errorf("pool %d lost its copies to a denied purge", pool)
+ }
+ }
+ if got := p.get(key); got != "v1" {
+ t.Errorf("reads %s after a denied purge, want v1", got)
+ }
+ // Control: an allowed purge deletes them all.
+ if res, err = purge(func([]ObjectInfo) error { return nil }); err != nil || !res.Complete {
+ t.Fatalf("allowed purge: %v %+v", err, res)
+ }
+ for pool := range z.serverPools {
+ if got := p.held(pool, key); got != nil {
+ t.Errorf("pool %d keeps %v", pool, got)
+ }
+ }
+}
diff --git a/cmd/erasure-server-pool-delete-all.go b/cmd/erasure-server-pool-delete-all.go
index 50c594270..42fc17101 100644
--- a/cmd/erasure-server-pool-delete-all.go
+++ b/cmd/erasure-server-pool-delete-all.go
@@ -26,6 +26,7 @@ import (
"sync"
"time"
+ xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
)
@@ -49,6 +50,9 @@ import (
// when any revision of an approved version there is transitioned or,
// when the bucket has Object Lock, protected by retention or a legal
// hold. It deletes on exactly the drives it read;
+// - a batch expiry then authorizes every copy of every approved version,
+// and every revision of one that a drive holds, for the submitter of the
+// job, and the purge deletes nothing unless all of them are allowed;
// - it deletes the oldest version first and the latest version last and
// alone, once every older version is gone from every pool. A purge that
// stops early keeps the latest version, so it never exposes an older
@@ -118,11 +122,14 @@ func deleteAllVersionID(versionID string) string {
}
// deleteAllInvalid returns why opts cannot ask for a purge of all versions.
-// The purge is approved either as an ILM expiry or by a batch match, and it
-// only deletes: it never creates a marker, addresses a single version,
-// moves data or deletes a prefix.
+// The purge is approved either as an ILM expiry or by a batch match, which
+// also authorizes the versions it deletes, and it only deletes: it never
+// creates a marker, addresses a single version, moves data or deletes a
+// prefix.
func (opts ObjectOptions) deleteAllInvalid() error {
switch {
+ case !opts.DeleteAllVersions:
+ return errors.New("only a purge of all versions authorizes the versions it deletes")
case opts.DeletePrefix || opts.DeletePrefixObject:
return errors.New("a purge of all versions is not a prefix delete")
case opts.VersionID != "":
@@ -137,6 +144,8 @@ func (opts ObjectOptions) deleteAllInvalid() error {
return errors.New("a purge of all versions takes no options of other deletes")
case opts.Expiration.Expire == (opts.DeleteAllLatest != nil):
return errors.New("a purge of all versions needs exactly one approval")
+ case (opts.DeleteAllLatest != nil) != (opts.deleteAllAuthorize != nil):
+ return errors.New("a batch purge of all versions, and only it, authorizes the versions it deletes")
}
return nil
}
@@ -297,12 +306,29 @@ func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, obje
}
}
}
+ // A batch expiry deletes as its submitter: every write it is about to
+ // delete, the accepted copies and every revision a drive holds, must be
+ // allowed with the version ID and tags it has now.
+ var writes *deleteAllWrites
+ if opts.deleteAllAuthorize != nil {
+ writes = &deleteAllWrites{bucket: bucket, name: name, seen: make(map[[2]string]bool, len(history))}
+ for _, v := range history {
+ for _, c := range v.copies {
+ writes.add(c.fi)
+ }
+ }
+ }
drives := make([][]StorageAPI, len(z.serverPools))
for pool := range z.serverPools {
if !held[pool] {
continue
}
- if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled); err != nil {
+ if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled, writes); err != nil {
+ return fail(err)
+ }
+ }
+ if opts.deleteAllAuthorize != nil {
+ if err := opts.deleteAllAuthorize(writes.versions); err != nil {
return fail(err)
}
}
@@ -329,6 +355,28 @@ func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, obje
return res.Latest, nil
}
+// deleteAllWrites collects, for the authorization of a batch purge, the
+// writes it deletes: one of each distinct version ID and tags.
+type deleteAllWrites struct {
+ bucket, name string
+ seen map[[2]string]bool
+ versions []ObjectInfo
+}
+
+// add adds the write fi unless w is nil or holds one with its version ID and
+// tags.
+func (w *deleteAllWrites) add(fi FileInfo) {
+ if w == nil {
+ return
+ }
+ key := [2]string{deleteAllVersionID(fi.VersionID), fi.Metadata[xhttp.AmzObjectTagging]}
+ if w.seen[key] {
+ return
+ }
+ w.seen[key] = true
+ w.versions = append(w.versions, fi.ToObjectInfo(w.bucket, w.name, true))
+}
+
// deleteAllRefuses returns why the purge must keep the key that holds the
// revision fi of an approved version: it is transitioned, whose remote
// object a later release hands to the tier cleanup, or, with Object Lock,
@@ -344,7 +392,8 @@ func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, l
}
// deleteAllDrives reads object on every drive of the set and checks each
-// revision of the versions ids that a drive holds with deleteAllRefuses.
+// revision of the versions ids that a drive holds with deleteAllRefuses, and
+// adds it to writes.
// A minority of drives can hold a revision other than the accepted one, and
// heal would later remove what the purge leaves on a drive, so a drive that
// cannot be read or decoded refuses the key. A drive without the object
@@ -353,7 +402,7 @@ func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, l
// handle names a drive, not a backend: a backend that replaces the drive
// behind it starts empty and receives this object only from a write or a
// heal of it, which take the object lock the purge holds.
-func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool) ([]StorageAPI, error) {
+func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool, writes *deleteAllWrites) ([]StorageAPI, error) {
disks := slices.Clone(er.getDisks())
raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
if err := ctx.Err(); err != nil {
@@ -381,6 +430,7 @@ func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object str
if err := deleteAllRefuses(ctx, bucket, object, fi, lockEnabled); err != nil {
return nil, err
}
+ writes.add(fi)
}
}
return disks, nil
diff --git a/cmd/erasure-server-pool-delete-all_test.go b/cmd/erasure-server-pool-delete-all_test.go
index b3c861cb1..eb6ef2ecf 100644
--- a/cmd/erasure-server-pool-delete-all_test.go
+++ b/cmd/erasure-server-pool-delete-all_test.go
@@ -332,8 +332,9 @@ func (p *purgeTest) matched(name string) expireObjInfo {
return expireObjInfo{ObjectInfo: oi, ExpireAll: true, DeleteMarkerCount: markers}
}
-// batch runs the deletes of a batch expiration job for walked, the keys its
-// walk matched, and returns the counts of the job.
+// batch runs the deletes of a batch expiration job of the root user for
+// walked, the keys its walk matched, as one batch of a walk that reached its
+// end, and returns the counts of the job.
func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo {
p.t.Helper()
globalBatchJobsMetrics.Lock()
@@ -341,7 +342,11 @@ func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo {
globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo)
}
globalBatchJobsMetrics.Unlock()
- job := BatchJobRequest{ID: "purge-" + mustGetUUID(), Started: time.Now(), Expire: &BatchJobExpire{Bucket: p.bucket}}
+ sub, err := newBatchJobSubmitter(globalActiveCred)
+ if err != nil {
+ p.t.Fatal(err)
+ }
+ job := BatchJobRequest{ID: "purge-" + mustGetUUID(), Started: time.Now(), Submitter: sub, Expire: &BatchJobExpire{Bucket: p.bucket}}
ri := &batchJobInfo{JobID: job.ID}
wk, err := workers.New(1)
if err != nil {
@@ -350,7 +355,8 @@ func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo {
ch := make(chan []expireObjInfo, 1)
ch <- walked
close(ch)
- batchObjsForDelete(p.t.Context(), job.Expire, ri, job, p.z, wk, ch)
+ refuse := func(err error) { p.t.Errorf("batch job refused: %v", err) }
+ batchObjsForDelete(p.t.Context(), refuse, func() bool { return true }, job.Expire, ri, job, p.z, wk, ch)
wk.Wait()
return ri
}
@@ -837,8 +843,8 @@ func TestPurgeAllVersionsKeepsProtectedWrites(t *testing.T) {
t.Errorf("%s holds %v, want both versions", name, got)
}
}
- if ri.Objects != 2 || ri.ObjectsFailed == 0 {
- t.Errorf("job counts objects=%d failed=%d, want 2 and the refusals", ri.Objects, ri.ObjectsFailed)
+ if ri.Objects != 2 || ri.KeysRefused != 3 || ri.ObjectsFailed != 0 {
+ t.Errorf("job counts objects=%d refused=%d failed=%d, want 2, 3 and 0", ri.Objects, ri.KeysRefused, ri.ObjectsFailed)
}
}
diff --git a/cmd/erasure-server-pool-lock-domain_test.go b/cmd/erasure-server-pool-lock-domain_test.go
index 13249b213..2097f731d 100644
--- a/cmd/erasure-server-pool-lock-domain_test.go
+++ b/cmd/erasure-server-pool-lock-domain_test.go
@@ -1013,9 +1013,13 @@ func TestLockDomainKeyRotationWaitsForWriter(t *testing.T) {
// The rotation's listing found an SSE-S3 object; it reads it again
// under the lock.
listed := ObjectInfo{Bucket: bucket, Name: key, UserDefined: map[string]string{crypto.MetaSealedKeyS3: "listed"}}
+ sub, err := newBatchJobSubmitter(globalActiveCred)
+ if err != nil {
+ t.Fatal(err)
+ }
waited, _, _ := ldOverlapRun(t,
func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") },
- func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed) })
+ func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed, sub) })
if !waited {
t.Errorf("key rotation ran while a PUT held its commit")
}
diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go
index 57cd23ce9..1df968d81 100644
--- a/cmd/erasure-server-pool.go
+++ b/cmd/erasure-server-pool.go
@@ -1234,7 +1234,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
if err = checkDelObjArgs(ctx, bucket, object); err != nil {
return objInfo, err
}
- if opts.DeleteAllVersions {
+ if opts.DeleteAllVersions || opts.deleteAllAuthorize != nil {
if err := opts.deleteAllInvalid(); err != nil {
return objInfo, InvalidArgument{Bucket: bucket, Object: object, Err: err}
}
@@ -1277,6 +1277,15 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
return z.deleteAllVersions(ctx, bucket, object, opts)
}
+ // A delete marker that names a version replaces that version in
+ // whichever pool holds it.
+ if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" && !z.SinglePool() {
+ goi, gerr := z.replicaObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: opts.VersionID, NoLock: true, NoAuditLog: true})
+ if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil {
+ return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err
+ }
+ }
+
// An ILM expiry is decided on the versions in one pool. Under the lock,
// check it with the versions in all pools; the sets then take it as
// decided, without a recheck by the versions each one holds. They only
diff --git a/cmd/iam.go b/cmd/iam.go
index 1566d81d0..3cd5fffc2 100644
--- a/cmd/iam.go
+++ b/cmd/iam.go
@@ -2549,7 +2549,12 @@ func (sys *IAMSys) IsAllowed(args policy.Args) bool {
}
return ok
}
+ return sys.isAllowedLocally(args)
+}
+// isAllowedLocally evaluates args with the local policies only, as IsAllowed
+// does when no authorization plugin is configured.
+func (sys *IAMSys) isAllowedLocally(args policy.Args) bool {
// Policies don't apply to the owner.
if args.IsOwner {
return true
diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go
index cd75dd0bf..2ad350169 100644
--- a/cmd/object-api-interface.go
+++ b/cmd/object-api-interface.go
@@ -113,10 +113,22 @@ type ObjectOptions struct {
// DeleteAllResult, when set, receives what a purge of all versions
// confirmed, also when it fails part way.
DeleteAllResult *deleteAllResult
+ // deleteAllAuthorize authorizes a batch purge of all versions, which
+ // needs it, on every copy of every version the purge approved and every
+ // revision of one that a drive holds, one of each distinct version ID
+ // and tags, before it deletes any; its error refuses the key and is
+ // returned. Set by batch expiration only, never from a request.
+ deleteAllAuthorize func(versions []ObjectInfo) error
// Pools-layer version resolver; the caller holds the shared object lock.
replicaObjectInfo GetObjectInfoFn
+ // RefuseLockedOverwrite refuses, under the destination's write lock, a
+ // write that would replace a stored version Object Lock protects: the
+ // version it writes, or the version a delete marker it writes names.
+ // It returns ObjectLocked. Set by batch replication pulls.
+ RefuseLockedOverwrite bool
+
Speedtest bool // object call specifically meant for SpeedTest code, set to 'true' when invoked by SpeedtestHandler.
// Use the maximum parity (N/2), used when saving server configuration files