From 57c2b0fb76ce1b8bec4d18cecfac06ce76741574 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 04:14:06 +0800 Subject: [PATCH 1/8] fix(batch): let a write refuse to replace a locked version Add ObjectOptions.RefuseLockedOverwrite. With it set, PutObject (before the rename) and CompleteMultipartUpload check, under the destination's write lock, the stored version the write would replace: the version it addresses, or the null version. DeleteObject checks the stored version that a delete marker it writes names; in a multi-pool deployment that version is resolved across pools. A version under retention or legal hold is refused with ObjectLocked. The check reads the version's own metadata, whatever the bucket configuration, and returns read errors. Nothing sets the option yet. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/bucket-object-lock.go | 47 +++++++++++++++++++++++++++++++++++++ cmd/erasure-multipart.go | 9 +++++++ cmd/erasure-object.go | 16 +++++++++++++ cmd/erasure-server-pool.go | 9 +++++++ cmd/object-api-interface.go | 6 +++++ 5 files changed, 87 insertions(+) diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index 397b039ef..f791d7a12 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -291,6 +291,53 @@ func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, g return nil } +// checkVersionLockedForOverwrite re-checks under the object write lock +// whether a write would replace the stored version versionID (the null +// version if empty) while Object Lock protects it. It reads the version's own +// metadata, whatever the bucket configuration, and a failed read is an error. +func checkVersionLockedForOverwrite(ctx context.Context, bucket, object, versionID string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error { + if isMinioMetaBucketName(bucket) { + return nil + } + if versionID == "" { + versionID = nullVersionID + } + objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{ + VersionID: versionID, + Transition: transition, + NoLock: true, + NoAuditLog: true, + }) + if err != nil { + // A delete marker is not protected. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || errors.As(err, &MethodNotAllowed{}) { + return nil + } + return err + } + if enforceRetentionForDeletion(ctx, objInfo) { + return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID} + } + return nil +} + +// refuseLockedMarkerOverwrite refuses a delete marker write that would replace +// the stored version it names while Object Lock protects it. goi and gerr are +// that version as read under the write lock. A new marker, or one replacing a +// marker, is allowed. +func refuseLockedMarkerOverwrite(ctx context.Context, bucket, object, versionID string, goi ObjectInfo, gerr error) error { + if gerr != nil { + if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) || errors.As(gerr, &MethodNotAllowed{}) { + return nil + } + return gerr + } + if enforceRetentionForDeletion(ctx, goi) { + return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID} + } + return nil +} + // checkPutObjectLockAllowed enforces object retention policy and legal hold policy // for requests with WORM headers // See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec. diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 6d717df26..12b662fe5 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -1687,6 +1687,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str defer lk.Unlock(lkctx) } + if opts.RefuseLockedOverwrite { + getObjectInfo := er.getObjectInfo + if opts.replicaObjectInfo != nil { + getObjectInfo = opts.replicaObjectInfo + } + if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil { + return oi, toObjectErr(err, bucket, object) + } + } if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement { getObjectInfo := er.getObjectInfo if opts.replicaObjectInfo != nil { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 15249cf48..92d40d810 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1676,6 +1676,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st return ObjectInfo{}, toObjectErr(err, bucket, object) } } + if opts.RefuseLockedOverwrite { + getObjectInfo := er.getObjectInfo + if opts.replicaObjectInfo != nil { + getObjectInfo = opts.replicaObjectInfo + } + if err := checkVersionLockedForOverwrite(ctx, bucket, object, fi.VersionID, getObjectInfo, opts.Transition); err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + } // Rename the successfully written temporary object to final location. onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum) @@ -2199,6 +2208,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } + // A delete marker that names a stored version replaces it. + if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" { + if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil { + return ObjectInfo{}, err + } + } + if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools { if gerr == nil { evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi) diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 57cd23ce9..efaf3654b 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1277,6 +1277,15 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob return z.deleteAllVersions(ctx, bucket, object, opts) } + // A delete marker that names a version replaces that version in + // whichever pool holds it. + if opts.RefuseLockedOverwrite && opts.DeleteMarker && opts.VersionID != "" && !z.SinglePool() { + goi, gerr := z.replicaObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: opts.VersionID, NoLock: true, NoAuditLog: true}) + if err := refuseLockedMarkerOverwrite(ctx, bucket, object, opts.VersionID, goi, gerr); err != nil { + return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err + } + } + // An ILM expiry is decided on the versions in one pool. Under the lock, // check it with the versions in all pools; the sets then take it as // decided, without a recheck by the versions each one holds. They only diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index cd75dd0bf..fd52d307c 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -117,6 +117,12 @@ type ObjectOptions struct { // Pools-layer version resolver; the caller holds the shared object lock. replicaObjectInfo GetObjectInfoFn + // RefuseLockedOverwrite refuses, under the destination's write lock, a + // write that would replace a stored version Object Lock protects: the + // version it writes, or the version a delete marker it writes names. + // It returns ObjectLocked. Set by batch replication pulls. + RefuseLockedOverwrite bool + Speedtest bool // object call specifically meant for SpeedTest code, set to 'true' when invoked by SpeedtestHandler. // Use the maximum parity (N/2), used when saving server configuration files From a717711b11ee66ba93057e33d05daf7c920bd3a2 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 04:14:06 +0800 Subject: [PATCH 2/8] fix(iam): separate local policy evaluation from the plugin decision IsAllowed returns the authorization plugin's decision when a plugin is configured, and evaluates the local policies otherwise. Move the local evaluation into isAllowedLocally so that a caller can combine it with the plugin's decision. IsAllowed is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/iam.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/iam.go b/cmd/iam.go index 1566d81d0..3cd5fffc2 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -2549,7 +2549,12 @@ func (sys *IAMSys) IsAllowed(args policy.Args) bool { } return ok } + return sys.isAllowedLocally(args) +} +// isAllowedLocally evaluates args with the local policies only, as IsAllowed +// does when no authorization plugin is configured. +func (sys *IAMSys) isAllowedLocally(args policy.Args) bool { // Policies don't apply to the owner. if args.IsOwner { return true From 303e3402920748b64e6672dd575a03e1832c75b0 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 04:14:06 +0800 Subject: [PATCH 3/8] fix(batch): authorize batch jobs as their submitter A batch job ran with the server's object access, whoever submitted it, and acted on versions that Object Lock protects. Submitter: StartBatchJob binds the authenticated credential to the job (type, access key, parent, identity revision and a keyed fingerprint). Submission, every start and every object resolve the binding against IAM and evaluate the action with the submitter's policies and session policy. A credential that is gone, disabled, expired or changed fails the job. A policy statement that would need request context the job does not have refuses the action. With an authorization plugin, the plugin and the session policy must both allow. A refusal is persisted as the job's failure reason and ends the job. A job stored without a binding fails and has to be submitted again; a definition holds one job type. Expire deletes each older version on its own, checking retention and legal hold and authorizing it under the delete's lock; a locked version is left and counted. Until the exact-key purge is integrated, a retainVersions: 0 key is refused when a version the walk saw is locked. Pull writes refuse to replace a locked destination version, use the destination's versioning, validate copied retention and apply default retention. Rotation leaves locked versions as they are. The status reports versionsSkippedLocked, keysRefused and the failure reason. Expire and rotation persist their cursor together with these counts, so a resumed job counts each version once. Describe returns a redacted copy of the job. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-authz.go | 742 ++++++++++++++++++ cmd/batch-authz_gen.go | 260 ++++++ cmd/batch-authz_gen_test.go | 123 +++ cmd/batch-expire.go | 547 ++++++++----- cmd/batch-handlers.go | 563 ++++++++++--- cmd/batch-handlers_gen.go | 162 +++- cmd/batch-progress.go | 187 +++++ cmd/batch-rotate.go | 203 ++++- ...ure-server-pool-delete-all-options_test.go | 6 +- cmd/erasure-server-pool-delete-all_test.go | 18 +- cmd/erasure-server-pool-lock-domain_test.go | 6 +- 11 files changed, 2438 insertions(+), 379 deletions(-) create mode 100644 cmd/batch-authz.go create mode 100644 cmd/batch-authz_gen.go create mode 100644 cmd/batch-authz_gen_test.go create mode 100644 cmd/batch-progress.go diff --git a/cmd/batch-authz.go b/cmd/batch-authz.go new file mode 100644 index 000000000..192fbb473 --- /dev/null +++ b/cmd/batch-authz.go @@ -0,0 +1,742 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/auth" + "github.com/pgsty/silo-pkg/v3/policy" + "github.com/pgsty/silo-pkg/v3/policy/condition" +) + +// A batch job acts on objects long after the request that submitted it has +// gone. It therefore records who submitted it and acts only as that +// principal: +// +// - StartBatchJob binds the authenticated credential to the job +// (batchJobSubmitter). Nothing in the submitted definition names it. +// - Every start of the job, and every object it touches, resolves the +// binding against IAM again. A credential that is gone, disabled, +// expired, revoked or edited since submission fails the job; a reused +// access key does not match the binding. +// - Each action is evaluated with the submitter's policies and session +// policy but without a request. A policy statement that would need +// request context the job does not have (source address, transport, +// headers and so on) refuses the action instead of being skipped. +// +// A job without a binding was queued by an earlier release. It is never +// authorized, whoever submitted it, and has to be submitted again. + +//go:generate msgp -file $GOFILE -unexported + +// batchJobSubmitter is the server-assigned identity of the credential that +// submitted a batch job. +type batchJobSubmitter struct { + Version int `msg:"v"` + Type string `msg:"t"` + AccessKey string `msg:"ak"` + // ParentUser is the parent of a derived credential. + ParentUser string `msg:"pu"` + // UpdatedAt and RevokedBefore are the revision of the identity record; + // any edit of the identity changes them. They are unset for root. + UpdatedAt time.Time `msg:"ua"` + RevokedBefore time.Time `msg:"rb"` + // Binding is a keyed fingerprint of the credential itself, see + // batchSubmitterBinding. + Binding []byte `msg:"bd"` +} + +const batchJobSubmitterVersion = 1 + +// Submitter types. A credential is classified in the order IsAllowed +// evaluates it. +const ( + batchSubmitterRoot = "root" + batchSubmitterUser = "user" + batchSubmitterTemporary = "temporary" + batchSubmitterService = "service-account" +) + +//msgp:ignore batchJobAuthzError + +// batchJobAuthzError refuses a batch job, or one of its actions, for its +// submitter. It ends the job: it is persisted as the job's failure and is +// never retried. +type batchJobAuthzError struct { + reason string +} + +func (e batchJobAuthzError) Error() string { + return e.reason +} + +func isBatchJobAuthzError(err error) bool { + var e batchJobAuthzError + return errors.As(err, &e) +} + +var ( + errBatchJobResubmit = batchJobAuthzError{reason: "the batch job does not record who submitted it; submit it again"} + errBatchJobChanged = batchJobAuthzError{reason: "the credential that submitted the batch job has changed since submission; submit it again"} + errBatchJobInvalid = batchJobAuthzError{reason: "the credential that submitted the batch job is no longer valid"} +) + +func batchSubmitterType(cred auth.Credentials) string { + switch { + case cred.IsTemp(): + return batchSubmitterTemporary + case cred.IsServiceAccount(): + return batchSubmitterService + default: + return batchSubmitterUser + } +} + +// batchSubmitterBinding is a fingerprint of one credential keyed with its own +// secret. It changes when the secret, the session token or the parent +// changes, and a new credential that reuses the access key does not have it. +func batchSubmitterBinding(typ string, cred auth.Credentials) []byte { + mac := hmac.New(sha256.New, []byte(cred.SecretKey)) + for _, s := range []string{"silo batch-job submitter v1", typ, cred.AccessKey, cred.ParentUser, cred.SessionToken} { + var n [8]byte + binary.BigEndian.PutUint64(n[:], uint64(len(s))) + mac.Write(n[:]) + mac.Write([]byte(s)) + } + return mac.Sum(nil) +} + +// identityRecord returns the stored identity of a non-root access key. +func (sys *IAMSys) identityRecord(accessKey string) (UserIdentity, bool, error) { + if !sys.Initialized() { + return UserIdentity{}, false, errServerNotInitialized + } + u, ok := sys.store.GetUser(accessKey) + return u, ok, nil +} + +// newBatchJobSubmitter binds the credential that authenticated a +// StartBatchJob request. +func newBatchJobSubmitter(cred auth.Credentials) (*batchJobSubmitter, error) { + root := globalActiveCred + if cred.AccessKey == "" { + return nil, errBatchJobInvalid + } + if cred.AccessKey == root.AccessKey { + if subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(root.SecretKey)) != 1 { + return nil, errBatchJobChanged + } + return &batchJobSubmitter{ + Version: batchJobSubmitterVersion, + Type: batchSubmitterRoot, + AccessKey: root.AccessKey, + Binding: batchSubmitterBinding(batchSubmitterRoot, root), + }, nil + } + u, ok, err := globalIAMSys.identityRecord(cred.AccessKey) + if err != nil { + return nil, err + } + // The record must be the one the request was authenticated with. + if !ok || u.Credentials.AccessKey != cred.AccessKey || u.Credentials.ParentUser != cred.ParentUser || + subtle.ConstantTimeCompare([]byte(u.Credentials.SecretKey), []byte(cred.SecretKey)) != 1 || + subtle.ConstantTimeCompare([]byte(u.Credentials.SessionToken), []byte(cred.SessionToken)) != 1 { + return nil, errBatchJobChanged + } + typ := batchSubmitterType(u.Credentials) + return &batchJobSubmitter{ + Version: batchJobSubmitterVersion, + Type: typ, + AccessKey: u.Credentials.AccessKey, + ParentUser: u.Credentials.ParentUser, + UpdatedAt: u.UpdatedAt, + RevokedBefore: u.RevokedBefore, + Binding: batchSubmitterBinding(typ, u.Credentials), + }, nil +} + +//msgp:ignore batchJobPrincipal batchAuthzRequest + +// batchJobPrincipal is a submitter resolved against current IAM state. +type batchJobPrincipal struct { + typ string + accessKey string + parentUser string + groups []string + claims map[string]any + owner bool + temp bool // evaluated as a temporary credential + service bool // evaluated as a service account +} + +// principal resolves the binding against current IAM state. It fails if the +// credential no longer exists, is no longer valid, or is no longer the one +// that was bound. +func (s *batchJobSubmitter) principal(ctx context.Context) (batchJobPrincipal, error) { + if s == nil || s.AccessKey == "" { + return batchJobPrincipal{}, errBatchJobResubmit + } + if s.Version != batchJobSubmitterVersion { + return batchJobPrincipal{}, batchJobAuthzError{reason: fmt.Sprintf("unsupported batch job submitter version %d; submit the job again", s.Version)} + } + root := globalActiveCred + if s.Type == batchSubmitterRoot { + if s.AccessKey != root.AccessKey || !hmac.Equal(s.Binding, batchSubmitterBinding(batchSubmitterRoot, root)) { + return batchJobPrincipal{}, errBatchJobChanged + } + if !globalAPIConfig.permitRootAccess() { + return batchJobPrincipal{}, errBatchJobInvalid + } + return batchJobPrincipal{typ: batchSubmitterRoot, accessKey: root.AccessKey, owner: true}, nil + } + switch s.Type { + case batchSubmitterUser, batchSubmitterTemporary, batchSubmitterService: + default: + return batchJobPrincipal{}, errBatchJobResubmit + } + if s.AccessKey == root.AccessKey { + return batchJobPrincipal{}, errBatchJobChanged + } + u, ok, err := globalIAMSys.CheckKey(ctx, s.AccessKey) + if err != nil { + return batchJobPrincipal{}, err + } + if !ok { + // Deleted, disabled, expired or revoked. + return batchJobPrincipal{}, errBatchJobInvalid + } + cred := u.Credentials + typ := batchSubmitterType(cred) + if typ != s.Type || cred.AccessKey != s.AccessKey || cred.ParentUser != s.ParentUser || + !u.UpdatedAt.Equal(s.UpdatedAt) || !u.RevokedBefore.Equal(s.RevokedBefore) || + !hmac.Equal(s.Binding, batchSubmitterBinding(typ, cred)) { + return batchJobPrincipal{}, errBatchJobChanged + } + if cred.IsExpired() || cred.Status == auth.AccountOff { + return batchJobPrincipal{}, errBatchJobInvalid + } + p := batchJobPrincipal{ + typ: typ, + accessKey: cred.AccessKey, + parentUser: cred.ParentUser, + groups: cred.Groups, + claims: map[string]any{}, + temp: cred.IsTemp(), + service: !cred.IsTemp() && cred.IsServiceAccount(), + } + if typ != batchSubmitterUser { + // Verify the token as a request would, with the current signing keys. + jwtClaims, err := extractJWTClaims(u) + if err != nil { + return batchJobPrincipal{}, errBatchJobInvalid + } + p.claims = jwtClaims.Map() + if err := decodeBatchSessionPolicy(p.claims); err != nil { + return batchJobPrincipal{}, errBatchJobInvalid + } + } + p.owner = cred.ParentUser == root.AccessKey && cred.AccessKey != siteReplicatorSvcAcc + if p.owner && !globalAPIConfig.permitRootAccess() { + return batchJobPrincipal{}, errBatchJobInvalid + } + if _, ok := p.claims[policy.SessionPolicyName]; ok { + p.owner = false + } + return p, nil +} + +// decodeBatchSessionPolicy decodes a session policy claim for evaluation. +// A request decodes it while verifying its token, except when an +// authorization plugin is configured; a batch job decodes it always, so the +// session policy also limits what the plugin allows. +func decodeBatchSessionPolicy(claims map[string]any) error { + sp, ok := claims[policy.SessionPolicyName] + if !ok { + return nil + } + s, ok := sp.(string) + if !ok { + return errAuthentication + } + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return errAuthentication + } + claims[sessionPolicyNameExtracted] = string(b) + return nil +} + +// batchAuthzRequest is one action a batch job is about to take. +type batchAuthzRequest struct { + action policy.Action + bucket string + object string // empty for a bucket action + // versionID is the version an object version action addresses. + versionID string + // tags are the current tags of the object version acted on. + tags string + // tagsUnknown is set when the job does not read the version it acts + // on, so s3:ExistingObjectTag is context it does not have. + tagsUnknown bool + // values are request values the job supplies itself, such as the + // listing prefix or the encryption and retention it writes, keyed by + // condition key name. + values map[string][]string +} + +// batchAuthzMarker marks the condition values of a batch job for an +// authorization plugin. Requests cannot set s3:authType. +const batchAuthzMarker = "batch-job" + +// Condition keys a batch job always provides, as a request would: identity, +// time, the addressed version and the stored tags. Keys a request takes from +// itself are provided only when the job sets them (see batchAuthzRequest +// values); every other key is request context the job does not have. +var batchServerConditionKeys = map[string]bool{ + condition.AWSCurrentTime.ToKey().String(): true, + condition.AWSEpochTime.ToKey().String(): true, + condition.AWSPrincipalType.ToKey().String(): true, + condition.AWSUserID.ToKey().String(): true, + condition.AWSUsername.ToKey().String(): true, + condition.AWSGroups.ToKey().String(): true, + condition.S3VersionID.ToKey().String(): true, + condition.ExistingObjectTag.ToKey().String(): true, +} + +// Request keys a job may supply, by the name they have in condition values. +var batchRequestConditionKeys = map[string]condition.KeyName{ + condition.S3Prefix.Name(): condition.S3Prefix, + condition.S3XAmzServerSideEncryption.Name(): condition.S3XAmzServerSideEncryption, + condition.S3XAmzServerSideEncryptionAwsKmsKeyID.Name(): condition.S3XAmzServerSideEncryptionAwsKmsKeyID, + condition.S3ObjectLockMode.Name(): condition.S3ObjectLockMode, + condition.S3ObjectLockRetainUntilDate.Name(): condition.S3ObjectLockRetainUntilDate, + condition.S3ObjectLockLegalHold.Name(): condition.S3ObjectLockLegalHold, + condition.RequestObjectTagKeys.Name(): condition.RequestObjectTagKeys, +} + +// suppliedConditionKeys returns the condition keys whose values req provides. +func (req batchAuthzRequest) suppliedConditionKeys() map[string]bool { + supplied := make(map[string]bool, len(batchServerConditionKeys)+len(req.values)) + for k := range batchServerConditionKeys { + supplied[k] = true + } + if req.tagsUnknown { + delete(supplied, condition.ExistingObjectTag.ToKey().String()) + } + for name := range req.values { + if key, ok := batchRequestConditionKeys[name]; ok { + supplied[key.ToKey().String()] = true + } + if strings.HasPrefix(name, condition.RequestObjectTag.Name()+"/") { + supplied[condition.RequestObjectTag.ToKey().String()] = true + } + } + return supplied +} + +func batchKeySupplied(supplied map[string]bool, name string) bool { + if i := strings.IndexByte(name, '/'); i >= 0 { + name = name[:i] + } + if strings.HasPrefix(name, "jwt:") || strings.HasPrefix(name, "ldap:") { + // Claims of the verified token, as a request has them. + return true + } + return supplied[name] +} + +// conditionValues are the values a request with the same identity would +// carry, without any value taken from the request itself. +func (p batchJobPrincipal) conditionValues(req batchAuthzRequest) map[string][]string { + now := UTCNow() + username := p.accessKey + if p.temp || p.service { + username = p.parentUser + } + principalType := "User" + if len(p.claims) > 0 { + principalType = "AssumedRole" + } + if username == globalActiveCred.AccessKey { + principalType = "Account" + } + // Identity and time, then the claims over them, as a request has them + // (an LDAP session's short username replaces its parent DN). + values := map[string][]string{ + "CurrentTime": {now.Format(time.RFC3339)}, + "EpochTime": {strconv.FormatInt(now.Unix(), 10)}, + "principaltype": {principalType}, + "userid": {username}, + "username": {username}, + } + for k, v := range p.claims { + if s, ok := v.(string); ok { + values[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{s} + } + } + // No claim stands for a value of the object or of the job, which + // follow and always win. + for k := range values { + if k == "versionid" || k == "prefix" || k == "authtype" || + strings.HasPrefix(k, "existingobjecttag/") || strings.HasPrefix(k, "requestobjecttag") || + strings.HasPrefix(k, "x-amz-") || strings.HasPrefix(k, "object-lock-") { + delete(values, k) + } + } + // For an authorization plugin: this is a batch job, not a request. + values["authType"] = []string{batchAuthzMarker} + if req.versionID != "" { + values["versionid"] = []string{req.versionID} + } + if req.tags != "" { + if t, err := tags.ParseObjectTags(req.tags); err == nil { + for k, v := range t.ToMap() { + values[pathJoin("ExistingObjectTag", k)] = []string{v} + } + } + } + for k, v := range req.values { + values[k] = v + } + if grpsVal, ok := p.claims["groups"]; ok { + if grpsIs, ok := grpsVal.([]any); ok { + grps := []string{} + for _, gI := range grpsIs { + if g, ok := gI.(string); ok { + grps = append(grps, g) + } + } + if len(grps) > 0 { + values["groups"] = grps + } + } + } + if _, ok := values["groups"]; !ok && len(p.groups) > 0 { + values["groups"] = p.groups + } + return values +} + +// authorize evaluates one action of the job as its submitter. +func (p batchJobPrincipal) authorize(req batchAuthzRequest) error { + args := policy.Args{ + AccountName: p.accessKey, + Groups: p.groups, + Action: req.action, + BucketName: req.bucket, + ObjectName: req.object, + ConditionValues: p.conditionValues(req), + IsOwner: p.owner, + Claims: p.claims, + } + denied := batchJobAuthzError{reason: batchDeniedReason(req)} + plugin := newGlobalAuthZPluginFn() + policies, err := globalIAMSys.batchLocalPolicies(p, args, plugin == nil) + if err != nil { + return denied + } + supplied := req.suppliedConditionKeys() + for _, pol := range policies { + for _, st := range pol.Statements { + if batchStatementNeedsRequest(st, args, supplied) { + return batchJobAuthzError{reason: denied.reason + ": a policy statement depends on request context that a batch job does not have"} + } + } + } + if plugin != nil { + ok, err := plugin.IsAllowed(args) + if err != nil { + authZLogIf(GlobalContext, err) + } + if !ok { + return denied + } + // The plugin does not see a session policy; it only limits further. + if has, allowed := p.sessionPolicyDecision(args); has && !allowed { + return denied + } + return nil + } + if !globalIAMSys.isAllowedLocally(args) { + return denied + } + return nil +} + +func batchDeniedReason(req batchAuthzRequest) string { + resource := req.bucket + if req.object != "" { + resource = pathJoin(req.bucket, req.object) + } + if req.versionID != "" { + resource += " (versionId " + req.versionID + ")" + } + return fmt.Sprintf("the batch job submitter is not allowed %s on %s", req.action, resource) +} + +// sessionPolicyDecision evaluates the session policy the way IsAllowed would +// for this credential. +func (p batchJobPrincipal) sessionPolicyDecision(args policy.Args) (bool, bool) { + switch { + case p.temp: + return isAllowedBySessionPolicy(args) + case p.service: + if p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType { + return false, false + } + return isAllowedBySessionPolicyForServiceAccount(args) + } + return false, false +} + +// batchLocalPolicies returns every local policy that can take part in the +// decision on args: the mapped policies that IsAllowed would merge when +// withMapped is set (no authorization plugin), and the session policy. +func (sys *IAMSys) batchLocalPolicies(p batchJobPrincipal, args policy.Args, withMapped bool) ([]policy.Policy, error) { + var out []policy.Policy + if withMapped && args.IsOwner { + // IsAllowed does not evaluate any policy for the owner. + return nil, nil + } + if withMapped { + var names []string + ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey + switch { + case p.typ == batchSubmitterRoot || ownerDerived: + case p.temp || p.service: + if roleArn := args.GetRoleArn(); roleArn != "" { + a, err := arn.Parse(roleArn) + if err != nil { + return nil, err + } + names = append(names, newMappedPolicy(sys.rolesMap[a]).toSlice()...) + } else { + mapped, err := sys.PolicyDBGet(p.parentUser, args.Groups...) + if err != nil { + return nil, err + } + names = append(names, mapped...) + // The claimed policies apply only when nothing is mapped. + if len(mapped) == 0 { + if claimed, ok := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID()); ok { + names = append(names, claimed.ToSlice()...) + } + } + } + default: + mapped, err := sys.PolicyDBGet(p.accessKey, args.Groups...) + if err != nil { + return nil, err + } + names = append(names, mapped...) + } + if len(names) > 0 { + out = append(out, sys.GetCombinedPolicy(names...)) + } + } + if p.temp || p.service { + if p.service && p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType { + return out, nil + } + if sp, ok := p.claims[sessionPolicyNameExtracted]; ok { + s, ok := sp.(string) + if !ok { + return nil, errAuthentication + } + parsed, err := policy.ParseConfig(strings.NewReader(s)) + if err != nil { + return nil, err + } + out = append(out, *parsed) + } + } + return out, nil +} + +// batchStatementNeedsRequest reports whether a statement that applies to +// args depends on a condition key that the job does not supply. Applying +// such a statement without its value could grant (Allow) or drop (Deny) +// what a request with that context would not, so the job refuses the action. +// Resources are matched without conditions; a statement is taken to apply +// whenever it might. +func batchStatementNeedsRequest(st policy.Statement, args policy.Args, supplied map[string]bool) bool { + if (!st.Actions.Match(args.Action) && !st.Actions.IsEmpty()) || st.NotActions.Match(args.Action) { + return false + } + // Policy variables in resources or condition values. + if data, err := json.Marshal(st); err == nil { + s := string(data) + for { + i := strings.Index(s, "${") + if i < 0 { + break + } + s = s[i+2:] + j := strings.IndexByte(s, '}') + if j < 0 { + break + } + if !batchKeySupplied(supplied, s[:j]) { + return true + } + s = s[j+1:] + } + } + if !batchStatementMayApply(st, args) { + return false + } + for key := range st.Conditions.Keys() { + if !batchKeySupplied(supplied, key.String()) { + return true + } + } + return false +} + +// batchStatementMayApply matches the resources of a statement, in both forms +// a bucket request can take. A statement whose resources the evaluator +// translates (S3 Tables and vectors actions cover S3 data actions) is taken +// to apply. +func batchStatementMayApply(st policy.Statement, args policy.Args) bool { + for _, actions := range []policy.ActionSet{st.Actions, st.NotActions} { + for action := range actions { + if policy.TableAction(action).IsValid() || policy.VectorsAction(action).IsValid() { + return true + } + } + } + for _, resources := range []policy.ResourceSet{st.Resources, st.NotResources} { + for r := range resources { + if r.Type != policy.ResourceARNS3 && r.Type != policy.ResourceARNAll { + return true + } + } + } + resources := []string{args.BucketName + "/" + strings.TrimPrefix(args.ObjectName, "/")} + if args.ObjectName == "" { + resources = []string{args.BucketName, args.BucketName + "/"} + } + for _, resource := range resources { + if len(st.Resources) > 0 && !st.Resources.Match(resource, args.ConditionValues) { + continue + } + if len(st.NotResources) > 0 && st.NotResources.Match(resource, args.ConditionValues) { + continue + } + return true + } + return false +} + +// authorize resolves the submitter and evaluates each request. +func (s *batchJobSubmitter) authorize(ctx context.Context, reqs ...batchAuthzRequest) error { + p, err := s.principal(ctx) + if err != nil { + return err + } + for _, req := range reqs { + if err := p.authorize(req); err != nil { + return err + } + } + return nil +} + +// startAuthzRequests are the bucket actions a job needs before it acts on any +// object: the listings it walks. Object actions are authorized per object. +func (j BatchJobRequest) startAuthzRequests() []batchAuthzRequest { + list := func(bucket string, prefixes []string) []batchAuthzRequest { + if len(prefixes) == 0 { + prefixes = []string{""} + } + var reqs []batchAuthzRequest + for _, prefix := range prefixes { + values := map[string][]string{condition.S3Prefix.Name(): {prefix}} + reqs = append(reqs, + batchAuthzRequest{action: policy.ListBucketAction, bucket: bucket, values: values}, + batchAuthzRequest{action: policy.ListBucketVersionsAction, bucket: bucket, values: values}, + ) + } + return reqs + } + // The same order as Type() and the job pool. + switch { + case j.Replicate != nil: + if !j.Replicate.RemoteToLocal() { + return list(j.Replicate.Source.Bucket, j.Replicate.Source.Prefix.F()) + } + case j.KeyRotate != nil: + return list(j.KeyRotate.Bucket, []string{j.KeyRotate.Prefix}) + case j.Expire != nil: + return list(j.Expire.Bucket, batchCanonicalPrefixes(j.Expire.Prefix.F())) + } + return nil +} + +// jobSections counts the job types a definition holds. A job is one of them. +func (j BatchJobRequest) jobSections() int { + n := 0 + for _, set := range []bool{j.Replicate != nil, j.KeyRotate != nil, j.Expire != nil} { + if set { + n++ + } + } + return n +} + +var errBatchJobSections = batchJobAuthzError{reason: "a batch job must define exactly one job type"} + +// refuse ends the job for its submitter. +func (ri *batchJobInfo) refuse(err error) { + ri.mu.Lock() + defer ri.mu.Unlock() + if ri.FailureReason == "" { + ri.FailureReason = err.Error() + } + ri.Complete = false + ri.Failed = true +} + +func (ri *batchJobInfo) refused() bool { + ri.mu.RLock() + defer ri.mu.RUnlock() + return ri.FailureReason != "" +} + +// startAuthorized checks the submitter of a job that is about to run. It +// returns done for a job that must not run now: one refused before, which +// stays refused, or one refused now, which is persisted as failed first. +// An error that is not a decision leaves the job to a later start. +func (ri *batchJobInfo) startAuthorized(ctx context.Context, api ObjectLayer, job BatchJobRequest) (done bool, err error) { + if !ri.refused() { + err := job.Submitter.authorize(ctx, job.startAuthzRequests()...) + if err == nil && job.jobSections() != 1 { + err = errBatchJobSections + } + if err == nil { + return false, nil + } + if !isBatchJobAuthzError(err) { + return true, err + } + ri.refuse(err) + batchLogIf(ctx, fmt.Errorf("batch job %s refused: %w", job.ID, err)) + } + globalBatchJobsMetrics.save(job.ID, ri) + return true, ri.updateAfter(ctx, api, 0, job) +} diff --git a/cmd/batch-authz_gen.go b/cmd/batch-authz_gen.go new file mode 100644 index 000000000..b5ebe3f64 --- /dev/null +++ b/cmd/batch-authz_gen.go @@ -0,0 +1,260 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *batchJobSubmitter) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "t": + z.Type, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + case "ak": + z.AccessKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AccessKey") + return + } + case "pu": + z.ParentUser, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ParentUser") + return + } + case "ua": + z.UpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + case "rb": + z.RevokedBefore, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "RevokedBefore") + return + } + case "bd": + z.Binding, err = dc.ReadBytes(z.Binding) + if err != nil { + err = msgp.WrapError(err, "Binding") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *batchJobSubmitter) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "v" + err = en.Append(0x87, 0xa1, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + // write "t" + err = en.Append(0xa1, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Type) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + // write "ak" + err = en.Append(0xa2, 0x61, 0x6b) + if err != nil { + return + } + err = en.WriteString(z.AccessKey) + if err != nil { + err = msgp.WrapError(err, "AccessKey") + return + } + // write "pu" + err = en.Append(0xa2, 0x70, 0x75) + if err != nil { + return + } + err = en.WriteString(z.ParentUser) + if err != nil { + err = msgp.WrapError(err, "ParentUser") + return + } + // write "ua" + err = en.Append(0xa2, 0x75, 0x61) + if err != nil { + return + } + err = en.WriteTime(z.UpdatedAt) + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + // write "rb" + err = en.Append(0xa2, 0x72, 0x62) + if err != nil { + return + } + err = en.WriteTime(z.RevokedBefore) + if err != nil { + err = msgp.WrapError(err, "RevokedBefore") + return + } + // write "bd" + err = en.Append(0xa2, 0x62, 0x64) + if err != nil { + return + } + err = en.WriteBytes(z.Binding) + if err != nil { + err = msgp.WrapError(err, "Binding") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *batchJobSubmitter) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "v" + o = append(o, 0x87, 0xa1, 0x76) + o = msgp.AppendInt(o, z.Version) + // string "t" + o = append(o, 0xa1, 0x74) + o = msgp.AppendString(o, z.Type) + // string "ak" + o = append(o, 0xa2, 0x61, 0x6b) + o = msgp.AppendString(o, z.AccessKey) + // string "pu" + o = append(o, 0xa2, 0x70, 0x75) + o = msgp.AppendString(o, z.ParentUser) + // string "ua" + o = append(o, 0xa2, 0x75, 0x61) + o = msgp.AppendTime(o, z.UpdatedAt) + // string "rb" + o = append(o, 0xa2, 0x72, 0x62) + o = msgp.AppendTime(o, z.RevokedBefore) + // string "bd" + o = append(o, 0xa2, 0x62, 0x64) + o = msgp.AppendBytes(o, z.Binding) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *batchJobSubmitter) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "t": + z.Type, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + case "ak": + z.AccessKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AccessKey") + return + } + case "pu": + z.ParentUser, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ParentUser") + return + } + case "ua": + z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + case "rb": + z.RevokedBefore, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RevokedBefore") + return + } + case "bd": + z.Binding, bts, err = msgp.ReadBytesBytes(bts, z.Binding) + if err != nil { + err = msgp.WrapError(err, "Binding") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *batchJobSubmitter) Msgsize() (s int) { + s = 1 + 2 + msgp.IntSize + 2 + msgp.StringPrefixSize + len(z.Type) + 3 + msgp.StringPrefixSize + len(z.AccessKey) + 3 + msgp.StringPrefixSize + len(z.ParentUser) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.BytesPrefixSize + len(z.Binding) + return +} diff --git a/cmd/batch-authz_gen_test.go b/cmd/batch-authz_gen_test.go new file mode 100644 index 000000000..15f9cdfa1 --- /dev/null +++ b/cmd/batch-authz_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalbatchJobSubmitter(t *testing.T) { + v := batchJobSubmitter{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgbatchJobSubmitter(b *testing.B) { + v := batchJobSubmitter{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgbatchJobSubmitter(b *testing.B) { + v := batchJobSubmitter{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalbatchJobSubmitter(b *testing.B) { + v := batchJobSubmitter{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodebatchJobSubmitter(t *testing.T) { + v := batchJobSubmitter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodebatchJobSubmitter Msgsize() is inaccurate") + } + + vn := batchJobSubmitter{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodebatchJobSubmitter(b *testing.B) { + v := batchJobSubmitter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodebatchJobSubmitter(b *testing.B) { + v := batchJobSubmitter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go index 17dcf556f..3f8b1e15a 100644 --- a/cmd/batch-expire.go +++ b/cmd/batch-expire.go @@ -27,6 +27,7 @@ import ( "net/http" "runtime" "strconv" + "sync/atomic" "time" "github.com/minio/minio-go/v7/pkg/tags" @@ -34,6 +35,7 @@ import ( xhttp "github.com/minio/minio/internal/http" xioutil "github.com/minio/minio/internal/ioutil" "github.com/pgsty/silo-pkg/v3/env" + "github.com/pgsty/silo-pkg/v3/policy" "github.com/pgsty/silo-pkg/v3/wildcard" "github.com/pgsty/silo-pkg/v3/workers" "github.com/pgsty/silo-pkg/v3/xtime" @@ -289,16 +291,6 @@ type BatchJobExpire struct { var _ yaml.Unmarshaler = &BatchJobExpire{} -// RedactSensitive will redact any sensitive information in b. -func (r *BatchJobExpire) RedactSensitive() { - if r == nil { - return - } - if r.NotificationCfg.Token != "" { - r.NotificationCfg.Token = redactedText - } -} - // UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information. func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error { type expireJob BatchJobExpire @@ -345,32 +337,6 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error { return nil } -// Expire expires object versions which have already matched supplied filter conditions -func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error { - opts := ObjectOptions{ - PrefixEnabledFn: vc.PrefixEnabled, - VersionSuspended: vc.Suspended(), - } - - allErrs := make([]error, 0, len(objsToDel)) - - for { - count := len(objsToDel) - if count == 0 { - break - } - if count > maxDeleteList { - count = maxDeleteList - } - _, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts) - allErrs = append(allErrs, errs...) - // Next batch of deletion - objsToDel = objsToDel[count:] - } - - return allErrs -} - const ( batchExpireName = "batch-expire.bin" batchExpireFormat = 1 @@ -381,22 +347,238 @@ const ( batchExpireJobDefaultRetryDelay = 250 * time.Millisecond ) -type objInfoCache map[string]*ObjectInfo - -func newObjInfoCache() objInfoCache { - return objInfoCache(make(map[string]*ObjectInfo)) +// expireVersionRef is a version of a key whose versions are all purged, as +// the walk saw it. +type expireVersionRef struct { + VersionID string + UserTags string + DeleteMarker bool + Locked bool } -func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) { - oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi +type expireObjInfo struct { + ObjectInfo + ExpireAll bool + DeleteMarkerCount int64 + // versions are the versions of an ExpireAll key the walk saw, the + // latest first. + versions []expireVersionRef } -func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) { - oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] - return oi, ok +// isErrObjectLocked reports a version that Object Lock protects. +func isErrObjectLocked(err error) bool { + var locked ObjectLocked + return errors.As(err, &locked) } -func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) { +func (r *BatchJobExpire) tagsFiltered() bool { + for _, rule := range r.Rules { + if len(rule.Tags) > 0 { + return true + } + } + return false +} + +// expireAuthzRequests are the actions the job takes on one version: it reads +// the version, and its tags when a rule filters by them, and deletes it. +func (r *BatchJobExpire) expireAuthzRequests(name, versionID, userTags string) []batchAuthzRequest { + if versionID == "" { + versionID = nullVersionID + } + reqs := []batchAuthzRequest{ + {action: policy.GetObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags}, + {action: policy.GetObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags}, + {action: policy.DeleteObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags}, + {action: policy.DeleteObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags}, + } + if r.tagsFiltered() { + reqs = append(reqs, + batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags}, + batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags}, + ) + } + return reqs +} + +// expireVersion deletes one version that is older than the versions the rule +// retains. Under the object's write lock, with the version as it is then, it +// leaves a version that Object Lock protects (ObjectLocked) and refuses one +// that the submitter may not delete (batchJobAuthzError). An unreadable +// version is not deleted. +func (r *BatchJobExpire) expireVersion(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, exp expireObjInfo) error { + p, err := job.Submitter.principal(ctx) + if err != nil { + return err + } + versionID := exp.VersionID + if versionID == "" { + // An empty version ID would add a delete marker instead. + versionID = nullVersionID + } + opts := ObjectOptions{ + VersionID: versionID, + Versioned: vc.PrefixEnabled(exp.Name), + VersionSuspended: vc.Suspended(), + } + opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) error { + // A delete marker reads as MethodNotAllowed; any other error, + // including a missing version, stops the delete. + marker := isErrMethodNotAllowed(gerr) && (goi.DeleteMarker || !goi.VersionPurgeStatus.Empty()) + if gerr != nil && !marker { + return gerr + } + if enforceRetentionForDeletion(ctx, goi) { + return ObjectLocked{Bucket: r.Bucket, Object: exp.Name, VersionID: versionID} + } + for _, req := range r.expireAuthzRequests(exp.Name, versionID, goi.UserTags) { + if err := p.authorize(req); err != nil { + return err + } + } + return nil + }) + _, err = api.DeleteObject(ctx, r.Bucket, exp.Name, opts) + return err +} + +// expireVersions deletes the versions of one batch entry and records the +// outcome in res. It returns an error that ends the job. +func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, delay time.Duration, res *batchCounts) error { + for attempts := 1; attempts <= retryAttempts; attempts++ { + if err := ctx.Err(); err != nil { + return err + } + stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) + err := r.expireVersion(ctx, api, vc, job, exp) + stopFn(exp, err) + switch { + case err == nil: + if exp.DeleteMarker { + ri.countRemoved(r.Bucket, 0, 1) + } else { + ri.countRemoved(r.Bucket, 1, 0) + } + return nil + case isBatchJobAuthzError(err): + return err + case isErrObjectLocked(err): + // Left in place, not retried and not a failure. + res.versionsSkipped++ + return nil + case isErrVersionNotFound(err) || isErrObjectNotFound(err): + // Removed concurrently. + return nil + } + batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", r.Bucket, exp.Name, exp.VersionID, err, attempts)) + if attempts < retryAttempts { + time.Sleep(delay) + } + } + if exp.DeleteMarker { + res.deleteMarkersFailed++ + } else { + res.objectsFailed++ + } + return nil +} + +// expireAllVersions purges every version of a key whose rule retains none, +// with the purge of exactly that key (erasureServerPools.deleteAllVersions), +// while the version the walk matched is still its latest. What the purge +// removed is counted at once; a key it refused because Object Lock protects +// a version is counted in res as refused, and the versions a failed purge +// left in res as failures. A failed purge is retried up to retryAttempts +// times, each time with the version the walk matched; a purge that stopped +// at its budget after removing versions continues without spending an +// attempt. A key whose latest version changed since the walk is not +// expired. It returns an error that ends the job. +// +// TODO(rebase onto the delete-all purge, claude/fix-delete-all): before the +// purge removes anything, authorize every version of the history it +// approved under its lock, with version-specific conditions +// (expireAuthzRequests). Until then the checks here use the versions the +// walk saw, and do not cover a version added after the walk; the purge +// itself refuses a key that holds a locked version. +func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error { + for _, v := range exp.versions { + if v.Locked { + res.keysRefused++ + return nil + } + } + var reqs []batchAuthzRequest + for _, v := range exp.versions { + reqs = append(reqs, r.expireAuthzRequests(exp.Name, v.VersionID, v.UserTags)...) + } + if err := job.Submitter.authorize(ctx, reqs...); err != nil { + return err + } + var objects, markers int64 + matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime} + for attempts := 1; attempts <= retryAttempts; attempts++ { + if err := ctx.Err(); err != nil { + return err + } + stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) + latest := matched + var purge deleteAllResult + _, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{ + DeleteAllVersions: true, + DeleteAllLatest: &latest, + DeleteAllResult: &purge, + }) + // Removed versions are gone for any later run: count them now. + if purge.Removed > 0 { + ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers)) + objects += int64(purge.Removed - purge.Markers) + markers += int64(purge.Markers) + } + stopFn(exp, err) + switch { + case err == nil: + return nil + case errors.Is(err, errDeleteAllChanged): + // Changed since the walk matched it: not expired. + return nil + case isErrObjectLocked(err): + // Refused as a whole: nothing removed, not retried and not a + // failure. + res.keysRefused++ + return nil + case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0: + attempts-- // progress + continue + } + if err := ctx.Err(); err != nil { + return err + } + batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts)) + if errors.Is(err, errDeleteAllTiered) { + break // refused; a later run of the job checks again + } + } + res.expireAllFailed(exp, objects, markers) + return nil +} + +// expireAllFailed counts the versions of info, as the walk saw them, that a +// failed purge of all its versions did not remove: those beyond the objects +// and markers it removed, at least one. +func (c *batchCounts) expireAllFailed(info expireObjInfo, objects, markers int64) { + failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0) + failedMarkers := max(info.DeleteMarkerCount-markers, 0) + if failedObjects+failedMarkers == 0 { + failedObjects = 1 + } + c.objectsFailed += failedObjects + c.deleteMarkersFailed += failedMarkers +} + +// batchObjsForDelete expires the batches of expireCh. walkDone reports, once +// expireCh is closed, whether the walk reached its end: only then is its last +// key known to be complete. +func batchObjsForDelete(ctx context.Context, refuse func(error), walkDone func() bool, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) { vc, _ := globalBucketVersioningSys.Get(r.Bucket) retryAttempts := job.Expire.Retry.Attempts if retryAttempts <= 0 { @@ -407,162 +589,53 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo delay = batchExpireJobDefaultRetryDelay } - var i int + progress := newBatchProgress(ri, r.Bucket) + defer func() { + wk.Wait() + if walkDone() { + progress.finish() + } + }() for toExpire := range expireCh { select { case <-ctx.Done(): return default: } - if i > 0 { + seq := progress.begin() + if seq > 0 { if wait := globalBatchConfig.ExpirationWait(); wait > 0 { time.Sleep(wait) } } - i++ wk.Take() - go func(toExpire []expireObjInfo) { + go func(seq uint64, toExpire []expireObjInfo) { defer wk.Give() - - toExpireAll := make([]expireObjInfo, 0, len(toExpire)) - toDel := make([]ObjectToDelete, 0, len(toExpire)) - oiCache := newObjInfoCache() + var res batchCounts for _, exp := range toExpire { + var err error if exp.ExpireAll { - toExpireAll = append(toExpireAll, exp) - continue + err = r.expireAllVersions(ctx, api, job, ri, exp, retryAttempts, &res) + } else { + err = r.expireVersions(ctx, api, vc, job, ri, exp, retryAttempts, delay, &res) } - // Cache ObjectInfo value via pointers for - // subsequent use to track objects which - // couldn't be deleted. - od := ObjectToDelete{ - ObjectV: ObjectV{ - ObjectName: exp.Name, - VersionID: exp.VersionID, - }, - } - toDel = append(toDel, od) - oiCache.Add(od, &exp.ObjectInfo) - } - - // Purge all versions of exactly each key while the version the - // walk matched is still its latest; see - // erasureServerPools.deleteAllVersions. - for _, exp := range toExpireAll { - if !expireAllVersions(ctx, ri, api, exp, retryAttempts) { - return - } - } - - // DeleteMultiple objects - toDelCopy := make([]ObjectToDelete, len(toDel)) - for attempts := 1; attempts <= retryAttempts; attempts++ { - select { - case <-ctx.Done(): - return - default: - } - - stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) - // Copying toDel to select from objects whose - // deletion failed - copy(toDelCopy, toDel) - var failed int - errs := r.Expire(ctx, api, vc, toDel) - // reslice toDel in preparation for next retry attempt - toDel = toDel[:0] - for i, err := range errs { - if err != nil { - stopFn(toDelCopy[i], err) - batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, - err, attempts)) - failed++ - if oi, ok := oiCache.Get(toDelCopy[i]); ok { - ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts) - } - if attempts != retryAttempts { - // retry - toDel = append(toDel, toDelCopy[i]) - } - } else { - stopFn(toDelCopy[i], nil) - if oi, ok := oiCache.Get(toDelCopy[i]); ok { - ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts) - } + if err != nil { + // The batch is not done; a later run does it again. + if isBatchJobAuthzError(err) { + refuse(err) } - } - - globalBatchJobsMetrics.save(ri.JobID, ri) - - if failed == 0 { - break - } - - // Add a delay between retry attempts - if attempts < retryAttempts { - time.Sleep(delay) + globalBatchJobsMetrics.save(ri.JobID, ri) + return } } - }(toExpire) + if n := len(toExpire); n > 0 { + progress.done(seq, toExpire[n-1].Name, res) + } + globalBatchJobsMetrics.save(ri.JobID, ri) + }(seq, toExpire) } } -// expireAllVersions purges all versions of the key exp, which the walk -// matched by its latest version, and counts what the purge removed. It -// retries a failed purge up to attempts times, each time with the version -// the walk matched; a purge that stopped at its budget continues without -// spending an attempt. It does not retry once the key no longer has that -// latest version, which is not an expiry, or once the purge refused the -// key. It returns false once ctx ends. -func expireAllVersions(ctx context.Context, ri *batchJobInfo, api ObjectLayer, exp expireObjInfo, attempts int) bool { - var objects, markers int64 - matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime} - for attempt := 1; attempt <= attempts; attempt++ { - select { - case <-ctx.Done(): - ri.trackExpireAll(exp, objects, markers, true) - return false - default: - } - stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempt) - latest := matched - var purge deleteAllResult - _, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{ - DeleteAllVersions: true, - DeleteAllLatest: &latest, - DeleteAllResult: &purge, - }) - objects += int64(purge.Removed - purge.Markers) - markers += int64(purge.Markers) - stopFn(exp, err) - switch { - case err == nil: - ri.trackExpireAll(exp, objects, markers, false) - return true - case errors.Is(err, errDeleteAllChanged): - // Changed since the walk matched it: not expired. - ri.trackExpireAll(exp, objects, markers, false) - return true - case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0: - attempt-- // progress - continue - } - batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempt)) - var locked ObjectLocked - if errors.Is(err, errDeleteAllTiered) || errors.As(err, &locked) { - break // refused; a later run of the job checks again - } - } - ri.trackExpireAll(exp, objects, markers, true) - return true -} - -type expireObjInfo struct { - ObjectInfo - ExpireAll bool - DeleteMarkerCount int64 -} - // Start the batch expiration job, resumes if there was a pending job via "job.ID" func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { ri := &batchJobInfo{ @@ -573,6 +646,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo if err := ri.loadOrInit(ctx, api, job); err != nil { return err } + if done, err := ri.startAuthorized(ctx, api, job); done || err != nil { + return err + } globalBatchJobsMetrics.save(job.ID, ri) lastObject := ri.Object @@ -592,27 +668,43 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo ctx, cancelCause := context.WithCancelCause(ctx) defer cancelCause(nil) + // refuse ends the job for its submitter: record why before stopping. + refuse := func(err error) { + ri.refuse(err) + cancelCause(err) + } + // The walk stops at a listing failure: work after it is left to a later + // run, so the cursor cannot move past what was not listed. + walkCtx, stopWalk := context.WithCancel(ctx) + defer stopWalk() results := make(chan itemOrErr[ObjectInfo], workerSize) go func() { - prefixes := r.Prefix.F() - if len(prefixes) == 0 { - prefixes = []string{""} - } - for _, prefix := range prefixes { + // One ascending walk over disjoint prefixes, which the cursor + // describes. + for _, prefix := range batchCanonicalPrefixes(r.Prefix.F()) { + if walkCtx.Err() != nil { + break + } prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize) - err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{ + err := api.Walk(walkCtx, r.Bucket, prefix, prefixResultCh, WalkOptions{ Marker: lastObject, LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions VersionsSort: WalkVersionsSortDesc, }) if err != nil { - cancelCause(err) + if walkCtx.Err() == nil { + cancelCause(err) + } xioutil.SafeClose(results) return } for result := range prefixResultCh { - results <- result + select { + case results <- result: + case <-walkCtx.Done(): + // Stopped: let the walk end without a reader. + } } } xioutil.SafeClose(results) @@ -620,7 +712,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo // Goroutine to periodically save batch-expire job's in-memory state saverQuitCh := make(chan struct{}) + saverDoneCh := make(chan struct{}) go func() { + defer close(saverDoneCh) saveTicker := time.NewTicker(10 * time.Second) defer saveTicker.Stop() quit := false @@ -647,9 +741,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo expireCh := make(chan []expireObjInfo, workerSize) expireDoneCh := make(chan struct{}) + var walkDone atomic.Bool go func() { defer close(expireDoneCh) - batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh) + batchObjsForDelete(ctx, refuse, walkDone.Load, r, ri, job, api, wk, expireCh) }() var ( @@ -661,6 +756,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo done bool ) deleteMarkerCountMap := map[string]int64{} + // pushToExpire is called between keys only, so that a batch holds all + // the versions of its keys and its last key is done with it. pushToExpire := func() { // set preObject deleteMarkerCount if len(toDel) > 0 { @@ -695,6 +792,12 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo if result.Err != nil { failed = true batchLogIf(ctx, result.Err) + stopWalk() + done = true + break + } + if !batchResumeAfter(lastObject, result.Item.Name) { + // Completed by the run that set the cursor. continue } if result.Item.DeleteMarker { @@ -730,12 +833,17 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo toDel = append(toDel, expireObjInfo{ ObjectInfo: result.Item, ExpireAll: true, + versions: []expireVersionRef{newExpireVersionRef(ctx, result.Item)}, }) continue } } else if prevObj.Name == result.Item.Name { if matchedFilter.Purge.RetainVersions == 0 { - continue // including latest version in toDel suffices, skipping other versions + // The latest version in toDel purges them all. + if n := len(toDel); n > 0 && toDel[n-1].Name == result.Item.Name { + toDel[n-1].versions = append(toDel[n-1].versions, newExpireVersionRef(ctx, result.Item)) + } + continue } versionsCount++ } else { @@ -752,7 +860,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo toDel = append(toDel, expireObjInfo{ ObjectInfo: result.Item, }) - pushToExpire() case <-ctx.Done(): done = true } @@ -761,16 +868,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo } } - if context.Cause(ctx) != nil { - xioutil.SafeClose(expireCh) - return context.Cause(ctx) - } - pushToExpire() - // Send any remaining objects downstream - if len(toDel) > 0 { - select { - case <-ctx.Done(): - case expireCh <- toDel: + if context.Cause(ctx) == nil { + walkDone.Store(!failed) + pushToExpire() + // Send any remaining objects downstream + if len(toDel) > 0 { + select { + case <-ctx.Done(): + case expireCh <- toDel: + } } } xioutil.SafeClose(expireCh) @@ -778,13 +884,29 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo <-expireDoneCh // waits for the expire goroutine to complete wk.Wait() // waits for all expire workers to retire - ri.Complete = !failed && !ri.expireFailures() - ri.Failed = failed || ri.expireFailures() - globalBatchJobsMetrics.save(job.ID, ri) - - // Close the saverQuitCh - this also triggers saving in-memory state - // immediately one last time before we exit this method. + // Persist the final state once the saver has stopped, so that its last + // save cannot overwrite it. xioutil.SafeClose(saverQuitCh) + <-saverDoneCh + refused := ri.refused() + cause := context.Cause(ctx) + if cause != nil && !refused { + saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) + batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job)) + cancel() + globalBatchJobsMetrics.save(job.ID, ri) + return cause + } + + ri.mu.Lock() + itemsFailed := ri.expireFailures() + ri.Complete = !refused && !failed && !itemsFailed + ri.Failed = refused || failed || itemsFailed + ri.mu.Unlock() + globalBatchJobsMetrics.save(job.ID, ri) + saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) + batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job)) + cancel() // Notify expire jobs final status to the configured endpoint buf, _ := json.Marshal(ri) @@ -795,6 +917,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo return nil } +func newExpireVersionRef(ctx context.Context, oi ObjectInfo) expireVersionRef { + return expireVersionRef{ + VersionID: oi.VersionID, + UserTags: oi.UserTags, + DeleteMarker: oi.DeleteMarker, + Locked: enforceRetentionForDeletion(ctx, oi), + } +} + //msgp:ignore batchExpireJobError type batchExpireJobError struct { Code string diff --git a/cmd/batch-handlers.go b/cmd/batch-handlers.go index dba3c6a20..28985f728 100644 --- a/cmd/batch-handlers.go +++ b/cmd/batch-handlers.go @@ -43,6 +43,8 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" + objectlock "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/config/batch" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/hash" @@ -72,32 +74,47 @@ type BatchJobRequest struct { Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"` KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"` Expire *BatchJobExpire `yaml:"expire" json:"expire"` - ctx context.Context `msg:"-"` + // Submitter is the credential the job acts as. The server assigns it + // at submission; a job definition cannot name it. + Submitter *batchJobSubmitter `yaml:"-" json:"-"` + ctx context.Context `msg:"-"` } -// RedactSensitive will redact any sensitive information in b. -func (j *BatchJobRequest) RedactSensitive() { - j.Replicate.RedactSensitive() - j.Expire.RedactSensitive() - j.KeyRotate.RedactSensitive() +// redacted returns a copy of the job for output outside the server: the +// describe response, logs and errors. The copy has no secret keys, session +// tokens or notification tokens, and no submitter binding; the job itself, +// which a run needs as it is, is not changed. +func (j BatchJobRequest) redacted() (BatchJobRequest, error) { + buf, err := j.MarshalMsg(nil) + if err != nil { + return BatchJobRequest{}, err + } + var c BatchJobRequest + if _, err = c.UnmarshalMsg(buf); err != nil { + return BatchJobRequest{}, err + } + redact := func(s *string) { + if *s != "" { + *s = redactedText + } + } + c.Submitter = nil + if r := c.Replicate; r != nil { + redact(&r.Source.Creds.SecretKey) + redact(&r.Source.Creds.SessionToken) + redact(&r.Target.Creds.SecretKey) + redact(&r.Target.Creds.SessionToken) + redact(&r.Flags.Notify.Token) + } + if r := c.KeyRotate; r != nil { + redact(&r.Flags.Notify.Token) + } + if r := c.Expire; r != nil { + redact(&r.NotificationCfg.Token) + } + return c, nil } -// RedactSensitive will redact any sensitive information in b. -func (r *BatchJobReplicateV1) RedactSensitive() { - if r == nil { - return - } - if r.Target.Creds.SecretKey != "" { - r.Target.Creds.SecretKey = redactedText - } - if r.Target.Creds.SessionToken != "" { - r.Target.Creds.SessionToken = redactedText - } -} - -// RedactSensitive will redact any sensitive information in b. -func (r *BatchJobKeyRotateV1) RedactSensitive() {} - func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error { if endpoint == "" { return nil @@ -140,8 +157,114 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token) } -// ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local. -func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error { +// batchPullObjectLock prepares the Object Lock metadata that a pull writes +// with a version, as a PUT would on the destination: Object Lock metadata +// copied from the source must be valid, a version without retention takes the +// destination bucket's default retention, and both are dated with the +// destination's clock. Outside a lock-enabled bucket the metadata has no +// meaning and is not written. It reports the retention and legal hold that +// the pull copies from the source. +func batchPullObjectLock(bucket string, meta map[string]string) (retention, legalHold bool, err error) { + cfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + return false, false, err + } + h := make(http.Header) + for k, v := range meta { + for _, name := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate, xhttp.AmzObjectLockLegalHold} { + if strings.EqualFold(k, name) { + h.Set(name, v) + delete(meta, k) + } + } + } + if !cfg.LockEnabled { + return false, false, nil + } + if h.Get(xhttp.AmzObjectLockMode) != "" || h.Get(xhttp.AmzObjectLockRetainUntilDate) != "" { + // A copied retention may have expired already. + mode, until, err := objectlock.ParseObjectLockRetentionHeaders(h, true) + if err != nil { + return false, false, err + } + meta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(mode) + meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(until.UTC()) + retention = true + } else if cfg.Validity > 0 { + now, err := objectlock.UTCNowNTP() + if err != nil { + return false, false, err + } + meta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(cfg.Mode) + meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(now.Add(cfg.Validity).UTC()) + } + if _, ok := h[xhttp.AmzObjectLockLegalHold]; ok { + hold, err := objectlock.ParseObjectLockLegalHoldHeaders(h) + if err != nil { + return false, false, err + } + meta[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(hold.Status) + legalHold = true + } + return retention, legalHold, nil +} + +// pullAuthzRequests are the writes of one version that a pull makes to the +// destination key: the version, with the tags, retention and legal hold it +// copies, and the replication permission for a version ID it keeps. +// +// The pull does not encrypt what it writes, so it supplies no encryption +// values, whatever the source reports; and it does not read the destination +// version it may replace, so the tags of that version are unknown. +func pullAuthzRequests(bucket, object, versionID string, keepsVersion bool, opts ObjectOptions, retention, legalHold bool) []batchAuthzRequest { + values := map[string][]string{} + meta := caseInsensitiveMap(opts.UserDefined) + if retention { + mode, _ := meta.Lookup(xhttp.AmzObjectLockMode) + until, _ := meta.Lookup(xhttp.AmzObjectLockRetainUntilDate) + values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockMode), "x-amz-")] = []string{mode} + values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), "x-amz-")] = []string{until} + } + if legalHold { + hold, _ := meta.Lookup(xhttp.AmzObjectLockLegalHold) + values[strings.TrimPrefix(strings.ToLower(xhttp.AmzObjectLockLegalHold), "x-amz-")] = []string{hold} + } + tagStr, hasTags := meta.Lookup(xhttp.AmzObjectTagging) + if hasTags && tagStr != "" { + if t, err := tags.ParseObjectTags(tagStr); err == nil { + m := t.ToMap() + keys := make([]string, 0, len(m)) + for k, v := range m { + values[pathJoin("RequestObjectTag", k)] = []string{v} + keys = append(keys, k) + } + values["RequestObjectTagKeys"] = keys + } + } + actions := []policy.Action{policy.PutObjectAction} + if keepsVersion { + actions = append(actions, policy.ReplicateObjectAction) + } + if hasTags && tagStr != "" { + actions = append(actions, policy.PutObjectTaggingAction) + } + if retention { + actions = append(actions, policy.PutObjectRetentionAction) + } + if legalHold { + actions = append(actions, policy.PutObjectLegalHoldAction) + } + reqs := make([]batchAuthzRequest, 0, len(actions)) + for _, action := range actions { + reqs = append(reqs, batchAuthzRequest{action: action, bucket: bucket, object: object, versionID: versionID, tagsUnknown: true, values: values}) + } + return reqs +} + +// ReplicateFromSource copies one version from the remote source to the local +// target. A write that would replace a version protected by Object Lock +// returns ObjectLocked and leaves it. +func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool, sub *batchJobSubmitter) error { srcBucket := r.Source.Bucket tgtBucket := r.Target.Bucket srcObject := srcObjInfo.Name @@ -150,11 +273,28 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name) } + p, err := sub.principal(ctx) + if err != nil { + return err + } versionID := srcObjInfo.VersionID if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { versionID = "" } if srcObjInfo.DeleteMarker { + markerVersion := versionID + if markerVersion == "" { + markerVersion = nullVersionID + } + for _, req := range []batchAuthzRequest{ + {action: policy.DeleteObjectAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true}, + {action: policy.DeleteObjectVersionAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true}, + {action: policy.ReplicateDeleteAction, bucket: tgtBucket, object: tgtObject, versionID: markerVersion, tagsUnknown: true}, + } { + if err := p.authorize(req); err != nil { + return err + } + } _, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{ VersionID: versionID, // Since we are preserving a delete marker, we have to make sure this is always true. @@ -164,22 +304,43 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec MTime: srcObjInfo.ModTime, DeleteMarker: srcObjInfo.DeleteMarker, ReplicationRequest: true, + // A new marker may hide a locked version; one that would replace + // it is refused. + RefuseLockedOverwrite: true, }) return err } - opts := ObjectOptions{ - VersionID: srcObjInfo.VersionID, - MTime: srcObjInfo.ModTime, - PreserveETag: srcObjInfo.ETag, - UserDefined: srcObjInfo.UserDefined, + userDefined := cloneMSS(srcObjInfo.UserDefined) + retention, legalHold, err := batchPullObjectLock(tgtBucket, userDefined) + if err != nil { + return err } - if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { - opts.VersionID = "" + opts := ObjectOptions{ + VersionID: versionID, + Versioned: globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject), + MTime: srcObjInfo.ModTime, + PreserveETag: srcObjInfo.ETag, + UserDefined: userDefined, + // Checked with the commit, under the destination's write lock. + RefuseLockedOverwrite: true, } if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) { opts.ServerSideEncryption = encrypt.NewSSE() } + // The version the write replaces or creates: the source's version ID it + // keeps, or the null version of a destination that does not version + // the key. A versioned destination gives a write without an ID a new one. + writeVersion := versionID + if writeVersion == "" && !opts.Versioned { + writeVersion = nullVersionID + } + for _, req := range pullAuthzRequests(tgtBucket, tgtObject, writeVersion, versionID != "", opts, retention, legalHold) { + if err := p.authorize(req); err != nil { + return err + } + } slc := strings.Split(srcObjInfo.ETag, "-") if len(slc) == 2 { partsCount, err := strconv.Atoi(slc[1]) @@ -298,6 +459,9 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay if ri.Complete { return nil } + if done, err := ri.startAuthorized(ctx, api, job); done || err != nil { + return err + } globalBatchJobsMetrics.save(job.ID, ri) retryAttempts := job.Replicate.Flags.Retry.Attempts @@ -411,7 +575,17 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay // one of source/target is s3, skip delete marker and all versions under the same object name. s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO - ctx, cancel := context.WithCancel(ctx) + ctx, cancelCause := context.WithCancelCause(ctx) + // refuse ends the job for its submitter: record why before stopping. + refuse := func(err error) { + ri.refuse(err) + cancelCause(err) + } + // Each run lists the source from the start and counts every locked + // collision again. + ri.mu.Lock() + ri.VersionsSkippedLocked = 0 + ri.mu.Unlock() objInfoCh := make(chan minio.ObjectInfo, 1) go func() { @@ -437,6 +611,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay skipReplicate := false for obj := range objInfoCh { + if context.Cause(ctx) != nil { + // Refused or canceled: schedule nothing more. + continue + } oi := toObjectInfo(r.Source.Bucket, obj.Key, obj) if !minioSrc { // Check if metadata filter was requested and it is expected to have @@ -484,7 +662,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay defer wk.Give() stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts) success := true - if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil { + if err := r.ReplicateFromSource(ctx, api, core, oi, retry, job.Submitter); err != nil { + if isBatchJobAuthzError(err) { + refuse(err) + return + } + // A locked destination version is left as it is: the + // job completes with it skipped, not replicated. + if isErrObjectLocked(err) { + stopFn(oi, err) + ri.countSkipped() + globalBatchJobsMetrics.save(job.ID, ri) + return + } // object must be deleted concurrently, allow these failures but do not count them if isErrVersionNotFound(err) || isErrObjectNotFound(err) { return @@ -495,7 +685,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay } else { stopFn(oi, nil) } - ri.trackCurrentBucketObject(r.Target.Bucket, oi, success, attempts) + ri.trackCurrentBucketObject(r.Target.Bucket, oi, success) globalBatchJobsMetrics.save(job.ID, ri) // persist in-memory state to disk after every 10secs. batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job)) @@ -507,9 +697,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay } wk.Wait() + if ri.refused() { + cancelCause(nil) + globalBatchJobsMetrics.save(job.ID, ri) + batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job)) + if err := r.Notify(GlobalContext, ri); err != nil { + batchLogIf(ctx, fmt.Errorf("unable to notify %v", err)) + } + return nil + } + ri.RetryAttempts = attempts - ri.Complete = ri.ObjectsFailed == 0 - ri.Failed = ri.ObjectsFailed > 0 + ri.Complete = ri.ObjectsFailed == 0 && ri.DeleteMarkersFailed == 0 + ri.Failed = !ri.Complete globalBatchJobsMetrics.save(job.ID, ri) // persist in-memory state to disk. @@ -519,12 +719,14 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay batchLogIf(ctx, fmt.Errorf("unable to notify %v", err)) } - cancel() + cancelCause(nil) if ri.Failed { ri.ObjectsFailed = 0 + ri.DeleteMarkersFailed = 0 ri.Bucket = "" ri.Object = "" ri.Objects = 0 + ri.DeleteMarkers = 0 ri.BytesFailed = 0 ri.BytesTransferred = 0 retry = true // indicate we are retrying.. @@ -580,7 +782,10 @@ func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo { return oi } -func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error { +func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string, sub *batchJobSubmitter) error { + // An entry the submitter may not read ends the archive and the job; an + // entry whose metadata cannot be read ends the archive. + var refused, readErr error input := make(chan minio.SnowballObject, 1) opts := minio.SnowballOptions{ Opts: minio.PutObjectOptions{ @@ -593,17 +798,41 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa SkipErrs: *r.Source.Snowball.SkipErrs, } + uploadCtx, cancelUpload := context.WithCancel(ctx) + defer cancelUpload() + producerDone := make(chan struct{}) go func() { + defer close(producerDone) defer xioutil.SafeClose(input) for _, entry := range entries { - gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket, + gr, err := objAPI.GetObjectNInfo(uploadCtx, r.Source.Bucket, entry.Name, nil, nil, ObjectOptions{ VersionID: entry.VersionID, }) if err != nil { batchLogIf(ctx, err) - continue + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || *r.Source.Snowball.SkipErrs { + continue + } + // Not sent and not skippable: the batch goes to the + // per-object path. + readErr = err + return + } + // Each entry is a decision of its own, for the submitter as it is, + // on the metadata of all copies of the version. + logical, lerr := batchAuthoritativeRead(uploadCtx, objAPI, r.Source.Bucket, entry.Name, gr.ObjInfo) + if lerr != nil { + // Not sent: the batch goes to the per-object path. + gr.Close() + readErr = lerr + return + } + refused = sub.authorize(uploadCtx, append(r.pushAuthzRequests(gr.ObjInfo), r.pushAuthzRequests(logical)...)...) + if refused != nil { + gr.Close() + return } if prefix != "" { @@ -625,6 +854,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa opts, _, err := batchReplicationOpts(ctx, "", gr.ObjInfo) if err != nil { + gr.Close() batchLogIf(ctx, err) continue } @@ -635,23 +865,75 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa } } - input <- snowballObj + select { + case input <- snowballObj: + case <-uploadCtx.Done(): + gr.Close() + return + } } }() // Collect and upload all entries. - return remoteClnt.PutObjectsSnowball(ctx, r.Target.Bucket, opts, input) + err := remoteClnt.PutObjectsSnowball(uploadCtx, r.Target.Bucket, opts, input) + cancelUpload() + <-producerDone + if refused != nil { + return refused + } + if readErr != nil { + return readErr + } + return err +} + +// pushAuthzRequests are the reads of one source version that a push copies to +// the target: the version itself, and its tags, retention and legal hold. +func (r *BatchJobReplicateV1) pushAuthzRequests(oi ObjectInfo) []batchAuthzRequest { + bucket, object, versionID := r.Source.Bucket, oi.Name, oi.VersionID + if versionID == "" { + versionID = nullVersionID + } + reqs := []batchAuthzRequest{ + {action: policy.GetObjectAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags}, + {action: policy.GetObjectVersionAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags}, + } + if oi.UserTags != "" { + reqs = append(reqs, + batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags}, + batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags}, + ) + } + meta := caseInsensitiveMap(oi.UserDefined) + _, mode := meta.Lookup(xhttp.AmzObjectLockMode) + _, until := meta.Lookup(xhttp.AmzObjectLockRetainUntilDate) + if mode || until { + reqs = append(reqs, batchAuthzRequest{action: policy.GetObjectRetentionAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags}) + } + if _, hold := meta.Lookup(xhttp.AmzObjectLockLegalHold); hold { + reqs = append(reqs, batchAuthzRequest{action: policy.GetObjectLegalHoldAction, bucket: bucket, object: object, versionID: versionID, tags: oi.UserTags}) + } + return reqs } // ReplicateToTarget read from source and replicate to configured target -func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool) error { +func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool, sub *batchJobSubmitter) error { srcBucket := r.Source.Bucket tgtBucket := r.Target.Bucket tgtPrefix := r.Target.Prefix srcObject := srcObjInfo.Name s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 + p, err := sub.principal(ctx) + if err != nil { + return err + } if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() { + for _, req := range r.pushAuthzRequests(srcObjInfo) { + if err := p.authorize(req); err != nil { + return err + } + } if retry && !s3Type { if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.StatObjectOptions{ VersionID: srcObjInfo.VersionID, @@ -707,6 +989,19 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL } defer rd.Close() objInfo := rd.ObjInfo + // Authorize the version as it is read, with the metadata of all its + // copies. + logical, err := batchAuthoritativeRead(ctx, api, srcBucket, srcObject, objInfo) + if err != nil { + return err + } + // What is sent comes from the opened copy; both it and the version as + // all copies describe it must be authorized. + for _, req := range append(r.pushAuthzRequests(objInfo), r.pushAuthzRequests(logical)...) { + if err := p.authorize(req); err != nil { + return err + } + } size, err := objInfo.GetActualSize() if err != nil { @@ -760,6 +1055,16 @@ type batchJobInfo struct { DeleteMarkersFailed int64 `json:"deleteMarkersFailed" msg:"dmf"` BytesTransferred int64 `json:"bytesTransferred" msg:"bt"` BytesFailed int64 `json:"bytesFailed" msg:"bf"` + + // Versions the job left in place because Object Lock protects them, + // and keys it refused as a whole for the same reason. A job that + // completes with them has not acted on those versions. + VersionsSkippedLocked int64 `json:"versionsSkippedLocked" msg:"vsl"` + KeysRefused int64 `json:"keysRefused" msg:"kr"` + + // FailureReason is set when the job was refused for its submitter. It + // ends the job: a job that has one does not run again. + FailureReason string `json:"failureReason,omitempty" msg:"fr"` } const ( @@ -901,40 +1206,31 @@ func (ri *batchJobInfo) clone() *batchJobInfo { BytesTransferred: ri.BytesTransferred, BytesFailed: ri.BytesFailed, Attempts: ri.Attempts, + + VersionsSkippedLocked: ri.VersionsSkippedLocked, + KeysRefused: ri.KeysRefused, + FailureReason: ri.FailureReason, } } -func (ri *batchJobInfo) countItem(size int64, dmarker, success bool, attempt int) { +// countItem counts the outcome of one item in the current attempt of a +// replicate job, which resets its counters before it retries. +func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) { if ri == nil { return } ri.Attempts++ - if success { - if dmarker { - ri.DeleteMarkers++ - } else { - ri.Objects++ - ri.BytesTransferred += size - } - if attempt > 1 { - if dmarker { - ri.DeleteMarkersFailed-- - } else { - ri.ObjectsFailed-- - ri.BytesFailed += size - } - } - } else { - if attempt > 1 { - // Only count first attempt - return - } - if dmarker { - ri.DeleteMarkersFailed++ - } else { - ri.ObjectsFailed++ - ri.BytesFailed += size - } + switch { + case success && dmarker: + ri.DeleteMarkers++ + case success: + ri.Objects++ + ri.BytesTransferred += size + case dmarker: + ri.DeleteMarkersFailed++ + default: + ri.ObjectsFailed++ + ri.BytesFailed += size } } @@ -1002,35 +1298,7 @@ func (ri *batchJobInfo) expireFailures() bool { return ri.ObjectsFailed > 0 || ri.DeleteMarkersFailed > 0 } -// trackExpireAll counts the objects and delete markers that an expiry of all -// versions of info removed and, when it failed, those the walk saw that it -// did not remove, at least one. Only batch-expire uses it, for a rule that -// expires all versions of an object. -func (ri *batchJobInfo) trackExpireAll(info expireObjInfo, objects, markers int64, failed bool) { - if ri == nil { - return - } - - ri.mu.Lock() - defer ri.mu.Unlock() - - ri.Objects += objects - ri.DeleteMarkers += markers - if !failed { - ri.Bucket = info.Bucket - ri.Object = info.Name - return - } - failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0) - failedMarkers := max(info.DeleteMarkerCount-markers, 0) - if failedObjects+failedMarkers == 0 { - failedObjects = 1 - } - ri.ObjectsFailed += failedObjects - ri.DeleteMarkersFailed += failedMarkers -} - -func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool, attempt int) { +func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool) { if ri == nil { return } @@ -1042,7 +1310,7 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, ri.Bucket = bucket ri.Object = info.Name } - ri.countItem(info.Size, info.DeleteMarker, success, attempt) + ri.countItem(info.Size, info.DeleteMarker, success) } func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInfo) { @@ -1056,7 +1324,7 @@ func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInf ri.Bucket = bucket for i := range batch { ri.Object = batch[i].Name - ri.countItem(batch[i].Size, batch[i].DeleteMarker, true, 1) + ri.countItem(batch[i].Size, batch[i].DeleteMarker, true) } } @@ -1073,6 +1341,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba if ri.Complete { return nil } + if done, err := ri.startAuthorized(ctx, api, job); done || err != nil { + return err + } globalBatchJobsMetrics.save(job.ID, ri) lastObject := ri.Object @@ -1180,6 +1451,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba walkCh = make(chan itemOrErr[ObjectInfo], 100) slowCh = make(chan itemOrErr[ObjectInfo], 100) ) + ctx, cancelCause := context.WithCancelCause(ctx) + // refuse ends the job for its submitter: record why before stopping. + refuse := func(err error) { + ri.refuse(err) + cancelCause(err) + } if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() { go func() { @@ -1201,7 +1478,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch) writeFn := func(batch []ObjectInfo) { if len(batch) > 0 { - if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil { + if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix, job.Submitter); err != nil { + if isBatchJobAuthzError(err) { + refuse(err) + return + } batchLogOnceIf(ctx, err, job.ID+"writeAsArchive") for _, b := range batch { slowCh <- itemOrErr[ObjectInfo]{Item: b} @@ -1250,7 +1531,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba if walkQuorum == "" { walkQuorum = "strict" } - ctx, cancelCause := context.WithCancelCause(ctx) // one of source/target is s3, skip delete marker and all versions under the same object name. s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 @@ -1281,6 +1561,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba skipReplicate := false for res := range slowCh { + if context.Cause(ctx) != nil { + // Refused or canceled: schedule nothing more. + continue + } if res.Err != nil { ri.Failed = true batchLogOnceIf(ctx, res.Err, job.ID+"res.Err") @@ -1300,7 +1584,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts) success := true - if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil { + if err := r.ReplicateToTarget(ctx, api, c, result, retry, job.Submitter); err != nil { + if isBatchJobAuthzError(err) { + refuse(err) + return + } if minio.ToErrorResponse(err).Code == "PreconditionFailed" { // pre-condition failed means we already have the object copied over. return @@ -1315,7 +1603,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba } else { stopFn(result, nil) } - ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts) + ri.trackCurrentBucketObject(r.Source.Bucket, result, success) globalBatchJobsMetrics.save(job.ID, ri) // persist in-memory state to disk after every 10secs. batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2") @@ -1326,13 +1614,22 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba }() } wk.Wait() + if ri.refused() { + cancelCause(nil) + globalBatchJobsMetrics.save(job.ID, ri) + batchLogOnceIf(ctx, ri.updateAfter(GlobalContext, api, 0, job), job.ID+"updateAfter3") + if err := r.Notify(GlobalContext, ri); err != nil { + batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify") + } + return nil + } // Do not need to retry if we can't list objects on source. if context.Cause(ctx) != nil { return context.Cause(ctx) } ri.RetryAttempts = attempts - ri.Complete = ri.ObjectsFailed == 0 - ri.Failed = ri.ObjectsFailed > 0 + ri.Complete = ri.ObjectsFailed == 0 && ri.DeleteMarkersFailed == 0 + ri.Failed = !ri.Complete globalBatchJobsMetrics.save(job.ID, ri) // persist in-memory state to disk. @@ -1345,9 +1642,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba cancelCause(nil) if ri.Failed { ri.ObjectsFailed = 0 + ri.DeleteMarkersFailed = 0 ri.Bucket = "" ri.Object = "" ri.Objects = 0 + ri.DeleteMarkers = 0 ri.BytesFailed = 0 ri.BytesTransferred = 0 retry = true // indicate we are retrying.. @@ -1549,8 +1848,12 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error { return errInvalidArgument } -func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) { - deleteConfig(ctx, api, getJobPath(j)) +func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) error { + err := deleteConfig(ctx, api, getJobPath(j)) + if errors.Is(err, errConfigNotFound) { + return nil + } + return err } func (j BatchJobRequest) getJobReportPath() (string, error) { @@ -1709,7 +2012,7 @@ func (a adminAPIHandlers) BatchJobStatus(w http.ResponseWriter, r *http.Request) return } - buf, err := json.Marshal(madmin.BatchJobStatus{LastMetric: ri.metric()}) + buf, err := json.Marshal(batchJobStatus{LastMetric: ri.statusMetric()}) if err != nil { batchLogIf(ctx, err) writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) @@ -1747,8 +2050,13 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques } // Remove sensitive fields. - req.RedactSensitive() - buf, err := yaml.Marshal(req) + out, err := req.redacted() + if err != nil { + batchLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + buf, err := yaml.Marshal(out) if err != nil { batchLogIf(ctx, err) writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) @@ -1806,6 +2114,27 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) } } + if job.jobSections() != 1 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, errBatchJobSections), r.URL) + return + } + + // The job acts as the credential that submits it, and only as far as + // that credential may act on the job's buckets. + submitter, err := newBatchJobSubmitter(creds) + if err == nil { + job.Submitter = submitter + err = submitter.authorize(ctx, job.startAuthzRequests()...) + } + if err != nil { + apiErr := toAPIError(ctx, err) + if isBatchJobAuthzError(err) { + apiErr = errorCodes.ToAPIErrWithErr(ErrAccessDenied, err) + } + writeErrorResponseJSON(ctx, w, apiErr, r.URL) + return + } + // Validate the incoming job request if err := job.Validate(ctx, objectAPI); err != nil { writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) @@ -1942,6 +2271,14 @@ func (j *BatchJobPool) cleanupReports(randomWait func() time.Duration) { continue } if (ri.Complete || ri.Failed) && time.Since(ri.LastUpdate) > oldJobsExpiration { + // Remove the job before its report, so that a finished + // job is never resumed without its outcome. + if ri.JobID != "" { + if err := (BatchJobRequest{ID: ri.JobID}).delete(ctx, j.objLayer); err != nil { + batchLogIf(ctx, err) + continue + } + } deleteConfig(ctx, j.objLayer, result.Item.Name) } } @@ -2082,7 +2419,7 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error { }) case j.jobCh <- req: default: - return fmt.Errorf("batch job queue is currently full please try again later %#v", req) + return fmt.Errorf("batch job queue is currently full please try again later (job %s)", req.ID) } return nil } diff --git a/cmd/batch-handlers_gen.go b/cmd/batch-handlers_gen.go index ad29932b4..d0e1d7086 100644 --- a/cmd/batch-handlers_gen.go +++ b/cmd/batch-handlers_gen.go @@ -179,6 +179,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) { return } } + case "Submitter": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Submitter") + return + } + z.Submitter = nil + } else { + if z.Submitter == nil { + z.Submitter = new(batchJobSubmitter) + } + err = z.Submitter.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Submitter") + return + } + } default: err = dc.Skip() if err != nil { @@ -192,9 +210,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 6 + // map header, size 7 // write "ID" - err = en.Append(0x86, 0xa2, 0x49, 0x44) + err = en.Append(0x87, 0xa2, 0x49, 0x44) if err != nil { return } @@ -274,15 +292,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) { return } } + // write "Submitter" + err = en.Append(0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72) + if err != nil { + return + } + if z.Submitter == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Submitter.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Submitter") + return + } + } return } // MarshalMsg implements msgp.Marshaler func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 6 + // map header, size 7 // string "ID" - o = append(o, 0x86, 0xa2, 0x49, 0x44) + o = append(o, 0x87, 0xa2, 0x49, 0x44) o = msgp.AppendString(o, z.ID) // string "User" o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72) @@ -323,6 +358,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) { return } } + // string "Submitter" + o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72) + if z.Submitter == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Submitter.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Submitter") + return + } + } return } @@ -413,6 +459,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + case "Submitter": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Submitter = nil + } else { + if z.Submitter == nil { + z.Submitter = new(batchJobSubmitter) + } + bts, err = z.Submitter.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Submitter") + return + } + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -445,6 +508,12 @@ func (z *BatchJobRequest) Msgsize() (s int) { } else { s += z.Expire.Msgsize() } + s += 10 + if z.Submitter == nil { + s += msgp.NilSize + } else { + s += z.Submitter.Msgsize() + } return } @@ -568,6 +637,24 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "BytesFailed") return } + case "vsl": + z.VersionsSkippedLocked, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "VersionsSkippedLocked") + return + } + case "kr": + z.KeysRefused, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "KeysRefused") + return + } + case "fr": + z.FailureReason, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FailureReason") + return + } default: err = dc.Skip() if err != nil { @@ -581,9 +668,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 17 + // map header, size 20 // write "v" - err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76) + err = en.Append(0xde, 0x0, 0x14, 0xa1, 0x76) if err != nil { return } @@ -752,15 +839,45 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "BytesFailed") return } + // write "vsl" + err = en.Append(0xa3, 0x76, 0x73, 0x6c) + if err != nil { + return + } + err = en.WriteInt64(z.VersionsSkippedLocked) + if err != nil { + err = msgp.WrapError(err, "VersionsSkippedLocked") + return + } + // write "kr" + err = en.Append(0xa2, 0x6b, 0x72) + if err != nil { + return + } + err = en.WriteInt64(z.KeysRefused) + if err != nil { + err = msgp.WrapError(err, "KeysRefused") + return + } + // write "fr" + err = en.Append(0xa2, 0x66, 0x72) + if err != nil { + return + } + err = en.WriteString(z.FailureReason) + if err != nil { + err = msgp.WrapError(err, "FailureReason") + return + } return } // MarshalMsg implements msgp.Marshaler func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 17 + // map header, size 20 // string "v" - o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76) + o = append(o, 0xde, 0x0, 0x14, 0xa1, 0x76) o = msgp.AppendInt(o, z.Version) // string "jid" o = append(o, 0xa3, 0x6a, 0x69, 0x64) @@ -810,6 +927,15 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) { // string "bf" o = append(o, 0xa2, 0x62, 0x66) o = msgp.AppendInt64(o, z.BytesFailed) + // string "vsl" + o = append(o, 0xa3, 0x76, 0x73, 0x6c) + o = msgp.AppendInt64(o, z.VersionsSkippedLocked) + // string "kr" + o = append(o, 0xa2, 0x6b, 0x72) + o = msgp.AppendInt64(o, z.KeysRefused) + // string "fr" + o = append(o, 0xa2, 0x66, 0x72) + o = msgp.AppendString(o, z.FailureReason) return } @@ -933,6 +1059,24 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "BytesFailed") return } + case "vsl": + z.VersionsSkippedLocked, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionsSkippedLocked") + return + } + case "kr": + z.KeysRefused, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "KeysRefused") + return + } + case "fr": + z.FailureReason, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailureReason") + return + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -947,6 +1091,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *batchJobInfo) Msgsize() (s int) { - s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.StringPrefixSize + len(z.FailureReason) return } diff --git a/cmd/batch-progress.go b/cmd/batch-progress.go new file mode 100644 index 000000000..968e8e0ef --- /dev/null +++ b/cmd/batch-progress.go @@ -0,0 +1,187 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "slices" + "strings" + "sync" + + "github.com/minio/madmin-go/v3" +) + +// batchCounts are outcomes of a batch job that a later run of the job would +// see again, and so must be persisted together with the cursor of the walk. +type batchCounts struct { + objects int64 + objectsFailed int64 + deleteMarkersFailed int64 + versionsSkipped int64 + keysRefused int64 +} + +func (c *batchCounts) add(o batchCounts) { + c.objects += o.objects + c.objectsFailed += o.objectsFailed + c.deleteMarkersFailed += o.deleteMarkersFailed + c.versionsSkipped += o.versionsSkipped + c.keysRefused += o.keysRefused +} + +//msgp:ignore batchProgress batchProgressItem batchCounts + +// batchProgress is the cursor of a walk whose items are done concurrently. +// +// The items of a walk are numbered in walk order. A key is done when every +// item of it, and every item before it, is done; only then does the cursor +// move past it and its counts become part of the job's counts. The cursor and +// the counts are persisted together, so a job resumed after the cursor +// neither skips an item that was not done nor counts an item twice. +type batchProgress struct { + mu sync.Mutex + ri *batchJobInfo + bucket string + issued uint64 + next uint64 + pending map[uint64]batchProgressItem + // key is the key of the item last committed, and held are the counts of + // its items so far. + key string + held batchCounts +} + +type batchProgressItem struct { + key string + counts batchCounts +} + +func newBatchProgress(ri *batchJobInfo, bucket string) *batchProgress { + return &batchProgress{ri: ri, bucket: bucket, pending: make(map[uint64]batchProgressItem)} +} + +// begin numbers the next item of the walk. +func (p *batchProgress) begin() uint64 { + p.mu.Lock() + defer p.mu.Unlock() + seq := p.issued + p.issued++ + return seq +} + +// done records item seq, of key, as done with counts. +func (p *batchProgress) done(seq uint64, key string, counts batchCounts) { + p.mu.Lock() + defer p.mu.Unlock() + p.pending[seq] = batchProgressItem{key: key, counts: counts} + for { + item, ok := p.pending[p.next] + if !ok { + return + } + delete(p.pending, p.next) + p.next++ + if item.key != p.key { + p.commitLocked() + p.key = item.key + } + p.held.add(item.counts) + } +} + +// finish commits the last key if every item of the walk is done. Call it +// once no item is in progress. +func (p *batchProgress) finish() { + p.mu.Lock() + defer p.mu.Unlock() + if p.next == p.issued { + p.commitLocked() + } +} + +func (p *batchProgress) commitLocked() { + if p.key == "" { + return + } + ri := p.ri + ri.mu.Lock() + ri.Bucket = p.bucket + ri.Object = p.key + ri.Objects += p.held.objects + ri.ObjectsFailed += p.held.objectsFailed + ri.DeleteMarkersFailed += p.held.deleteMarkersFailed + ri.VersionsSkippedLocked += p.held.versionsSkipped + ri.KeysRefused += p.held.keysRefused + ri.mu.Unlock() + p.held = batchCounts{} +} + +// countRemoved counts versions a job removed. A removed version is gone for +// any later run, so it is counted at once, not at the cursor. +func (ri *batchJobInfo) countRemoved(bucket string, objects, markers int64) { + ri.mu.Lock() + defer ri.mu.Unlock() + ri.Bucket = bucket + ri.Objects += objects + ri.DeleteMarkers += markers +} + +// countSkipped counts a version a job left in place because Object Lock +// protects it, for a job that is counted per run rather than at a cursor. +func (ri *batchJobInfo) countSkipped() { + ri.mu.Lock() + defer ri.mu.Unlock() + ri.VersionsSkippedLocked++ +} + +// batchCanonicalPrefixes orders the prefixes of a walk and drops a prefix +// that another one covers. The keys of the walk are then visited once each, +// in one ascending order, which a single cursor can describe. +func batchCanonicalPrefixes(prefixes []string) []string { + if len(prefixes) == 0 { + return []string{""} + } + sorted := slices.Clone(prefixes) + slices.Sort(sorted) + out := sorted[:0] + for _, p := range sorted { + if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) { + continue + } + out = append(out, p) + } + return out +} + +// batchResumeAfter reports whether a resumed walk returned a key after the +// cursor. The cursor key is done; the walk forwards conservatively and +// returns it again. Keys compare as the walk orders them, by name, so a +// directory object comes before the keys under it. +func batchResumeAfter(cursor, name string) bool { + return cursor == "" || name > cursor +} + +// batchJobStatus is the status response: madmin.BatchJobStatus with the +// fields this server adds. The added fields are additive; a client that does +// not know them ignores them. +type batchJobStatus struct { + LastMetric batchJobStatusMetric +} + +type batchJobStatusMetric struct { + madmin.JobMetric + VersionsSkippedLocked int64 `json:"versionsSkippedLocked,omitempty"` + KeysRefused int64 `json:"keysRefused,omitempty"` + FailureReason string `json:"failureReason,omitempty"` +} + +func (ri *batchJobInfo) statusMetric() batchJobStatusMetric { + ri.mu.RLock() + defer ri.mu.RUnlock() + return batchJobStatusMetric{ + JobMetric: ri.metric(), + VersionsSkippedLocked: ri.VersionsSkippedLocked, + KeysRefused: ri.KeysRefused, + FailureReason: ri.FailureReason, + } +} diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go index 3ebbc844d..1f7306ff1 100644 --- a/cmd/batch-rotate.go +++ b/cmd/batch-rotate.go @@ -35,6 +35,7 @@ import ( xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/kms" "github.com/pgsty/silo-pkg/v3/env" + "github.com/pgsty/silo-pkg/v3/policy" "github.com/pgsty/silo-pkg/v3/workers" ) @@ -161,8 +162,68 @@ func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token) } -// KeyRotate rotates encryption key of an object -func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error { +// batchAuthoritativeVersion returns the version obj as all its copies +// describe it. With several pools a version can have copies in more than one +// pool, and a read returns one of them; the merged copy carries the newest +// Object Lock and tag state. The caller holds the object's write lock. +func batchAuthoritativeVersion(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) { + z, ok := api.(*erasureServerPools) + if !ok || z.SinglePool() { + return obj, nil + } + versionID := obj.VersionID + if versionID == "" { + versionID = nullVersionID + } + return z.replicaObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{VersionID: versionID, NoLock: true, NoAuditLog: true}) +} + +// batchAuthoritativeRead is batchAuthoritativeVersion for a version that has +// been opened for reading. It takes the object's read lock itself, since an +// opened reader may hold none. +func batchAuthoritativeRead(ctx context.Context, api ObjectLayer, bucket, object string, obj ObjectInfo) (ObjectInfo, error) { + z, ok := api.(*erasureServerPools) + if !ok || z.SinglePool() { + return obj, nil + } + lk := z.NewNSLock(bucket, encodeDirObject(object)) + lkctx, err := lk.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + defer lk.RUnlock(lkctx) + return batchAuthoritativeVersion(lkctx.Context(), api, bucket, object, obj) +} + +// rotateAuthzRequests are the actions of rotating one version: it is read and +// written again in place with the new key. +func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzRequest { + versionID := oi.VersionID + if versionID == "" { + versionID = nullVersionID + } + // The encryption the rewrite leaves: rotation keeps the kind of the + // version (rotateKey), and SSE-KMS takes the requested key. + values := map[string][]string{} + switch { + case crypto.S3KMS.IsEncrypted(oi.UserDefined): + values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS} + if r.Encryption.Type == ssekms && r.Encryption.Key != "" { + values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key} + } + case crypto.S3.IsEncrypted(oi.UserDefined): + values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionAES} + } + return []batchAuthzRequest{ + {action: policy.GetObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags}, + {action: policy.GetObjectVersionAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags}, + {action: policy.PutObjectAction, bucket: r.Bucket, object: oi.Name, versionID: versionID, tags: oi.UserTags, values: values}, + } +} + +// KeyRotate rotates encryption key of an object. A version that Object Lock +// protects is not rewritten: it returns ObjectLocked. +func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo, sub *batchJobSubmitter) error { srcBucket := r.Bucket srcObject := objInfo.Name @@ -180,7 +241,8 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject) - lock := api.NewObjectNSLock(r.Bucket, objInfo.Name) + // Lock the name writers lock: the stored name of a directory object. + lock := api.NewObjectNSLock(r.Bucket, encodeDirObject(objInfo.Name)) lkctx, err := lock.GetLock(ctx, globalOperationTimeout) if err != nil { return err @@ -198,6 +260,24 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob if err != nil { return err } + // Decide on the version as it is under the lock that the rewrite holds, + // before any key is requested, with the metadata of all its copies. + logical, err := batchAuthoritativeVersion(ctx, api, r.Bucket, objInfo.Name, obj) + if err != nil { + return err + } + if enforceRetentionForDeletion(ctx, logical) { + return ObjectLocked{Bucket: r.Bucket, Object: obj.Name, VersionID: obj.VersionID} + } + p, err := sub.principal(ctx) + if err != nil { + return err + } + for _, req := range r.rotateAuthzRequests(logical) { + if err := p.authorize(req); err != nil { + return err + } + } oi := obj.Clone() var ( newKeyID string @@ -260,6 +340,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba if ri.Complete { return nil } + if done, err := ri.startAuthorized(ctx, api, job); done || err != nil { + return err + } globalBatchJobsMetrics.save(job.ID, ri) lastObject := ri.Object @@ -273,8 +356,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba delay = batchKeyRotateJobDefaultRetryDelay } - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - selectObj := func(info FileInfo) (ok bool) { if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan { // skip all objects that are newer than specified older duration @@ -355,65 +436,92 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba return err } - ctx, cancel := context.WithCancel(ctx) + ctx, cancelCause := context.WithCancelCause(ctx) + defer cancelCause(nil) + // refuse ends the job for its submitter: record why before stopping. + refuse := func(err error) { + ri.refuse(err) + cancelCause(err) + } + progress := newBatchProgress(ri, r.Bucket) results := make(chan itemOrErr[ObjectInfo], 100) if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{ Marker: lastObject, Filter: selectObj, }); err != nil { - cancel() // Do not need to retry if we can't list objects on source. return err } failed := false for res := range results { + if context.Cause(ctx) != nil { + // Refused or canceled: schedule nothing more. + continue + } if res.Err != nil { failed = true batchLogIf(ctx, res.Err) break } result := res.Item + if !batchResumeAfter(lastObject, result.Name) { + // Completed by the run that set the cursor. + continue + } sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined) sseS3 := crypto.S3.IsEncrypted(result.UserDefined) if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed continue } + seq := progress.begin() wk.Take() go func() { defer wk.Give() + var counts batchCounts for attempts := 1; attempts <= retryAttempts; attempts++ { stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts) - success := true - if err := r.KeyRotate(ctx, api, result); err != nil { - stopFn(result, err) - batchLogIf(ctx, err) - success = false - if attempts >= retryAttempts { - auditOptions := AuditLogOptions{ - Event: "KeyRotate", - APIName: "StartBatchJob", - Bucket: result.Bucket, - Object: result.Name, - VersionID: result.VersionID, - Error: err.Error(), - } - auditLogInternal(ctx, auditOptions) - } - } else { - stopFn(result, nil) + err := r.KeyRotate(ctx, api, result, job.Submitter) + stopFn(result, err) + if err == nil { + counts.objects = 1 + break } - ri.trackCurrentBucketObject(r.Bucket, result, success, attempts) - globalBatchJobsMetrics.save(job.ID, ri) - // persist in-memory state to disk after every 10secs. - batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job)) - if success { + if isBatchJobAuthzError(err) { + refuse(err) + return + } + if isErrObjectLocked(err) { + // Left as it is, not retried and not a failure. + counts.versionsSkipped = 1 + break + } + if ctx.Err() != nil { + // Not done; a later run rotates it. + return + } + batchLogIf(ctx, err) + if attempts >= retryAttempts { + auditOptions := AuditLogOptions{ + Event: "KeyRotate", + APIName: "StartBatchJob", + Bucket: result.Bucket, + Object: result.Name, + VersionID: result.VersionID, + Error: err.Error(), + } + auditLogInternal(ctx, auditOptions) + counts.objectsFailed = 1 break } if delay > 0 { - time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay))) + time.Sleep(delay + time.Duration(rand.Float64()*float64(delay))) } } + progress.done(seq, result.Name, counts) + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk after every 10secs. + batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job)) if wait := globalBatchConfig.KeyRotationWait(); wait > 0 { time.Sleep(wait) @@ -421,18 +529,31 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba }() } wk.Wait() - - ri.Complete = !failed && ri.ObjectsFailed == 0 - ri.Failed = failed || ri.ObjectsFailed > 0 - globalBatchJobsMetrics.save(job.ID, ri) - // persist in-memory state to disk. - batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job)) - - if err := r.Notify(ctx, ri); err != nil { - batchLogIf(ctx, fmt.Errorf("unable to notify %v", err)) + cause := context.Cause(ctx) + if !failed && cause == nil { + // The walk ended: its last key is done. + progress.finish() + } + if cause != nil && !ri.refused() { + // Stopped before the end: keep the work after the cursor pending. + globalBatchJobsMetrics.save(job.ID, ri) + batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job)) + return cause } - cancel() + if !ri.refused() { + ri.mu.Lock() + ri.Complete = !failed && ri.ObjectsFailed == 0 + ri.Failed = failed || ri.ObjectsFailed > 0 + ri.mu.Unlock() + } + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk. + batchLogIf(ctx, ri.updateAfter(GlobalContext, api, 0, job)) + + if err := r.Notify(GlobalContext, ri); err != nil { + batchLogIf(ctx, fmt.Errorf("unable to notify %v", err)) + } return nil } diff --git a/cmd/erasure-server-pool-delete-all-options_test.go b/cmd/erasure-server-pool-delete-all-options_test.go index 96039ffae..4b64eeaf3 100644 --- a/cmd/erasure-server-pool-delete-all-options_test.go +++ b/cmd/erasure-server-pool-delete-all-options_test.go @@ -149,7 +149,11 @@ func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) { // A marker that an expiration job failed to remove fails the job. func TestPurgeAllVersionsMarkerFailureFailsJob(t *testing.T) { ri := &batchJobInfo{} - ri.trackExpireAll(expireObjInfo{ObjectInfo: ObjectInfo{NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0, true) + var res batchCounts + res.expireAllFailed(expireObjInfo{ObjectInfo: ObjectInfo{Name: "key", NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0) + progress := newBatchProgress(ri, "bucket") + progress.done(progress.begin(), "key", res) + progress.finish() if ri.ObjectsFailed != 0 || ri.DeleteMarkersFailed != 1 || !ri.expireFailures() { t.Errorf("failed=%d/%d failures=%t", ri.ObjectsFailed, ri.DeleteMarkersFailed, ri.expireFailures()) } diff --git a/cmd/erasure-server-pool-delete-all_test.go b/cmd/erasure-server-pool-delete-all_test.go index b3c861cb1..eb6ef2ecf 100644 --- a/cmd/erasure-server-pool-delete-all_test.go +++ b/cmd/erasure-server-pool-delete-all_test.go @@ -332,8 +332,9 @@ func (p *purgeTest) matched(name string) expireObjInfo { return expireObjInfo{ObjectInfo: oi, ExpireAll: true, DeleteMarkerCount: markers} } -// batch runs the deletes of a batch expiration job for walked, the keys its -// walk matched, and returns the counts of the job. +// batch runs the deletes of a batch expiration job of the root user for +// walked, the keys its walk matched, as one batch of a walk that reached its +// end, and returns the counts of the job. func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo { p.t.Helper() globalBatchJobsMetrics.Lock() @@ -341,7 +342,11 @@ func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo { globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo) } globalBatchJobsMetrics.Unlock() - job := BatchJobRequest{ID: "purge-" + mustGetUUID(), Started: time.Now(), Expire: &BatchJobExpire{Bucket: p.bucket}} + sub, err := newBatchJobSubmitter(globalActiveCred) + if err != nil { + p.t.Fatal(err) + } + job := BatchJobRequest{ID: "purge-" + mustGetUUID(), Started: time.Now(), Submitter: sub, Expire: &BatchJobExpire{Bucket: p.bucket}} ri := &batchJobInfo{JobID: job.ID} wk, err := workers.New(1) if err != nil { @@ -350,7 +355,8 @@ func (p *purgeTest) batch(walked ...expireObjInfo) *batchJobInfo { ch := make(chan []expireObjInfo, 1) ch <- walked close(ch) - batchObjsForDelete(p.t.Context(), job.Expire, ri, job, p.z, wk, ch) + refuse := func(err error) { p.t.Errorf("batch job refused: %v", err) } + batchObjsForDelete(p.t.Context(), refuse, func() bool { return true }, job.Expire, ri, job, p.z, wk, ch) wk.Wait() return ri } @@ -837,8 +843,8 @@ func TestPurgeAllVersionsKeepsProtectedWrites(t *testing.T) { t.Errorf("%s holds %v, want both versions", name, got) } } - if ri.Objects != 2 || ri.ObjectsFailed == 0 { - t.Errorf("job counts objects=%d failed=%d, want 2 and the refusals", ri.Objects, ri.ObjectsFailed) + if ri.Objects != 2 || ri.KeysRefused != 3 || ri.ObjectsFailed != 0 { + t.Errorf("job counts objects=%d refused=%d failed=%d, want 2, 3 and 0", ri.Objects, ri.KeysRefused, ri.ObjectsFailed) } } diff --git a/cmd/erasure-server-pool-lock-domain_test.go b/cmd/erasure-server-pool-lock-domain_test.go index 13249b213..2097f731d 100644 --- a/cmd/erasure-server-pool-lock-domain_test.go +++ b/cmd/erasure-server-pool-lock-domain_test.go @@ -1013,9 +1013,13 @@ func TestLockDomainKeyRotationWaitsForWriter(t *testing.T) { // The rotation's listing found an SSE-S3 object; it reads it again // under the lock. listed := ObjectInfo{Bucket: bucket, Name: key, UserDefined: map[string]string{crypto.MetaSealedKeyS3: "listed"}} + sub, err := newBatchJobSubmitter(globalActiveCred) + if err != nil { + t.Fatal(err) + } waited, _, _ := ldOverlapRun(t, func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") }, - func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed) }) + func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed, sub) }) if !waited { t.Errorf("key rotation ran while a PUT held its commit") } From ae54718d8465b9e455c74527716d8e1390d2cfcf Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 04:14:06 +0800 Subject: [PATCH 4/8] test(batch): cover submitter authorization and locked versions Submission and start with changed, expired or restricted credentials, session policies with and without an authorization plugin, request context in policy conditions, per-object denies, jobs without a recorded submitter and describe output; expire, pull, push and rotation over versions under retention or legal hold, on one and sixteen drives; cursor and counter persistence; and the cases found in review: destination versions and tags of pulls, encryption values, S3 Tables statements, copies in several pools, claims, prefixes, listing errors and retry accounting. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-authz-review2_test.go | 356 ++++++++++++++++++++++ cmd/batch-authz-review3_test.go | 140 +++++++++ cmd/batch-authz-review4_test.go | 157 ++++++++++ cmd/batch-authz-review5_test.go | 142 +++++++++ cmd/batch-authz-review_test.go | 287 ++++++++++++++++++ cmd/batch-object-lock_test.go | 479 ++++++++++++++++++++++++++++++ cmd/batch-submitter-authz_test.go | 456 ++++++++++++++++++++++++++++ 7 files changed, 2017 insertions(+) create mode 100644 cmd/batch-authz-review2_test.go create mode 100644 cmd/batch-authz-review3_test.go create mode 100644 cmd/batch-authz-review4_test.go create mode 100644 cmd/batch-authz-review5_test.go create mode 100644 cmd/batch-authz-review_test.go create mode 100644 cmd/batch-object-lock_test.go create mode 100644 cmd/batch-submitter-authz_test.go diff --git a/cmd/batch-authz-review2_test.go b/cmd/batch-authz-review2_test.go new file mode 100644 index 000000000..5692d1d52 --- /dev/null +++ b/cmd/batch-authz-review2_test.go @@ -0,0 +1,356 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "net/http" + "strings" + "testing" + "time" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/pgsty/silo-pkg/v3/policy" +) + +// A pull into a suspended bucket replaces the null version. Its policy +// decision must name that version even when the source is unversioned S3. +func TestBatchReviewPullImplicitNullVersionAuthorization(t *testing.T) { + f := newBatchAuthzFixture(t) + _, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`Suspended`)) + mustIAM(t, err) + data := []byte("original null version") + before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: true}) + mustIAM(t, err) + batchAuthzPolicy(f.ctx, t, f.sys, "null-deny", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"null"}}}`) + cred := iamAdminUser(f.ctx, t, f.sys, "null-deny-user", "null-deny") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + job := batchPullJob(f.bucket) + job.Source.Type = BatchJobReplicateResourceS3 + err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{ + Name: before.Name, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}, + }, false, sub) + if !isBatchJobAuthzError(err) { + t.Errorf("null version policy must refuse the write, got %v", err) + } + after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: nullVersionID}) + if gerr != nil || before.ETag != after.ETag { + t.Errorf("denied null version must remain unchanged: read=%v unchanged=%v", gerr, before.ETag == after.ETag) + } +} + +type batchReviewPrefixWalkFailure struct{ ObjectLayer } + +func (a batchReviewPrefixWalkFailure) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error { + if prefix != "a/" { + return a.ObjectLayer.Walk(ctx, bucket, prefix, results, opts) + } + go func() { + defer close(results) + results <- itemOrErr[ObjectInfo]{Err: errors.New("test: incomplete first prefix")} + }() + return nil +} + +type batchReviewWalkCutoff struct { + ObjectLayer + cutoff string +} + +func (a batchReviewWalkCutoff) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error { + inner := make(chan itemOrErr[ObjectInfo], 1) + if err := a.ObjectLayer.Walk(ctx, bucket, prefix, inner, opts); err != nil { + close(results) + return err + } + go func() { + defer close(results) + stopped := false + for result := range inner { + if stopped { + continue + } + if result.Err == nil && result.Item.Name >= a.cutoff { + stopped = true + result = itemOrErr[ObjectInfo]{Err: errors.New("test: listing stopped before remaining child keys")} + } + select { + case results <- result: + case <-ctx.Done(): + stopped = true + } + } + }() + return nil +} + +// Listing a directory prefix returns the directory object before its child +// keys. A durable cursor for that directory must preserve unfinished children. +func TestBatchReviewDirectoryCursorKeepsChildrenPending(t *testing.T) { + f := newBatchAuthzFixture(t) + for range 12 { + batchPutVersion(t, f.obj, f.bucket, "a/", nil) + } + var pending string + for i := 0; i < 13; i++ { + older := f.putVersions(t, fmt.Sprintf("a/key-%02d", i)) + if i == 11 { + pending = older + } + } + rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire: + apiVersion: v1 + bucket: %s + prefix: ["a/"] + rules: + - type: object + purge: + retainVersions: 1 +`, f.bucket)) + if job == nil { + t.Fatalf("submit: %d %s", rec.Code, rec.Body.String()) + } + _ = job.Expire.Start(f.ctx, batchReviewWalkCutoff{ObjectLayer: f.obj, cutoff: "a/key-11"}, *job) + ri := f.jobInfo(t, job) + if ri.Object != "a/" { + t.Fatalf("control: the completed directory batch must be the cursor, got %q", ri.Object) + } + mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job)) + if f.versionExists(t, "a/key-11", pending) { + t.Error("a directory cursor must not suppress the unvisited child key on resume") + } +} + +// Later prefixes must not advance the completion cursor past an earlier +// prefix whose listing failed. This state is produced by Start itself. +func TestBatchReviewExpireFailedPrefixKeepsWorkPending(t *testing.T) { + f := newBatchAuthzFixture(t) + older := f.putVersions(t, "a/todo") + for i := 0; i < 13; i++ { + f.putVersions(t, fmt.Sprintf("z/key-%02d", i)) + } + rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire: + apiVersion: v1 + bucket: %s + prefix: ["z/", "a/"] + rules: + - type: object + purge: + retainVersions: 1 +`, f.bucket)) + if job == nil { + t.Fatalf("submit: %d %s", rec.Code, rec.Body.String()) + } + _ = job.Expire.Start(f.ctx, batchReviewPrefixWalkFailure{f.obj}, *job) + ri := f.jobInfo(t, job) + if ri.Object >= "a/todo" { + t.Errorf("cursor must not cross an incomplete prefix, got %q", ri.Object) + } + mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job)) + if f.versionExists(t, "a/todo", older) { + t.Error("a fresh Start must revisit the key omitted by the failed listing") + } +} + +// A repeated marker replication error must remain a failure on the second +// attempt, including after the retry loop resets its counters. +func TestBatchReviewMarkerRetryRemainsFailed(t *testing.T) { + f := newBatchAuthzFixture(t) + batchPutVersion(t, f.obj, f.bucket, "marker", nil) + _, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true}) + mustIAM(t, err) + oldTransport := globalRemoteTargetTransport + globalRemoteTargetTransport = batchLockTransport(func(r *http.Request) (*http.Response, error) { + code, body := http.StatusForbidden, `AccessDeniedtest refusal` + if r.URL.Query().Has("location") { + code, body = http.StatusOK, `us-east-1` + } + return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + }) + t.Cleanup(func() { globalRemoteTargetTransport = oldTransport }) + job := BatchJobRequest{ + ID: "replicate-marker-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t), + Replicate: &BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}}, + }, + } + job.Replicate.Flags.Retry.Attempts = 2 + job.Replicate.Flags.Retry.Delay = time.Millisecond + mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job)) + ri := f.jobInfo(t, &job) + if ri.Complete || !ri.Failed || ri.DeleteMarkersFailed != 1 { + t.Errorf("both marker attempts failed: complete=%v failed=%v markerFailures=%d attempts=%d", ri.Complete, ri.Failed, ri.DeleteMarkersFailed, ri.RetryAttempts) + } +} + +// A current parent mapping suppresses the OpenID policy-name fallback. +// Background-condition validation must inspect the same effective policies. +func TestBatchReviewMappedPolicySuppressesClaimFallback(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess) + batchAuthzPolicy(f.ctx, t, f.sys, "unused-claim-policy", `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`) + parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "mapped-full") + cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{ + parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), iamPolicyClaimNameOpenID(): "unused-claim-policy", + }, globalActiveCred.SecretKey) + mustIAM(t, err) + cred.ParentUser = parent.AccessKey + _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "") + mustIAM(t, err) + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + p, err := sub.principal(f.ctx) + mustIAM(t, err) + req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "mapped/key", versionID: mustGetUUID()} + if !f.sys.isAllowedLocally(policy.Args{AccountName: p.accessKey, Claims: p.claims, Action: req.action, BucketName: req.bucket, ObjectName: req.object, ConditionValues: p.conditionValues(req)}) { + t.Fatal("control: the current mapped policy must allow this read") + } + if err := sub.authorize(f.ctx, req); err != nil { + t.Errorf("an unused fallback policy must not refuse a mapped-policy grant: %v", err) + } +} + +func TestBatchReviewBindingAndExpiringServiceControls(t *testing.T) { + f := newBatchAuthzFixture(t) + root := batchRootSubmitter(t) + priorRoot := globalActiveCred + globalActiveCred.SecretKey = "different-test-root-secret" + _, err := root.principal(f.ctx) + globalActiveCred = priorRoot + if !isBatchJobAuthzError(err) { + t.Errorf("root-secret change must invalidate the binding: %v", err) + } + missing := *root + missing.Binding = nil + if _, err := missing.principal(f.ctx); !isBatchJobAuthzError(err) { + t.Errorf("missing fingerprint must refuse the principal: %v", err) + } + batchAuthzPolicy(f.ctx, t, f.sys, "service-parent-full", batchAuthzFullAccess) + parent := iamAdminUser(f.ctx, t, f.sys, "service-parent", "service-parent-full") + sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`)) + mustIAM(t, err) + expires := UTCNow().Add(time.Hour) + cred, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp, expiration: &expires}) + mustIAM(t, err) + stored, ok := f.sys.GetUser(f.ctx, cred.AccessKey) + if !ok { + t.Fatal("created service account must resolve") + } + cred = stored.Credentials + if !cred.IsTemp() || !cred.IsServiceAccount() { + t.Fatal("fixture must be an expiring service account") + } + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "allowed"}); err != nil { + t.Errorf("expiring service account must retain its allowed read: %v", err) + } + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectAction, bucket: f.bucket, object: "allowed"}); !isBatchJobAuthzError(err) { + t.Errorf("expiring service account must remain session-policy restricted: %v", err) + } +} + +// An interrupted cross-pool copy can leave the same version in both pools. +// A newer legal-hold field in either copy must prevent key rotation. +func TestBatchReviewRotationUsesAuthoritativePoolHold(t *testing.T) { + z, bucket := consistencyPools(t) + const name = "rotation/split-hold" + previousKMS := GlobalKMS + GlobalKMS = kms.NewStub("batch-rotation-key") + t.Cleanup(func() { GlobalKMS = previousKMS }) + dek, err := GlobalKMS.GenerateKey(t.Context(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{bucket: bucket + "/" + name}}) + mustIAM(t, err) + objectKey := crypto.GenerateKey(dek.Plaintext, nil) + sealed := objectKey.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), bucket, name) + first := crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed) + first[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "OFF" + first[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + second := maps.Clone(first) + second[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON" + second[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano) + ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("protected payload"), objectKey)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first}) + putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second}) + logical, err := z.replicaObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if !enforceRetentionForDeletion(t.Context(), logical) { + t.Fatal("control: the authoritative logical version must be held") + } + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + err = r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)) + if !isErrObjectLocked(err) { + t.Errorf("rotation must refuse the authoritative legal hold, got %v", err) + } + after, gerr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, gerr) + if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] { + t.Error("held logical version must not have its key rewritten") + } +} + +// An encryption condition must describe what rotation actually commits, +// even when the requested encryption type differs from the existing type. +func TestBatchReviewRotationEncryptionConditionMatchesResult(t *testing.T) { + f := newBatchAuthzFixture(t) + previousKMS := GlobalKMS + GlobalKMS = kms.NewStub("required-key") + t.Cleanup(func() { GlobalKMS = previousKMS }) + const name = "rotation/encryption-condition" + meta := make(map[string]string) + key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, name, meta, nil) + mustIAM(t, err) + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("encrypted payload"), key)) + mustIAM(t, err) + before, err := f.obj.PutObject(f.ctx, f.bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, UserDefined: meta}) + mustIAM(t, err) + batchAuthzPolicy(f.ctx, t, f.sys, "kms-write-only", ` +{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}, +{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`) + cred := iamAdminUser(f.ctx, t, f.sys, "kms-write-user", "kms-write-only") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + r := BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "required-key"}} + err = r.KeyRotate(f.ctx, f.obj, before, sub) + after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, gerr) + if err == nil && !crypto.S3KMS.IsEncrypted(after.UserDefined) { + t.Error("rotation allowed by an aws:kms-only write policy must produce SSE-KMS or refuse the operation") + } + if err != nil && before.UserDefined[crypto.MetaSealedKeyS3] != after.UserDefined[crypto.MetaSealedKeyS3] { + t.Error("a refused rotation must leave the version unchanged") + } +} + +// An explicit null marker collision must retain a held null data version. +func TestBatchReviewPullMarkerNullLock(t *testing.T) { + f := newBatchAuthzFixture(t) + meta := batchLockMeta("LEGALHOLD") + before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, strings.NewReader("held null"), 9, "", ""), ObjectOptions{VersionID: nullVersionID, Versioned: true, UserDefined: meta}) + mustIAM(t, err) + job := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket}, + } + err = job.ReplicateFromSource(f.ctx, f.obj, nil, ObjectInfo{ + Name: before.Name, VersionID: nullVersionID, ModTime: UTCNow(), DeleteMarker: true, + }, false, batchRootSubmitter(t)) + if !isErrObjectLocked(err) { + t.Errorf("marker collision with protected null data must return ObjectLocked, got %v", err) + } + if !batchVersionExists(t, f.obj, f.bucket, before.Name, nullVersionID) { + t.Fatal("protected null data must remain present") + } +} diff --git a/cmd/batch-authz-review3_test.go b/cmd/batch-authz-review3_test.go new file mode 100644 index 000000000..d6d9d9313 --- /dev/null +++ b/cmd/batch-authz-review3_test.go @@ -0,0 +1,140 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "fmt" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio/internal/auth" + xhttp "github.com/minio/minio/internal/http" + "github.com/pgsty/silo-pkg/v3/policy" +) + +// A read policy must use the newest tag state of the logical version even +// when an interrupted movement left an older copy in the preferred pool. +func TestBatchReviewPushUsesAuthoritativePoolTags(t *testing.T) { + z, bucket := consistencyPools(t) + f := newBatchAuthzFixture(t) + const name = "push/tagged" + first := map[string]string{ + xhttp.AmzObjectTagging: "classification=public", + ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano), + } + second := map[string]string{ + xhttp.AmzObjectTagging: "classification=private", + ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano), + } + before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first}) + putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second}) + logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if logical.UserTags != "classification=private" { + t.Fatal("control: logical version must have the newer private tag") + } + batchAuthzPolicy(f.ctx, t, f.sys, "pooled-push", fmt.Sprintf(` +{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s/*"]}, +{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket)) + cred := iamAdminUser(f.ctx, t, f.sys, "pooled-push-user", "pooled-push") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: bucket, object: name, versionID: before.VersionID, tags: logical.UserTags}); !isBatchJobAuthzError(err) { + t.Fatal("control: authoritative tags must deny the read") + } + writes := 0 + target, err := minio.NewCore("target.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + if r.Body != nil { + _, _ = io.Copy(io.Discard, r.Body) + _ = r.Body.Close() + } + if r.Method == http.MethodPut { + writes++ + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil + }), + }) + mustIAM(t, err) + r := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + } + err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub) + if !isBatchJobAuthzError(err) || writes != 0 { + t.Errorf("newer authoritative tags must deny push: err=%v writes=%d", err, writes) + } +} + +// Verified token claims must not replace the actual version being authorized. +func TestBatchReviewVersionConditionResistsClaimCollision(t *testing.T) { + f := newBatchAuthzFixture(t) + deniedVersion := mustGetUUID() + batchAuthzPolicy(f.ctx, t, f.sys, "version-claims", batchAuthzFullAccess+fmt.Sprintf(`, +{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, deniedVersion)) + parent := iamAdminUser(f.ctx, t, f.sys, "version-claims-parent", "version-claims") + cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{ + parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), "versionid": "an-unrelated-claim-value", + }, globalActiveCred.SecretKey) + mustIAM(t, err) + cred.ParentUser = parent.AccessKey + _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "") + mustIAM(t, err) + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + req := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: deniedVersion} + if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) { + t.Errorf("a token claim must not replace the actual denied version: %v", err) + } +} + +// Retry totals describe a pass, including markers already successful before +// another item caused that pass to retry. +func TestBatchReviewMarkerSuccessCountResetsOnRetry(t *testing.T) { + f := newBatchAuthzFixture(t) + batchPutVersion(t, f.obj, f.bucket, "marker", nil) + _, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true}) + mustIAM(t, err) + oldTransport := remoteInstanceTransport + remoteInstanceTransport = atomic.Value{} + setRemoteInstanceTransport(batchLockTransport(func(r *http.Request) (*http.Response, error) { + code, body := http.StatusForbidden, `AccessDeniedtest refusal` + h := make(http.Header) + switch { + case r.URL.Query().Has("location"): + code, body = http.StatusOK, `us-east-1` + case r.Method == http.MethodDelete: + code, body = http.StatusNoContent, "" + case r.Method == http.MethodHead && r.URL.Query().Get("versionId") != "": + code, body = http.StatusMethodNotAllowed, "" + h.Set("X-Amz-Delete-Marker", "true") + } + return &http.Response{StatusCode: code, Header: h, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil + })) + t.Cleanup(func() { remoteInstanceTransport = oldTransport }) + job := BatchJobRequest{ + ID: "marker-success-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t), + Replicate: &BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}}, + }, + } + job.Replicate.Flags.Retry.Attempts = 2 + job.Replicate.Flags.Retry.Delay = time.Millisecond + mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job)) + ri := f.jobInfo(t, &job) + if !ri.Failed || ri.RetryAttempts != 2 { + t.Fatalf("control: data failure must cause two attempts: failed=%v attempts=%d", ri.Failed, ri.RetryAttempts) + } + if ri.DeleteMarkers != 1 { + t.Errorf("one successful marker must be counted once in the final attempt, got %d", ri.DeleteMarkers) + } +} diff --git a/cmd/batch-authz-review4_test.go b/cmd/batch-authz-review4_test.go new file mode 100644 index 000000000..a4b3fb02a --- /dev/null +++ b/cmd/batch-authz-review4_test.go @@ -0,0 +1,157 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio/internal/auth" + xhttp "github.com/minio/minio/internal/http" + "github.com/pgsty/silo-pkg/v3/policy" +) + +// A tag removal in a newer copy must govern both the authorization and the +// metadata sent by a push. The submitter can read data but cannot read tags. +func TestBatchReviewPushSendsOnlyAuthorizedMetadata(t *testing.T) { + z, bucket := consistencyPools(t) + f := newBatchAuthzFixture(t) + const name = "push/removed-tags" + first := map[string]string{ + xhttp.AmzObjectTagging: "classification=private", + ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano), + } + second := map[string]string{ + xhttp.AmzObjectTagging: "", + ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano), + } + before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first}) + putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second}) + logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if logical.UserTags != "" { + t.Fatal("control: the authoritative version must have no tags") + } + batchAuthzPolicy(f.ctx, t, f.sys, "data-without-tags", fmt.Sprintf(` +{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"]}, +{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket, bucket)) + cred := iamAdminUser(f.ctx, t, f.sys, "data-without-tags-user", "data-without-tags") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: name, versionID: before.VersionID}); !isBatchJobAuthzError(err) { + t.Fatal("control: reading tags must be denied") + } + writes := 0 + var sentTags string + target, err := minio.NewCore("target.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + if r.Body != nil { + _, _ = io.Copy(io.Discard, r.Body) + _ = r.Body.Close() + } + if r.Method == http.MethodPut { + writes++ + sentTags = r.Header.Get(xhttp.AmzObjectTagging) + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil + }), + }) + mustIAM(t, err) + r := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + } + err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub) + if err != nil && !isBatchJobAuthzError(err) { + t.Fatalf("unexpected replication failure: %v", err) + } + if writes > 0 && sentTags != logical.UserTags { + t.Errorf("push must send only metadata covered by authorization: writes=%d sentTags=%q authorizedTags=%q err=%v", writes, sentTags, logical.UserTags, err) + } +} + +// LDAP's short username and its parent DN are distinct identities. The +// background merge must not turn a deny for the verified short name into +// a non-matching statement by replacing it with the parent DN. +func TestBatchReviewLDAPUsernameDeny(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "ldap-short-name-deny", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"ldap:username":"alice"}}}`) + const dn = "uid=alice,ou=people,dc=example,dc=org" + cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{ + "exp": UTCNow().Add(time.Hour).Unix(), ldapUser: dn, ldapActualUser: dn, ldapUserN: "alice", + }, globalActiveCred.SecretKey) + mustIAM(t, err) + cred.ParentUser = dn + _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "ldap-short-name-deny") + mustIAM(t, err) + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + p, err := sub.principal(f.ctx) + mustIAM(t, err) + cred.Claims = p.claims + req, err := http.NewRequest(http.MethodDelete, "https://localhost/batch-authz/data", nil) + mustIAM(t, err) + args := policy.Args{ + AccountName: cred.AccessKey, Claims: p.claims, Action: policy.DeleteObjectVersionAction, + BucketName: f.bucket, ObjectName: "data", ConditionValues: getConditionValues(req, "", cred), + } + if args.ConditionValues["username"][0] != "alice" || f.sys.isAllowedLocally(args) { + t.Fatal("control: the request evaluator must deny the verified LDAP short username") + } + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: mustGetUUID()}); !isBatchJobAuthzError(err) { + t.Errorf("batch authorization must preserve the LDAP username deny or fail closed, got %v", err) + } +} + +// Failure to read authoritative metadata is incomplete work, even if one +// pool's copy can still be opened. In particular, skipErrs=false must not +// turn an unreadable entry into a successful archive. +func TestBatchReviewSnowballAuthoritativeReadFailure(t *testing.T) { + z, bucket := consistencyPools(t) + const name = "archive/unreadable-copy" + before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true}) + set := z.serverPools[1].sets[0] + getDisks := set.getDisks + disks := append([]StorageAPI(nil), getDisks()...) + for i := range disks { + disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: name} + } + set.getDisks = func() []StorageAPI { return disks } + defer func() { set.getDisks = getDisks }() + gr, err := z.GetObjectNInfo(t.Context(), bucket, name, nil, nil, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + _, logicalErr := batchAuthoritativeRead(t.Context(), z, bucket, name, gr.ObjInfo) + gr.Close() + if logicalErr == nil { + t.Fatal("control: the authoritative read must fail") + } + remote, err := minio.New("archive.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + if r.Body != nil { + _, _ = io.Copy(io.Discard, r.Body) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil + }), + }) + mustIAM(t, err) + r := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + } + r.Source.Snowball.InMemory = ptr(true) + r.Source.Snowball.Compress = ptr(false) + r.Source.Snowball.SkipErrs = ptr(false) + if err := r.writeAsArchive(t.Context(), z, remote, []ObjectInfo{before}, "", batchRootSubmitter(t)); err == nil { + t.Error("archive with an unreadable authoritative version must not report success") + } +} diff --git a/cmd/batch-authz-review5_test.go b/cmd/batch-authz-review5_test.go new file mode 100644 index 000000000..ccc2520e9 --- /dev/null +++ b/cmd/batch-authz-review5_test.go @@ -0,0 +1,142 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "context" + "fmt" + "io" + "maps" + "net/http" + "strings" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +// A suspended destination does not keep a source version ID. Authorization +// must therefore describe the null destination version that the write really +// replaces, and must not let a source ID escape that model. +func TestBatchReviewPullSuspendedVersionBinding(t *testing.T) { + f := newBatchAuthzFixture(t) + _, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`Suspended`)) + mustIAM(t, err) + + sourceVersion := mustGetUUID() + batchAuthzPolicy(f.ctx, t, f.sys, "pull-source-version-deny", batchAuthzFullAccess+fmt.Sprintf(`, +{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, sourceVersion)) + cred := iamAdminUser(f.ctx, t, f.sys, "pull-source-version-user", "pull-source-version-deny") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + job := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket}, + } + err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{ + Name: "pull/suspended-version", VersionID: sourceVersion, ModTime: UTCNow(), ETag: "batchetag", + UserDefined: map[string]string{}, + }, false, sub) + if !isBatchJobAuthzError(err) { + t.Fatalf("a source version ID must not evade the destination deny: %v", err) + } + if _, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/suspended-version", ObjectOptions{VersionID: sourceVersion}); gerr == nil { + t.Fatal("a denied suspended-destination write created the source version") + } +} + +// A failed source open in a non-skipping snowball must be reported to the +// batch rather than silently omitted from a successful archive. +type batchReviewOpenFailure struct { + ObjectLayer + name string +} + +func (a batchReviewOpenFailure) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) { + if object == a.name { + return nil, fmt.Errorf("test: source open failed") + } + return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts) +} + +func TestBatchReviewSnowballSourceOpenFailure(t *testing.T) { + f := newBatchAuthzFixture(t) + name := "archive/open-failure" + entry := batchPutVersion(t, f.obj, f.bucket, name, nil) + remote, err := minio.New("archive.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + if r.Body != nil { + _, _ = io.Copy(io.Discard, r.Body) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil + }), + }) + mustIAM(t, err) + r := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + } + r.Source.Snowball.InMemory = ptr(true) + r.Source.Snowball.Compress = ptr(false) + r.Source.Snowball.SkipErrs = ptr(false) + api := batchReviewOpenFailure{ObjectLayer: f.obj, name: name} + if err := r.writeAsArchive(t.Context(), api, remote, []ObjectInfo{entry}, "", batchRootSubmitter(t)); err == nil { + t.Fatal("a source open failure must not be reported as a successful archive") + } +} + +// Rotation must apply tag conditions to the logical version, including +// a more recent tag update stored in a pool other than the selected copy. +func TestBatchReviewRotationAuthoritativeTags(t *testing.T) { + z, bucket := consistencyPools(t) + f := newBatchAuthzFixture(t) + previousKMS := GlobalKMS + GlobalKMS = kms.NewStub("batch-round5-rotation-key") + t.Cleanup(func() { GlobalKMS = previousKMS }) + batchAuthzPolicy(f.ctx, t, f.sys, "round5-rotation-tags", fmt.Sprintf(` +{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]}, +{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket)) + cred := iamAdminUser(f.ctx, t, f.sys, "round5-rotation-user", "round5-rotation-tags") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + for _, classification := range []string{"private", "public"} { + t.Run(classification, func(t *testing.T) { + name := "rotation/authoritative-" + classification + first := make(map[string]string) + key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil) + mustIAM(t, err) + first[xhttp.AmzObjectTagging] = "classification=public" + first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + second := maps.Clone(first) + second[xhttp.AmzObjectTagging] = "classification=" + classification + second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano) + ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("test encrypted payload"), key)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first}) + putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second}) + logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if logical.UserTags != "classification="+classification { + t.Fatal("control: the second pool must govern the logical tags") + } + job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + err = job.KeyRotate(f.ctx, z, before, sub) + after, readErr := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, readErr) + changed := after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] + if classification == "private" { + if !isBatchJobAuthzError(err) || changed { + t.Fatalf("authoritative tag deny must prevent rewrapping: err=%v changed=%v", err, changed) + } + } else if err != nil || !changed { + t.Fatalf("authorized unlocked version must rotate: err=%v changed=%v", err, changed) + } + }) + } +} diff --git a/cmd/batch-authz-review_test.go b/cmd/batch-authz-review_test.go new file mode 100644 index 000000000..c69ee43e4 --- /dev/null +++ b/cmd/batch-authz-review_test.go @@ -0,0 +1,287 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + polplugin "github.com/minio/minio/internal/config/policy/plugin" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + xnet "github.com/pgsty/silo-pkg/v3/net" + "github.com/pgsty/silo-pkg/v3/policy" +) + +func batchPullJob(bucket string) BatchJobReplicateV1 { + return BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + } +} + +// A pull does not read the destination version it may replace, so a policy +// that conditions the write on that version's tags refuses it; a version +// condition sees the version the pull writes. +func TestBatchPullDestinationConditions(t *testing.T) { + for _, tc := range []struct { + name string + action policy.Action + marker bool + condition func(versionID string) string + }{ + {"tags-put", policy.ReplicateObjectAction, false, func(string) string { + return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}` + }}, + {"tags-marker", policy.ReplicateDeleteAction, true, func(string) string { + return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}` + }}, + {"version-put", policy.PutObjectAction, false, func(v string) string { + return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newBatchAuthzFixture(t) + before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"}) + batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`, +{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID))) + cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + job := batchPullJob(f.bucket) + err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{ + Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag", + DeleteMarker: tc.marker, UserDefined: map[string]string{}, + }, false, sub) + if !isBatchJobAuthzError(err) { + t.Errorf("pull returned %v, want a refusal", err) + } + after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID}) + if err != nil || after.ETag != before.ETag { + t.Errorf("refused pull changed the destination version: %v", err) + } + }) + } +} + +// A pull does not encrypt what it writes; encryption headers reported by the +// source do not satisfy a destination policy that requires encryption. +func TestBatchPullEncryptionCondition(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", ` +{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}, +{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`) + cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + job := batchPullJob(f.bucket) + version := mustGetUUID() + err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{ + Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag", + UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"}, + }, false, sub) + if err == nil { + oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version}) + mustIAM(t, gerr) + if !crypto.S3KMS.IsEncrypted(oi.UserDefined) { + t.Fatal("a pull was allowed by an encryption condition it did not meet") + } + } +} + +// A conditional Deny on an S3 Tables action covers the S3 data actions it +// implies, under the tables resource form; the job refuses rather than skip it. +func TestBatchJobTableConditionalDeny(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`) + cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context") + if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) { + t.Fatal("control: with the request context the Deny applies") + } + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) { + t.Fatalf("authorize returned %v, want a refusal", err) + } +} + +type batchArchiveRevocation struct { + ObjectLayer + revoke func() +} + +func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) { + if object == "archive/second" { + a.revoke() + } + return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts) +} + +// Each snowball entry is authorized for the submitter as it is then. +func TestBatchPushSnowballEntryRevalidation(t *testing.T) { + f := newBatchAuthzFixture(t) + cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/") + mustIAM(t, err) + previous := newGlobalAuthZPluginFn() + setGlobalAuthZPlugin(polplugin.New(polplugin.Args{ + URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() }, + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil + }), + })) + t.Cleanup(func() { setGlobalAuthZPlugin(previous) }) + entries := []ObjectInfo{ + batchPutVersion(t, f.obj, f.bucket, "archive/first", nil), + batchPutVersion(t, f.obj, f.bucket, "archive/second", nil), + } + remote, err := minio.New("archive.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + if r.Body != nil { + _, _ = io.Copy(io.Discard, r.Body) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil + }), + }) + mustIAM(t, err) + job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}} + job.Source.Snowball.InMemory = ptr(true) + job.Source.Snowball.Compress = ptr(false) + job.Source.Snowball.SkipErrs = ptr(false) + api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }} + if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) { + t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err) + } +} + +// A definition holds one job type; one with several is refused, so that what +// is authorized is what runs. +func TestBatchJobSeveralSections(t *testing.T) { + f := newBatchAuthzFixture(t) + older := f.putVersions(t, "data/a") + job := f.expireYAML("") + `keyrotate: + apiVersion: v1 + bucket: other + encryption: + type: sse-s3 +` + rec, queued := f.submit(t, globalActiveCred, job) + if queued != nil { + f.run(t, queued) + } + if rec.Code != http.StatusBadRequest { + t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String()) + } + if !f.versionExists(t, "data/a", older) { + t.Fatal("a definition with two job types ran") + } +} + +// Prefixes are walked once each in one order: a cursor in a later prefix is +// after every key of the earlier ones, and an overlapping prefix is not +// walked twice. +func TestBatchExpirePrefixes(t *testing.T) { + if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" { + t.Fatalf("canonical prefixes %q", got) + } + if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" { + t.Fatalf("canonical prefixes %q", got) + } + f := newBatchAuthzFixture(t) + done := f.putVersions(t, "a/done") + todo := f.putVersions(t, "z/todo") + batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD")) + batchPutVersion(t, f.obj, f.bucket, "x/data", nil) + rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire: + apiVersion: v1 + bucket: %s + prefix: ["z/", "x/", "a/", "x/"] + rules: + - type: object + purge: + retainVersions: 1 +`, f.bucket)) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + // A run that completed a/ and stopped. + ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"} + mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job)) + f.run(t, job) + if !f.versionExists(t, "a/done", done) { + t.Error("resume did again a key before the cursor") + } + if f.versionExists(t, "z/todo", todo) { + t.Error("resume skipped a later prefix") + } + if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 { + t.Errorf("a locked version under two equal prefixes was counted %d times", got) + } +} + +type batchPartialWalk struct { + ObjectLayer + version ObjectInfo + cancel context.CancelFunc + read chan struct{} +} + +func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error { + go func() { + defer close(results) + results <- itemOrErr[ObjectInfo]{Item: a.version} + if a.cancel != nil { + <-a.read + a.cancel() + return + } + results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")} + }() + return nil +} + +func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) { + if a.read != nil { + close(a.read) + } + return a.version, nil +} + +// A rotation whose walk fails or is canceled within a key keeps that key +// after its cursor and does not report completion. +func TestBatchKeyRotationIncompleteWalk(t *testing.T) { + for _, canceled := range []bool{false, true} { + t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) { + f := newBatchAuthzFixture(t) + ctx, cancel := context.WithCancel(f.ctx) + defer cancel() + meta := batchLockMeta("LEGALHOLD") + meta[crypto.MetaSealedKeyS3] = "test-sealed-key" + api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}} + if canceled { + api.cancel, api.read = cancel, make(chan struct{}) + } + job := BatchJobRequest{ + ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t), + KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}, + } + _ = job.KeyRotate.Start(ctx, api, job) + ri := f.jobInfo(t, &job) + if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 { + t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked) + } + }) + } +} diff --git a/cmd/batch-object-lock_test.go b/cmd/batch-object-lock_test.go new file mode 100644 index 000000000..dc3096c0b --- /dev/null +++ b/cmd/batch-object-lock_test.go @@ -0,0 +1,479 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/auth" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/pgsty/silo-pkg/v3/policy" +) + +var batchLockKinds = []string{"COMPLIANCE", "GOVERNANCE", "LEGALHOLD"} + +func batchLockMeta(kind string) map[string]string { + m := map[string]string{} + switch kind { + case "COMPLIANCE", "GOVERNANCE": + m[strings.ToLower(xhttp.AmzObjectLockMode)] = kind + m[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(UTCNow().Add(24 * time.Hour)) + case "LEGALHOLD": + m[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON" + } + return m +} + +func batchPutVersion(t *testing.T, obj ObjectLayer, bucket, key string, meta map[string]string) ObjectInfo { + t.Helper() + data := []byte("batch lock " + key) + oi, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), + ObjectOptions{Versioned: true, UserDefined: meta}) + if err != nil { + t.Fatal(err) + } + return oi +} + +func batchVersionExists(t *testing.T, obj ObjectLayer, bucket, key, versionID string) bool { + t.Helper() + _, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: versionID}) + switch { + case err == nil: + return true + case isErrVersionNotFound(err) || isErrObjectNotFound(err): + return false + } + t.Fatal(err) + return false +} + +// batchJobStatusJSON returns the status response of a job as decoded JSON. +func batchJobStatusJSON(t *testing.T, jobID string) map[string]any { + t.Helper() + cred := globalActiveCred + req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/status-job?jobId="+jobID, 0, nil, cred.AccessKey, cred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + if err = req.ParseForm(); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + req = req.WithContext(newContext(req, rec, "BatchJobStatus")) + adminAPIHandlers{}.BatchJobStatus(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var out struct{ LastMetric map[string]any } + if err = json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + return out.LastMetric +} + +// Batch expiration keeps versions under retention or legal hold, on both +// purge paths, and reports them; unlocked versions are still expired. +func TestBatchExpireObjectLock(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, endpoints: []string{"PutObject"}, + objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + defer r5Capacity(obj.(*erasureServerPools))() + enableBucketObjectLock(t, bucket) + f := &batchAuthzFixture{ctx: t.Context(), obj: obj, bucket: bucket} + batchTestJobPool(t.Context(), t, obj) + + type version struct{ key, id string } + var locked, unlocked []version + for _, retain := range []int{0, 1} { + for _, kind := range batchLockKinds { + key := fmt.Sprintf("r%d/%s", retain, kind) + older := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind)) + batchPutVersion(t, obj, bucket, key, nil) + locked = append(locked, version{key, older.VersionID}) + } + key := fmt.Sprintf("r%d/plain", retain) + older := batchPutVersion(t, obj, bucket, key, nil) + batchPutVersion(t, obj, bucket, key, nil) + unlocked = append(unlocked, version{key, older.VersionID}) + } + + var jobs []*BatchJobRequest + for _, retain := range []int{0, 1} { + job := fmt.Sprintf(`expire: + apiVersion: v1 + bucket: %s + prefix: r%d/ + rules: + - type: object + purge: + retainVersions: %d +`, bucket, retain, retain) + rec, queued := f.submit(t, globalActiveCred, job) + if queued == nil { + t.Fatalf("%s: submission status %d: %s", instance, rec.Code, rec.Body.String()) + } + // Running a job again, as a resumed job does, must not count + // the versions it left a second time. + f.run(t, queued) + f.run(t, queued) + jobs = append(jobs, queued) + } + + for _, v := range locked { + if !batchVersionExists(t, obj, bucket, v.key, v.id) { + t.Errorf("%s: %s: locked version %s was expired", instance, v.key, v.id) + } + } + for _, v := range unlocked { + if batchVersionExists(t, obj, bucket, v.key, v.id) { + t.Errorf("%s: %s: control: unlocked version %s was not expired", instance, v.key, v.id) + } + } + for i, field := range []string{"keysRefused", "versionsSkippedLocked"} { + m := batchJobStatusJSON(t, jobs[i].ID) + if got, _ := m[field].(float64); got != float64(len(batchLockKinds)) { + t.Errorf("%s: retain %d: %s = %v, want %d", instance, i, field, m[field], len(batchLockKinds)) + } + if m["complete"] != true || m["failed"] != false { + t.Errorf("%s: retain %d: complete=%v failed=%v", instance, i, m["complete"], m["failed"]) + } + } + }, + }) +} + +type batchLockTransport func(*http.Request) (*http.Response, error) + +func (f batchLockTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// batchLockRemote is a source that serves replacement for every read. +func batchLockRemote(t *testing.T, replacement string) *minio.Core { + t.Helper() + core, err := minio.NewCore("source.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{ + "Content-Length": []string{strconv.Itoa(len(replacement))}, + "Last-Modified": []string{UTCNow().Format(http.TimeFormat)}, + "Etag": []string{"\"batchetag\""}, + }, Body: io.NopCloser(strings.NewReader(replacement)), ContentLength: int64(len(replacement)), Request: r}, nil + }), + }) + if err != nil { + t.Fatal(err) + } + return core +} + +func batchRootSubmitter(t *testing.T) *batchJobSubmitter { + t.Helper() + sub, err := newBatchJobSubmitter(globalActiveCred) + if err != nil { + t.Fatal(err) + } + return sub +} + +// A pull leaves a destination version under Object Lock as it is, whether a +// single PUT, a multipart completion or a delete marker would replace it. +func TestBatchPullObjectLock(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, endpoints: []string{"PutObject"}, + objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + defer r5Capacity(obj.(*erasureServerPools))() + enableBucketObjectLock(t, bucket) + const replacement = "replacement from remote" + core := batchLockRemote(t, replacement) + sub := batchRootSubmitter(t) + job := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + } + for _, kind := range batchLockKinds { + for _, path := range []string{"put", "multipart", "marker", "directory"} { + key := "pull/" + kind + "/" + path + if path == "directory" { + key += "/" + } + before := batchPutVersion(t, obj, bucket, key, batchLockMeta(kind)) + src := ObjectInfo{Name: key, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}} + switch path { + case "multipart": + src.ETag = "batchetag-1" + case "marker": + src.DeleteMarker = true + } + err := job.ReplicateFromSource(t.Context(), obj, core, src, false, sub) + if !isErrObjectLocked(err) { + t.Errorf("%s: %s: pull returned %v, want ObjectLocked", instance, key, err) + } + after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID}) + if gerr != nil { + t.Errorf("%s: %s: locked version lost: %v", instance, key, gerr) + continue + } + if !enforceRetentionForDeletion(t.Context(), after) || after.ETag != before.ETag { + t.Errorf("%s: %s: locked version was replaced", instance, key) + } + } + } + + // Controls: a new version, and a new delete marker that hides a + // locked version without replacing it. + locked := batchPutVersion(t, obj, bucket, "pull/new", batchLockMeta("COMPLIANCE")) + newVersion := mustGetUUID() + if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{ + Name: "pull/new", VersionID: newVersion, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{}, + }, false, sub); err != nil { + t.Fatalf("%s: new version: %v", instance, err) + } + if got := readCopyChecksumObject(t, obj, bucket, "pull/new", ObjectOptions{VersionID: newVersion}); string(got) != replacement { + t.Errorf("%s: new version content %q", instance, got) + } + marker := mustGetUUID() + if err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{ + Name: "pull/new", VersionID: marker, ModTime: UTCNow(), DeleteMarker: true, + }, false, sub); err != nil { + t.Fatalf("%s: new marker: %v", instance, err) + } + if !batchVersionExists(t, obj, bucket, "pull/new", locked.VersionID) { + t.Errorf("%s: a new marker removed a locked version", instance) + } + }, + }) +} + +// A pull applies retention as a PUT to the destination would: copied +// retention is validated, and a version without one gets the bucket default. +func TestBatchPullRetentionMetadata(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, endpoints: []string{"PutObject"}, + objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + defer r5Capacity(obj.(*erasureServerPools))() + meta, err := globalBucketMetadataSys.Get(bucket) + if err != nil { + t.Fatal(err) + } + meta.ObjectLockConfigXML = []byte(`EnabledGOVERNANCE1`) + meta.VersioningConfigXML = enabledBucketVersioningConfig + if err = meta.parseAllConfigs(t.Context(), obj); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(bucket, meta) + core := batchLockRemote(t, "retention content") + sub := batchRootSubmitter(t) + job := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + } + pull := func(key string, meta map[string]string) (ObjectInfo, error) { + vid := mustGetUUID() + err := job.ReplicateFromSource(t.Context(), obj, core, ObjectInfo{Name: key, VersionID: vid, ModTime: UTCNow().Add(-time.Hour), ETag: "batchetag", UserDefined: meta}, false, sub) + if err != nil { + return ObjectInfo{}, err + } + return obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: vid}) + } + oi, err := pull("retention/default", map[string]string{}) + if err != nil { + t.Fatal(err) + } + ret := objectlock.GetObjectRetentionMeta(oi.UserDefined) + if ret.Mode != objectlock.RetGovernance || !ret.RetainUntilDate.After(UTCNow().Add(23*time.Hour)) { + t.Errorf("%s: default retention not applied with the destination's time: %+v", instance, ret) + } + until := UTCNow().Add(48 * time.Hour) + oi, err = pull("retention/copied", map[string]string{ + xhttp.AmzObjectLockMode: "COMPLIANCE", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until), + }) + if err != nil { + t.Fatal(err) + } + if ret = objectlock.GetObjectRetentionMeta(oi.UserDefined); ret.Mode != objectlock.RetCompliance || !ret.RetainUntilDate.Equal(until.Truncate(time.Millisecond)) { + t.Errorf("%s: copied retention: %+v", instance, ret) + } + if _, err = pull("retention/invalid", map[string]string{xhttp.AmzObjectLockMode: "FOREVER", xhttp.AmzObjectLockRetainUntilDate: amztime.ISO8601Format(until)}); err == nil { + t.Errorf("%s: invalid copied retention was accepted", instance) + } + }, + }) +} + +// Key rotation does not rewrite a version under Object Lock. +func TestBatchKeyRotationObjectLock(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, endpoints: []string{"PutObject", "GetObject"}, + objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + defer r5Capacity(obj.(*erasureServerPools))() + enableBucketObjectLock(t, bucket) + oldKMS := GlobalKMS + GlobalKMS = kms.NewStub("batch-lock-key") + defer func() { GlobalKMS = oldKMS }() + sub := batchRootSubmitter(t) + job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + for _, kind := range append([]string{"none"}, batchLockKinds...) { + key := "rotate/" + kind + headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES} + switch kind { + case "COMPLIANCE", "GOVERNANCE": + headers[xhttp.AmzObjectLockMode] = kind + headers[xhttp.AmzObjectLockRetainUntilDate] = UTCNow().Add(24 * time.Hour).Format(time.RFC3339) + case "LEGALHOLD": + headers[xhttp.AmzObjectLockLegalHold] = "ON" + } + putCopyChecksumSource(t, router, cred, bucket, key, []byte("rotate "+kind), headers) + before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}) + if err != nil || !crypto.S3.IsEncrypted(before.UserDefined) { + t.Fatalf("%s: %s: encrypted control: %v", instance, key, err) + } + err = job.KeyRotate(t.Context(), obj, before, sub) + after, gerr := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: before.VersionID}) + if gerr != nil { + t.Fatal(gerr) + } + if kind == "none" { + if err != nil || after.ModTime.Equal(before.ModTime) { + t.Errorf("%s: control: unlocked version not rotated: %v", instance, err) + } + continue + } + var locked ObjectLocked + if !errors.As(err, &locked) || !after.ModTime.Equal(before.ModTime) { + t.Errorf("%s: %s: rotation of a locked version returned %v; rewritten=%v", instance, key, err, !after.ModTime.Equal(before.ModTime)) + } + } + }, + }) +} + +// A push reads locked versions as any authorized read does, and a submitter +// that may not read a version does not send it. +func TestBatchPushReads(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, endpoints: []string{"PutObject"}, + objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) { + defer r5Capacity(obj.(*erasureServerPools))() + enableBucketObjectLock(t, bucket) + var puts []string + target, err := minio.NewCore("target.invalid", &minio.Options{ + Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""), + Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) { + if r.Body != nil { + io.Copy(io.Discard, r.Body) + r.Body.Close() + } + if r.Method == http.MethodPut { + puts = append(puts, r.URL.Path) + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": []string{"\"pushetag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil + }), + }) + if err != nil { + t.Fatal(err) + } + job := BatchJobReplicateV1{ + Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket}, + Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote-bucket"}, + } + src := batchPutVersion(t, obj, bucket, "push/locked", batchLockMeta("COMPLIANCE")) + if err := job.ReplicateToTarget(t.Context(), obj, target, src, false, batchRootSubmitter(t)); err != nil { + t.Fatalf("%s: authorized push of a locked version: %v", instance, err) + } + if len(puts) != 1 { + t.Fatalf("%s: authorized push sent %d writes", instance, len(puts)) + } + + // A user that may list but not read. + p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:StartBatchJob"]},{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::` + bucket + `"]}]}`)) + if err != nil { + t.Fatal(err) + } + if _, err = globalIAMSys.SetPolicy(t.Context(), "batch-list-only", *p); err != nil { + t.Fatal(err) + } + user := auth.Credentials{AccessKey: "batch-list-user", SecretKey: "batch-list-user-secret"} + if _, err = globalIAMSys.CreateUser(t.Context(), user.AccessKey, madmin.AddOrUpdateUserReq{SecretKey: user.SecretKey, Status: madmin.AccountEnabled}); err != nil { + t.Fatal(err) + } + if _, err = globalIAMSys.PolicyDBSet(t.Context(), user.AccessKey, "batch-list-only", regUser, false); err != nil { + t.Fatal(err) + } + sub, err := newBatchJobSubmitter(globalIAMSysUser(t, user.AccessKey)) + if err != nil { + t.Fatal(err) + } + err = job.ReplicateToTarget(t.Context(), obj, target, src, false, sub) + if !isBatchJobAuthzError(err) || len(puts) != 1 { + t.Fatalf("%s: unauthorized push returned %v and sent %d writes", instance, err, len(puts)-1) + } + }, + }) +} + +// globalIAMSysUser returns the stored credential of accessKey, as a request +// authenticated with it has. +func globalIAMSysUser(t *testing.T, accessKey string) auth.Credentials { + t.Helper() + u, ok := globalIAMSys.store.GetUser(accessKey) + if !ok { + t.Fatalf("no user %s", accessKey) + } + return u.Credentials +} + +// The cursor of a walk done out of order moves only past keys whose items +// are all done, and counts each item once. +func TestBatchProgressCursor(t *testing.T) { + ri := &batchJobInfo{} + p := newBatchProgress(ri, "bucket") + items := []string{"a", "a", "b", "c", "c"} + seqs := make([]uint64, len(items)) + for i := range items { + seqs[i] = p.begin() + } + skip := batchCounts{versionsSkipped: 1} + p.done(seqs[1], "a", skip) + p.done(seqs[3], "c", skip) + if ri.Object != "" || ri.VersionsSkippedLocked != 0 { + t.Fatalf("cursor moved before the first item: %q %d", ri.Object, ri.VersionsSkippedLocked) + } + p.done(seqs[0], "a", skip) + p.done(seqs[2], "b", skip) + // a is done; b is done but c may still have items. + if ri.Object != "b" && ri.Object != "a" { + t.Fatalf("cursor %q", ri.Object) + } + if ri.Object == "a" && ri.VersionsSkippedLocked != 2 { + t.Fatalf("counts %d at cursor a", ri.VersionsSkippedLocked) + } + // c's last item never finishes, as when a run stops: the cursor stays + // before c and c is not counted. + p.finish() + if ri.Object != "b" || ri.VersionsSkippedLocked != 3 { + t.Fatalf("interrupted: cursor %q counts %d, want b 3", ri.Object, ri.VersionsSkippedLocked) + } + p.done(seqs[4], "c", skip) + p.finish() + if ri.Object != "c" || ri.VersionsSkippedLocked != 5 { + t.Fatalf("finished: cursor %q counts %d, want c 5", ri.Object, ri.VersionsSkippedLocked) + } +} diff --git a/cmd/batch-submitter-authz_test.go b/cmd/batch-submitter-authz_test.go new file mode 100644 index 000000000..64a16d97c --- /dev/null +++ b/cmd/batch-submitter-authz_test.go @@ -0,0 +1,456 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/pgsty/silo-pkg/v3/policy" + "go.yaml.in/yaml/v3" +) + +// batchAuthzFixture runs batch jobs submitted through the admin API, one job +// at a time and in the test's goroutine. +type batchAuthzFixture struct { + ctx context.Context + sys *IAMSys + obj ObjectLayer + bucket string +} + +func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture { + t.Helper() + ctx, sys := iamAdminFixture(t) + obj := newObjectLayerFn() + batchTestJobPool(ctx, t, obj) + f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"} + mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true})) + _, err := globalBucketMetadataSys.Update(ctx, f.bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + return f +} + +// batchTestJobPool queues submitted jobs without running them. +func batchTestJobPool(ctx context.Context, t *testing.T, obj ObjectLayer) { + t.Helper() + prevPool := globalBatchJobPool + globalBatchJobPool = &BatchJobPool{ + ctx: ctx, + objLayer: obj, + jobCh: make(chan *BatchJobRequest, 10), + jobCancelers: make(map[string]context.CancelFunc), + } + globalBatchJobsMetrics.Lock() + globalBatchJobsMetrics.metrics = make(map[string]*batchJobInfo) + globalBatchJobsMetrics.Unlock() + t.Cleanup(func() { globalBatchJobPool = prevPool }) +} + +// putVersions writes two versions of key and returns the older one. +func (f *batchAuthzFixture) putVersions(t *testing.T, key string) string { + t.Helper() + var older string + for i := range 2 { + data := []byte(fmt.Sprintf("version %d", i)) + oi, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true}) + mustIAM(t, err) + if i == 0 { + older = oi.VersionID + } + } + return older +} + +func (f *batchAuthzFixture) versionExists(t *testing.T, key, versionID string) bool { + t.Helper() + _, err := f.obj.GetObjectInfo(f.ctx, f.bucket, key, ObjectOptions{VersionID: versionID}) + if err == nil { + return true + } + if isErrVersionNotFound(err) || isErrObjectNotFound(err) { + return false + } + t.Fatal(err) + return false +} + +func (f *batchAuthzFixture) expireYAML(prefix string) string { + return fmt.Sprintf(`expire: + apiVersion: v1 + bucket: %s + prefix: %q + rules: + - type: object + purge: + retainVersions: 1 +`, f.bucket, prefix) +} + +// submit sends a job definition to StartBatchJob as cred. An accepted job is +// returned from the queue, not run. +func (f *batchAuthzFixture) submit(t *testing.T, cred auth.Credentials, job string) (*httptest.ResponseRecorder, *BatchJobRequest) { + t.Helper() + headers := map[string]string{} + if cred.IsTemp() && !cred.IsServiceAccount() { + headers["X-Amz-Security-Token"] = cred.SessionToken + } + body := []byte(job) + req, err := newTestSignedRequestV4(http.MethodPost, "http://localhost/minio/admin/v3/start-job", int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers) + mustIAM(t, err) + mustIAM(t, req.ParseForm()) + rec := httptest.NewRecorder() + req = req.WithContext(newContext(req, rec, "StartBatchJob")) + adminAPIHandlers{}.StartBatchJob(rec, req) + if rec.Code != http.StatusOK { + return rec, nil + } + select { + case queued := <-globalBatchJobPool.jobCh: + return rec, queued + default: + t.Fatal("accepted job was not queued") + return rec, nil + } +} + +func (f *batchAuthzFixture) run(t *testing.T, job *BatchJobRequest) { + t.Helper() + ctx := job.ctx + if ctx == nil { + ctx = f.ctx + } + switch { + case job.Expire != nil: + if err := job.Expire.Start(ctx, f.obj, *job); err != nil { + t.Logf("start: %v", err) + } + default: + t.Fatalf("unexpected job type %s", job.Type()) + } + ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())} + if err := ri.load(f.ctx, f.obj, *job); err == nil { + t.Logf("job %s: %+v", job.ID, ri.clone()) + } +} + +func (f *batchAuthzFixture) jobInfo(t *testing.T, job *BatchJobRequest) *batchJobInfo { + t.Helper() + ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type())} + mustIAM(t, ri.load(f.ctx, f.obj, *job)) + return ri +} + +func batchAuthzPolicy(ctx context.Context, t *testing.T, sys *IAMSys, name, statements string) { + t.Helper() + p, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[` + statements + `]}`)) + mustIAM(t, err) + _, err = sys.SetPolicy(ctx, name, *p) + mustIAM(t, err) +} + +const batchAuthzFullAccess = `{"Effect":"Allow","Action":["admin:StartBatchJob"]}, +{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}` + +// A credential that may start batch jobs but has no access to the bucket +// cannot delete through a job. +func TestBatchJobSubmitterWithoutDataAccess(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-only", `{"Effect":"Allow","Action":["admin:StartBatchJob","admin:ListBatchJobs"]}`) + cred := iamAdminUser(f.ctx, t, f.sys, "batch-only-user", "batch-only") + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job != nil { + f.run(t, job) + } + if !f.versionExists(t, "data/a", older) { + t.Fatal("a job submitted without data access deleted a version") + } + if rec.Code != http.StatusForbidden { + t.Fatalf("submission status %d, want 403: %s", rec.Code, rec.Body.String()) + } +} + +// A job acts only while the credential that submitted it is the same valid +// credential. +func TestBatchJobSubmitterChangedBeforeStart(t *testing.T) { + for _, tc := range []struct { + name string + change func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) + }{ + {"unchanged", nil}, + {"deleted", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) { + mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) + }}, + {"disabled", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) { + _, err := f.sys.SetUserStatus(f.ctx, cred.AccessKey, madmin.AccountDisabled) + mustIAM(t, err) + }}, + {"secret-changed", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) { + mustIAM(t, f.sys.SetUserSecretKey(f.ctx, cred.AccessKey, "batch-authz-changed-secret")) + }}, + {"access-key-reused", func(t *testing.T, f *batchAuthzFixture, cred auth.Credentials) { + mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) + time.Sleep(10 * time.Millisecond) + iamAdminUser(f.ctx, t, f.sys, cred.AccessKey, "batch-full") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full") + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + if tc.change != nil { + tc.change(t, f, cred) + } + f.run(t, job) + exists := f.versionExists(t, "data/a", older) + if tc.change == nil && exists { + t.Fatal("control: an authorized job did not expire the older version") + } + if tc.change != nil && !exists { + t.Fatal("a job ran as a credential that changed after submission") + } + }) + } +} + +// An expired temporary credential does not keep its job running. +func TestBatchJobSubmitterExpired(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full") + cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(2 * time.Second).Unix()}, globalActiveCred.SecretKey) + mustIAM(t, err) + cred.ParentUser = parent.AccessKey + _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "") + mustIAM(t, err) + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + time.Sleep(time.Until(cred.Expiration) + time.Second) + f.run(t, job) + if !f.versionExists(t, "data/a", older) { + t.Fatal("a job ran as an expired credential") + } +} + +// A session policy limits the job as it limits requests, also when an +// authorization plugin decides the rest. +func TestBatchJobSessionPolicyDeny(t *testing.T) { + session := `{"Version":"2012-10-17","Statement":[ +{"Effect":"Allow","Action":["admin:StartBatchJob"]}, +{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}, +{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}` + for _, tc := range []struct { + name string + plugin bool + }{{"local", false}, {"plugin-allows", true}} { + t.Run(tc.name, func(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full") + cred := iamAdminServiceAccount(f.ctx, t, f.sys, parent.AccessKey, "batch-restricted", session) + if tc.plugin { + iamAdminPlugin(t, true) + } + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job != nil { + f.run(t, job) + } else { + t.Logf("submission status %d: %s", rec.Code, rec.Body.String()) + } + if !f.versionExists(t, "data/a", older) { + t.Fatal("a job deleted a version its session policy denies") + } + }) + } +} + +// A temporary credential's session policy limits its job too. +func TestBatchJobTemporarySessionPolicyDeny(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full") + session := `{"Version":"2012-10-17","Statement":[ +{"Effect":"Allow","Action":["admin:StartBatchJob"]}, +{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions","s3:GetObject","s3:GetObjectVersion","s3:DeleteObject"],"Resource":["arn:aws:s3:::batch-authz","arn:aws:s3:::batch-authz/*"]}]}` + cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{ + parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), + policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(session)), + }, globalActiveCred.SecretKey) + mustIAM(t, err) + cred.ParentUser = parent.AccessKey + _, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "") + mustIAM(t, err) + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job != nil { + f.run(t, job) + } else { + t.Logf("submission status %d: %s", rec.Code, rec.Body.String()) + } + if !f.versionExists(t, "data/a", older) { + t.Fatal("a job deleted a version its session policy does not allow") + } +} + +// A Deny that depends on request context the job does not have refuses the +// action instead of being skipped. +func TestBatchJobAbsentContextDeny(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-context", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`) + cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-context") + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job != nil { + f.run(t, job) + } else { + t.Logf("submission status %d: %s", rec.Code, rec.Body.String()) + } + if !f.versionExists(t, "data/a", older) { + t.Fatal("a job skipped a Deny that depends on request context") + } +} + +// Each object is authorized: a Deny under an allowed prefix keeps its objects. +func TestBatchJobPerObjectDeny(t *testing.T) { + f := newBatchAuthzFixture(t) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-keep", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/data/keep*"]}`) + cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-keep") + kept := f.putVersions(t, "data/keep") + rec, job := f.submit(t, cred, f.expireYAML("data/")) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + f.run(t, job) + if !f.versionExists(t, "data/keep", kept) { + t.Fatal("a job deleted a version that a per-object Deny protects") + } + if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete { + t.Fatalf("refused job: complete=%v failed=%v", ri.Complete, ri.Failed) + } +} + +// A job queued without its submitter, as an earlier release stored it, fails +// and stays failed across another start. +func TestBatchJobWithoutSubmitter(t *testing.T) { + f := newBatchAuthzFixture(t) + older := f.putVersions(t, "data/a") + job := &BatchJobRequest{} + mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job)) + job.ID = "expire-legacy" + getKeySeparator() + "0" + job.User = globalActiveCred.AccessKey + job.Started = time.Now() + mustIAM(t, job.save(f.ctx, f.obj)) + for range 2 { + loaded := &BatchJobRequest{} + mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job))) + f.run(t, loaded) + if !f.versionExists(t, "data/a", older) { + t.Fatal("a job without a recorded submitter deleted a version") + } + if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete { + t.Fatalf("job without submitter: complete=%v failed=%v", ri.Complete, ri.Failed) + } + } +} + +// Root and an authorized user can still run jobs. +func TestBatchJobAuthorizedControls(t *testing.T) { + for _, root := range []bool{true, false} { + t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) { + f := newBatchAuthzFixture(t) + cred := globalActiveCred + if !root { + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full") + } + older := f.putVersions(t, "data/a") + rec, job := f.submit(t, cred, f.expireYAML("")) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + f.run(t, job) + if f.versionExists(t, "data/a", older) { + t.Fatal("an authorized job did not expire the older version") + } + if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed { + t.Fatalf("authorized job: complete=%v failed=%v", ri.Complete, ri.Failed) + } + }) + } +} + +// The describe output has no secrets of the job. +func TestBatchJobDescribeRedactsSecrets(t *testing.T) { + f := newBatchAuthzFixture(t) + secrets := []string{"describe-source-secret", "describe-source-token", "describe-target-secret", "describe-target-token", "describe-notify-token"} + for _, job := range []BatchJobRequest{ + {ID: "replicate-describe" + getKeySeparator() + "0", Replicate: &BatchJobReplicateV1{ + APIVersion: batchReplJobAPIVersion, + Source: BatchJobReplicateSource{Bucket: "remote", Endpoint: "https://remote.invalid", Creds: BatchJobReplicateCredentials{ + AccessKey: "describe-source-access", SecretKey: secrets[0], SessionToken: secrets[1], + }}, + Target: BatchJobReplicateTarget{Bucket: f.bucket, Creds: BatchJobReplicateCredentials{ + AccessKey: "describe-target-access", SecretKey: secrets[2], SessionToken: secrets[3], + }}, + Flags: BatchJobReplicateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}}, + }}, + {ID: "expire-describe" + getKeySeparator() + "0", Expire: &BatchJobExpire{ + APIVersion: batchExpireAPIVersion, Bucket: f.bucket, + NotificationCfg: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}, + }}, + {ID: "keyrotate-describe" + getKeySeparator() + "0", KeyRotate: &BatchJobKeyRotateV1{ + APIVersion: batchKeyRotateAPIVersion, Bucket: f.bucket, + Flags: BatchJobKeyRotateFlags{Notify: BatchJobNotification{Endpoint: "https://notify.invalid", Token: secrets[4]}}, + }}, + } { + data, err := job.MarshalMsg(nil) + mustIAM(t, err) + mustIAM(t, saveConfig(f.ctx, f.obj, getJobPath(job), data)) + cred := globalActiveCred + req, err := newTestSignedRequestV4(http.MethodGet, "http://localhost/minio/admin/v3/describe-job?jobId="+job.ID, 0, nil, cred.AccessKey, cred.SecretKey, nil) + mustIAM(t, err) + mustIAM(t, req.ParseForm()) + rec := httptest.NewRecorder() + req = req.WithContext(newContext(req, rec, "DescribeBatchJob")) + adminAPIHandlers{}.DescribeBatchJob(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: describe status %d: %s", job.ID, rec.Code, rec.Body.String()) + } + for _, s := range secrets { + if strings.Contains(rec.Body.String(), s) { + t.Errorf("%s: describe output contains %s", job.ID, s) + } + } + if job.Replicate != nil && !strings.Contains(rec.Body.String(), "describe-source-access") { + t.Errorf("%s: describe output lost the access key", job.ID) + } + // The stored job keeps what a run needs. + stored := &BatchJobRequest{} + mustIAM(t, stored.load(f.ctx, f.obj, getJobPath(job))) + if stored.Replicate != nil && stored.Replicate.Source.Creds.SecretKey != secrets[0] { + t.Errorf("%s: describe changed the stored job", job.ID) + } + } +} From 9fe32b77432291b8bb1d898bc80c6dbc6ee98372 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 07:10:32 +0800 Subject: [PATCH 5/8] fix(batch): authorize the versions that a purge of all versions deletes A batch expiration that keeps no versions authorized the versions its walk saw, and the purge then deleted the versions it read under its lock. A version written after the walk, or tags changed on one since, was deleted without being authorized for the submitter. The purge takes an authorization that a batch purge must set and no other delete may. Under the object lock, once it has approved the history and checked every drive, and before its first delete, it hands the writes it is about to delete to the job: every copy of every approved version and every revision of one that a drive holds, one per distinct version ID and tags, as they are then. The job evaluates the expire actions on each for the submitter. A denial deletes nothing and ends the job as refused; the time it takes counts against the purge's budget. A locked version still refuses the whole key, counted as a refused key. The checks on the walk's snapshot are removed. Reviewed adversarially; the review's regressions are included. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-expire-purge-authz_test.go | 475 ++++++++++++++++++ cmd/batch-expire.go | 99 ++-- ...ure-server-pool-delete-all-options_test.go | 97 +++- cmd/erasure-server-pool-delete-all.go | 62 ++- cmd/erasure-server-pool.go | 2 +- cmd/object-api-interface.go | 6 + 6 files changed, 661 insertions(+), 80 deletions(-) create mode 100644 cmd/batch-expire-purge-authz_test.go diff --git a/cmd/batch-expire-purge-authz_test.go b/cmd/batch-expire-purge-authz_test.go new file mode 100644 index 000000000..20f6815af --- /dev/null +++ b/cmd/batch-expire-purge-authz_test.go @@ -0,0 +1,475 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/auth" + xhttp "github.com/minio/minio/internal/http" + "github.com/pgsty/silo-pkg/v3/policy" +) + +// A batch expiration that keeps no versions purges each key it matched under +// the purge's lock. What it authorizes for the submitter, and checks for +// Object Lock, are the versions the purge deletes then, not the versions the +// walk saw before. + +// batchPurgeHook runs change once, after the walk of a job and before its +// first purge of all versions. +type batchPurgeHook struct { + ObjectLayer + once sync.Once + change func() +} + +func (h *batchPurgeHook) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + if opts.DeleteAllVersions { + h.once.Do(h.change) + } + return h.ObjectLayer.DeleteObject(ctx, bucket, object, opts) +} + +func (f *batchAuthzFixture) purgeYAML(prefix string) string { + return fmt.Sprintf(`expire: + apiVersion: v1 + bucket: %s + prefix: %q + rules: + - type: object + purge: + retainVersions: 0 +`, f.bucket, prefix) +} + +// runChanged submits a purge of prefix as cred and runs it, with change made +// between its walk and its purge, and returns the job's final state. +func (f *batchAuthzFixture) runChanged(t *testing.T, cred auth.Credentials, prefix string, change func()) *batchJobInfo { + t.Helper() + rec, job := f.submit(t, cred, f.purgeYAML(prefix)) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + hook := &batchPurgeHook{ObjectLayer: f.obj, change: change} + if err := job.Expire.Start(f.ctx, hook, *job); err != nil { + t.Logf("start: %v", err) + } + return f.jobInfo(t, job) +} + +// putOlder writes a version of key older than every version it has, as a +// replicated or pulled version can be, with metadata meta. +func (f *batchAuthzFixture) putOlder(t *testing.T, key, versionID string, meta map[string]string) { + t.Helper() + data := []byte("older " + key) + _, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), + ObjectOptions{Versioned: true, VersionID: versionID, MTime: UTCNow().Add(-time.Hour), UserDefined: meta}) + mustIAM(t, err) +} + +func batchPurgeVersionIDs(t *testing.T, f *batchAuthzFixture, key string) []string { + t.Helper() + var ids []string + marker, vmarker := "", "" + for { + res, err := f.obj.ListObjectVersions(f.ctx, f.bucket, key, marker, vmarker, "", 1000) + mustIAM(t, err) + for _, o := range res.Objects { + if o.Name == key { + ids = append(ids, o.VersionID) + } + } + if !res.IsTruncated { + return ids + } + marker, vmarker = res.NextMarker, res.NextVersionIDMarker + } +} + +// A version that the submitter may not expire, because of its tags or its +// version ID, is refused although the walk did not see it so: the job fails +// and the key keeps all its versions. +func TestBatchExpirePurgeAuthorizesCurrentVersions(t *testing.T) { + for _, tc := range []struct { + name string + action string // the action a conditional Deny denies + deny string // its condition + // change returns the version IDs that change made. + change func(t *testing.T, f *batchAuthzFixture, key, older string) []string + }{ + { + name: "tagged-after-walk", + action: "s3:GetObjectVersion", + deny: `{"StringEquals":{"s3:ExistingObjectTag/keep":["yes"]}}`, + change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string { + _, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older}) + mustIAM(t, err) + return nil + }, + }, + { + name: "version-after-walk", + action: "s3:DeleteObjectVersion", + deny: `{"StringEquals":{"s3:versionid":["%s"]}}`, + change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string { + f.putOlder(t, key, batchPurgeDeniedVersion, nil) + return []string{batchPurgeDeniedVersion} + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + f := newBatchAuthzFixture(t) + deny := tc.deny + if strings.Contains(deny, "%s") { + deny = fmt.Sprintf(deny, batchPurgeDeniedVersion) + } + batchAuthzPolicy(f.ctx, t, f.sys, "batch-purge", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["`+tc.action+`"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":`+deny+`}`) + cred := iamAdminUser(f.ctx, t, f.sys, "batch-purge-user", "batch-purge") + const key = "purge/key" + older := f.putVersions(t, key) + want := batchPurgeVersionIDs(t, f, key) + ri := f.runChanged(t, cred, "purge/", func() { + want = append(want, tc.change(t, f, key, older)...) + }) + for _, id := range want { + if !f.versionExists(t, key, id) { + t.Errorf("version %s was purged although the submitter may not delete it", id) + } + } + if !ri.Failed || ri.Complete || ri.FailureReason == "" { + t.Errorf("job complete=%v failed=%v reason=%q, want a refused job", ri.Complete, ri.Failed, ri.FailureReason) + } + if ri.Objects != 0 || ri.KeysRefused != 0 { + t.Errorf("job counts objects=%d refused=%d", ri.Objects, ri.KeysRefused) + } + }) + } +} + +const batchPurgeDeniedVersion = "6c4b2f8e-1d3a-4e5b-9c7d-0a1b2c3d4e5f" + +// A version that Object Lock protects refuses the whole key, also when it +// was added after the walk. The job completes with the key counted as +// refused, not failed. +func TestBatchExpirePurgeRefusesLockedAfterWalk(t *testing.T) { + for _, kind := range batchLockKinds { + t.Run(kind, func(t *testing.T) { + f := newBatchAuthzFixture(t) + enableBucketObjectLock(t, f.bucket) + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full") + const key = "purge/locked" + f.putVersions(t, key) + want := batchPurgeVersionIDs(t, f, key) + locked := mustGetUUID() + ri := f.runChanged(t, cred, "purge/", func() { + f.putOlder(t, key, locked, batchLockMeta(kind)) + want = append(want, locked) + }) + for _, id := range want { + if !f.versionExists(t, key, id) { + t.Errorf("version %s of a key with a locked version was purged", id) + } + } + if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 { + t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key", + ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects) + } + }) + } +} + +// Controls: a purge by root and by an authorized user removes every version, +// including one added after the walk. +func TestBatchExpirePurgeAuthorizedControls(t *testing.T) { + for _, root := range []bool{true, false} { + t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) { + f := newBatchAuthzFixture(t) + cred := globalActiveCred + if !root { + batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess) + cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full") + } + const key = "purge/key" + f.putVersions(t, key) + added := mustGetUUID() + ri := f.runChanged(t, cred, "purge/", func() { + f.putOlder(t, key, added, map[string]string{xhttp.AmzObjectTagging: "keep=yes"}) + }) + if ids := batchPurgeVersionIDs(t, f, key); len(ids) != 0 { + t.Errorf("an authorized purge kept %v", ids) + } + if !ri.Complete || ri.Failed || ri.Objects != 3 || ri.KeysRefused != 0 { + t.Errorf("job complete=%v failed=%v objects=%d refused=%d, want complete with 3 objects", + ri.Complete, ri.Failed, ri.Objects, ri.KeysRefused) + } + }) + } +} + +// Cases found in the sixth review round. + +// batchPurgeBudgetChange runs change once the first purge stopped at its +// budget after removing versions. +type batchPurgeBudgetChange struct { + ObjectLayer + change func() + calls int +} + +func (h *batchPurgeBudgetChange) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + oi, err := h.ObjectLayer.DeleteObject(ctx, bucket, object, opts) + if opts.DeleteAllVersions { + h.calls++ + if h.calls == 1 && errors.Is(err, errDeleteAllBudget) && opts.DeleteAllResult.Removed > 0 { + h.change() + } + } + return oi, err +} + +// A purge that continues after its budget authorizes the versions again, as +// they are then; the refusal and the versions already removed persist +// across another start. +func TestBatchExpirePurgeBudgetReauthorizes(t *testing.T) { + f := newBatchAuthzFixture(t) + const key = "purge/budget" + oldest := batchPutVersion(t, f.obj, f.bucket, key, nil) + middle := batchPutVersion(t, f.obj, f.bucket, key, nil) + latest := batchPutVersion(t, f.obj, f.bucket, key, nil) + batchAuthzPolicy(f.ctx, t, f.sys, "review6-budget-deny", batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`) + cred := iamAdminUser(f.ctx, t, f.sys, "review6-budget-user", "review6-budget-deny") + rec, job := f.submit(t, cred, f.purgeYAML("purge/")) + if job == nil { + t.Fatalf("submit: %d %s", rec.Code, rec.Body.String()) + } + oldBudget := deleteAllBudget + deleteAllBudget.versions = 1 + t.Cleanup(func() { deleteAllBudget = oldBudget }) + hook := &batchPurgeBudgetChange{ObjectLayer: f.obj, change: func() { + _, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: middle.VersionID}) + mustIAM(t, err) + }} + mustIAM(t, job.Expire.Start(f.ctx, hook, *job)) + ri := f.jobInfo(t, job) + if hook.calls != 2 || !ri.Failed || ri.Complete || ri.FailureReason == "" || ri.Objects != 1 || ri.KeysRefused != 0 { + t.Fatalf("continuation: calls=%d complete=%v failed=%v reason=%q objects=%d refused=%d", hook.calls, ri.Complete, ri.Failed, ri.FailureReason, ri.Objects, ri.KeysRefused) + } + if f.versionExists(t, key, oldest.VersionID) || !f.versionExists(t, key, middle.VersionID) || !f.versionExists(t, key, latest.VersionID) { + t.Fatal("only the first authorized version may be removed") + } + _, err := f.obj.DeleteObjectTags(f.ctx, f.bucket, key, ObjectOptions{VersionID: middle.VersionID}) + mustIAM(t, err) + mustIAM(t, job.Expire.Start(f.ctx, hook, *job)) + after := f.jobInfo(t, job) + if hook.calls != 2 || after.FailureReason != ri.FailureReason || after.Objects != 1 || after.Complete { + t.Fatal("persisted refusal must prevent a fresh purge after the tags change") + } +} + +// The purge's lock excludes tag writers from its authorization through its +// last delete, for a directory key in a lock domain of its own on one pool +// and for a key in a second pool. +func TestBatchExpirePurgeExcludesTagWriter(t *testing.T) { + for _, mode := range []string{ldDsync, "two-pools"} { + t.Run(mode, func(t *testing.T) { + var z *erasureServerPools + var bucket string + pool, key := 0, "purge/tags/" + if mode == "two-pools" { + z, bucket = consistencyPools(t) + pool = 1 + } else { + z, bucket = ldPool(t, mode) + for i := 0; ; i++ { + key = fmt.Sprintf("purge/tags-%d/", i) + if z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) == 1 { + break + } + } + } + before := putConsistencyObject(t, z, bucket, encodeDirObject(key), pool, "", ObjectOptions{Versioned: true}) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + entered, release := make(chan struct{}), make(chan struct{}) + var released atomic.Bool + unblock := func() { + if released.CompareAndSwap(false, true) { + close(release) + } + } + defer unblock() + purged := make(chan error, 1) + go func() { + _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{ + DeleteAllVersions: true, + DeleteAllLatest: &deleteAllMatch{VersionID: before.VersionID, ModTime: before.ModTime}, + deleteAllAuthorize: func(versions []ObjectInfo) error { + close(entered) + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }, + }) + purged <- err + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("purge never reached authorization") + } + written := make(chan error, 1) + go func() { + _, err := z.PutObjectTags(ctx, bucket, key, "keep=yes", ObjectOptions{VersionID: before.VersionID}) + written <- err + }() + select { + case err := <-written: + t.Errorf("tag writer crossed the purge's authorization lock: %v", err) + case <-time.After(ldWait): + unblock() + err := <-written + if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + t.Errorf("writer must observe the completed purge: %v", err) + } + } + unblock() + mustIAM(t, <-purged) + }) + } +} + +// batchTagWriteFault fails metadata updates of one key, so that a tag write +// reaches only the drives it does not wrap. +type batchTagWriteFault struct { + StorageAPI + bucket, object string +} + +func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object string, fi FileInfo, opts UpdateMetadataOpts) error { + if bucket == d.bucket && object == d.object { + return errDiskNotFound + } + return d.StorageAPI.UpdateMetadata(ctx, bucket, object, fi, opts) +} + +// A failed tag write can leave a revision that the submitter may not expire +// on a minority of drives. The purge removes it with the version, so it is +// authorized too. +func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + z := purgeSinglePool(t, 16) + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + f := &batchAuthzFixture{ctx: t.Context(), sys: globalIAMSys, obj: z, bucket: "review6-batch-authz"} + mustIAM(t, z.MakeBucket(f.ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true})) + batchTestJobPool(f.ctx, t, z) + const key = "purge/minority-tags" + older := f.putVersions(t, key) + batchAuthzPolicy(f.ctx, t, f.sys, "review6-minority-deny", strings.ReplaceAll(batchAuthzFullAccess+`, +{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`, "batch-authz", f.bucket)) + cred := iamAdminUser(f.ctx, t, f.sys, "review6-minority-user", "review6-minority-deny") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectVersionAction, bucket: f.bucket, object: key, versionID: older, tags: "keep=yes"}); !isBatchJobAuthzError(err) { + t.Fatal("control: this tag revision must deny the read needed for expiration") + } + set := z.serverPools[0].getHashedSet(key) + original := set.getDisks + disks := append([]StorageAPI(nil), original()...) + wrapped := append([]StorageAPI(nil), disks...) + for i := 1; i < len(wrapped); i++ { + wrapped[i] = batchTagWriteFault{StorageAPI: wrapped[i], bucket: f.bucket, object: key} + } + set.getDisks = func() []StorageAPI { return wrapped } + _, err = f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older, UserDefined: map[string]string{ + ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano), + }}) + set.getDisks = original + if err == nil { + t.Fatal("control: the tag write must fail its quorum") + } + fi, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}) + mustIAM(t, err) + if fi.Metadata[xhttp.AmzObjectTagging] != "keep=yes" { + t.Fatal("control: the successful drive must hold the denied tag revision") + } + history, err := z.deleteAllHistory(f.ctx, f.bucket, key) + mustIAM(t, err) + for _, v := range history { + for _, c := range v.copies { + if c.fi.Metadata[xhttp.AmzObjectTagging] != "" { + t.Fatal("control: the quorum history must still have the original tags") + } + } + } + ri := f.runChanged(t, cred, "purge/", func() {}) + if !ri.Failed || ri.Complete || !strings.Contains(ri.FailureReason, "is not allowed") { + t.Errorf("denied physical revision must refuse the job: complete=%v failed=%v reason=%q", ri.Complete, ri.Failed, ri.FailureReason) + } + if _, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}); err != nil { + t.Errorf("denied physical revision was removed: %v", err) + } + if got := batchPurgeVersionIDs(t, f, key); len(got) != 2 { + t.Errorf("denial must keep the whole key; remaining versions=%v", got) + } +} + +// Cases found in the seventh review round. + +// The time the authorization takes counts against the purge's budget: a +// late allow and a deny both delete nothing. +func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) { + z := purgeSinglePool(t, 16) + p := newPurgeTest(t, z, "review7-purge-auth-time", MakeBucketOptions{VersioningEnabled: true}, "") + const key = "purge/key" + p.version(0, key, "older", 2) + p.version(0, key, "latest", 1) + exp := p.matched(key) + oldBudget := deleteAllBudget + deleteAllBudget.duration = 10 * time.Millisecond + t.Cleanup(func() { deleteAllBudget = oldBudget }) + denied := errors.New("review7: revision denied") + for _, decision := range []error{nil, denied} { + called := false + var res deleteAllResult + _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{ + DeleteAllVersions: true, + DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}, + DeleteAllResult: &res, + deleteAllAuthorize: func(versions []ObjectInfo) error { + called = true + if len(versions) < 2 { + t.Error("authorization did not receive both approved versions") + } + time.Sleep(20 * time.Millisecond) + return decision + }, + }) + want := decision + if want == nil { + want = errDeleteAllBudget + } + if !called || !errors.Is(err, want) || res.Started || res.Complete || res.Removed != 0 { + t.Errorf("late authorization: called=%v err=%v result=%+v", called, err, res) + } + if got := p.held(0, key); len(got) != 2 { + t.Fatalf("purge removed versions after late authorization: %v", got) + } + } +} diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go index 3f8b1e15a..3ddf86450 100644 --- a/cmd/batch-expire.go +++ b/cmd/batch-expire.go @@ -347,22 +347,10 @@ const ( batchExpireJobDefaultRetryDelay = 250 * time.Millisecond ) -// expireVersionRef is a version of a key whose versions are all purged, as -// the walk saw it. -type expireVersionRef struct { - VersionID string - UserTags string - DeleteMarker bool - Locked bool -} - type expireObjInfo struct { ObjectInfo ExpireAll bool DeleteMarkerCount int64 - // versions are the versions of an ExpireAll key the walk saw, the - // latest first. - versions []expireVersionRef } // isErrObjectLocked reports a version that Object Lock protects. @@ -485,35 +473,20 @@ func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc // expireAllVersions purges every version of a key whose rule retains none, // with the purge of exactly that key (erasureServerPools.deleteAllVersions), -// while the version the walk matched is still its latest. What the purge -// removed is counted at once; a key it refused because Object Lock protects -// a version is counted in res as refused, and the versions a failed purge -// left in res as failures. A failed purge is retried up to retryAttempts -// times, each time with the version the walk matched; a purge that stopped -// at its budget after removing versions continues without spending an -// attempt. A key whose latest version changed since the walk is not -// expired. It returns an error that ends the job. +// while the version the walk matched is still its latest. Under its lock, +// before it deletes anything, the purge refuses the key when Object Lock +// protects a version and has every write it would delete, each copy and +// each revision a drive holds, authorized for the submitter, with the +// version ID and tags it has then. // -// TODO(rebase onto the delete-all purge, claude/fix-delete-all): before the -// purge removes anything, authorize every version of the history it -// approved under its lock, with version-specific conditions -// (expireAuthzRequests). Until then the checks here use the versions the -// walk saw, and do not cover a version added after the walk; the purge -// itself refuses a key that holds a locked version. +// What the purge removed is counted at once; a refused key is counted in res +// as refused, and the versions a failed purge left in res as failures. A +// failed purge is retried up to retryAttempts times, each time with the +// version the walk matched; a purge that stopped at its budget after +// removing versions continues without spending an attempt. A key whose +// latest version changed since the walk is not expired. It returns an error +// that ends the job: an authorization failure or the end of ctx. func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error { - for _, v := range exp.versions { - if v.Locked { - res.keysRefused++ - return nil - } - } - var reqs []batchAuthzRequest - for _, v := range exp.versions { - reqs = append(reqs, r.expireAuthzRequests(exp.Name, v.VersionID, v.UserTags)...) - } - if err := job.Submitter.authorize(ctx, reqs...); err != nil { - return err - } var objects, markers int64 matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime} for attempts := 1; attempts <= retryAttempts; attempts++ { @@ -523,11 +496,15 @@ func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) latest := matched var purge deleteAllResult - _, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{ - DeleteAllVersions: true, - DeleteAllLatest: &latest, - DeleteAllResult: &purge, - }) + p, err := job.Submitter.principal(ctx) + if err == nil { + _, err = api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{ + DeleteAllVersions: true, + DeleteAllLatest: &latest, + DeleteAllResult: &purge, + deleteAllAuthorize: r.expireAllAuthorize(p, exp.Name), + }) + } // Removed versions are gone for any later run: count them now. if purge.Removed > 0 { ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers)) @@ -538,6 +515,8 @@ func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, switch { case err == nil: return nil + case isBatchJobAuthzError(err): + return err case errors.Is(err, errDeleteAllChanged): // Changed since the walk matched it: not expired. return nil @@ -562,6 +541,22 @@ func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, return nil } +// expireAllAuthorize returns the authorization of a purge of all versions of +// name for p: each version the purge would delete, as it holds it, needs the +// actions of expiring it (expireAuthzRequests). +func (r *BatchJobExpire) expireAllAuthorize(p batchJobPrincipal, name string) func([]ObjectInfo) error { + return func(versions []ObjectInfo) error { + for _, v := range versions { + for _, req := range r.expireAuthzRequests(name, v.VersionID, v.UserTags) { + if err := p.authorize(req); err != nil { + return err + } + } + } + return nil + } +} + // expireAllFailed counts the versions of info, as the walk saw them, that a // failed purge of all its versions did not remove: those beyond the objects // and markers it removed, at least one. @@ -833,17 +828,12 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo toDel = append(toDel, expireObjInfo{ ObjectInfo: result.Item, ExpireAll: true, - versions: []expireVersionRef{newExpireVersionRef(ctx, result.Item)}, }) continue } } else if prevObj.Name == result.Item.Name { if matchedFilter.Purge.RetainVersions == 0 { - // The latest version in toDel purges them all. - if n := len(toDel); n > 0 && toDel[n-1].Name == result.Item.Name { - toDel[n-1].versions = append(toDel[n-1].versions, newExpireVersionRef(ctx, result.Item)) - } - continue + continue // including latest version in toDel suffices, skipping other versions } versionsCount++ } else { @@ -917,15 +907,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo return nil } -func newExpireVersionRef(ctx context.Context, oi ObjectInfo) expireVersionRef { - return expireVersionRef{ - VersionID: oi.VersionID, - UserTags: oi.UserTags, - DeleteMarker: oi.DeleteMarker, - Locked: enforceRetentionForDeletion(ctx, oi), - } -} - //msgp:ignore batchExpireJobError type batchExpireJobError struct { Code string diff --git a/cmd/erasure-server-pool-delete-all-options_test.go b/cmd/erasure-server-pool-delete-all-options_test.go index 4b64eeaf3..4dfac1b79 100644 --- a/cmd/erasure-server-pool-delete-all-options_test.go +++ b/cmd/erasure-server-pool-delete-all-options_test.go @@ -25,6 +25,8 @@ import ( "sync/atomic" "testing" "time" + + xhttp "github.com/minio/minio/internal/http" ) // The parts of the purge of all versions that need its own options. @@ -65,16 +67,22 @@ func TestPurgeAllVersionsOptions(t *testing.T) { p.version(0, "ea/key", "v1", 5) match := &deleteAllMatch{} expire := ExpirationOptions{Expire: true} + allow := func([]ObjectInfo) error { return nil } for name, opts := range map[string]ObjectOptions{ "no approval": {DeleteAllVersions: true}, - "two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match}, - "prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true}, - "prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true}, - "version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()}, - "movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true}, - "marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true}, - "versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true}, - "suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true}, + "two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match, deleteAllAuthorize: allow}, + // A batch purge deletes as the submitter of its job, an ILM purge + // as the server, and no other delete takes an authorization. + "batch unauthorized": {DeleteAllVersions: true, DeleteAllLatest: match}, + "ilm authorized": {DeleteAllVersions: true, Expiration: expire, deleteAllAuthorize: allow}, + "authorized delete": {deleteAllAuthorize: allow}, + "prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true}, + "prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true}, + "version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()}, + "movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true}, + "marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true}, + "versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true}, + "suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true}, } { if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(InvalidArgument)) { t.Errorf("%s: %v", name, err) @@ -134,9 +142,10 @@ func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) { t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() }) var res deleteAllResult _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{ - DeleteAllVersions: true, - DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}, - DeleteAllResult: &res, + DeleteAllVersions: true, + DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}, + DeleteAllResult: &res, + deleteAllAuthorize: func([]ObjectInfo) error { return nil }, }) if !errors.Is(err, errDeleteAllBudget) || res.Started || res.Removed != 0 { t.Errorf("purge after its budget ran out on reads: err=%v %+v", err, res) @@ -213,9 +222,10 @@ func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) { }) var res deleteAllResult if _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{ - DeleteAllVersions: true, - DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}, - DeleteAllResult: &res, + DeleteAllVersions: true, + DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}, + DeleteAllResult: &res, + deleteAllAuthorize: func([]ObjectInfo) error { return nil }, }); err != nil || !res.Complete { t.Fatalf("purge: %v %+v", err, res) } @@ -223,3 +233,62 @@ func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) { t.Errorf("reads=%d deletes on the replacement handle=%d", reads.Load(), deletes.Load()) } } + +// A batch purge has every copy of every version it approved authorized, as +// the copy is now, before it deletes any; an authorization error deletes +// nothing and is returned as it is. +func TestPurgeAllVersionsAuthorizesEveryCopy(t *testing.T) { + z, _ := consistencyPools(t) + p := newPurgeTest(t, z, "purge-authorize", MakeBucketOptions{VersioningEnabled: true}, "") + const key = "bx/authorize" + p.put(0, key, "old", daysAgo(6), ObjectOptions{}) + p.put(1, key, "new", daysAgo(5), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: "copy=new"}}) + tagged := p.put(0, key, "v1", daysAgo(4), ObjectOptions{Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "keep=yes"}}) + exp := p.matched(key) + denied := errors.New("denied") + var seen []string // the distinct versions and tags authorized + purge := func(authorize func([]ObjectInfo) error) (deleteAllResult, error) { + var res deleteAllResult + _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{ + DeleteAllVersions: true, + DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}, + DeleteAllResult: &res, + deleteAllAuthorize: authorize, + }) + return res, err + } + res, err := purge(func(versions []ObjectInfo) error { + for _, v := range versions { + if got := v.VersionID + " " + v.UserTags; !slices.Contains(seen, got) { + seen = append(seen, got) + } + } + return denied + }) + if !errors.Is(err, denied) || res.Started || res.Removed != 0 { + t.Errorf("denied purge: err=%v %+v", err, res) + } + want := []string{"null ", "null copy=new", tagged.VersionID + " keep=yes"} + slices.Sort(seen) + slices.Sort(want) + if !slices.Equal(seen, want) { + t.Errorf("authorized %q, want every copy %q", seen, want) + } + for pool := range z.serverPools { + if got := p.held(pool, key); len(got) == 0 { + t.Errorf("pool %d lost its copies to a denied purge", pool) + } + } + if got := p.get(key); got != "v1" { + t.Errorf("reads %s after a denied purge, want v1", got) + } + // Control: an allowed purge deletes them all. + if res, err = purge(func([]ObjectInfo) error { return nil }); err != nil || !res.Complete { + t.Fatalf("allowed purge: %v %+v", err, res) + } + for pool := range z.serverPools { + if got := p.held(pool, key); got != nil { + t.Errorf("pool %d keeps %v", pool, got) + } + } +} diff --git a/cmd/erasure-server-pool-delete-all.go b/cmd/erasure-server-pool-delete-all.go index 50c594270..42fc17101 100644 --- a/cmd/erasure-server-pool-delete-all.go +++ b/cmd/erasure-server-pool-delete-all.go @@ -26,6 +26,7 @@ import ( "sync" "time" + xhttp "github.com/minio/minio/internal/http" "github.com/pgsty/silo-pkg/v3/sync/errgroup" ) @@ -49,6 +50,9 @@ import ( // when any revision of an approved version there is transitioned or, // when the bucket has Object Lock, protected by retention or a legal // hold. It deletes on exactly the drives it read; +// - a batch expiry then authorizes every copy of every approved version, +// and every revision of one that a drive holds, for the submitter of the +// job, and the purge deletes nothing unless all of them are allowed; // - it deletes the oldest version first and the latest version last and // alone, once every older version is gone from every pool. A purge that // stops early keeps the latest version, so it never exposes an older @@ -118,11 +122,14 @@ func deleteAllVersionID(versionID string) string { } // deleteAllInvalid returns why opts cannot ask for a purge of all versions. -// The purge is approved either as an ILM expiry or by a batch match, and it -// only deletes: it never creates a marker, addresses a single version, -// moves data or deletes a prefix. +// The purge is approved either as an ILM expiry or by a batch match, which +// also authorizes the versions it deletes, and it only deletes: it never +// creates a marker, addresses a single version, moves data or deletes a +// prefix. func (opts ObjectOptions) deleteAllInvalid() error { switch { + case !opts.DeleteAllVersions: + return errors.New("only a purge of all versions authorizes the versions it deletes") case opts.DeletePrefix || opts.DeletePrefixObject: return errors.New("a purge of all versions is not a prefix delete") case opts.VersionID != "": @@ -137,6 +144,8 @@ func (opts ObjectOptions) deleteAllInvalid() error { return errors.New("a purge of all versions takes no options of other deletes") case opts.Expiration.Expire == (opts.DeleteAllLatest != nil): return errors.New("a purge of all versions needs exactly one approval") + case (opts.DeleteAllLatest != nil) != (opts.deleteAllAuthorize != nil): + return errors.New("a batch purge of all versions, and only it, authorizes the versions it deletes") } return nil } @@ -297,12 +306,29 @@ func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, obje } } } + // A batch expiry deletes as its submitter: every write it is about to + // delete, the accepted copies and every revision a drive holds, must be + // allowed with the version ID and tags it has now. + var writes *deleteAllWrites + if opts.deleteAllAuthorize != nil { + writes = &deleteAllWrites{bucket: bucket, name: name, seen: make(map[[2]string]bool, len(history))} + for _, v := range history { + for _, c := range v.copies { + writes.add(c.fi) + } + } + } drives := make([][]StorageAPI, len(z.serverPools)) for pool := range z.serverPools { if !held[pool] { continue } - if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled); err != nil { + if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled, writes); err != nil { + return fail(err) + } + } + if opts.deleteAllAuthorize != nil { + if err := opts.deleteAllAuthorize(writes.versions); err != nil { return fail(err) } } @@ -329,6 +355,28 @@ func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, obje return res.Latest, nil } +// deleteAllWrites collects, for the authorization of a batch purge, the +// writes it deletes: one of each distinct version ID and tags. +type deleteAllWrites struct { + bucket, name string + seen map[[2]string]bool + versions []ObjectInfo +} + +// add adds the write fi unless w is nil or holds one with its version ID and +// tags. +func (w *deleteAllWrites) add(fi FileInfo) { + if w == nil { + return + } + key := [2]string{deleteAllVersionID(fi.VersionID), fi.Metadata[xhttp.AmzObjectTagging]} + if w.seen[key] { + return + } + w.seen[key] = true + w.versions = append(w.versions, fi.ToObjectInfo(w.bucket, w.name, true)) +} + // deleteAllRefuses returns why the purge must keep the key that holds the // revision fi of an approved version: it is transitioned, whose remote // object a later release hands to the tier cleanup, or, with Object Lock, @@ -344,7 +392,8 @@ func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, l } // deleteAllDrives reads object on every drive of the set and checks each -// revision of the versions ids that a drive holds with deleteAllRefuses. +// revision of the versions ids that a drive holds with deleteAllRefuses, and +// adds it to writes. // A minority of drives can hold a revision other than the accepted one, and // heal would later remove what the purge leaves on a drive, so a drive that // cannot be read or decoded refuses the key. A drive without the object @@ -353,7 +402,7 @@ func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, l // handle names a drive, not a backend: a backend that replaces the drive // behind it starts empty and receives this object only from a write or a // heal of it, which take the object lock the purge holds. -func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool) ([]StorageAPI, error) { +func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool, writes *deleteAllWrites) ([]StorageAPI, error) { disks := slices.Clone(er.getDisks()) raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, false) if err := ctx.Err(); err != nil { @@ -381,6 +430,7 @@ func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object str if err := deleteAllRefuses(ctx, bucket, object, fi, lockEnabled); err != nil { return nil, err } + writes.add(fi) } } return disks, nil diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index efaf3654b..1df968d81 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1234,7 +1234,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob if err = checkDelObjArgs(ctx, bucket, object); err != nil { return objInfo, err } - if opts.DeleteAllVersions { + if opts.DeleteAllVersions || opts.deleteAllAuthorize != nil { if err := opts.deleteAllInvalid(); err != nil { return objInfo, InvalidArgument{Bucket: bucket, Object: object, Err: err} } diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index fd52d307c..2ad350169 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -113,6 +113,12 @@ type ObjectOptions struct { // DeleteAllResult, when set, receives what a purge of all versions // confirmed, also when it fails part way. DeleteAllResult *deleteAllResult + // deleteAllAuthorize authorizes a batch purge of all versions, which + // needs it, on every copy of every version the purge approved and every + // revision of one that a drive holds, one of each distinct version ID + // and tags, before it deletes any; its error refuses the key and is + // returned. Set by batch expiration only, never from a request. + deleteAllAuthorize func(versions []ObjectInfo) error // Pools-layer version resolver; the caller holds the shared object lock. replicaObjectInfo GetObjectInfoFn From 21b6711562e9def52e8606943f297ca182c25a5c Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 09:04:13 +0800 Subject: [PATCH 6/8] fix(batch): rotate exactly the copy that was read and authorized Key rotation authorized the version as the copies in all pools merge it, but rewrote the copy that GetObjectInfo returned, routed by the pools layer to the pool of the latest version of the key. Where the copies differ, it rewrote metadata it had not authorized; where another pool held another write of the same version ID, it replaced that write's sealed key without re-encrypting its data, which no longer decrypted. The rewrite also stored the metadata without the tags and the expiry, which a read returns apart from it. Rotation now authorizes both the copy it rewrites and the merged version, and rewrites exactly that copy in its own pool. A version read from a pool that a decommission or a rebalance is moving is left to a later run, checked again right before the write, so that the source of the move does not change under it. The tags and the expiry are kept. A pool-state change during the commit itself affects every write of the pools layer and is left to a fix there. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-authz-review6_test.go | 493 ++++++++++++++++++++++++++++++++ cmd/batch-rotate.go | 74 ++++- 2 files changed, 555 insertions(+), 12 deletions(-) create mode 100644 cmd/batch-authz-review6_test.go diff --git a/cmd/batch-authz-review6_test.go b/cmd/batch-authz-review6_test.go new file mode 100644 index 000000000..447183e92 --- /dev/null +++ b/cmd/batch-authz-review6_test.go @@ -0,0 +1,493 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +// Rotation cases found in review. + +// Rotation rewrites the copy GetObjectInfo returns, which can carry another +// tag revision than the merged version of all pools: both are authorized. +func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) { + z, bucket := consistencyPools(t) + f := newBatchAuthzFixture(t) + previousKMS := GlobalKMS + GlobalKMS = kms.NewStub("review6-rotation-copy-key") + t.Cleanup(func() { GlobalKMS = previousKMS }) + batchAuthzPolicy(f.ctx, t, f.sys, "review6-rotation-copy-deny", fmt.Sprintf(` +{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]}, +{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket)) + cred := iamAdminUser(f.ctx, t, f.sys, "review6-rotation-copy-user", "review6-rotation-copy-deny") + sub, err := newBatchJobSubmitter(cred) + mustIAM(t, err) + const name = "rotation/written-copy" + first := make(map[string]string) + key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil) + mustIAM(t, err) + first[xhttp.AmzObjectTagging] = "classification=private" + first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + second := maps.Clone(first) + second[xhttp.AmzObjectTagging] = "classification=public" + second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano) + ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("review6 rotation payload"), key)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first}) + putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second}) + logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if logical.UserTags != "classification=public" { + t.Fatalf("control: merged tags=%q, want public", logical.UserTags) + } + job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + if err = job.KeyRotate(f.ctx, z, before, sub); !isBatchJobAuthzError(err) { + t.Fatalf("rotation of a copy the submitter may not read returned %v", err) + } + after, err := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] { + t.Fatal("a refused rotation rewrote the copy") + } +} + +// Two pools hold different writes of the null version, and a newer version +// of the key is in the second one. Rotation rewrites the copy it read and +// authorized, in the first pool, not the copy the latest version of the key +// would route a write to; the other write keeps its key and stays readable. +func TestBatchReviewRotationRewritesTheCopyItRead(t *testing.T) { + for _, otherLatest := range []bool{false, true} { + name := "addressed-version-is-latest" + if otherLatest { + name = "another-version-is-latest" + } + t.Run(name, func(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + z, _ := consistencyPools(t) + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + const bucket = "review7-rotation-copies" + mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true})) + _, versionErr := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, versionErr) + oldKMS := GlobalKMS + GlobalKMS = kms.NewStub("review7-rotation-key") + t.Cleanup(func() { GlobalKMS = oldKMS }) + const object = "rotation/null-copy" + const payload = "rotation must preserve this data" + readCopy := func(pool int) ([]byte, error) { + gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: nullVersionID}) + if err != nil { + return nil, err + } + defer gr.Close() + return io.ReadAll(gr) + } + var before [2]ObjectInfo + var keys [2]crypto.ObjectKey + kinds := []crypto.Type{crypto.S3, crypto.S3KMS} + for pool, kind := range kinds { + meta := make(map[string]string) + key, err := newEncryptMetadata(t.Context(), kind, "review7-rotation-key", nil, bucket, object, meta, nil) + mustIAM(t, err) + keys[pool] = key + meta[xhttp.AmzObjectTagging] = "classification=public" + if pool == 1 { + meta[xhttp.AmzObjectTagging] = "classification=private" + } + meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Duration(pool+2) * time.Hour).Format(time.RFC3339Nano) + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key)) + mustIAM(t, err) + putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{ + Versioned: true, VersionID: nullVersionID, UserDefined: meta, + MTime: UTCNow().Add(-time.Duration(pool+2) * time.Hour), + }) + // As a read returns it, without the write's internal fields. + before[pool], err = z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID}) + mustIAM(t, err) + data, err := readCopy(pool) + if err != nil || string(data) != payload { + t.Fatalf("control: pool %d was not readable before rotation: %v", pool, err) + } + } + if otherLatest { + putConsistencyObject(t, z, bucket, object, 1, "later version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)}) + } + selected, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID, Versioned: true}) + mustIAM(t, err) + if selected.VersionID != nullVersionID || !selected.ModTime.Equal(before[0].ModTime) || !crypto.S3.IsEncrypted(selected.UserDefined) { + t.Fatal("control: the addressed null version must select the SSE-S3 write in pool zero") + } + logical, err := batchAuthoritativeVersion(t.Context(), z, bucket, object, selected) + mustIAM(t, err) + if logical.UserTags != "classification=public" || selected.UserTags != logical.UserTags { + t.Fatal("control: both current authorization inputs must have public tags") + } + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review7-rotation-key"}} + sub := batchRootSubmitter(t) + if otherLatest { + batchAuthzPolicy(t.Context(), t, globalIAMSys, "review7-rotate-public", fmt.Sprintf(` +{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]}, +{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket)) + cred := iamAdminUser(t.Context(), t, globalIAMSys, "review7-rotate-user", "review7-rotate-public") + sub, err = newBatchJobSubmitter(cred) + mustIAM(t, err) + if err := sub.authorize(t.Context(), r.rotateAuthzRequests(before[1])...); !isBatchJobAuthzError(err) { + t.Fatal("control: the older write in pool one must be denied") + } + } + err = r.KeyRotate(t.Context(), z, selected, sub) + if err != nil { + t.Logf("rotation refused: %v", err) + } + for pool, want := range keys { + after, readErr := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID}) + mustIAM(t, readErr) + if otherLatest && pool == 1 && (!maps.Equal(after.UserDefined, before[pool].UserDefined) || !after.ModTime.Equal(before[pool].ModTime) || after.UserTags != before[pool].UserTags) { + t.Error("rotation rewrote a copy denied by the submitter's tag-conditioned policy") + } + kind, _ := crypto.IsEncrypted(after.UserDefined) + var got crypto.ObjectKey + switch kind { + case crypto.S3: + got, readErr = crypto.S3.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object) + case crypto.S3KMS: + got, readErr = crypto.S3KMS.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object) + default: + t.Fatalf("pool %d: rotation lost encryption metadata", pool) + } + mustIAM(t, readErr) + if got != want { + t.Errorf("pool %d: rotation changed the data key without reencrypting its ciphertext (rotation error: %v)", pool, err) + } + data, readErr := readCopy(pool) + if readErr != nil || string(data) != payload { + t.Errorf("pool %d: rotation left unreadable ciphertext: %v", pool, readErr) + } + } + }) + } +} + +// A version in a pool that a decommission or a rebalance is moving is not +// rewritten there: only the source holds it, so the rotation is left to a +// later run. +func TestBatchReviewRotationLeavesMovingSource(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + z, _ := consistencyPools(t) + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + const bucket = "review8-rotation-draining" + mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true})) + _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + oldKMS := GlobalKMS + GlobalKMS = kms.NewStub("review8-rotation-key") + t.Cleanup(func() { GlobalKMS = oldKMS }) + const object = "rotation/draining" + meta := make(map[string]string) + key, err := newEncryptMetadata(t.Context(), crypto.S3, "review8-rotation-key", nil, bucket, object, meta, nil) + mustIAM(t, err) + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("draining rotation"), key)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)}) + beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + + z.poolMetaMutex.Lock() + z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{} + z.poolMetaMutex.Unlock() + + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}} + err = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t)) + if !errors.Is(err, errBatchRotateMoving) { + t.Errorf("rotation of a version only a draining pool holds returned %v", err) + } + after, readErr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, readErr) + if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] { + t.Fatal("rotation rewrote a draining pool") + } +} + +// Once a move copied a version, the rotation leaves it to a later run, so the +// source stays as the move's cleanup expects it. +func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) { + for _, tc := range []struct { + kind string + rotate bool + }{{"decommission", false}, {"decommission", true}, {"rebalance", false}, {"rebalance", true}} { + name := tc.kind + "/without-rotation" + if tc.rotate { + name = tc.kind + "/with-rotation" + } + t.Run(name, func(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + z, _ := migrationTestPools(t) + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + const bucket = "review8-rotation-migration" + mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true})) + _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + oldKMS := GlobalKMS + GlobalKMS = kms.NewStub("review8-rotation-key") + t.Cleanup(func() { GlobalKMS = oldKMS }) + const object = "rotation/migration" + meta := make(map[string]string) + key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil) + mustIAM(t, err) + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation and migration"), key)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)}) + beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + move := migrationTestMover(t, z, 0, tc.kind) + mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, beforeInfo))) + if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); err != nil { + t.Fatalf("control: migration did not copy the version: %v", err) + } + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}} + var rotationErr error + if tc.rotate { + moved, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + rotationErr = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t)) + if !errors.Is(rotationErr, errBatchRotateMoving) { + t.Errorf("rotation during a move returned %v", rotationErr) + } + after, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if !maps.Equal(after.UserDefined, moved.UserDefined) || after.UserTags != moved.UserTags { + t.Error("rotation during a move rewrote the moved copy") + } + } + after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if !after.ModTime.Equal(beforeInfo.ModTime) || !maps.Equal(after.UserDefined, beforeInfo.UserDefined) { + t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr) + } + snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}} + if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil { + t.Fatalf("rotation prevented migration cleanup: %v", err) + } + if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + t.Fatalf("migration left a source version behind: %v", err) + } + }) + } +} + +// The rewrite keeps the tags and the expiry of the version, which a read +// returns apart from its metadata. +func TestBatchReviewRotationKeepsTagsAndExpiry(t *testing.T) { + for _, mode := range []string{"single-pool", "multiple-pools"} { + t.Run(mode, func(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + var z *erasureServerPools + if mode == "single-pool" { + z = purgeSinglePool(t, 4) + } else { + z, _ = consistencyPools(t) + } + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + const bucket = "review8-rotation-metadata" + mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true})) + _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + oldKMS := GlobalKMS + GlobalKMS = kms.NewStub("review8-rotation-metadata-key") + t.Cleanup(func() { GlobalKMS = oldKMS }) + const object = "rotation/tagged" + meta := make(map[string]string) + key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil) + mustIAM(t, err) + meta[xhttp.AmzObjectTagging] = "classification=private" + meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano) + meta["expires"] = UTCNow().Add(time.Hour).Format(http.TimeFormat) + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("metadata preservation"), key)) + mustIAM(t, err) + written := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta}) + before, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: written.VersionID}) + mustIAM(t, err) + if before.UserTags == "" || before.Expires.IsZero() { + t.Fatal("control: stored version is missing the metadata to preserve") + } + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + mustIAM(t, r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))) + after, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if after.UserDefined[crypto.MetaSealedKeyS3] == before.UserDefined[crypto.MetaSealedKeyS3] { + t.Error("control: rotation did not rewrap the key") + } + if after.UserTags != before.UserTags { + t.Errorf("rotation changed stored tags: got %q, want %q", after.UserTags, before.UserTags) + } + if !after.Expires.Equal(before.Expires) { + t.Errorf("rotation changed Expires: got %v, want %v", after.Expires, before.Expires) + } + }) + } +} + +// Two pools hold the same write with different tags, with or without +// tagging timestamps. A rotation, also during a move, leaves the tags that +// the merged version has. +func TestBatchReviewRotationKeepsMergedTags(t *testing.T) { + for _, tc := range []struct { + ordered, moving bool + }{{false, false}, {false, true}, {true, false}, {true, true}} { + t.Run(fmt.Sprintf("timestamp=%t/moving=%t", tc.ordered, tc.moving), func(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + z, _ := consistencyPools(t) + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + const bucket = "review9-rotation-tag-tie" + const object = "rotation/tag-tie" + mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true})) + _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + previousKMS := GlobalKMS + GlobalKMS = kms.NewStub("review9-tag-tie-key") + t.Cleanup(func() { GlobalKMS = previousKMS }) + meta := make(map[string]string) + key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil) + mustIAM(t, err) + meta[xhttp.AmzObjectTagging] = "classification=private" + if tc.ordered { + meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano) + } + other := maps.Clone(meta) + other[xhttp.AmzObjectTagging] = "classification=public" + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("same write"), key)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)}) + putConsistencyObject(t, z, bucket, object, 1, string(body), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: other}) + prior, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if prior.UserTags != "classification=private" { + t.Fatalf("control: logical tags=%q", prior.UserTags) + } + if tc.moving { + migrationTestMover(t, z, 0, "decommission") + } + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + rotationErr := r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)) + after, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if after.UserTags != prior.UserTags { + t.Fatalf("rotation changed logical tags from %q to %q (rotation error: %v)", prior.UserTags, after.UserTags, rotationErr) + } + }) + } +} + +// batchRotateFlipDisk starts a decommission of pool 0 at the flipAt-th +// version read, after the rotation's first read of the version. +type batchRotateFlipDisk struct { + StorageAPI + z *erasureServerPools + reads *atomic.Int32 + flipAt int32 + flipped *atomic.Bool +} + +func (d batchRotateFlipDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) { + n := d.reads.Add(1) + if n == d.flipAt && d.flipped.CompareAndSwap(false, true) { + d.z.poolMetaMutex.Lock() + d.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{} + d.z.poolMetaMutex.Unlock() + } + return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts) +} + +// A move that starts while the rotation authorizes and rewraps the version +// defers it: the state is checked again right before the write, where the +// pools layer checks it. +func TestBatchReviewRotationRechecksPoolBeforeWrite(t *testing.T) { + resetTestGlobals() + t.Cleanup(resetTestGlobals) + z, _ := consistencyPools(t) + globalIAMSys.initStore(z, nil) + mustIAM(t, globalIAMSys.Load(t.Context(), true)) + const bucket = "review11-rotation-window" + const object = "rotation/window" + mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true})) + _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig) + mustIAM(t, err) + oldKMS := GlobalKMS + GlobalKMS = kms.NewStub("review11-rotation-key") + t.Cleanup(func() { GlobalKMS = oldKMS }) + + meta := make(map[string]string) + key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil) + mustIAM(t, err) + body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation window"), key)) + mustIAM(t, err) + before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{ + Versioned: true, + UserDefined: meta, + MTime: UTCNow().Add(-time.Hour), + }) + beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + + var reads atomic.Int32 + var flipped atomic.Bool + firstPass := int32(0) + for _, pool := range z.serverPools { + firstPass += int32(pool.SetDriveCount()) + } + var restore []func() + for _, pool := range z.serverPools { + restore = append(restore, conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { + return batchRotateFlipDisk{ + StorageAPI: d, + z: z, + reads: &reads, + flipAt: firstPass + 1, + flipped: &flipped, + } + })) + } + t.Cleanup(func() { + for _, fn := range restore { + fn() + } + }) + + r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + gotErr := r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t)) + if !flipped.Load() { + t.Fatal("fault injection did not occur after the initial rotation read") + } + if !errors.Is(gotErr, errBatchRotateMoving) { + t.Errorf("rotation did not defer after the pool became moving during pre-work: %v", gotErr) + } + after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}) + mustIAM(t, err) + if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] { + t.Error("rotation rewrote the pool after it became moving during pre-work") + } +} diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go index 1f7306ff1..b8599fd1d 100644 --- a/cmd/batch-rotate.go +++ b/cmd/batch-rotate.go @@ -20,6 +20,7 @@ package cmd import ( "context" "encoding/base64" + "errors" "fmt" "maps" "math/rand" @@ -256,7 +257,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob VersionSuspended: versionSuspended, NoLock: true, } - obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts) + obj, pool, err := batchRotateRead(ctx, api, r.Bucket, objInfo.Name, opts) if err != nil { return err } @@ -273,9 +274,13 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob if err != nil { return err } - for _, req := range r.rotateAuthzRequests(logical) { - if err := p.authorize(req); err != nil { - return err + // Authorize the copy that is read and rewritten, and the version as all + // its copies describe it. + for _, v := range []ObjectInfo{obj, logical} { + for _, req := range r.rotateAuthzRequests(v) { + if err := p.authorize(req); err != nil { + return err + } } } oi := obj.Clone() @@ -305,16 +310,61 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob oi.metadataOnly = true oi.keyRotation = true maps.Copy(oi.UserDefined, encMetadata) - if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{ - VersionID: oi.VersionID, - }, ObjectOptions{ - VersionID: oi.VersionID, - NoLock: true, - }); err != nil { - return err + // The rewrite replaces the stored metadata with UserDefined, which a + // read returns without the tags and the expiry: keep them. + if oi.UserTags != "" { + oi.UserDefined[xhttp.AmzObjectTagging] = oi.UserTags } + if !oi.Expires.IsZero() { + oi.UserDefined["expires"] = oi.ExpiresStr() + } + srcOpts := ObjectOptions{VersionID: oi.VersionID} + dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true} + if z, ok := api.(*erasureServerPools); ok && pool >= 0 { + // Rewrite the copy that was read and authorized. The pools layer + // would route the rewrite by the latest version of the key, which + // can be in a pool that holds another write of this version ID. + // + // Check the pool's state again where the pools layer checks it, + // right before the write: a move can have started since the read. + if z.IsSuspended(pool) || z.IsPoolRebalancing(pool) { + return errBatchRotateMoving + } + dstOpts.replicaObjectInfo = z.replicaObjectInfo + name := encodeDirObject(oi.Name) + _, err = z.serverPools[pool].CopyObject(ctx, r.Bucket, name, r.Bucket, name, oi, srcOpts, dstOpts) + } else { + _, err = api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, srcOpts, dstOpts) + } + return err +} - return nil +// errBatchRotateMoving: the version is read from a pool that a decommission +// or a rebalance is moving. +var errBatchRotateMoving = errors.New("the version is being moved to another pool; rotate it once the move is done") + +// batchRotateRead reads the version that a rotation rewrites, as +// GetObjectInfo does. With several pools it also returns the pool of the +// copy it read, otherwise -1. The caller holds the object lock. +// +// A copy in a pool that a decommission or a rebalance is moving is the +// source of that move, which must not change under it, and rewriting a copy +// in another pool would change which copy the version's metadata comes +// from. Such a version is left to a later run (errBatchRotateMoving). +func batchRotateRead(ctx context.Context, api ObjectLayer, bucket, object string, opts ObjectOptions) (ObjectInfo, int, error) { + z, ok := api.(*erasureServerPools) + if !ok || z.SinglePool() { + oi, err := api.GetObjectInfo(ctx, bucket, object, opts) + return oi, -1, err + } + if err := checkGetObjArgs(ctx, bucket, object); err != nil { + return ObjectInfo{}, -1, err + } + oi, idx, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts) + if err == nil && (z.IsSuspended(idx) || z.IsPoolRebalancing(idx)) { + return ObjectInfo{}, -1, errBatchRotateMoving + } + return oi, idx, err } const ( From adc824087b6e94dc7d980ed8de260d29ce387f55 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 09:04:39 +0800 Subject: [PATCH 7/8] fix(batch): fail an expiration run whose listing fails at once A Walk that returned an error before any result cancelled the run like an ordinary cancellation: the job was saved as neither complete nor failed. The listing failure is now recorded first, and the run is saved as failed with its cursor where it was. Start still returns the error, so the job is left to a later run, which lists again. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-authz-review6_test.go | 29 ++++++++++++++++++- cmd/batch-expire-purge-authz_test.go | 42 ++++++++++++++++++++++++++++ cmd/batch-expire.go | 10 +++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/cmd/batch-authz-review6_test.go b/cmd/batch-authz-review6_test.go index 447183e92..63303b733 100644 --- a/cmd/batch-authz-review6_test.go +++ b/cmd/batch-authz-review6_test.go @@ -20,7 +20,7 @@ import ( "github.com/minio/minio/internal/kms" ) -// Rotation cases found in review. +// Rotation and listing cases found in review. // Rotation rewrites the copy GetObjectInfo returns, which can carry another // tag revision than the merged version of all pools: both are authorized. @@ -65,6 +65,33 @@ func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) { } } +// batchWalkError fails every listing before it returns a result. +type batchWalkError struct { + ObjectLayer + err error +} + +func (w batchWalkError) Walk(context.Context, string, string, chan<- itemOrErr[ObjectInfo], WalkOptions) error { + return w.err +} + +// A listing that fails before it returns anything fails the run, as a +// listing error in the results does, and the job is left to a later run. +func TestBatchReviewExpireWalkErrorFailsRun(t *testing.T) { + f := newBatchAuthzFixture(t) + rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/")) + if job == nil { + t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String()) + } + listErr := errors.New("review6: listing failed") + if err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job); !errors.Is(err, listErr) { + t.Fatalf("start returned %v, want %v", err, listErr) + } + if ri := f.jobInfo(t, job); !ri.Failed || ri.Complete { + t.Fatalf("failed listing: complete=%v failed=%v", ri.Complete, ri.Failed) + } +} + // Two pools hold different writes of the null version, and a newer version // of the key is in the second one. Rotation rewrites the copy it read and // authorized, in the first pool, not the copy the latest version of the key diff --git a/cmd/batch-expire-purge-authz_test.go b/cmd/batch-expire-purge-authz_test.go index 20f6815af..259f725d9 100644 --- a/cmd/batch-expire-purge-authz_test.go +++ b/cmd/batch-expire-purge-authz_test.go @@ -473,3 +473,45 @@ func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) { } } } + +// A listing that failed before any result fails the run without moving the +// cursor or refusing the job; a later run completes it, in both modes. +func TestBatchExpireWalkFailureResumes(t *testing.T) { + for _, purgeAll := range []bool{false, true} { + name := "retain-one" + if purgeAll { + name = "retain-zero" + } + t.Run(name, func(t *testing.T) { + f := newBatchAuthzFixture(t) + const key = "data/key" + older := f.putVersions(t, key) + definition := f.expireYAML("data/") + if purgeAll { + definition = f.purgeYAML("data/") + } + rec, job := f.submit(t, globalActiveCred, definition) + if job == nil { + t.Fatalf("submit: %d %s", rec.Code, rec.Body.String()) + } + listErr := errors.New("review7: transient listing failure") + err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job) + if !errors.Is(err, listErr) { + t.Fatalf("listing error: %v", err) + } + ri := f.jobInfo(t, job) + if !ri.Failed || ri.Complete || ri.FailureReason != "" || ri.Object != "" || !f.versionExists(t, key, older) { + t.Fatalf("listing failure changed progress or became terminal: failed=%v complete=%v reason=%q cursor=%q", ri.Failed, ri.Complete, ri.FailureReason, ri.Object) + } + mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job)) + ri = f.jobInfo(t, job) + wantRemoved := int64(1) + if purgeAll { + wantRemoved = 2 + } + if ri.Failed || !ri.Complete || ri.FailureReason != "" || ri.Objects != wantRemoved || f.versionExists(t, key, older) { + t.Fatalf("resume did not complete: failed=%v complete=%v reason=%q removed=%d", ri.Failed, ri.Complete, ri.FailureReason, ri.Objects) + } + }) + } +} diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go index 3ddf86450..77c444dfc 100644 --- a/cmd/batch-expire.go +++ b/cmd/batch-expire.go @@ -673,6 +673,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo // run, so the cursor cannot move past what was not listed. walkCtx, stopWalk := context.WithCancel(ctx) defer stopWalk() + // walkFailed: a listing failed before it returned any result. + var walkFailed atomic.Bool results := make(chan itemOrErr[ObjectInfo], workerSize) go func() { // One ascending walk over disjoint prefixes, which the cursor @@ -689,6 +691,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo }) if err != nil { if walkCtx.Err() == nil { + walkFailed.Store(true) cancelCause(err) } xioutil.SafeClose(results) @@ -881,6 +884,13 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo refused := ri.refused() cause := context.Cause(ctx) if cause != nil && !refused { + if walkFailed.Load() { + // This run failed; a later run lists again from the cursor. + ri.mu.Lock() + ri.Complete = false + ri.Failed = true + ri.mu.Unlock() + } saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job)) cancel() From 77ad2fc12b4d1dd5424c612d52955e15390b7d68 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 09:26:32 +0800 Subject: [PATCH 8/8] test(batch): pin the parity of the drive-revision purge test TestBatchExpirePurgeAuthorizesDriveRevisions writes a failed tag revision to one drive of sixteen and needs the read quorum of the metadata to stay below sixteen. Tests that run newTestConfig on a single-drive layer, such as TestIsReqAuthenticated, leave a storage class of parity 0 behind, which resetTestGlobals does not restore. With parity 0 the quorum history of the key is unreadable, the purge correctly refuses, and the test failed in the full ./cmd run while passing alone. Set the default parity for the test, as other erasure tests do, and restore it afterwards. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-expire-purge-authz_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/batch-expire-purge-authz_test.go b/cmd/batch-expire-purge-authz_test.go index 259f725d9..6c8b14dfd 100644 --- a/cmd/batch-expire-purge-authz_test.go +++ b/cmd/batch-expire-purge-authz_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/storageclass" xhttp "github.com/minio/minio/internal/http" "github.com/pgsty/silo-pkg/v3/policy" ) @@ -373,6 +374,12 @@ func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object s func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) { resetTestGlobals() t.Cleanup(resetTestGlobals) + // One drive of sixteen holds the failed tag write, so the read quorum of + // the metadata must stay below sixteen. Earlier tests can leave a + // storage class of parity 0 behind (newTestConfig on a single drive). + previous := globalStorageClass + globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}}) + t.Cleanup(func() { globalStorageClass.Update(previous) }) z := purgeSinglePool(t, 16) globalIAMSys.initStore(z, nil) mustIAM(t, globalIAMSys.Load(t.Context(), true))