fix(ilm): purge only the expired key's own versions in delete-all

An expiry of all versions (ExpiredObjectAllVersions, DelMarkerExpiration,
and batch expiration with the default purge) asked the pools layer for a
prefix delete. The pools layer rebuilt the options, so the request ran as
a recursive delete of the key's directory in every erasure set of every
pool. Object a/b is stored inside the directory of a, so every key under
a/ was removed with it, however new, with one audit entry and one
notification for a alone. The rebuilt options also skipped the lifecycle
recheck, so on a single pool a version written after the decision was
deleted too, and tiered versions lost their metadata without free
versions.

The expiry worker, the migration expiry and batch expiration now ask for
a purge with the in-memory ObjectOptions.DeleteAllVersions:

- DeleteObject hands it to the purge under the object lock, before the
  all-pools check. ILM decides with the unchanged lifecycleEventsAllPools,
  on a single pool too; batch needs the version its walk matched to still
  be the latest. A second read of each pool's authoritative history
  records every copy, older writes of the same ID included, and must
  equal the approval. Anything else deletes nothing.
- Before deleting, the purge reads every drive of each set that holds a
  copy. It refuses the key when a drive cannot be read, when any revision
  of an approved version there is transitioned, or, with Object Lock,
  when retention or a legal hold protects one. It then deletes on exactly
  the drive handles it checked.
- Versions are deleted by ID with the drives' version delete, which only
  removes the object directory once it is empty: oldest first, for each
  version older writes before the copy reads choose, and the latest last,
  only once every older version is gone from every pool. No marker,
  replication state or free version is written. The purge stops at the
  first error, repeats a quorum-successful delete on the drives that
  missed it while the lock is held, and stops before the latest version
  at a budget of 1000 versions or one minute, reads included.
- The ILM audit entry records how many versions were removed. A purge
  that failed after it began to delete is audited as partial and sends
  no removal event. Batch counts come from the purge; a key whose latest
  version changed is not expired, and a failed delete marker fails the
  job.

The option is rejected with conflicting options and by every layer below
the pools. Force delete and internal prefix deletes keep DeletePrefix.
Delete-all of a transitioned latest version keeps its current path, and
a key with tiered older versions is refused. On a single pool the purge
relies on the lock-domain fix for writer exclusion.

Reviewed adversarially in two design rounds and four patch rounds; the
last approved with nits, which are applied.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/data-scanner.go

@@ -1229,10 +1229,16 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
 	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 
+	// Purge every version of exactly this key, by version ID and under the
+	// object lock, after the pools approved it on the versions they hold;
+	// see erasureServerPools.deleteAllVersions.
+	var purge deleteAllResult
 	if lcEvent.Action.DeleteAll() {
-		opts.DeletePrefix = true
-		// use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-		opts.DeletePrefixObject = true
+		opts = ObjectOptions{
+			Expiration:        ExpirationOptions{Expire: true},
+			DeleteAllVersions: true,
+			DeleteAllResult:   &purge,
+		}
 	}
 
 	// ILM-created delete markers and version purges must carry the same
@@ -1263,13 +1269,16 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		if lcEvent.Action != lifecycle.NoneAction {
 			numVersions := uint64(1)
 			if lcEvent.Action.DeleteAll() {
-				numVersions = uint64(obj.NumVersions)
+				numVersions = uint64(purge.Removed)
 			}
 			timeILM(numVersions)
 		}
 	}()
 
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
+	if lcEvent.Action.DeleteAll() && (err != nil || purge.Removed == 0) {
+		return deleteAllNotCompleted(ctx, obj, lcEvent, src, purge, err, traceFn)
+	}
 	if err != nil {
 		if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			traceFn(ILMExpiry, nil, nil)
@@ -1310,6 +1319,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	tags["version-id"] = dobj.VersionID
+	if lcEvent.Action.DeleteAll() {
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+	}
 
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
@@ -1336,6 +1348,57 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	return true
 }
 
+// ilmVersionsRemoved is the audit tag with the number of versions that an
+// expiry of all versions removed; ilmPartial marks one that began to delete
+// and did not finish.
+const (
+	ilmVersionsRemoved = "ilm-versions-removed"
+	ilmPartial         = "ilm-partial"
+)
+
+// deleteAllNotCompleted reports an expiry of all versions of obj that did
+// not remove them all. A purge that had nothing to remove, or that failed
+// before it deleted anything, reports what a rejected expiry does. One that
+// failed or stopped after it began to delete is audited as partial, with the
+// versions it removed, but sends no removal event: the key is still there.
+func deleteAllNotCompleted(ctx context.Context, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, purge deleteAllResult, err error, traceFn func(event string, metadata map[string]string, err error)) bool {
+	switch {
+	case err == nil: // there was nothing to remove
+		traceFn(ILMExpiry, nil, nil)
+		return false
+	case purge.Started:
+		tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+		tags["version-id"] = purge.Latest.VersionID
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+		tags[ilmPartial] = "true"
+		auditLogInternal(ctx, AuditLogOptions{
+			Event:     ILMExpiry,
+			APIName:   "ILMExpiry",
+			Bucket:    obj.Bucket,
+			Object:    obj.Name,
+			VersionID: purge.Latest.VersionID,
+			Error:     err.Error(),
+			Tags:      tags,
+		})
+		traceFn(ILMExpiry, tags, err)
+		if errors.Is(err, errDeleteAllBudget) {
+			return false // a long history; the next expiry continues
+		}
+	case errors.Is(err, errDeleteAllTiered):
+		// Kept until tiered versions can be purged; see deleteAllVersions.
+		traceFn(ILMExpiry, nil, err)
+		return false
+	case errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err):
+		traceFn(ILMExpiry, nil, nil)
+		return false
+	default:
+		traceFn(ILMExpiry, nil, err)
+	}
+	// Assume it is still there.
+	ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
+	return false
+}
+
 // Apply object, object version, restored object or restored object version action on the given object
 func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
 	globalExpiryState.enqueueByDays(obj, event, src)