Merge exact-key purge for lifecycle delete-all

ExpiredObjectAllVersions, DelMarkerExpiration and batch expiry purge only the expired key's own versions, not other keys under the same prefix.

cmd/batch-expire.go

@@ -445,31 +445,13 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 				oiCache.Add(od, &exp.ObjectInfo)
 			}
 
-			// DeleteObject(deletePrefix: true) to expire all versions of an object
+			// Purge all versions of exactly each key while the version the
+			// walk matched is still its latest; see
+			// erasureServerPools.deleteAllVersions.
 			for _, exp := range toExpireAll {
-				var success bool
-				for attempts := 1; attempts <= retryAttempts; attempts++ {
-					select {
-					case <-ctx.Done():
-						ri.trackMultipleObjectVersions(exp, success)
-						return
-					default:
-					}
-					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
-						DeletePrefix:       true,
-						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-					})
-					if err != nil {
-						stopFn(exp, err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
-					} else {
-						stopFn(exp, err)
-						success = true
-						break
-					}
+				if !expireAllVersions(ctx, ri, api, exp, retryAttempts) {
+					return
 				}
-				ri.trackMultipleObjectVersions(exp, success)
 			}
 
 			// DeleteMultiple objects
@@ -525,6 +507,56 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 	}
 }
 
+// expireAllVersions purges all versions of the key exp, which the walk
+// matched by its latest version, and counts what the purge removed. It
+// retries a failed purge up to attempts times, each time with the version
+// the walk matched; a purge that stopped at its budget continues without
+// spending an attempt. It does not retry once the key no longer has that
+// latest version, which is not an expiry, or once the purge refused the
+// key. It returns false once ctx ends.
+func expireAllVersions(ctx context.Context, ri *batchJobInfo, api ObjectLayer, exp expireObjInfo, attempts int) bool {
+	var objects, markers int64
+	matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
+	for attempt := 1; attempt <= attempts; attempt++ {
+		select {
+		case <-ctx.Done():
+			ri.trackExpireAll(exp, objects, markers, true)
+			return false
+		default:
+		}
+		stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempt)
+		latest := matched
+		var purge deleteAllResult
+		_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
+			DeleteAllVersions: true,
+			DeleteAllLatest:   &latest,
+			DeleteAllResult:   &purge,
+		})
+		objects += int64(purge.Removed - purge.Markers)
+		markers += int64(purge.Markers)
+		stopFn(exp, err)
+		switch {
+		case err == nil:
+			ri.trackExpireAll(exp, objects, markers, false)
+			return true
+		case errors.Is(err, errDeleteAllChanged):
+			// Changed since the walk matched it: not expired.
+			ri.trackExpireAll(exp, objects, markers, false)
+			return true
+		case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
+			attempt-- // progress
+			continue
+		}
+		batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempt))
+		var locked ObjectLocked
+		if errors.Is(err, errDeleteAllTiered) || errors.As(err, &locked) {
+			break // refused; a later run of the job checks again
+		}
+	}
+	ri.trackExpireAll(exp, objects, markers, true)
+	return true
+}
+
 type expireObjInfo struct {
 	ObjectInfo
 	ExpireAll         bool
@@ -746,8 +778,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	<-expireDoneCh // waits for the expire goroutine to complete
 	wk.Wait()      // waits for all expire workers to retire
 
-	ri.Complete = !failed && ri.ObjectsFailed == 0
-	ri.Failed = failed || ri.ObjectsFailed > 0
+	ri.Complete = !failed && !ri.expireFailures()
+	ri.Failed = failed || ri.expireFailures()
 	globalBatchJobsMetrics.save(job.ID, ri)
 
 	// Close the saverQuitCh - this also triggers saving in-memory state

cmd/batch-handlers.go

@@ -996,10 +996,17 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 	return nil
 }
 
-// Note: to be used only with batch jobs that affect multiple versions through
-// a single action. e.g batch-expire has an option to expire all versions of an
-// object which matches the given filters.
-func (ri *batchJobInfo) trackMultipleObjectVersions(info expireObjInfo, success bool) {
+// expireFailures reports whether an expiration job failed to remove an
+// object or a delete marker it matched.
+func (ri *batchJobInfo) expireFailures() bool {
+	return ri.ObjectsFailed > 0 || ri.DeleteMarkersFailed > 0
+}
+
+// trackExpireAll counts the objects and delete markers that an expiry of all
+// versions of info removed and, when it failed, those the walk saw that it
+// did not remove, at least one. Only batch-expire uses it, for a rule that
+// expires all versions of an object.
+func (ri *batchJobInfo) trackExpireAll(info expireObjInfo, objects, markers int64, failed bool) {
 	if ri == nil {
 		return
 	}
@@ -1007,15 +1014,20 @@ func (ri *batchJobInfo) trackMultipleObjectVersions(info expireObjInfo, success
 	ri.mu.Lock()
 	defer ri.mu.Unlock()
 
-	if success {
+	ri.Objects += objects
+	ri.DeleteMarkers += markers
+	if !failed {
 		ri.Bucket = info.Bucket
 		ri.Object = info.Name
-		ri.Objects += int64(info.NumVersions) - info.DeleteMarkerCount
-		ri.DeleteMarkers += info.DeleteMarkerCount
-	} else {
-		ri.ObjectsFailed += int64(info.NumVersions) - info.DeleteMarkerCount
-		ri.DeleteMarkersFailed += info.DeleteMarkerCount
+		return
 	}
+	failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
+	failedMarkers := max(info.DeleteMarkerCount-markers, 0)
+	if failedObjects+failedMarkers == 0 {
+		failedObjects = 1
+	}
+	ri.ObjectsFailed += failedObjects
+	ri.DeleteMarkersFailed += failedMarkers
 }
 
 func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool, attempt int) {

cmd/data-scanner.go

@@ -1229,10 +1229,16 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
 	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 
+	// Purge every version of exactly this key, by version ID and under the
+	// object lock, after the pools approved it on the versions they hold;
+	// see erasureServerPools.deleteAllVersions.
+	var purge deleteAllResult
 	if lcEvent.Action.DeleteAll() {
-		opts.DeletePrefix = true
-		// use prefix delete on exact object (this is an optimization to avoid fan-out calls)
-		opts.DeletePrefixObject = true
+		opts = ObjectOptions{
+			Expiration:        ExpirationOptions{Expire: true},
+			DeleteAllVersions: true,
+			DeleteAllResult:   &purge,
+		}
 	}
 
 	// ILM-created delete markers and version purges must carry the same
@@ -1263,13 +1269,16 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		if lcEvent.Action != lifecycle.NoneAction {
 			numVersions := uint64(1)
 			if lcEvent.Action.DeleteAll() {
-				numVersions = uint64(obj.NumVersions)
+				numVersions = uint64(purge.Removed)
 			}
 			timeILM(numVersions)
 		}
 	}()
 
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
+	if lcEvent.Action.DeleteAll() && (err != nil || purge.Removed == 0) {
+		return deleteAllNotCompleted(ctx, obj, lcEvent, src, purge, err, traceFn)
+	}
 	if err != nil {
 		if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			traceFn(ILMExpiry, nil, nil)
@@ -1310,6 +1319,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	tags["version-id"] = dobj.VersionID
+	if lcEvent.Action.DeleteAll() {
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+	}
 
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
@@ -1336,6 +1348,57 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	return true
 }
 
+// ilmVersionsRemoved is the audit tag with the number of versions that an
+// expiry of all versions removed; ilmPartial marks one that began to delete
+// and did not finish.
+const (
+	ilmVersionsRemoved = "ilm-versions-removed"
+	ilmPartial         = "ilm-partial"
+)
+
+// deleteAllNotCompleted reports an expiry of all versions of obj that did
+// not remove them all. A purge that had nothing to remove, or that failed
+// before it deleted anything, reports what a rejected expiry does. One that
+// failed or stopped after it began to delete is audited as partial, with the
+// versions it removed, but sends no removal event: the key is still there.
+func deleteAllNotCompleted(ctx context.Context, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, purge deleteAllResult, err error, traceFn func(event string, metadata map[string]string, err error)) bool {
+	switch {
+	case err == nil: // there was nothing to remove
+		traceFn(ILMExpiry, nil, nil)
+		return false
+	case purge.Started:
+		tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+		tags["version-id"] = purge.Latest.VersionID
+		tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
+		tags[ilmPartial] = "true"
+		auditLogInternal(ctx, AuditLogOptions{
+			Event:     ILMExpiry,
+			APIName:   "ILMExpiry",
+			Bucket:    obj.Bucket,
+			Object:    obj.Name,
+			VersionID: purge.Latest.VersionID,
+			Error:     err.Error(),
+			Tags:      tags,
+		})
+		traceFn(ILMExpiry, tags, err)
+		if errors.Is(err, errDeleteAllBudget) {
+			return false // a long history; the next expiry continues
+		}
+	case errors.Is(err, errDeleteAllTiered):
+		// Kept until tiered versions can be purged; see deleteAllVersions.
+		traceFn(ILMExpiry, nil, err)
+		return false
+	case errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err):
+		traceFn(ILMExpiry, nil, nil)
+		return false
+	default:
+		traceFn(ILMExpiry, nil, err)
+	}
+	// Assume it is still there.
+	ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
+	return false
+}
+
 // Apply object, object version, restored object or restored object version action on the given object
 func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
 	globalExpiryState.enqueueByDays(obj, event, src)

cmd/erasure-object.go

@@ -1773,6 +1773,13 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	if opts.DeleteAllVersions {
+		errs := make([]error, len(objects))
+		for i := range errs {
+			errs[i] = errDeleteAllUnsupported
+		}
+		return make([]DeletedObject, len(objects)), errs
+	}
 	if !opts.NoAuditLog {
 		for _, obj := range objects {
 			auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectName, &er)
@@ -2023,6 +2030,9 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if opts.DeleteAllVersions {
+		return objInfo, errDeleteAllUnsupported
+	}
 	if !opts.NoAuditLog {
 		auditObjectErasureSet(ctx, "DeleteObject", object, &er)
 	}

cmd/erasure-server-pool-delete-all-options_test.go

@@ -0,0 +1,221 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// The parts of the purge of all versions that need its own options.
+
+// A purge stops at its budget before the latest version.
+func TestPurgeAllVersionsStopsAtBudget(t *testing.T) {
+	z := purgeSinglePool(t, 16)
+	p := newPurgeTest(t, z, "purge-budget", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
+	prev := deleteAllBudget
+	deleteAllBudget.versions = 2
+	t.Cleanup(func() { deleteAllBudget = prev })
+	const key = "ea/long"
+	for i, days := range []float64{8, 7, 6, 5} {
+		p.version(0, key, fmt.Sprintf("v%d", i+1), days)
+	}
+	o := observePurges(t)
+	p.expire(0, key)
+	if got := p.held(0, key); !slices.Equal(got, []string{"hot", "hot"}) {
+		t.Errorf("holds %v after a purge stopped at its budget, want the two newest versions", got)
+	}
+	if got := p.get(key); got != "v4" {
+		t.Errorf("reads %s, want v4", got)
+	}
+	events, traces := o.drain()
+	if len(events) != 0 || len(traces) != 1 || traces[0]["ilm-partial"] != "true" || traces[0]["ilm-versions-removed"] != "2" {
+		t.Errorf("stopped purge sent %v and traced %v", events, traces)
+	}
+	p.expire(0, key)
+	if got := p.held(0, key); got != nil {
+		t.Errorf("keeps %v once the purge continued", got)
+	}
+}
+
+// The option admits no other delete.
+func TestPurgeAllVersionsOptions(t *testing.T) {
+	z := purgeSinglePool(t, 16)
+	p := newPurgeTest(t, z, "purge-options", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
+	p.version(0, "ea/key", "v1", 5)
+	match := &deleteAllMatch{}
+	expire := ExpirationOptions{Expire: true}
+	for name, opts := range map[string]ObjectOptions{
+		"no approval":   {DeleteAllVersions: true},
+		"two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match},
+		"prefix":        {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true},
+		"prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true},
+		"version":       {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()},
+		"movement":      {DeleteAllVersions: true, Expiration: expire, DataMovement: true},
+		"marker":        {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true},
+		"versioned":     {DeleteAllVersions: true, Expiration: expire, Versioned: true},
+		"suspended":     {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true},
+	} {
+		if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(InvalidArgument)) {
+			t.Errorf("%s: %v", name, err)
+		}
+	}
+	opts := ObjectOptions{DeleteAllVersions: true, Expiration: expire}
+	if _, err := z.serverPools[0].DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(NotImplemented)) {
+		t.Errorf("erasure sets: %v", err)
+	}
+	if _, errs := z.DeleteObjects(t.Context(), p.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: "ea/key"}}}, opts); !errors.As(errs[0], new(NotImplemented)) {
+		t.Errorf("bulk delete: %v", errs[0])
+	}
+	if got := p.held(0, "ea/key"); !slices.Equal(got, []string{"hot"}) {
+		t.Errorf("holds %v after rejected requests", got)
+	}
+	// An absent key is a success that removes nothing.
+	var res deleteAllResult
+	if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/absent", ObjectOptions{DeleteAllVersions: true, Expiration: expire, DeleteAllResult: &res}); err != nil || res.Removed != 0 || !res.Complete {
+		t.Errorf("absent key: %v %+v", err, res)
+	}
+}
+
+type purgeSlowDisk struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d purgeSlowDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+	if bucket == d.bucket && object == d.object {
+		time.Sleep(20 * time.Millisecond)
+	}
+	return d.StorageAPI.ReadXL(ctx, bucket, object, data)
+}
+
+// The time budget covers the reads and checks before the first delete.
+func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) {
+	z := purgeSinglePool(t, 16)
+	p := newPurgeTest(t, z, "purge-budget-reads", MakeBucketOptions{VersioningEnabled: true}, "")
+	const key = "bx/slow"
+	p.version(0, key, "v1", 5)
+	exp := p.matched(key)
+	prev := deleteAllBudget
+	deleteAllBudget.duration = 10 * time.Millisecond
+	t.Cleanup(func() { deleteAllBudget = prev })
+	pool := z.serverPools[0]
+	pool.erasureDisksMu.Lock()
+	prior := pool.erasureDisks
+	wrapped := make([][]StorageAPI, len(prior))
+	for i, disks := range prior {
+		wrapped[i] = make([]StorageAPI, len(disks))
+		for j, disk := range disks {
+			wrapped[i][j] = purgeSlowDisk{StorageAPI: disk, bucket: p.bucket, object: key}
+		}
+	}
+	pool.erasureDisks = wrapped
+	pool.erasureDisksMu.Unlock()
+	t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() })
+	var res deleteAllResult
+	_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
+		DeleteAllVersions: true,
+		DeleteAllLatest:   &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+		DeleteAllResult:   &res,
+	})
+	if !errors.Is(err, errDeleteAllBudget) || res.Started || res.Removed != 0 {
+		t.Errorf("purge after its budget ran out on reads: err=%v %+v", err, res)
+	}
+	if got := p.held(0, key); !slices.Equal(got, []string{"hot"}) {
+		t.Errorf("holds %v", got)
+	}
+}
+
+// A marker that an expiration job failed to remove fails the job.
+func TestPurgeAllVersionsMarkerFailureFailsJob(t *testing.T) {
+	ri := &batchJobInfo{}
+	ri.trackExpireAll(expireObjInfo{ObjectInfo: ObjectInfo{NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0, true)
+	if ri.ObjectsFailed != 0 || ri.DeleteMarkersFailed != 1 || !ri.expireFailures() {
+		t.Errorf("failed=%d/%d failures=%t", ri.ObjectsFailed, ri.DeleteMarkersFailed, ri.expireFailures())
+	}
+}
+
+// purgeSwapDisk replaces itself in its slot once the purge has read it
+// twice, for the history and for the drive check, as a reconnect does.
+type purgeSwapDisk struct {
+	StorageAPI
+	bucket, object string
+	reads          *atomic.Int32
+	swap           func()
+}
+
+func (d purgeSwapDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+	rf, err := d.StorageAPI.ReadXL(ctx, bucket, object, data)
+	if bucket == d.bucket && object == d.object && d.reads.Add(1) == 2 {
+		d.swap()
+	}
+	return rf, err
+}
+
+// purgeCountDisk counts the version deletes it receives.
+type purgeCountDisk struct {
+	StorageAPI
+	deletes *atomic.Int32
+}
+
+func (d purgeCountDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
+	d.deletes.Add(1)
+	return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
+}
+
+// The purge deletes on the drive handles it checked, not on a handle that
+// replaced one of them since.
+func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) {
+	z := purgeSinglePool(t, 16)
+	p := newPurgeTest(t, z, "purge-checked-drives", MakeBucketOptions{VersioningEnabled: true}, "")
+	const key = "bx/swap"
+	p.version(0, key, "v1", 5)
+	exp := p.matched(key)
+	sets := z.serverPools[0]
+	index := sets.getHashedSetIndex(key)
+	var reads, deletes atomic.Int32
+	sets.erasureDisksMu.Lock()
+	original := sets.erasureDisks[index][0]
+	sets.erasureDisks[index][0] = purgeSwapDisk{StorageAPI: original, bucket: p.bucket, object: key, reads: &reads, swap: func() {
+		sets.erasureDisksMu.Lock()
+		sets.erasureDisks[index][0] = purgeCountDisk{StorageAPI: original, deletes: &deletes}
+		sets.erasureDisksMu.Unlock()
+	}}
+	sets.erasureDisksMu.Unlock()
+	t.Cleanup(func() {
+		sets.erasureDisksMu.Lock()
+		sets.erasureDisks[index][0] = original
+		sets.erasureDisksMu.Unlock()
+	})
+	var res deleteAllResult
+	if _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
+		DeleteAllVersions: true,
+		DeleteAllLatest:   &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
+		DeleteAllResult:   &res,
+	}); err != nil || !res.Complete {
+		t.Fatalf("purge: %v %+v", err, res)
+	}
+	if reads.Load() != 2 || deletes.Load() != 0 {
+		t.Errorf("reads=%d deletes on the replacement handle=%d", reads.Load(), deletes.Load())
+	}
+}

cmd/erasure-server-pool-delete-all.go

@@ -0,0 +1,459 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pgsty/silo-pkg/v3/sync/errgroup"
+)
+
+// An expiry of all versions of a key (ExpiredObjectAllVersions,
+// DelMarkerExpiration, a batch expiry that keeps no versions) removes the
+// versions of that key and nothing else. On the drives, object a/b lives
+// inside the directory of object a, next to the metadata and data of a. The
+// purge therefore never deletes that directory: it deletes the versions of
+// the key one by one, by version ID, with the version delete of the drives,
+// which removes the directory only once it is empty.
+//
+// It decides and deletes under the object lock that DeleteObject holds, on
+// the authoritative history of the key in all pools (objectVersions):
+//   - an ILM expiry needs the latest version to expire with all versions as
+//     the scanner evaluates it on all of them (lifecycleEventsAllPools); a
+//     batch expiry needs the version it matched to be the latest still;
+//   - it deletes exactly the versions it approved, with every copy of them
+//     in every pool and the older writes of their IDs, and nothing else;
+//   - before it deletes anything it reads every drive of every set that
+//     holds a copy, and refuses the whole key when a drive cannot be read or
+//     when any revision of an approved version there is transitioned or,
+//     when the bucket has Object Lock, protected by retention or a legal
+//     hold. It deletes on exactly the drives it read;
+//   - it deletes the oldest version first and the latest version last and
+//     alone, once every older version is gone from every pool. A purge that
+//     stops early keeps the latest version, so it never exposes an older
+//     one, and the next expiry continues on a fresh history.
+
+var (
+	// errDeleteAllChanged: the version a batch expiry matched is no longer
+	// the latest version of the key.
+	errDeleteAllChanged = fmt.Errorf("%w: the latest version changed", errLifecycleNotApplicable)
+	// errDeleteAllTiered: a version of the key is in a remote tier. Removing
+	// all versions is left to a later release, which hands the remote
+	// objects to the tier cleanup.
+	errDeleteAllTiered = fmt.Errorf("%w: a version is transitioned", errLifecycleNotApplicable)
+	// errDeleteAllBudget: the purge stopped at its budget, before the
+	// latest version.
+	errDeleteAllBudget = errors.New("purge of all versions stopped at its budget")
+	// errDeleteAllUnsupported: only a single-object delete of the pools
+	// purges all versions. Any other layer would turn the request into
+	// another delete.
+	errDeleteAllUnsupported = NotImplemented{Message: "purge of all versions is a single-object delete of the pools"}
+)
+
+// deleteAllBudget bounds one purge, which holds the object lock while it
+// runs. It stops before the next version once it removed versions versions
+// or ran for duration, its reads and checks included.
+var deleteAllBudget = struct {
+	versions int
+	duration time.Duration
+}{versions: 1000, duration: time.Minute}
+
+// deleteAllRetries bounds how often a version delete that reached its write
+// quorum is repeated on the drives that failed it.
+const deleteAllRetries = 2
+
+// deleteAllMatch identifies the latest version that a batch expiry matched.
+type deleteAllMatch struct {
+	VersionID string
+	ModTime   time.Time
+}
+
+// matches reports whether fi is the version m identifies.
+func (m deleteAllMatch) matches(fi FileInfo) bool {
+	return deleteAllVersionID(m.VersionID) == deleteAllVersionID(fi.VersionID) && m.ModTime.Equal(fi.ModTime)
+}
+
+// deleteAllResult is what a purge of all versions confirmed.
+type deleteAllResult struct {
+	// Latest is the latest version the purge approved.
+	Latest ObjectInfo
+	// Removed counts the versions, delete markers included, that are gone
+	// from every pool, Markers the delete markers among them. Free versions
+	// and further copies of a version do not count.
+	Removed, Markers int
+	// Started: the purge began to delete. A failure after that can have
+	// removed copies that Removed does not count.
+	Started bool
+	// Complete: every approved version is gone, the latest included.
+	Complete bool
+}
+
+// deleteAllVersionID returns versionID, "null" for the null version.
+func deleteAllVersionID(versionID string) string {
+	if versionID == "" {
+		return nullVersionID
+	}
+	return versionID
+}
+
+// deleteAllInvalid returns why opts cannot ask for a purge of all versions.
+// The purge is approved either as an ILM expiry or by a batch match, and it
+// only deletes: it never creates a marker, addresses a single version,
+// moves data or deletes a prefix.
+func (opts ObjectOptions) deleteAllInvalid() error {
+	switch {
+	case opts.DeletePrefix || opts.DeletePrefixObject:
+		return errors.New("a purge of all versions is not a prefix delete")
+	case opts.VersionID != "":
+		return errors.New("a purge of all versions addresses no single version")
+	case opts.DataMovement:
+		return errors.New("a purge of all versions moves no data")
+	case opts.DeleteMarker || opts.Versioned || opts.VersionSuspended:
+		return errors.New("a purge of all versions creates no delete marker")
+	case opts.ReplicationRequest || opts.InclFreeVersions || opts.Transition != (TransitionOptions{}) ||
+		opts.CheckPrecondFn != nil || opts.EvalMetadataFn != nil || opts.EvalRetentionBypassFn != nil ||
+		opts.Expiration.CheckedAllPools:
+		return errors.New("a purge of all versions takes no options of other deletes")
+	case opts.Expiration.Expire == (opts.DeleteAllLatest != nil):
+		return errors.New("a purge of all versions needs exactly one approval")
+	}
+	return nil
+}
+
+// deleteAllCopy is the copy of a version in a pool: the revision of it that
+// the authoritative history of the set of the key in that pool accepted.
+// Copies of one write share its ModTime, since data movement keeps it and
+// writes a new data dir. Other writes of a version ID, such as null versions
+// written in several pools, have another ModTime.
+type deleteAllCopy struct {
+	pool int
+	fi   FileInfo
+}
+
+// deleteAllVersion is a version of the key, "null" for the null version, and
+// its copies in all pools, the newest write first and, on equal times, the
+// lower pool first. The first copy is the one reads choose.
+type deleteAllVersion struct {
+	id     string
+	copies []deleteAllCopy
+}
+
+// deleteAllHistory returns the versions of object in all pools, newest first,
+// with the copies of each, from the authoritative history of each pool. It
+// fails when a pool cannot tell the history.
+func (z *erasureServerPools) deleteAllHistory(ctx context.Context, bucket, object string) ([]deleteAllVersion, error) {
+	poolVersions := make([][]FileInfo, len(z.serverPools))
+	errs := make([]error, len(z.serverPools))
+	var wg sync.WaitGroup
+	for i, pool := range z.serverPools {
+		wg.Add(1)
+		go func(i int, set *erasureObjects) {
+			defer wg.Done()
+			poolVersions[i], errs[i] = set.objectVersions(ctx, bucket, object)
+		}(i, pool.getHashedSet(object))
+	}
+	wg.Wait()
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var all []deleteAllCopy
+	for pool, versions := range poolVersions {
+		for _, fi := range versions {
+			if fi.ModTime.IsZero() {
+				// A write that cannot be ordered cannot be deleted in order.
+				return nil, toObjectErr(errErasureReadQuorum, bucket, object)
+			}
+			all = append(all, deleteAllCopy{pool: pool, fi: fi})
+		}
+	}
+	// The order of lifecycleEventsAllPools and of reads.
+	sort.SliceStable(all, func(i, j int) bool { return all[i].fi.ModTime.After(all[j].fi.ModTime) })
+	index := make(map[string]int, len(all))
+	var history []deleteAllVersion
+	for _, c := range all {
+		id := deleteAllVersionID(c.fi.VersionID)
+		if i, ok := index[id]; ok {
+			history[i].copies = append(history[i].copies, c)
+			continue
+		}
+		index[id] = len(history)
+		history = append(history, deleteAllVersion{id: id, copies: []deleteAllCopy{c}})
+	}
+	return history, nil
+}
+
+// deleteAllApproved reports whether history holds exactly the versions that
+// the ILM evaluation approved, in the same order, the same writes chosen.
+func deleteAllApproved(history []deleteAllVersion, approved []ObjectInfo) bool {
+	if len(history) != len(approved) {
+		return false
+	}
+	for i, v := range history {
+		got := v.copies[0].fi.ToObjectInfo("", "", false)
+		want := approved[i]
+		if v.id != deleteAllVersionID(want.VersionID) || !got.ModTime.Equal(want.ModTime) ||
+			got.DeleteMarker != want.DeleteMarker || got.ETag != want.ETag || got.Size != want.Size {
+			return false
+		}
+	}
+	return true
+}
+
+// deleteAllVersions deletes every version of object, whose expiry of all
+// versions opts describe, and nothing else, see the comment above. The
+// caller holds the object lock. It returns the approved latest version, and
+// what it confirmed in opts.DeleteAllResult, also on an error. A key without
+// versions is a success that removes nothing.
+func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	res := opts.DeleteAllResult
+	if res == nil {
+		res = &deleteAllResult{}
+	}
+	*res = deleteAllResult{}
+	start := time.Now()
+	name := decodeDirObject(object)
+	fail := func(err error) (ObjectInfo, error) {
+		if isErrReadQuorum(err) {
+			err = InsufficientWriteQuorum{}
+		}
+		return ObjectInfo{Bucket: bucket, Name: name}, err
+	}
+
+	lr, err := globalBucketObjectLockSys.Get(bucket)
+	if err != nil {
+		return fail(err)
+	}
+
+	// Decide.
+	var approved []ObjectInfo
+	if opts.Expiration.Expire {
+		var all lifecycleAllPools
+		all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, object)
+		if all.err != nil || len(all.versions) > 0 {
+			if err := all.expiryApplies(bucket, object, ObjectOptions{DeletePrefix: true}); err != nil {
+				return fail(err)
+			}
+		}
+		approved = all.versions
+	}
+	// The placement of the approved versions. This read may only confirm
+	// them: whatever differs from the approval refuses the key.
+	history, err := z.deleteAllHistory(ctx, bucket, object)
+	if err != nil {
+		return fail(err)
+	}
+	if opts.Expiration.Expire && !deleteAllApproved(history, approved) {
+		return fail(InsufficientWriteQuorum{})
+	}
+	if len(history) == 0 {
+		res.Complete = true
+		return ObjectInfo{Bucket: bucket, Name: name}, nil
+	}
+	if opts.DeleteAllLatest != nil && !opts.DeleteAllLatest.matches(history[0].copies[0].fi) {
+		return fail(errDeleteAllChanged)
+	}
+	if opts.Expiration.Expire {
+		res.Latest = approved[0]
+	} else {
+		vc, _ := globalBucketVersioningSys.Get(bucket)
+		res.Latest = history[0].copies[0].fi.ToObjectInfo(bucket, object, vc != nil && vc.Versioned(name))
+	}
+	res.Latest.Bucket, res.Latest.Name = bucket, name
+
+	// Check every write before deleting any: the accepted revisions, then
+	// every revision on every drive that the purge will delete on.
+	ids := make(map[string]bool, len(history))
+	held := make([]bool, len(z.serverPools))
+	for _, v := range history {
+		ids[v.id] = true
+		for _, c := range v.copies {
+			held[c.pool] = true
+			if err := deleteAllRefuses(ctx, bucket, object, c.fi, lr.LockEnabled); err != nil {
+				return fail(err)
+			}
+		}
+	}
+	drives := make([][]StorageAPI, len(z.serverPools))
+	for pool := range z.serverPools {
+		if !held[pool] {
+			continue
+		}
+		if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled); err != nil {
+			return fail(err)
+		}
+	}
+
+	// Delete the oldest version first. The latest one is the last and is
+	// only reached once every older one is gone from every pool.
+	for i := len(history) - 1; i >= 0; i-- {
+		if err := ctx.Err(); err != nil {
+			return fail(err)
+		}
+		if res.Removed >= deleteAllBudget.versions || time.Since(start) >= deleteAllBudget.duration {
+			return fail(errDeleteAllBudget)
+		}
+		res.Started = true
+		if err := z.deleteAllCopies(ctx, bucket, object, history[i], drives); err != nil {
+			return fail(err)
+		}
+		res.Removed++
+		if history[i].copies[0].fi.Deleted {
+			res.Markers++
+		}
+	}
+	res.Complete = true
+	return res.Latest, nil
+}
+
+// deleteAllRefuses returns why the purge must keep the key that holds the
+// revision fi of an approved version: it is transitioned, whose remote
+// object a later release hands to the tier cleanup, or, with Object Lock,
+// protected by retention or a legal hold.
+func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, lockEnabled bool) error {
+	if fi.TransitionStatus != "" || fi.TransitionTier != "" || fi.TransitionedObjName != "" {
+		return errDeleteAllTiered
+	}
+	if lockEnabled && enforceRetentionForDeletion(ctx, fi.ToObjectInfo(bucket, object, true)) {
+		return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: deleteAllVersionID(fi.VersionID)}
+	}
+	return nil
+}
+
+// deleteAllDrives reads object on every drive of the set and checks each
+// revision of the versions ids that a drive holds with deleteAllRefuses.
+// A minority of drives can hold a revision other than the accepted one, and
+// heal would later remove what the purge leaves on a drive, so a drive that
+// cannot be read or decoded refuses the key. A drive without the object
+// holds nothing to check. It returns the drive handles it read, which the
+// purge deletes on; a handle that a reconnect installs later is not used. A
+// handle names a drive, not a backend: a backend that replaces the drive
+// behind it starts empty and receives this object only from a write or a
+// heal of it, which take the object lock the purge holds.
+func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool) ([]StorageAPI, error) {
+	disks := slices.Clone(er.getDisks())
+	raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	for i := range disks {
+		if errors.Is(errs[i], errFileNotFound) {
+			continue
+		}
+		if errs[i] != nil {
+			return nil, toObjectErr(errErasureReadQuorum, bucket, object)
+		}
+		var xl xlMetaV2
+		if err := xl.LoadOrConvert(raws[i].Buf); err != nil {
+			return nil, toObjectErr(errErasureReadQuorum, bucket, object)
+		}
+		versions, err := xl.ListVersions(bucket, object, false)
+		if err != nil {
+			return nil, toObjectErr(errErasureReadQuorum, bucket, object)
+		}
+		for _, fi := range versions {
+			if fi.TierFreeVersion() || !ids[deleteAllVersionID(fi.VersionID)] {
+				continue
+			}
+			if err := deleteAllRefuses(ctx, bucket, object, fi, lockEnabled); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return disks, nil
+}
+
+// deleteAllCopies deletes every copy of the version v: the older writes of
+// its ID first, the copy that reads choose last, so that a failure leaves
+// what a read returned before. It deletes on the drives that drives holds
+// for the pool of a copy.
+func (z *erasureServerPools) deleteAllCopies(ctx context.Context, bucket, object string, v deleteAllVersion, drives [][]StorageAPI) error {
+	for i := len(v.copies) - 1; i >= 0; i-- {
+		pool := v.copies[i].pool
+		set := z.serverPools[pool].getHashedSet(object)
+		if err := set.purgeVersion(ctx, bucket, object, v.id, drives[pool]); err != nil {
+			return toObjectErr(err, bucket, decodeDirObject(object), v.id)
+		}
+	}
+	return nil
+}
+
+// purgeVersion physically removes the version versionID of object from
+// disks, the drives of the set that deleteAllDrives checked: no delete
+// marker, no replication state, no free version. A drive that does not hold
+// the version counts as done, so that a retry is idempotent. When the delete
+// reaches its write quorum but fails on some drives, it is repeated on those
+// while the caller still holds the object lock; what remains is left to
+// heal. The caller holds the object lock.
+func (er erasureObjects) purgeVersion(ctx context.Context, bucket, object, versionID string, disks []StorageAPI) error {
+	writeQuorum := len(disks)/2 + 1
+	fi := FileInfo{Name: object, VersionID: versionID}
+	// The purge checked that no drive it deletes on holds a transitioned
+	// revision. A free version would hand a remote object to the tier
+	// cleanup, so none is made.
+	fi.SetSkipTierFreeVersion()
+
+	errs := make([]error, len(disks))
+	for attempt := 0; attempt <= deleteAllRetries; attempt++ {
+		g := errgroup.WithNErrs(len(disks))
+		for index := range disks {
+			if attempt > 0 && (errs[index] == nil || disks[index] == nil) {
+				continue
+			}
+			g.Go(func() error {
+				if disks[index] == nil {
+					return errDiskNotFound
+				}
+				err := disks[index].DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{})
+				if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
+					return nil
+				}
+				return err
+			}, index)
+		}
+		for index, err := range g.Wait() {
+			if attempt == 0 || (errs[index] != nil && disks[index] != nil) {
+				errs[index] = err
+			}
+		}
+		if err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum); err != nil {
+			er.addPartial(bucket, object, versionID)
+			return err
+		}
+		if !slices.ContainsFunc(errs, func(err error) bool { return err != nil }) {
+			return nil
+		}
+		retry := false
+		for index, err := range errs {
+			retry = retry || err != nil && disks[index] != nil
+		}
+		if !retry {
+			break
+		}
+	}
+	er.addPartial(bucket, object, versionID)
+	return nil
+}

cmd/erasure-server-pool.go

@@ -1234,6 +1234,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	if err = checkDelObjArgs(ctx, bucket, object); err != nil {
 		return objInfo, err
 	}
+	if opts.DeleteAllVersions {
+		if err := opts.deleteAllInvalid(); err != nil {
+			return objInfo, InvalidArgument{Bucket: bucket, Object: object, Err: err}
+		}
+	}
 
 	if !opts.DeletePrefix { // DeletePrefix handles dir object encoding differently.
 		object = encodeDirObject(object)
@@ -1266,6 +1271,12 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	// A purge of all versions approves and deletes on the history it reads
+	// itself. The checks below decide by other options and would discard it.
+	if opts.DeleteAllVersions {
+		return z.deleteAllVersions(ctx, bucket, object, opts)
+	}
+
 	// An ILM expiry is decided on the versions in one pool. Under the lock,
 	// check it with the versions in all pools; the sets then take it as
 	// decided, without a recheck by the versions each one holds. They only
@@ -1514,6 +1525,12 @@ func deletesData(obj ObjectToDelete, opts ObjectOptions) bool {
 func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
 	derrs := make([]error, len(objects))
 	dobjects := make([]DeletedObject, len(objects))
+	if opts.DeleteAllVersions {
+		for i := range derrs {
+			derrs[i] = errDeleteAllUnsupported
+		}
+		return dobjects, derrs
+	}
 	objSets := set.NewStringSet()
 	for i := range derrs {
 		objects[i].ObjectName = encodeDirObject(objects[i].ObjectName)

cmd/erasure-sets.go

@@ -763,6 +763,9 @@ func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix st
 
 // DeleteObject - deletes an object from the hashedSet based on the object name.
 func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if opts.DeleteAllVersions {
+		return objInfo, errDeleteAllUnsupported
+	}
 	if opts.DeletePrefix && !opts.DeletePrefixObject {
 		err := s.deletePrefix(ctx, bucket, object)
 		return ObjectInfo{}, err

cmd/object-api-interface.go

@@ -103,6 +103,17 @@ type ObjectOptions struct {
 	DeletePrefix                        bool      // set true to enforce a prefix deletion, only application for DeleteObject API,
 	DeletePrefixObject                  bool      // set true when object's erasure set is resolvable by object name (using getHashedSetIndex)
 
+	// DeleteAllVersions asks DeleteObject to purge every version of exactly
+	// this key by version ID, for an ILM or batch expiry of all versions;
+	// see erasureServerPools.deleteAllVersions. In memory only.
+	DeleteAllVersions bool
+	// DeleteAllLatest is the latest version that a batch expiry matched.
+	// The purge of all versions only runs while it is still the latest.
+	DeleteAllLatest *deleteAllMatch
+	// DeleteAllResult, when set, receives what a purge of all versions
+	// confirmed, also when it fails part way.
+	DeleteAllResult *deleteAllResult
+
 	// Pools-layer version resolver; the caller holds the shared object lock.
 	replicaObjectInfo GetObjectInfoFn