fix(ilm): purge only the expired key's own versions in delete-all

An expiry of all versions (ExpiredObjectAllVersions, DelMarkerExpiration,
and batch expiration with the default purge) asked the pools layer for a
prefix delete. The pools layer rebuilt the options, so the request ran as
a recursive delete of the key's directory in every erasure set of every
pool. Object a/b is stored inside the directory of a, so every key under
a/ was removed with it, however new, with one audit entry and one
notification for a alone. The rebuilt options also skipped the lifecycle
recheck, so on a single pool a version written after the decision was
deleted too, and tiered versions lost their metadata without free
versions.

The expiry worker, the migration expiry and batch expiration now ask for
a purge with the in-memory ObjectOptions.DeleteAllVersions:

- DeleteObject hands it to the purge under the object lock, before the
  all-pools check. ILM decides with the unchanged lifecycleEventsAllPools,
  on a single pool too; batch needs the version its walk matched to still
  be the latest. A second read of each pool's authoritative history
  records every copy, older writes of the same ID included, and must
  equal the approval. Anything else deletes nothing.
- Before deleting, the purge reads every drive of each set that holds a
  copy. It refuses the key when a drive cannot be read, when any revision
  of an approved version there is transitioned, or, with Object Lock,
  when retention or a legal hold protects one. It then deletes on exactly
  the drive handles it checked.
- Versions are deleted by ID with the drives' version delete, which only
  removes the object directory once it is empty: oldest first, for each
  version older writes before the copy reads choose, and the latest last,
  only once every older version is gone from every pool. No marker,
  replication state or free version is written. The purge stops at the
  first error, repeats a quorum-successful delete on the drives that
  missed it while the lock is held, and stops before the latest version
  at a budget of 1000 versions or one minute, reads included.
- The ILM audit entry records how many versions were removed. A purge
  that failed after it began to delete is audited as partial and sends
  no removal event. Batch counts come from the purge; a key whose latest
  version changed is not expired, and a failed delete marker fails the
  job.

The option is rejected with conflicting options and by every layer below
the pools. Force delete and internal prefix deletes keep DeletePrefix.
Delete-all of a transitioned latest version keeps its current path, and
a key with tiered older versions is refused. On a single pool the purge
relies on the lock-domain fix for writer exclusion.

Reviewed adversarially in two design rounds and four patch rounds; the
last approved with nits, which are applied.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-05 03:24:42 +08:00
parent 9f7b49369d
commit 2b228f678c
10 changed files with 1896 additions and 39 deletions
+57 -25
View File
@@ -445,31 +445,13 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
oiCache.Add(od, &exp.ObjectInfo)
}
// DeleteObject(deletePrefix: true) to expire all versions of an object
// Purge all versions of exactly each key while the version the
// walk matched is still its latest; see
// erasureServerPools.deleteAllVersions.
for _, exp := range toExpireAll {
var success bool
for attempts := 1; attempts <= retryAttempts; attempts++ {
select {
case <-ctx.Done():
ri.trackMultipleObjectVersions(exp, success)
return
default:
}
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
DeletePrefix: true,
DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
})
if err != nil {
stopFn(exp, err)
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
} else {
stopFn(exp, err)
success = true
break
}
if !expireAllVersions(ctx, ri, api, exp, retryAttempts) {
return
}
ri.trackMultipleObjectVersions(exp, success)
}
// DeleteMultiple objects
@@ -525,6 +507,56 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
}
}
// expireAllVersions purges all versions of the key exp, which the walk
// matched by its latest version, and counts what the purge removed. It
// retries a failed purge up to attempts times, each time with the version
// the walk matched; a purge that stopped at its budget continues without
// spending an attempt. It does not retry once the key no longer has that
// latest version, which is not an expiry, or once the purge refused the
// key. It returns false once ctx ends.
func expireAllVersions(ctx context.Context, ri *batchJobInfo, api ObjectLayer, exp expireObjInfo, attempts int) bool {
var objects, markers int64
matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
for attempt := 1; attempt <= attempts; attempt++ {
select {
case <-ctx.Done():
ri.trackExpireAll(exp, objects, markers, true)
return false
default:
}
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempt)
latest := matched
var purge deleteAllResult
_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &latest,
DeleteAllResult: &purge,
})
objects += int64(purge.Removed - purge.Markers)
markers += int64(purge.Markers)
stopFn(exp, err)
switch {
case err == nil:
ri.trackExpireAll(exp, objects, markers, false)
return true
case errors.Is(err, errDeleteAllChanged):
// Changed since the walk matched it: not expired.
ri.trackExpireAll(exp, objects, markers, false)
return true
case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
attempt-- // progress
continue
}
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempt))
var locked ObjectLocked
if errors.Is(err, errDeleteAllTiered) || errors.As(err, &locked) {
break // refused; a later run of the job checks again
}
}
ri.trackExpireAll(exp, objects, markers, true)
return true
}
type expireObjInfo struct {
ObjectInfo
ExpireAll bool
@@ -746,8 +778,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
<-expireDoneCh // waits for the expire goroutine to complete
wk.Wait() // waits for all expire workers to retire
ri.Complete = !failed && ri.ObjectsFailed == 0
ri.Failed = failed || ri.ObjectsFailed > 0
ri.Complete = !failed && !ri.expireFailures()
ri.Failed = failed || ri.expireFailures()
globalBatchJobsMetrics.save(job.ID, ri)
// Close the saverQuitCh - this also triggers saving in-memory state
+22 -10
View File
@@ -996,10 +996,17 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
return nil
}
// Note: to be used only with batch jobs that affect multiple versions through
// a single action. e.g batch-expire has an option to expire all versions of an
// object which matches the given filters.
func (ri *batchJobInfo) trackMultipleObjectVersions(info expireObjInfo, success bool) {
// expireFailures reports whether an expiration job failed to remove an
// object or a delete marker it matched.
func (ri *batchJobInfo) expireFailures() bool {
return ri.ObjectsFailed > 0 || ri.DeleteMarkersFailed > 0
}
// trackExpireAll counts the objects and delete markers that an expiry of all
// versions of info removed and, when it failed, those the walk saw that it
// did not remove, at least one. Only batch-expire uses it, for a rule that
// expires all versions of an object.
func (ri *batchJobInfo) trackExpireAll(info expireObjInfo, objects, markers int64, failed bool) {
if ri == nil {
return
}
@@ -1007,15 +1014,20 @@ func (ri *batchJobInfo) trackMultipleObjectVersions(info expireObjInfo, success
ri.mu.Lock()
defer ri.mu.Unlock()
if success {
ri.Objects += objects
ri.DeleteMarkers += markers
if !failed {
ri.Bucket = info.Bucket
ri.Object = info.Name
ri.Objects += int64(info.NumVersions) - info.DeleteMarkerCount
ri.DeleteMarkers += info.DeleteMarkerCount
} else {
ri.ObjectsFailed += int64(info.NumVersions) - info.DeleteMarkerCount
ri.DeleteMarkersFailed += info.DeleteMarkerCount
return
}
failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
failedMarkers := max(info.DeleteMarkerCount-markers, 0)
if failedObjects+failedMarkers == 0 {
failedObjects = 1
}
ri.ObjectsFailed += failedObjects
ri.DeleteMarkersFailed += failedMarkers
}
func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool, attempt int) {
+67 -4
View File
@@ -1229,10 +1229,16 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
// Purge every version of exactly this key, by version ID and under the
// object lock, after the pools approved it on the versions they hold;
// see erasureServerPools.deleteAllVersions.
var purge deleteAllResult
if lcEvent.Action.DeleteAll() {
opts.DeletePrefix = true
// use prefix delete on exact object (this is an optimization to avoid fan-out calls)
opts.DeletePrefixObject = true
opts = ObjectOptions{
Expiration: ExpirationOptions{Expire: true},
DeleteAllVersions: true,
DeleteAllResult: &purge,
}
}
// ILM-created delete markers and version purges must carry the same
@@ -1263,13 +1269,16 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
if lcEvent.Action != lifecycle.NoneAction {
numVersions := uint64(1)
if lcEvent.Action.DeleteAll() {
numVersions = uint64(obj.NumVersions)
numVersions = uint64(purge.Removed)
}
timeILM(numVersions)
}
}()
dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
if lcEvent.Action.DeleteAll() && (err != nil || purge.Removed == 0) {
return deleteAllNotCompleted(ctx, obj, lcEvent, src, purge, err, traceFn)
}
if err != nil {
if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
traceFn(ILMExpiry, nil, nil)
@@ -1310,6 +1319,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
tags := newLifecycleAuditEvent(src, lcEvent).Tags()
tags["version-id"] = dobj.VersionID
if lcEvent.Action.DeleteAll() {
tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
}
// Send audit for the lifecycle delete operation
auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
@@ -1336,6 +1348,57 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
return true
}
// ilmVersionsRemoved is the audit tag with the number of versions that an
// expiry of all versions removed; ilmPartial marks one that began to delete
// and did not finish.
const (
ilmVersionsRemoved = "ilm-versions-removed"
ilmPartial = "ilm-partial"
)
// deleteAllNotCompleted reports an expiry of all versions of obj that did
// not remove them all. A purge that had nothing to remove, or that failed
// before it deleted anything, reports what a rejected expiry does. One that
// failed or stopped after it began to delete is audited as partial, with the
// versions it removed, but sends no removal event: the key is still there.
func deleteAllNotCompleted(ctx context.Context, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc, purge deleteAllResult, err error, traceFn func(event string, metadata map[string]string, err error)) bool {
switch {
case err == nil: // there was nothing to remove
traceFn(ILMExpiry, nil, nil)
return false
case purge.Started:
tags := newLifecycleAuditEvent(src, lcEvent).Tags()
tags["version-id"] = purge.Latest.VersionID
tags[ilmVersionsRemoved] = strconv.Itoa(purge.Removed)
tags[ilmPartial] = "true"
auditLogInternal(ctx, AuditLogOptions{
Event: ILMExpiry,
APIName: "ILMExpiry",
Bucket: obj.Bucket,
Object: obj.Name,
VersionID: purge.Latest.VersionID,
Error: err.Error(),
Tags: tags,
})
traceFn(ILMExpiry, tags, err)
if errors.Is(err, errDeleteAllBudget) {
return false // a long history; the next expiry continues
}
case errors.Is(err, errDeleteAllTiered):
// Kept until tiered versions can be purged; see deleteAllVersions.
traceFn(ILMExpiry, nil, err)
return false
case errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err):
traceFn(ILMExpiry, nil, nil)
return false
default:
traceFn(ILMExpiry, nil, err)
}
// Assume it is still there.
ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
return false
}
// Apply object, object version, restored object or restored object version action on the given object
func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
globalExpiryState.enqueueByDays(obj, event, src)
+10
View File
@@ -1774,6 +1774,13 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
// into smaller bulks if some object names are found to be duplicated in the delete list, splitting
// into smaller bulks will avoid holding twice the write lock of the duplicated object names.
func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
if opts.DeleteAllVersions {
errs := make([]error, len(objects))
for i := range errs {
errs[i] = errDeleteAllUnsupported
}
return make([]DeletedObject, len(objects)), errs
}
if !opts.NoAuditLog {
for _, obj := range objects {
auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectName, &er)
@@ -2024,6 +2031,9 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
// any error as it is not necessary for the handler to reply back a
// response to the client request.
func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
if opts.DeleteAllVersions {
return objInfo, errDeleteAllUnsupported
}
if !opts.NoAuditLog {
auditObjectErasureSet(ctx, "DeleteObject", object, &er)
}
@@ -0,0 +1,221 @@
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"errors"
"fmt"
"slices"
"sync/atomic"
"testing"
"time"
)
// The parts of the purge of all versions that need its own options.
// A purge stops at its budget before the latest version.
func TestPurgeAllVersionsStopsAtBudget(t *testing.T) {
z := purgeSinglePool(t, 16)
p := newPurgeTest(t, z, "purge-budget", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
prev := deleteAllBudget
deleteAllBudget.versions = 2
t.Cleanup(func() { deleteAllBudget = prev })
const key = "ea/long"
for i, days := range []float64{8, 7, 6, 5} {
p.version(0, key, fmt.Sprintf("v%d", i+1), days)
}
o := observePurges(t)
p.expire(0, key)
if got := p.held(0, key); !slices.Equal(got, []string{"hot", "hot"}) {
t.Errorf("holds %v after a purge stopped at its budget, want the two newest versions", got)
}
if got := p.get(key); got != "v4" {
t.Errorf("reads %s, want v4", got)
}
events, traces := o.drain()
if len(events) != 0 || len(traces) != 1 || traces[0]["ilm-partial"] != "true" || traces[0]["ilm-versions-removed"] != "2" {
t.Errorf("stopped purge sent %v and traced %v", events, traces)
}
p.expire(0, key)
if got := p.held(0, key); got != nil {
t.Errorf("keeps %v once the purge continued", got)
}
}
// The option admits no other delete.
func TestPurgeAllVersionsOptions(t *testing.T) {
z := purgeSinglePool(t, 16)
p := newPurgeTest(t, z, "purge-options", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
p.version(0, "ea/key", "v1", 5)
match := &deleteAllMatch{}
expire := ExpirationOptions{Expire: true}
for name, opts := range map[string]ObjectOptions{
"no approval": {DeleteAllVersions: true},
"two approvals": {DeleteAllVersions: true, Expiration: expire, DeleteAllLatest: match},
"prefix": {DeleteAllVersions: true, Expiration: expire, DeletePrefix: true},
"prefix object": {DeleteAllVersions: true, Expiration: expire, DeletePrefixObject: true},
"version": {DeleteAllVersions: true, Expiration: expire, VersionID: mustGetUUID()},
"movement": {DeleteAllVersions: true, Expiration: expire, DataMovement: true},
"marker": {DeleteAllVersions: true, Expiration: expire, DeleteMarker: true},
"versioned": {DeleteAllVersions: true, Expiration: expire, Versioned: true},
"suspended": {DeleteAllVersions: true, Expiration: expire, VersionSuspended: true},
} {
if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(InvalidArgument)) {
t.Errorf("%s: %v", name, err)
}
}
opts := ObjectOptions{DeleteAllVersions: true, Expiration: expire}
if _, err := z.serverPools[0].DeleteObject(t.Context(), p.bucket, "ea/key", opts); !errors.As(err, new(NotImplemented)) {
t.Errorf("erasure sets: %v", err)
}
if _, errs := z.DeleteObjects(t.Context(), p.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: "ea/key"}}}, opts); !errors.As(errs[0], new(NotImplemented)) {
t.Errorf("bulk delete: %v", errs[0])
}
if got := p.held(0, "ea/key"); !slices.Equal(got, []string{"hot"}) {
t.Errorf("holds %v after rejected requests", got)
}
// An absent key is a success that removes nothing.
var res deleteAllResult
if _, err := z.DeleteObject(t.Context(), p.bucket, "ea/absent", ObjectOptions{DeleteAllVersions: true, Expiration: expire, DeleteAllResult: &res}); err != nil || res.Removed != 0 || !res.Complete {
t.Errorf("absent key: %v %+v", err, res)
}
}
type purgeSlowDisk struct {
StorageAPI
bucket, object string
}
func (d purgeSlowDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
if bucket == d.bucket && object == d.object {
time.Sleep(20 * time.Millisecond)
}
return d.StorageAPI.ReadXL(ctx, bucket, object, data)
}
// The time budget covers the reads and checks before the first delete.
func TestPurgeAllVersionsBudgetCoversReads(t *testing.T) {
z := purgeSinglePool(t, 16)
p := newPurgeTest(t, z, "purge-budget-reads", MakeBucketOptions{VersioningEnabled: true}, "")
const key = "bx/slow"
p.version(0, key, "v1", 5)
exp := p.matched(key)
prev := deleteAllBudget
deleteAllBudget.duration = 10 * time.Millisecond
t.Cleanup(func() { deleteAllBudget = prev })
pool := z.serverPools[0]
pool.erasureDisksMu.Lock()
prior := pool.erasureDisks
wrapped := make([][]StorageAPI, len(prior))
for i, disks := range prior {
wrapped[i] = make([]StorageAPI, len(disks))
for j, disk := range disks {
wrapped[i][j] = purgeSlowDisk{StorageAPI: disk, bucket: p.bucket, object: key}
}
}
pool.erasureDisks = wrapped
pool.erasureDisksMu.Unlock()
t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() })
var res deleteAllResult
_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
})
if !errors.Is(err, errDeleteAllBudget) || res.Started || res.Removed != 0 {
t.Errorf("purge after its budget ran out on reads: err=%v %+v", err, res)
}
if got := p.held(0, key); !slices.Equal(got, []string{"hot"}) {
t.Errorf("holds %v", got)
}
}
// A marker that an expiration job failed to remove fails the job.
func TestPurgeAllVersionsMarkerFailureFailsJob(t *testing.T) {
ri := &batchJobInfo{}
ri.trackExpireAll(expireObjInfo{ObjectInfo: ObjectInfo{NumVersions: 2}, DeleteMarkerCount: 1}, 1, 0, true)
if ri.ObjectsFailed != 0 || ri.DeleteMarkersFailed != 1 || !ri.expireFailures() {
t.Errorf("failed=%d/%d failures=%t", ri.ObjectsFailed, ri.DeleteMarkersFailed, ri.expireFailures())
}
}
// purgeSwapDisk replaces itself in its slot once the purge has read it
// twice, for the history and for the drive check, as a reconnect does.
type purgeSwapDisk struct {
StorageAPI
bucket, object string
reads *atomic.Int32
swap func()
}
func (d purgeSwapDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
rf, err := d.StorageAPI.ReadXL(ctx, bucket, object, data)
if bucket == d.bucket && object == d.object && d.reads.Add(1) == 2 {
d.swap()
}
return rf, err
}
// purgeCountDisk counts the version deletes it receives.
type purgeCountDisk struct {
StorageAPI
deletes *atomic.Int32
}
func (d purgeCountDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
d.deletes.Add(1)
return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
}
// The purge deletes on the drive handles it checked, not on a handle that
// replaced one of them since.
func TestPurgeAllVersionsDeletesOnCheckedDrives(t *testing.T) {
z := purgeSinglePool(t, 16)
p := newPurgeTest(t, z, "purge-checked-drives", MakeBucketOptions{VersioningEnabled: true}, "")
const key = "bx/swap"
p.version(0, key, "v1", 5)
exp := p.matched(key)
sets := z.serverPools[0]
index := sets.getHashedSetIndex(key)
var reads, deletes atomic.Int32
sets.erasureDisksMu.Lock()
original := sets.erasureDisks[index][0]
sets.erasureDisks[index][0] = purgeSwapDisk{StorageAPI: original, bucket: p.bucket, object: key, reads: &reads, swap: func() {
sets.erasureDisksMu.Lock()
sets.erasureDisks[index][0] = purgeCountDisk{StorageAPI: original, deletes: &deletes}
sets.erasureDisksMu.Unlock()
}}
sets.erasureDisksMu.Unlock()
t.Cleanup(func() {
sets.erasureDisksMu.Lock()
sets.erasureDisks[index][0] = original
sets.erasureDisksMu.Unlock()
})
var res deleteAllResult
if _, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
DeleteAllVersions: true,
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
DeleteAllResult: &res,
}); err != nil || !res.Complete {
t.Fatalf("purge: %v %+v", err, res)
}
if reads.Load() != 2 || deletes.Load() != 0 {
t.Errorf("reads=%d deletes on the replacement handle=%d", reads.Load(), deletes.Load())
}
}
+459
View File
@@ -0,0 +1,459 @@
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"errors"
"fmt"
"slices"
"sort"
"sync"
"time"
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
)
// An expiry of all versions of a key (ExpiredObjectAllVersions,
// DelMarkerExpiration, a batch expiry that keeps no versions) removes the
// versions of that key and nothing else. On the drives, object a/b lives
// inside the directory of object a, next to the metadata and data of a. The
// purge therefore never deletes that directory: it deletes the versions of
// the key one by one, by version ID, with the version delete of the drives,
// which removes the directory only once it is empty.
//
// It decides and deletes under the object lock that DeleteObject holds, on
// the authoritative history of the key in all pools (objectVersions):
// - an ILM expiry needs the latest version to expire with all versions as
// the scanner evaluates it on all of them (lifecycleEventsAllPools); a
// batch expiry needs the version it matched to be the latest still;
// - it deletes exactly the versions it approved, with every copy of them
// in every pool and the older writes of their IDs, and nothing else;
// - before it deletes anything it reads every drive of every set that
// holds a copy, and refuses the whole key when a drive cannot be read or
// when any revision of an approved version there is transitioned or,
// when the bucket has Object Lock, protected by retention or a legal
// hold. It deletes on exactly the drives it read;
// - it deletes the oldest version first and the latest version last and
// alone, once every older version is gone from every pool. A purge that
// stops early keeps the latest version, so it never exposes an older
// one, and the next expiry continues on a fresh history.
var (
// errDeleteAllChanged: the version a batch expiry matched is no longer
// the latest version of the key.
errDeleteAllChanged = fmt.Errorf("%w: the latest version changed", errLifecycleNotApplicable)
// errDeleteAllTiered: a version of the key is in a remote tier. Removing
// all versions is left to a later release, which hands the remote
// objects to the tier cleanup.
errDeleteAllTiered = fmt.Errorf("%w: a version is transitioned", errLifecycleNotApplicable)
// errDeleteAllBudget: the purge stopped at its budget, before the
// latest version.
errDeleteAllBudget = errors.New("purge of all versions stopped at its budget")
// errDeleteAllUnsupported: only a single-object delete of the pools
// purges all versions. Any other layer would turn the request into
// another delete.
errDeleteAllUnsupported = NotImplemented{Message: "purge of all versions is a single-object delete of the pools"}
)
// deleteAllBudget bounds one purge, which holds the object lock while it
// runs. It stops before the next version once it removed versions versions
// or ran for duration, its reads and checks included.
var deleteAllBudget = struct {
versions int
duration time.Duration
}{versions: 1000, duration: time.Minute}
// deleteAllRetries bounds how often a version delete that reached its write
// quorum is repeated on the drives that failed it.
const deleteAllRetries = 2
// deleteAllMatch identifies the latest version that a batch expiry matched.
type deleteAllMatch struct {
VersionID string
ModTime time.Time
}
// matches reports whether fi is the version m identifies.
func (m deleteAllMatch) matches(fi FileInfo) bool {
return deleteAllVersionID(m.VersionID) == deleteAllVersionID(fi.VersionID) && m.ModTime.Equal(fi.ModTime)
}
// deleteAllResult is what a purge of all versions confirmed.
type deleteAllResult struct {
// Latest is the latest version the purge approved.
Latest ObjectInfo
// Removed counts the versions, delete markers included, that are gone
// from every pool, Markers the delete markers among them. Free versions
// and further copies of a version do not count.
Removed, Markers int
// Started: the purge began to delete. A failure after that can have
// removed copies that Removed does not count.
Started bool
// Complete: every approved version is gone, the latest included.
Complete bool
}
// deleteAllVersionID returns versionID, "null" for the null version.
func deleteAllVersionID(versionID string) string {
if versionID == "" {
return nullVersionID
}
return versionID
}
// deleteAllInvalid returns why opts cannot ask for a purge of all versions.
// The purge is approved either as an ILM expiry or by a batch match, and it
// only deletes: it never creates a marker, addresses a single version,
// moves data or deletes a prefix.
func (opts ObjectOptions) deleteAllInvalid() error {
switch {
case opts.DeletePrefix || opts.DeletePrefixObject:
return errors.New("a purge of all versions is not a prefix delete")
case opts.VersionID != "":
return errors.New("a purge of all versions addresses no single version")
case opts.DataMovement:
return errors.New("a purge of all versions moves no data")
case opts.DeleteMarker || opts.Versioned || opts.VersionSuspended:
return errors.New("a purge of all versions creates no delete marker")
case opts.ReplicationRequest || opts.InclFreeVersions || opts.Transition != (TransitionOptions{}) ||
opts.CheckPrecondFn != nil || opts.EvalMetadataFn != nil || opts.EvalRetentionBypassFn != nil ||
opts.Expiration.CheckedAllPools:
return errors.New("a purge of all versions takes no options of other deletes")
case opts.Expiration.Expire == (opts.DeleteAllLatest != nil):
return errors.New("a purge of all versions needs exactly one approval")
}
return nil
}
// deleteAllCopy is the copy of a version in a pool: the revision of it that
// the authoritative history of the set of the key in that pool accepted.
// Copies of one write share its ModTime, since data movement keeps it and
// writes a new data dir. Other writes of a version ID, such as null versions
// written in several pools, have another ModTime.
type deleteAllCopy struct {
pool int
fi FileInfo
}
// deleteAllVersion is a version of the key, "null" for the null version, and
// its copies in all pools, the newest write first and, on equal times, the
// lower pool first. The first copy is the one reads choose.
type deleteAllVersion struct {
id string
copies []deleteAllCopy
}
// deleteAllHistory returns the versions of object in all pools, newest first,
// with the copies of each, from the authoritative history of each pool. It
// fails when a pool cannot tell the history.
func (z *erasureServerPools) deleteAllHistory(ctx context.Context, bucket, object string) ([]deleteAllVersion, error) {
poolVersions := make([][]FileInfo, len(z.serverPools))
errs := make([]error, len(z.serverPools))
var wg sync.WaitGroup
for i, pool := range z.serverPools {
wg.Add(1)
go func(i int, set *erasureObjects) {
defer wg.Done()
poolVersions[i], errs[i] = set.objectVersions(ctx, bucket, object)
}(i, pool.getHashedSet(object))
}
wg.Wait()
for _, err := range errs {
if err != nil {
return nil, err
}
}
var all []deleteAllCopy
for pool, versions := range poolVersions {
for _, fi := range versions {
if fi.ModTime.IsZero() {
// A write that cannot be ordered cannot be deleted in order.
return nil, toObjectErr(errErasureReadQuorum, bucket, object)
}
all = append(all, deleteAllCopy{pool: pool, fi: fi})
}
}
// The order of lifecycleEventsAllPools and of reads.
sort.SliceStable(all, func(i, j int) bool { return all[i].fi.ModTime.After(all[j].fi.ModTime) })
index := make(map[string]int, len(all))
var history []deleteAllVersion
for _, c := range all {
id := deleteAllVersionID(c.fi.VersionID)
if i, ok := index[id]; ok {
history[i].copies = append(history[i].copies, c)
continue
}
index[id] = len(history)
history = append(history, deleteAllVersion{id: id, copies: []deleteAllCopy{c}})
}
return history, nil
}
// deleteAllApproved reports whether history holds exactly the versions that
// the ILM evaluation approved, in the same order, the same writes chosen.
func deleteAllApproved(history []deleteAllVersion, approved []ObjectInfo) bool {
if len(history) != len(approved) {
return false
}
for i, v := range history {
got := v.copies[0].fi.ToObjectInfo("", "", false)
want := approved[i]
if v.id != deleteAllVersionID(want.VersionID) || !got.ModTime.Equal(want.ModTime) ||
got.DeleteMarker != want.DeleteMarker || got.ETag != want.ETag || got.Size != want.Size {
return false
}
}
return true
}
// deleteAllVersions deletes every version of object, whose expiry of all
// versions opts describe, and nothing else, see the comment above. The
// caller holds the object lock. It returns the approved latest version, and
// what it confirmed in opts.DeleteAllResult, also on an error. A key without
// versions is a success that removes nothing.
func (z *erasureServerPools) deleteAllVersions(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
res := opts.DeleteAllResult
if res == nil {
res = &deleteAllResult{}
}
*res = deleteAllResult{}
start := time.Now()
name := decodeDirObject(object)
fail := func(err error) (ObjectInfo, error) {
if isErrReadQuorum(err) {
err = InsufficientWriteQuorum{}
}
return ObjectInfo{Bucket: bucket, Name: name}, err
}
lr, err := globalBucketObjectLockSys.Get(bucket)
if err != nil {
return fail(err)
}
// Decide.
var approved []ObjectInfo
if opts.Expiration.Expire {
var all lifecycleAllPools
all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, object)
if all.err != nil || len(all.versions) > 0 {
if err := all.expiryApplies(bucket, object, ObjectOptions{DeletePrefix: true}); err != nil {
return fail(err)
}
}
approved = all.versions
}
// The placement of the approved versions. This read may only confirm
// them: whatever differs from the approval refuses the key.
history, err := z.deleteAllHistory(ctx, bucket, object)
if err != nil {
return fail(err)
}
if opts.Expiration.Expire && !deleteAllApproved(history, approved) {
return fail(InsufficientWriteQuorum{})
}
if len(history) == 0 {
res.Complete = true
return ObjectInfo{Bucket: bucket, Name: name}, nil
}
if opts.DeleteAllLatest != nil && !opts.DeleteAllLatest.matches(history[0].copies[0].fi) {
return fail(errDeleteAllChanged)
}
if opts.Expiration.Expire {
res.Latest = approved[0]
} else {
vc, _ := globalBucketVersioningSys.Get(bucket)
res.Latest = history[0].copies[0].fi.ToObjectInfo(bucket, object, vc != nil && vc.Versioned(name))
}
res.Latest.Bucket, res.Latest.Name = bucket, name
// Check every write before deleting any: the accepted revisions, then
// every revision on every drive that the purge will delete on.
ids := make(map[string]bool, len(history))
held := make([]bool, len(z.serverPools))
for _, v := range history {
ids[v.id] = true
for _, c := range v.copies {
held[c.pool] = true
if err := deleteAllRefuses(ctx, bucket, object, c.fi, lr.LockEnabled); err != nil {
return fail(err)
}
}
}
drives := make([][]StorageAPI, len(z.serverPools))
for pool := range z.serverPools {
if !held[pool] {
continue
}
if drives[pool], err = z.serverPools[pool].getHashedSet(object).deleteAllDrives(ctx, bucket, object, ids, lr.LockEnabled); err != nil {
return fail(err)
}
}
// Delete the oldest version first. The latest one is the last and is
// only reached once every older one is gone from every pool.
for i := len(history) - 1; i >= 0; i-- {
if err := ctx.Err(); err != nil {
return fail(err)
}
if res.Removed >= deleteAllBudget.versions || time.Since(start) >= deleteAllBudget.duration {
return fail(errDeleteAllBudget)
}
res.Started = true
if err := z.deleteAllCopies(ctx, bucket, object, history[i], drives); err != nil {
return fail(err)
}
res.Removed++
if history[i].copies[0].fi.Deleted {
res.Markers++
}
}
res.Complete = true
return res.Latest, nil
}
// deleteAllRefuses returns why the purge must keep the key that holds the
// revision fi of an approved version: it is transitioned, whose remote
// object a later release hands to the tier cleanup, or, with Object Lock,
// protected by retention or a legal hold.
func deleteAllRefuses(ctx context.Context, bucket, object string, fi FileInfo, lockEnabled bool) error {
if fi.TransitionStatus != "" || fi.TransitionTier != "" || fi.TransitionedObjName != "" {
return errDeleteAllTiered
}
if lockEnabled && enforceRetentionForDeletion(ctx, fi.ToObjectInfo(bucket, object, true)) {
return ObjectLocked{Bucket: bucket, Object: decodeDirObject(object), VersionID: deleteAllVersionID(fi.VersionID)}
}
return nil
}
// deleteAllDrives reads object on every drive of the set and checks each
// revision of the versions ids that a drive holds with deleteAllRefuses.
// A minority of drives can hold a revision other than the accepted one, and
// heal would later remove what the purge leaves on a drive, so a drive that
// cannot be read or decoded refuses the key. A drive without the object
// holds nothing to check. It returns the drive handles it read, which the
// purge deletes on; a handle that a reconnect installs later is not used. A
// handle names a drive, not a backend: a backend that replaces the drive
// behind it starts empty and receives this object only from a write or a
// heal of it, which take the object lock the purge holds.
func (er erasureObjects) deleteAllDrives(ctx context.Context, bucket, object string, ids map[string]bool, lockEnabled bool) ([]StorageAPI, error) {
disks := slices.Clone(er.getDisks())
raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
if err := ctx.Err(); err != nil {
return nil, err
}
for i := range disks {
if errors.Is(errs[i], errFileNotFound) {
continue
}
if errs[i] != nil {
return nil, toObjectErr(errErasureReadQuorum, bucket, object)
}
var xl xlMetaV2
if err := xl.LoadOrConvert(raws[i].Buf); err != nil {
return nil, toObjectErr(errErasureReadQuorum, bucket, object)
}
versions, err := xl.ListVersions(bucket, object, false)
if err != nil {
return nil, toObjectErr(errErasureReadQuorum, bucket, object)
}
for _, fi := range versions {
if fi.TierFreeVersion() || !ids[deleteAllVersionID(fi.VersionID)] {
continue
}
if err := deleteAllRefuses(ctx, bucket, object, fi, lockEnabled); err != nil {
return nil, err
}
}
}
return disks, nil
}
// deleteAllCopies deletes every copy of the version v: the older writes of
// its ID first, the copy that reads choose last, so that a failure leaves
// what a read returned before. It deletes on the drives that drives holds
// for the pool of a copy.
func (z *erasureServerPools) deleteAllCopies(ctx context.Context, bucket, object string, v deleteAllVersion, drives [][]StorageAPI) error {
for i := len(v.copies) - 1; i >= 0; i-- {
pool := v.copies[i].pool
set := z.serverPools[pool].getHashedSet(object)
if err := set.purgeVersion(ctx, bucket, object, v.id, drives[pool]); err != nil {
return toObjectErr(err, bucket, decodeDirObject(object), v.id)
}
}
return nil
}
// purgeVersion physically removes the version versionID of object from
// disks, the drives of the set that deleteAllDrives checked: no delete
// marker, no replication state, no free version. A drive that does not hold
// the version counts as done, so that a retry is idempotent. When the delete
// reaches its write quorum but fails on some drives, it is repeated on those
// while the caller still holds the object lock; what remains is left to
// heal. The caller holds the object lock.
func (er erasureObjects) purgeVersion(ctx context.Context, bucket, object, versionID string, disks []StorageAPI) error {
writeQuorum := len(disks)/2 + 1
fi := FileInfo{Name: object, VersionID: versionID}
// The purge checked that no drive it deletes on holds a transitioned
// revision. A free version would hand a remote object to the tier
// cleanup, so none is made.
fi.SetSkipTierFreeVersion()
errs := make([]error, len(disks))
for attempt := 0; attempt <= deleteAllRetries; attempt++ {
g := errgroup.WithNErrs(len(disks))
for index := range disks {
if attempt > 0 && (errs[index] == nil || disks[index] == nil) {
continue
}
g.Go(func() error {
if disks[index] == nil {
return errDiskNotFound
}
err := disks[index].DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{})
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
return nil
}
return err
}, index)
}
for index, err := range g.Wait() {
if attempt == 0 || (errs[index] != nil && disks[index] != nil) {
errs[index] = err
}
}
if err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum); err != nil {
er.addPartial(bucket, object, versionID)
return err
}
if !slices.ContainsFunc(errs, func(err error) bool { return err != nil }) {
return nil
}
retry := false
for index, err := range errs {
retry = retry || err != nil && disks[index] != nil
}
if !retry {
break
}
}
er.addPartial(bucket, object, versionID)
return nil
}
File diff suppressed because it is too large Load Diff
+17
View File
@@ -1234,6 +1234,11 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
if err = checkDelObjArgs(ctx, bucket, object); err != nil {
return objInfo, err
}
if opts.DeleteAllVersions {
if err := opts.deleteAllInvalid(); err != nil {
return objInfo, InvalidArgument{Bucket: bucket, Object: object, Err: err}
}
}
if !opts.DeletePrefix { // DeletePrefix handles dir object encoding differently.
object = encodeDirObject(object)
@@ -1248,6 +1253,12 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
// A purge of all versions approves and deletes on the history it reads
// itself. The checks below decide by other options and would discard it.
if opts.DeleteAllVersions {
return z.deleteAllVersions(ctx, bucket, object, opts)
}
// An ILM expiry is decided on the versions in one pool. Under the lock,
// check it with the versions in all pools; the sets then take it as
// decided, without a recheck by the versions each one holds. They only
@@ -1496,6 +1507,12 @@ func deletesData(obj ObjectToDelete, opts ObjectOptions) bool {
func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
derrs := make([]error, len(objects))
dobjects := make([]DeletedObject, len(objects))
if opts.DeleteAllVersions {
for i := range derrs {
derrs[i] = errDeleteAllUnsupported
}
return dobjects, derrs
}
objSets := set.NewStringSet()
for i := range derrs {
objects[i].ObjectName = encodeDirObject(objects[i].ObjectName)
+3
View File
@@ -763,6 +763,9 @@ func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix st
// DeleteObject - deletes an object from the hashedSet based on the object name.
func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
if opts.DeleteAllVersions {
return objInfo, errDeleteAllUnsupported
}
if opts.DeletePrefix && !opts.DeletePrefixObject {
err := s.deletePrefix(ctx, bucket, object)
return ObjectInfo{}, err
+11
View File
@@ -103,6 +103,17 @@ type ObjectOptions struct {
DeletePrefix bool // set true to enforce a prefix deletion, only application for DeleteObject API,
DeletePrefixObject bool // set true when object's erasure set is resolvable by object name (using getHashedSetIndex)
// DeleteAllVersions asks DeleteObject to purge every version of exactly
// this key by version ID, for an ILM or batch expiry of all versions;
// see erasureServerPools.deleteAllVersions. In memory only.
DeleteAllVersions bool
// DeleteAllLatest is the latest version that a batch expiry matched.
// The purge of all versions only runs while it is still the latest.
DeleteAllLatest *deleteAllMatch
// DeleteAllResult, when set, receives what a purge of all versions
// confirmed, also when it fails part way.
DeleteAllResult *deleteAllResult
// Pools-layer version resolver; the caller holds the shared object lock.
replicaObjectInfo GetObjectInfoFn