minio/cmd/erasure-server-pool-delete-all-reasons_test.go

// Copyright (c) 2026 PGSTY
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"bytes"
	"errors"
	"slices"
	"strings"
	"sync"
	"testing"

	madmin "github.com/minio/madmin-go/v3"
	"github.com/minio/minio/internal/bucket/lifecycle"
)

// An expiry of all versions that does not remove the key is counted by why,
// traced with that reason, and sends no removal event. The scanner does not
// queue one that the purge would refuse for a tiered version it holds, and
// it accounts the versions of a key under such an expiry at their sizes.

// incompleteDelta runs f and returns by how much each reason of
// expiry_incomplete_total grew.
func incompleteDelta(f func()) map[string]int64 {
	before := globalILMIncomplete.snapshot()
	f()
	after := globalILMIncomplete.snapshot()
	delta := map[string]int64{}
	for r, n := range after {
		if d := n - before[r]; d != 0 {
			delta[r] = d
		}
	}
	return delta
}

func TestPurgeAllVersionsRefusalReasons(t *testing.T) {
	z := purgeSinglePool(t, 16)
	p := newPurgeTest(t, z, "purge-reasons", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
	o := observePurges(t)
	expect := func(what string, ops []expiryOp, reason, result string) map[string]string {
		t.Helper()
		o.drain()
		delta := incompleteDelta(func() { p.run(ops) })
		if delta[reason] != 1 {
			t.Errorf("%s: counted %v, want one %s", what, delta, reason)
		}
		events, traces := o.drain()
		if len(events) != 0 {
			t.Errorf("%s: sent %v", what, events)
		}
		if len(traces) != 1 || traces[0]["result"] != result || traces[0]["reason"] != reason {
			t.Fatalf("%s: traced %v, want one %s expiry for %s", what, traces, result, reason)
		}
		return traces[0]
	}

	// A version in a remote tier: refused, the key kept.
	p.tiered(0, "ea/tiered", "v1", 6)
	p.version(0, "ea/tiered", "v2", 5)
	expect("tiered", []expiryOp{p.deleteAllTask(0, "ea/tiered", lifecycle.DeleteAllVersionsAction)}, ilmIncompleteTierUnsupported, "refused")
	if got := p.held(0, "ea/tiered"); !slices.Equal(got, []string{"hot", "tiered"}) {
		t.Errorf("tiered key holds %v", got)
	}

	// A drive that cannot be read: failed, the key kept.
	p.version(0, "ea/unreadable", "v1", 5)
	ops := []expiryOp{p.deleteAllTask(0, "ea/unreadable", lifecycle.DeleteAllVersionsAction)}
	sets := z.serverPools[0]
	index := sets.getHashedSetIndex("ea/unreadable")
	sets.erasureDisksMu.Lock()
	prior := sets.erasureDisks[index][0]
	sets.erasureDisks[index][0] = authorityDisk{StorageAPI: prior, bucket: p.bucket, key: "ea/unreadable", failRead: true}
	sets.erasureDisksMu.Unlock()
	expect("unreadable drive", ops, ilmIncompleteHistoryUnavailable, "failed")
	sets.erasureDisksMu.Lock()
	sets.erasureDisks[index][0] = prior
	sets.erasureDisksMu.Unlock()
	if got := p.held(0, "ea/unreadable"); !slices.Equal(got, []string{"hot"}) {
		t.Errorf("key behind an unreadable drive holds %v", got)
	}

	// The budget: partial, with the versions removed so far; the next
	// expiry continues.
	for i, label := range []string{"v1", "v2", "v3"} {
		p.version(0, "ea/budget", label, float64(6-i))
	}
	saved := deleteAllBudget
	deleteAllBudget.versions = 1
	tr := expect("budget", []expiryOp{p.deleteAllTask(0, "ea/budget", lifecycle.DeleteAllVersionsAction)}, ilmIncompleteBudget, "partial")
	deleteAllBudget = saved
	if tr[ilmPartial] != "true" || tr[ilmVersionsRemoved] != "1" {
		t.Errorf("budget: traced %v, want a partial expiry of 1 version", tr)
	}
	if got := p.held(0, "ea/budget"); !slices.Equal(got, []string{"hot", "hot"}) {
		t.Errorf("after the budget the key holds %v, want the two newer versions", got)
	}
	p.expire(0, "ea/budget")
	if got := p.held(0, "ea/budget"); got != nil {
		t.Errorf("the next expiry left %v", got)
	}

	// Only a batch expiry names the latest version it matched; a changed
	// one is a refusal too.
	for err, want := range map[error]string{
		errDeleteAllChanged:            ilmIncompleteChanged,
		errDeleteAllTiered:             ilmIncompleteTierUnsupported,
		errDeleteAllBudget:             ilmIncompleteBudget,
		errDeleteAllHistoryUnavailable: ilmIncompleteHistoryUnavailable,
		errLifecycleNotApplicable:      ilmIncompleteNotApplicable,
	} {
		reason, counted := ilmExpiryNotDone(err)
		if !counted || reason != want {
			t.Errorf("%v: reason %q counted %v, want %q", err, reason, counted, want)
		}
		if refused := ilmExpiryRefused(reason); refused != (want == ilmIncompleteChanged || want == ilmIncompleteTierUnsupported || want == ilmIncompleteNotApplicable) {
			t.Errorf("%s: refused %v", reason, refused)
		}
	}
	// To an API caller the unreadable drive is a write quorum error.
	if !isErrWriteQuorum(errDeleteAllHistoryUnavailable) || toAPIErrorCode(t.Context(), errDeleteAllHistoryUnavailable) != ErrSlowDownWrite {
		t.Errorf("%v is not a write quorum error to callers", errDeleteAllHistoryUnavailable)
	}
}

// A drive whose metadata does not decode refuses the purge, as an
// unreadable one does. Healing each surviving version restores the metadata
// from a quorum before the next purge can remove the key.
func TestPurgeAllVersionsRefusesCorruptDrive(t *testing.T) {
	z := purgeSinglePool(t, 16)
	p := newPurgeTest(t, z, "purge-corrupt", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
	ctx := t.Context()
	const key = "ea/corrupt"
	p.put(0, key, "null", daysAgo(7), ObjectOptions{})
	p.version(0, key, "v1", 6)
	p.version(0, key, "v2", 5)
	ops := []expiryOp{p.deleteAllTask(0, key, lifecycle.DeleteAllVersionsAction)}

	disks := z.serverPools[0].getHashedSet(key).getDisks()
	bad := disks[len(disks)-1]
	metaPath := pathJoin(key, xlStorageFormatFile)
	good, err := bad.ReadAll(ctx, p.bucket, metaPath)
	if err != nil {
		t.Fatal(err)
	}
	buf := slices.Clone(good)
	meta, _, err := isIndexedMetaV2(buf)
	if err != nil {
		t.Fatal(err)
	}
	meta[len(meta)-1] ^= 1
	if err := bad.WriteAll(ctx, p.bucket, metaPath, buf); err != nil {
		t.Fatal(err)
	}
	if raw, err := bad.ReadXL(ctx, p.bucket, key, false); err != nil {
		t.Fatalf("the corrupted metadata is not read: %v", err)
	} else if err := (&xlMetaV2{}).LoadOrConvert(raw.Buf); !errors.Is(err, errFileCorrupt) {
		t.Fatalf("CRC classification: %v", err)
	} else if got := toStorageErr(errors.New(err.Error())); got != errFileCorrupt {
		t.Fatalf("CRC classification after storage RPC: %v", got)
	}

	var healMu sync.Mutex
	queued := make(map[string]bool)
	hook := func(op PartialOperation) {
		if op.Bucket == p.bucket && op.Object == key {
			healMu.Lock()
			queued[op.VersionID] = true
			healMu.Unlock()
		}
	}
	previousHook := mrfPartialOpHook.Swap(&hook)
	t.Cleanup(func() { mrfPartialOpHook.Store(previousHook) })
	delta := incompleteDelta(func() { p.run(ops) })
	if delta[ilmIncompleteHistoryUnavailable] != 1 {
		t.Errorf("counted %v, want one %s", delta, ilmIncompleteHistoryUnavailable)
	}
	// The purge names the cause, and callers see a write quorum error.
	_, err = z.DeleteObject(ctx, p.bucket, key, ObjectOptions{DeleteAllVersions: true, Expiration: ExpirationOptions{Expire: true}})
	if !errors.Is(err, errDeleteAllHistoryUnavailable) || toAPIErrorCode(ctx, err) != ErrSlowDownWrite {
		t.Errorf("purge on a corrupt drive: %v, want %v as SlowDownWrite", err, errDeleteAllHistoryUnavailable)
	}
	if got := p.get(key); got != "v2" {
		t.Fatalf("the refused purge left %s readable", got)
	}

	versions, err := getFileInfoVersions(good, p.bucket, key, false)
	if err != nil {
		t.Fatal(err)
	}
	mrfPartialOpHook.Store(previousHook)
	healMu.Lock()
	for _, fi := range versions.Versions {
		if !queued[deleteAllVersionID(fi.VersionID)] {
			t.Errorf("no MRF heal for %q: %v", fi.VersionID, queued)
		}
	}
	if queued[""] {
		t.Error("MRF used latest instead of explicit null version")
	}
	healMu.Unlock()
	for _, mode := range []madmin.HealScanMode{madmin.HealNormalScan, madmin.HealDeepScan} {
		if err := bad.WriteAll(ctx, p.bucket, metaPath, buf); err != nil {
			t.Fatal(err)
		}
		if _, err := z.HealObject(ctx, p.bucket, key, nullVersionID, madmin.HealOpts{DryRun: true, ScanMode: mode}); err != nil {
			t.Fatal(err)
		}
		unchanged, err := bad.ReadAll(ctx, p.bucket, metaPath)
		if err != nil || !bytes.Equal(unchanged, buf) {
			t.Fatalf("dry-run changed corrupt metadata: %v", err)
		}
		for _, fi := range versions.Versions {
			id := fi.VersionID
			if id == "" {
				id = nullVersionID
			}
			if _, err := z.HealObject(ctx, p.bucket, key, id, madmin.HealOpts{ScanMode: mode}); err != nil {
				t.Fatalf("heal %s: %v", id, err)
			}
			if _, err := bad.ReadVersion(ctx, "", p.bucket, key, id, ReadOptions{}); err != nil {
				t.Fatalf("repaired version %s unreadable: %v", id, err)
			}
		}
		if got := p.get(key); got != "v2" {
			t.Fatalf("heal changed latest data: %s", got)
		}
	}
	p.run([]expiryOp{p.deleteAllTask(0, key, lifecycle.DeleteAllVersionsAction)})
	if got := p.held(0, key); got != nil {
		t.Errorf("the purge after the repair left %v", got)
	}
}

// Half of the drives of an eight-drive set hold metadata that does not
// decode. The key still reads and heals from the other half, but the drives
// cannot tell its history, so the purge refuses the key before it checks the
// drives. It still queues a heal of every version the decodable drives hold,
// the null version by its explicit ID, so that the next purge finds the key
// whole; the drives refuse nothing once healed.
func TestPurgeAllVersionsHealsUndecodableHistory(t *testing.T) {
	z := purgeSinglePool(t, 8)
	p := newPurgeTest(t, z, "purge-undecodable", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
	ctx := t.Context()
	const key = "ea/undecodable"
	p.put(0, key, "null", daysAgo(7), ObjectOptions{})
	p.version(0, key, "v1", 6)
	p.version(0, key, "v2", 5)
	ops := []expiryOp{p.deleteAllTask(0, key, lifecycle.DeleteAllVersionsAction)}

	disks := z.serverPools[0].getHashedSet(key).getDisks()
	metaPath := pathJoin(key, xlStorageFormatFile)
	good, err := disks[0].ReadAll(ctx, p.bucket, metaPath)
	if err != nil {
		t.Fatal(err)
	}
	corrupt := slices.Clone(good)
	meta, _, err := isIndexedMetaV2(corrupt)
	if err != nil {
		t.Fatal(err)
	}
	meta[len(meta)-1] ^= 1
	bad := disks[len(disks)/2:]
	for _, disk := range bad {
		if err := disk.WriteAll(ctx, p.bucket, metaPath, corrupt); err != nil {
			t.Fatal(err)
		}
	}
	if got := p.get(key); got != "v2" {
		t.Fatalf("fixture: with half of the drives undecodable the key reads %s, want v2", got)
	}

	var healMu sync.Mutex
	queued := make(map[string]bool)
	hook := func(op PartialOperation) {
		if op.Bucket == p.bucket && op.Object == key {
			healMu.Lock()
			queued[op.VersionID] = true
			healMu.Unlock()
		}
	}
	previousHook := mrfPartialOpHook.Swap(&hook)
	t.Cleanup(func() { mrfPartialOpHook.Store(previousHook) })
	delta := incompleteDelta(func() { p.run(ops) })
	if delta[ilmIncompleteHistoryUnavailable] != 1 {
		t.Errorf("counted %v, want one %s", delta, ilmIncompleteHistoryUnavailable)
	}
	_, err = z.DeleteObject(ctx, p.bucket, key, ObjectOptions{DeleteAllVersions: true, Expiration: ExpirationOptions{Expire: true}})
	if !isErrWriteQuorum(err) {
		t.Errorf("purge on an undecodable history: %v, want a write quorum error", err)
	}
	if got := p.get(key); got != "v2" {
		t.Fatalf("the refused purge left %s readable", got)
	}
	mrfPartialOpHook.Store(previousHook)

	versions, err := getFileInfoVersions(good, p.bucket, key, false)
	if err != nil {
		t.Fatal(err)
	}
	healMu.Lock()
	for _, fi := range versions.Versions {
		if !queued[deleteAllVersionID(fi.VersionID)] {
			t.Errorf("no MRF heal for %q: %v", fi.VersionID, queued)
		}
	}
	if queued[""] {
		t.Error("MRF used latest instead of explicit null version")
	}
	healMu.Unlock()

	for _, fi := range versions.Versions {
		id := deleteAllVersionID(fi.VersionID)
		if _, err := z.HealObject(ctx, p.bucket, key, id, madmin.HealOpts{}); err != nil {
			t.Fatalf("heal %s: %v", id, err)
		}
		for i, disk := range bad {
			if _, err := disk.ReadVersion(ctx, "", p.bucket, key, id, ReadOptions{}); err != nil {
				t.Fatalf("version %s on healed drive %d: %v", id, i, err)
			}
		}
	}
	if got := p.get(key); got != "v2" {
		t.Fatalf("heal changed latest data: %s", got)
	}
	p.run([]expiryOp{p.deleteAllTask(0, key, lifecycle.DeleteAllVersionsAction)})
	if got := p.held(0, key); got != nil {
		t.Errorf("the purge after the heal left %v", got)
	}
}

// The scanner of a pool that holds a tiered version of a key whose latest
// version is hot does not queue the expiry of all versions, which the purge
// would refuse, and counts that it skipped it. It accounts every version of
// a key under an expiry of all versions at its size, queued or not.
func TestScannerSkipsDeleteAllForTieredKey(t *testing.T) {
	z := purgeSinglePool(t, 16)
	p := newPurgeTest(t, z, "purge-scan-tiered", MakeBucketOptions{VersioningEnabled: true}, purgeRules)
	ctx := t.Context()

	p.tiered(0, "ea/tiered", "v1", 6)
	p.version(0, "ea/tiered", "v2", 5)
	p.tiered(0, "dm/tiered", "v1", 6)
	p.marker(0, "dm/tiered", 5)
	// A tiered latest version takes the transitioned expiry.
	p.version(0, "ea/tiered-latest", "v1", 6)
	p.tiered(0, "ea/tiered-latest", "v2", 5)
	// A free version is not a tiered version.
	fv := p.tiered(0, "ea/free", "v1", 6)
	if _, err := z.DeleteObject(ctx, p.bucket, "ea/free", ObjectOptions{Versioned: true, VersionID: fv.VersionID}); err != nil {
		t.Fatal(err)
	}
	p.version(0, "ea/free", "v2", 5)
	p.version(0, "ea/hot", "v1", 6)
	p.version(0, "ea/hot", "v2", 5)

	for _, tc := range []struct {
		key    string
		queued bool
	}{
		{"ea/tiered", false},
		{"dm/tiered", false},
		{"ea/tiered-latest", true},
		{"ea/free", true},
		{"ea/hot", true},
	} {
		var ops []expiryOp
		var sizes map[string]int64
		delta := incompleteDelta(func() { ops, sizes = p.scan(0, tc.key) })
		if got := queuesDeleteAll(ops); got != tc.queued {
			t.Errorf("%s: queued an expiry of all versions %v, want %v: %v", tc.key, got, tc.queued, ops)
		}
		skipped := int64(0)
		if !tc.queued {
			skipped = 1
		}
		if delta[ilmIncompleteTierSkipped] != skipped {
			t.Errorf("%s: counted %v, want %d %s", tc.key, delta, skipped, ilmIncompleteTierSkipped)
		}
		for _, oi := range p.scanned(0, tc.key) {
			size, _ := oi.GetActualSize()
			if got := sizes[deleteAllVersionID(oi.VersionID)]; got != size {
				t.Errorf("%s: version %s accounted at %d, want its size %d", tc.key, oi.VersionID, got, size)
			}
		}
	}

	// The skipped keys keep their versions and remote objects, and the
	// queued hot key is purged.
	remotes := p.remotes()
	for _, key := range []string{"ea/tiered", "dm/tiered", "ea/free", "ea/hot"} {
		if ops := p.scanQueue(0, key); len(ops) > 0 {
			p.run(ops)
		}
	}
	for key, want := range map[string][]string{
		"ea/tiered": {"hot", "tiered"},
		"dm/tiered": {"DM", "tiered"},
		"ea/free":   {"free"},
		"ea/hot":    nil,
	} {
		if got := p.held(0, key); !slices.Equal(got, want) {
			t.Errorf("%s holds %v, want %v", key, got, want)
		}
	}
	if got := p.remotes(); got != remotes {
		t.Errorf("tier holds %d remote objects, want %d", got, remotes)
	}
}

// A lifecycle expiry never deletes a prefix: the old combination of options
// that removed the keys under a key is refused, and a force delete of a
// prefix still works.
func TestPoolsDeleteObjectRejectsExpirePrefix(t *testing.T) {
	z := purgeSinglePool(t, 16)
	p := newPurgeTest(t, z, "purge-prefix-expire", MakeBucketOptions{VersioningEnabled: true}, "")
	ctx := t.Context()
	p.version(0, "pfx/key", "v1", 5)
	p.version(0, "pfx/key/child", "v1", 5)
	for _, opts := range []ObjectOptions{
		{DeletePrefix: true, Expiration: ExpirationOptions{Expire: true}},
		{DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}},
	} {
		_, err := z.DeleteObject(ctx, p.bucket, "pfx/key", opts)
		var invalid InvalidArgument
		if !errors.As(err, &invalid) || !strings.Contains(err.Error(), "never deletes a prefix") {
			t.Errorf("%+v: err = %v, want InvalidArgument", opts, err)
		}
	}
	for _, key := range []string{"pfx/key", "pfx/key/child"} {
		if got := p.held(0, key); !slices.Equal(got, []string{"hot"}) {
			t.Errorf("%s holds %v after a refused delete", key, got)
		}
	}
	if _, err := z.DeleteObject(ctx, p.bucket, "pfx/key", ObjectOptions{DeletePrefix: true}); err != nil {
		t.Fatalf("force delete of the prefix: %v", err)
	}
	for _, key := range []string{"pfx/key", "pfx/key/child"} {
		if got := p.get(key); got != "-" {
			t.Errorf("%s reads %s after a force delete", key, got)
		}
	}
}