// Copyright (c) 2026 Denis Babchuk // // This file is part of Silo Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "fmt" "io" "slices" "strings" "testing" "time" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/versioning" ) // The scanner and the walks of decommission and rebalance evaluate ILM on the // versions of a key in one pool: IsLatest, NumVersions and SuccessorModTime // count only these. A key can have versions in several pools: two concurrent // first writes, overwrites that went to another pool. The expiry they decide // runs in every pool. // splitILM lays out the versions of keys in the two pools of migrationILM. type splitILM struct { *migrationILM ids map[string]string // object and label -> version ID bodies map[string]string // body -> label // replication is the replication configuration the scanner runs with. replication replicationConfig } func newSplitILM(t *testing.T, status versioning.State, rules string) *splitILM { t.Helper() return &splitILM{migrationILM: newMigrationILM(t, status, false, rules), ids: map[string]string{}, bodies: map[string]string{}} } // splitVersion is a version of a key in a test layout. type splitVersion struct { pools []int // the pools that hold a copy of it label string days float64 // its age marker bool tiered bool // its data is in a remote tier null bool // a null version (unversioned or suspended bucket) size int // its size, at least the size of its label } func (s *splitILM) write(object string, v splitVersion) { s.t.Helper() ctx := s.t.Context() mtime := UTCNow().Add(-time.Duration(v.days * float64(24*time.Hour))) versionID := mustGetUUID() if v.null { versionID = "" } for _, pool := range v.pools { if v.marker { _, err := s.z.serverPools[pool].DeleteObject(ctx, s.bucket, object, ObjectOptions{Versioned: true, VersionID: versionID, DeleteMarker: true, MTime: mtime}) if err != nil { s.t.Fatal(err) } continue } opts := ObjectOptions{Versioned: !v.null, VersionID: versionID, MTime: mtime} if v.null && s.status() == versioning.Suspended { opts = ObjectOptions{VersionSuspended: true, MTime: mtime} } body := object + " " + v.label if len(body) < v.size { body += strings.Repeat(".", v.size-len(body)) } putConsistencyObject(s.t, s.z, s.bucket, object, pool, body, opts) s.bodies[body] = v.label if v.tiered { vid := versionID if vid == "" { vid = nullVersionID } infos, errs := readAllFileInfo(ctx, s.z.serverPools[pool].getHashedSet(object).getDisks(), "", s.bucket, object, vid, false, false) if errs[0] != nil { s.t.Fatal(errs[0]) } fi := infos[0] fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote-"+v.label if err := s.z.serverPools[pool].DecomTieredObject(ctx, s.bucket, object, fi, ObjectOptions{}); err != nil { s.t.Fatal(err) } } } if v.marker { s.markers[s.key(object, versionID)] = v.label } else { s.labels[s.key(object, versionID)] = v.label } s.ids[object+" "+v.label] = versionID s.track(object) } func (s *splitILM) status() versioning.State { vc, _ := globalBucketVersioningSys.Get(s.bucket) if vc == nil { return "" } return vc.Status } // scan applies ILM to every object in a pool as the scanner of a drive of // that pool does, from the xl.meta of that drive, and runs the queued // expiries as an expiry worker does. func (s *splitILM) scan(pool int) { s.t.Helper() ctx := s.t.Context() lc, err := globalLifecycleSys.Get(s.bucket) if err != nil { s.t.Fatal(err) } lr, err := globalBucketObjectLockSys.Get(s.bucket) if err != nil { s.t.Fatal(err) } vc, _ := globalBucketVersioningSys.Get(s.bucket) for _, object := range s.objects { disk := s.z.serverPools[pool].getHashedSet(object).getDisks()[0] rf, err := disk.ReadXL(ctx, s.bucket, object, false) if err != nil { continue // not in this pool } fivs, err := getFileInfoVersions(rf.Buf, s.bucket, object, false) if err != nil { s.t.Fatal(err) } objInfos := make([]ObjectInfo, len(fivs.Versions)) for i, fi := range fivs.Versions { objInfos[i] = fi.ToObjectInfo(s.bucket, object, vc != nil && vc.Versioned(object)) } item := scannerItem{bucket: s.bucket, objectName: object, lifeCycle: lc, replication: s.replication} item.applyActions(ctx, s.z, objInfos, lr, &sizeSummary{}, nil) } s.runQueue() } // head applies ILM to a version as a HEAD or GET with its versionId does. func (s *splitILM) head(object, label string) lifecycle.Action { s.t.Helper() ctx := s.t.Context() oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{VersionID: s.ids[object+" "+label]}) if err != nil { s.t.Fatal(err) } lc, err := globalLifecycleSys.Get(s.bucket) if err != nil { s.t.Fatal(err) } lr, err := globalBucketObjectLockSys.Get(s.bucket) if err != nil { s.t.Fatal(err) } evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi) if evt.Action.Delete() { applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi) } s.runQueue() return evt.Action } // runQueue runs the queued expiries with an expiry worker. func (s *splitILM) runQueue() { done := make(chan struct{}) go func() { defer close(done) globalExpiryState.Worker(s.queue) }() s.queue <- nil // the worker returns once it ran every expiry queued before <-done } // get returns the label of the version a GET of object reads, or "-" when it // reads as deleted. func (s *splitILM) get(object string) string { s.t.Helper() ctx := s.t.Context() oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{}) if err != nil { if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrMethodNotAllowed(err) { return "-" } return fmt.Sprintf("<%v>", err) } if oi.VersionID != "" && oi.VersionID != nullVersionID { return s.label(object, oi.VersionID, false) } // Copies of a null version in two pools are told apart by their data. gr, err := s.z.GetObjectNInfo(ctx, s.bucket, object, nil, nil, ObjectOptions{}) if err != nil { return fmt.Sprintf("<%v>", err) } defer gr.Close() data, err := io.ReadAll(gr) if err != nil { s.t.Fatal(err) } if l, ok := s.bodies[string(data)]; ok { return l } return "?" + string(data) } // views returns the versions of each object in each pool, newest first. func (s *splitILM) views() []map[string][]string { views := make([]map[string][]string, len(s.z.serverPools)) for pool := range s.z.serverPools { views[pool] = s.view(pool) } return views } type splitKey struct { name string versions []splitVersion before string // what a GET reads before ILM ran after string // what a GET reads once ILM ran want [][]string // versions in each pool once ILM ran, nil to skip } // run lays out keys, runs two scanner cycles over all pools and checks that // a GET of each key never reads anything but before or after, and the // versions each pool holds at the end. The second cycle must not change // anything. func (s *splitILM) run(keys []splitKey) { s.t.Helper() for _, k := range keys { for _, v := range k.versions { s.write(k.name, v) } if got := s.get(k.name); got != k.before { s.t.Fatalf("%s: fixture reads %s, want %s", k.name, got, k.before) } } var first []map[string][]string for cycle := 1; cycle <= 2; cycle++ { for pool := range s.z.serverPools { s.scan(pool) for _, k := range keys { if got := s.get(k.name); got != k.before && got != k.after { s.t.Errorf("%s: reads %s after the scanner of pool %d (cycle %d), want %s", k.name, got, pool, cycle, k.after) } } } if cycle == 1 { first = s.views() } } last := s.views() for _, k := range keys { if got := s.get(k.name); got != k.after { s.t.Errorf("%s: reads %s once ILM ran, want %s", k.name, got, k.after) } for pool := range last { if !slices.Equal(first[pool][k.name], last[pool][k.name]) { s.t.Errorf("%s: the second scanner cycle changed pool %d from %v to %v", k.name, pool, first[pool][k.name], last[pool][k.name]) } if k.want != nil && !slices.Equal(last[pool][k.name], k.want[pool]) { s.t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, last[pool][k.name], k.want[pool]) } } } } const splitILMRules = "expEnabledexp-" + "1" + "eavEnabledeav-" + "1true" + "edmEnablededm-" + "true" + "dmeEnableddme-" + "1" + "ncvEnabledncv-" + "1" + "exsEnabledexs-100" + "1" + "nnvEnablednnv-" + "11" func splitAt(label string, days float64, pools ...int) splitVersion { return splitVersion{pools: pools, label: label, days: days} } func splitMarker(label string, days float64, pools ...int) splitVersion { return splitVersion{pools: pools, label: label, days: days, marker: true} } func splitTiered(label string, days float64, pools ...int) splitVersion { return splitVersion{pools: pools, label: label, days: days, tiered: true} } func splitNull(label string, days float64, pools ...int) splitVersion { return splitVersion{pools: pools, label: label, days: days, null: true} } // splitILMOnePool are keys in one pool. ILM applies to them as usual. func splitILMOnePool(pool int) []splitKey { in := func(vs ...string) [][]string { want := [][]string{nil, nil} want[pool] = vs return want } return []splitKey{ {"exp-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")}, {"exp-one-tiered", []splitVersion{splitAt("v1", 5, pool), splitTiered("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")}, {"eav-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in()}, {"edm-one", []splitVersion{splitMarker("dm", 3, pool)}, "-", "-", in()}, {"dme-one", []splitVersion{splitAt("v1", 5, pool), splitMarker("dm", 3, pool)}, "-", "-", in()}, {"ncv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 0.01, pool)}, "v3", "v3", in("v3", "v2")}, {"nnv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 3, pool), splitAt("v4", 2, pool)}, "v4", "v4", in("v4", "v3")}, } } func TestPoolsLifecycleSplitObject(t *testing.T) { t.Run("versioned", func(t *testing.T) { s := newSplitILM(t, versioning.Enabled, splitILMRules) s.run([]splitKey{ // Expiration with Days applies to the current version v2 only. {"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, {"exp-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}}, {"exp-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, // v2 is old enough: one delete marker hides the key. {"exp-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}}, // The rule applies to v2 only by its size. {"exs-big-in-1", []splitVersion{splitAt("v1", 5, 0), {pools: []int{1}, label: "v2", days: 4, size: 200}}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}}, // The marker hides v1 and is not a marker without noncurrent // versions; v1 is not the current version. {"exp-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}}, // ExpiredObjectAllVersions applies to the current version v2 only. {"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, {"eav-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}}, {"eav-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, {"eav-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{nil, nil}}, // A copy of v1 in both pools, as an interrupted move leaves it. {"eav-copies", []splitVersion{splitAt("v1", 5, 0, 1)}, "v1", "-", [][]string{nil, nil}}, // The marker hides v1 in the other pool. {"edm-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}}, {"edm-marker-in-1", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{{"v1"}, {"dm"}}}, {"edm-copies", []splitVersion{splitMarker("dm", 3, 0, 1)}, "-", "-", [][]string{nil, nil}}, // v2 in the other pool is the current version, the marker does // not expire. {"dme-new-in-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"dm"}, {"v2"}}}, {"dme-new-in-0", []splitVersion{splitAt("v2", 0.01, 0), splitMarker("dm", 3, 1)}, "v2", "v2", [][]string{{"v2"}, {"dm"}}}, // The marker is the current version and expires with v1. {"dme-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{nil, nil}}, // v1 is noncurrent since v2 was written, v2 since v3 was. {"ncv-split", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1), splitAt("v3", 2, 0)}, "v3", "v3", [][]string{{"v3"}, {"v2"}}}, }) }) t.Run("versioned/one-pool", func(t *testing.T) { for pool := range 2 { s := newSplitILM(t, versioning.Enabled, splitILMRules) s.run(splitILMOnePool(pool)) } }) t.Run("versioned/head-version", func(t *testing.T) { s := newSplitILM(t, versioning.Enabled, splitILMRules) keys := []splitKey{ {"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, {"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, } for _, k := range keys { for _, v := range k.versions { s.write(k.name, v) } act := s.head(k.name, "v1") t.Logf("%s: HEAD of v1 evaluates %s", k.name, act) if got := s.get(k.name); got != k.after { t.Errorf("%s: reads %s after a HEAD of v1, want %s", k.name, got, k.after) } views := s.views() for pool := range views { if !slices.Equal(views[pool][k.name], k.want[pool]) { t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, views[pool][k.name], k.want[pool]) } } } }) t.Run("suspended", func(t *testing.T) { s := newSplitILM(t, versioning.Suspended, splitILMRules) s.run([]splitKey{ {"exp-null-old-in-0", []splitVersion{splitNull("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, }) }) t.Run("unversioned", func(t *testing.T) { s := newSplitILM(t, "", splitILMRules) s.run([]splitKey{ {"eav-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil}, {"exp-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil}, }) views := s.views() for object, want := range map[string]int{"eav-old-in-0": 1, "exp-old-in-0": 0} { if len(views[0][object]) != want || len(views[1][object]) != 1 { t.Errorf("%s: pools hold %v and %v, want %d and 1 copies", object, views[0][object], views[1][object], want) } } }) } // walk moves pool 0 out with a decommission or a rebalance pass, as move // does, but reports a decommission that did not complete instead of stopping // the test, so the reads are still checked. func (s *splitILM) walk(kind string) { s.t.Helper() ctx := s.t.Context() if kind == "rebalance" { migrationTestWalk(s.t, s.z, s.bucket, 0, kind) s.z.rebalMu.Lock() s.z.rebalMeta.PoolStats[0].Info.Status = rebalCompleted s.z.rebalMu.Unlock() return } if err := s.z.StartDecommission(ctx, 0); err != nil { s.t.Fatal(err) } s.z.doDecommissionInRoutine(ctx, 0) s.z.poolMetaMutex.RLock() info := *s.z.poolMeta.Pools[0].Decommission s.z.poolMetaMutex.RUnlock() if !info.Complete || info.Failed || info.ItemsDecommissionFailed > 0 { s.t.Errorf("decommission did not complete: complete=%t failed=%t itemsFailed=%d; pool 0 holds %v", info.Complete, info.Failed, info.ItemsDecommissionFailed, s.view(0)) } } // Rules that expire a whole object (DelMarkerExpiration, // ExpiredObjectAllVersions) or a delete marker without noncurrent versions // (ExpiredObjectDeleteMarker) depend on every version of the object. A walk // sees the versions in its own pool only. func TestPoolsMigrationLifecycleWholeObject(t *testing.T) { recent := 1.0 / 24 objects := []struct { name string versions []splitVersion // oldest first before string // what a GET reads before the walk after string // what a GET reads once ILM caught up want []string // versions in pool 1 once ILM caught up, newest first }{ // The marker expires and takes v1 with it. {"dme-one-pool", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", nil}, {"dme-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", nil}, // v2 is the current version, the marker does not expire. {"dme-newer-in-pool-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "dm"}}, // v2 expires and takes v1 with it. {"eav-one-pool", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "-", nil}, {"eav-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitAt("v2", 4, 0)}, "v2", "-", nil}, // v2 is the current version and does not expire. {"eav-newer-in-pool-1", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "v1"}}, // The marker hides v1, it is not a marker without noncurrent versions. {"edm-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}}, {"keep-marker", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}}, {"keep-versions", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "v2", []string{"v2", "v1"}}, } for _, kind := range []string{"decommission", "rebalance"} { t.Run(kind, func(t *testing.T) { s := newSplitILM(t, versioning.Enabled, splitILMRules) for _, o := range objects { for _, v := range o.versions { s.write(o.name, v) } if got := s.get(o.name); got != o.before { t.Fatalf("%s: fixture reads %s, want %s", o.name, got, o.before) } } s.walk(kind) for _, o := range objects { if got := s.get(o.name); got != o.before && got != o.after { t.Errorf("%s: reads %s after the %s, before ILM caught up; want %s or %s", o.name, got, kind, o.before, o.after) } } s.runQueue() s.scan(1) want := map[string][]string{} for _, o := range objects { want[o.name] = o.want if got := s.get(o.name); got != o.after { t.Errorf("%s: reads %s once ILM caught up, want %s", o.name, got, o.after) } } s.check(want) }) } }