// Copyright (c) 2026 Feng Ruohang // // This file is part of Silo Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "fmt" "net/http" "net/http/httptest" "sync/atomic" "testing" "time" "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/versioning" xhttp "github.com/minio/minio/internal/http" ) // An explicit-version read uses ReadVersion. Only the added lifecycle scan // uses ReadXL, so its errors/absence can be injected without failing the read. type lifecycleReadDisk struct { StorageAPI bucket, object string scans *atomic.Int64 err error } func (d lifecycleReadDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) { if bucket == d.bucket && object == d.object { d.scans.Add(1) if d.err != nil { return RawFileInfo{}, d.err } } return d.StorageAPI.ReadXL(ctx, bucket, object, data) } func lifecycleReadDisks(t *testing.T, s *splitILM, object string, failedPool int, err error) *atomic.Int64 { t.Helper() var scans atomic.Int64 for p, pool := range s.z.serverPools { pool.erasureDisksMu.Lock() previous := pool.erasureDisks wrapped := make([][]StorageAPI, len(previous)) for i, disks := range previous { wrapped[i] = append([]StorageAPI(nil), disks...) for j, disk := range disks { var injected error if p == failedPool { injected = err } wrapped[i][j] = lifecycleReadDisk{StorageAPI: disk, bucket: s.bucket, object: object, scans: &scans, err: injected} } } pool.erasureDisks = wrapped pool.erasureDisksMu.Unlock() t.Cleanup(func() { pool.erasureDisksMu.Lock() pool.erasureDisks = previous pool.erasureDisksMu.Unlock() }) } return &scans } // Real signed GET and HEAD requests use the author's two-pool storage fixture. // Checking the queue as well as the response catches both false hiding and // accidentally authorizing the latest version instead of the version read. func TestPoolsLifecycleReadHTTP(t *testing.T) { const bothRules = `bothEnabledboth-` + `11` for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} { t.Run(string(state), func(t *testing.T) { s := newSplitILM(t, state, splitILMRules+bothRules) router := initTestAPIEndPoints(s.z, nil) for _, tc := range []struct { name, prefix string latestAge float64 readLatest bool implicit bool action lifecycle.Action local lifecycle.Action fault error wantScan bool // skipped is the reason the recheck counts and traces when // it serves the read instead of the local expiry. skipped string }{ {name: "old-version-new-latest", prefix: "exp-", latestAge: 0.01, local: lifecycle.DeleteAction, wantScan: true, skipped: ilmIncompleteNotApplicable}, {name: "old-version-expired-latest", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, wantScan: true, skipped: ilmIncompleteNotApplicable}, {name: "old-version-delete-all-latest", prefix: "eav-", latestAge: 3, local: lifecycle.DeleteAllVersionsAction, wantScan: true, skipped: ilmIncompleteNotApplicable}, {name: "expired-current", prefix: "exp-", latestAge: 3, readLatest: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true}, {name: "expired-current-implicit", prefix: "exp-", latestAge: 3, readLatest: true, implicit: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true}, {name: "expired-noncurrent", prefix: "both-", latestAge: 3, local: lifecycle.DeleteAction, action: lifecycle.DeleteVersionAction, wantScan: true}, {name: "extra-read-quorum-error", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errDiskNotFound, wantScan: true, skipped: ilmIncompleteHistoryUnavailable}, {name: "read-version-missing-from-recheck", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errFileNotFound, wantScan: true, skipped: ilmIncompleteNotApplicable}, {name: "local-none-global-expired", prefix: "ncv-", latestAge: 3, local: lifecycle.NoneAction}, } { for _, null := range []bool{false, true} { for oldPool := range 2 { t.Run(fmt.Sprintf("%s/null=%t/old-pool=%d", tc.name, null, oldPool), func(t *testing.T) { key := fmt.Sprintf("%s%s-%t-%d", tc.prefix, tc.name, null, oldPool) old := splitAt("v1", 5, oldPool) old.null = null s.write(key, old) s.write(key, splitAt("v2", tc.latestAge, 1-oldPool)) label := "v1" if tc.readLatest { label = "v2" } vid := s.ids[key+" "+label] if vid == "" { vid = nullVersionID } oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: state == versioning.Enabled, VersionSuspended: state == versioning.Suspended}) if err != nil { t.Fatal(err) } lc, err := globalLifecycleSys.Get(s.bucket) if err != nil { t.Fatal(err) } if local := lc.Eval(oi.ToLifecycleOpts()); local.Action != tc.local { t.Fatalf("fixture local event=%v, want %v", local.Action, tc.local) } versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key) if err != nil { t.Fatal(err) } index := 1 if tc.readLatest { index = 0 } if tc.local == lifecycle.NoneAction && events[index].Action != lifecycle.DeleteVersionAction { t.Fatal("fixture must be globally expired but locally NoneAction") } scans := lifecycleReadDisks(t, s, key, oldPool, tc.fault) observer := observePurges(t) for _, method := range []string{http.MethodGet, http.MethodHead} { t.Run(method, func(t *testing.T) { scans.Store(0) requestID := vid if tc.implicit { requestID = "" } observer.drain() var rec *httptest.ResponseRecorder delta := incompleteDelta(func() { rec = consistencyRequest(t, router, method, s.bucket, key, requestID) }) // A read the recheck serves instead of the local // expiry is counted and traced as skipped, by why. _, traces := observer.drain() var skipped []map[string]string for _, tr := range traces { if tr["result"] == "skipped" { skipped = append(skipped, tr) } } switch { case tc.skipped == "" && (len(delta) != 0 || len(skipped) != 0): t.Errorf("counted %v and traced %v, want nothing skipped", delta, skipped) case tc.skipped != "" && (len(delta) != 1 || delta[tc.skipped] != 1): t.Errorf("counted %v, want one %s", delta, tc.skipped) case tc.skipped != "" && (len(skipped) != 1 || skipped[0]["reason"] != tc.skipped || s.key(key, skipped[0]["version-id"]) != s.key(key, vid)): t.Errorf("traced %v, want one skipped expiry of %s for %s", skipped, vid, tc.skipped) } want := http.StatusOK if tc.action.Delete() { want = http.StatusNotFound } if rec.Code != want { t.Errorf("status=%d want=%d body=%s", rec.Code, want, rec.Body.String()) } if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" "+label { t.Errorf("body=%q, want requested version %s", rec.Body.String(), label) } if !tc.implicit && (scans.Load() > 0) != tc.wantScan { t.Errorf("ReadXL calls=%d, want scan=%t", scans.Load(), tc.wantScan) } if tc.action.Delete() { select { case op := <-s.queue: task, ok := op.(expiryTask) src := lcEventSrc_s3GetObject if method == http.MethodHead { src = lcEventSrc_s3HeadObject } global := versions[index] if !ok || task.src != src || task.event != events[index] || task.event.Action != tc.action || task.objInfo.VersionID != global.VersionID || task.objInfo.IsLatest != global.IsLatest || task.objInfo.NumVersions != global.NumVersions || !task.objInfo.SuccessorModTime.Equal(global.SuccessorModTime) { t.Errorf("wrong queued expiry: %+v; want %s with global event %+v", task, vid, events[index]) } default: t.Error("missing expiry task") } } if n := len(s.queue); n != 0 { t.Errorf("unexpected expiry tasks: %d", n) for len(s.queue) > 0 { <-s.queue } } }) } }) } } } }) } } func TestPoolsLifecycleReadAuthorizationAndClose(t *testing.T) { s := newSplitILM(t, versioning.Enabled, splitILMRules) initConfigSubsystem(t.Context(), s.z) globalIAMSys.Init(t.Context(), s.z, globalEtcdClient, 2*time.Second) denied := newObjectAttributesAuthzUser(t, "lifecycle-read", s.bucket, `"s3:PutObject"`) router := initTestAPIEndPoints(s.z, nil) for _, expire := range []bool{false, true} { key := fmt.Sprintf("exp-reader-%t", expire) v := splitAt("v1", 5, 0) v.size = 1 << 20 // a non-inline reader retains the object read lock s.write(key, v) if !expire { s.write(key, splitAt("v2", 0.01, 1)) } vid := s.ids[key+" v1"] oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true}) if err != nil || oi.Inlined { t.Fatalf("fixture must retain a reader lock: inlined=%t err=%v", oi.Inlined, err) } scans := lifecycleReadDisks(t, s, key, -1, nil) for _, method := range []string{http.MethodGet, http.MethodHead} { req, err := newTestSignedRequestV4(method, getGetObjectURL("", s.bucket, key)+"?versionId="+vid, 0, nil, denied.AccessKey, denied.SecretKey, nil) if err != nil { t.Fatal(err) } rec := httptest.NewRecorder() router.ServeHTTP(rec, req) if rec.Code != http.StatusForbidden || scans.Load() != 0 || len(s.queue) != 0 { t.Fatalf("denied %s: status=%d scans=%d queued=%d", method, rec.Code, scans.Load(), len(s.queue)) } } rec := consistencyRequest(t, router, http.MethodGet, s.bucket, key, vid) want := http.StatusOK if expire { want = http.StatusNotFound } if rec.Code != want { t.Errorf("authorized GET status=%d want=%d", rec.Code, want) } // The reader owns the multi-pool read lock until Close, including the // early 404 path. A writer must acquire it immediately after the handler. lk := s.z.NewNSLock(s.bucket, key) lkctx, err := lk.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second)) if err != nil { t.Fatalf("GET leaked its reader/lock: %v", err) } lk.Unlock(lkctx) for len(s.queue) > 0 { <-s.queue } } } func TestPoolsLifecycleReadBoundaries(t *testing.T) { const transitionRules = `transitionEnabledtransition-` + `1WARM` for _, tc := range []struct { name string state versioning.State null bool singlePool bool restored bool transition bool action lifecycle.Action wantScan bool }{ {name: "single-pool", state: versioning.Enabled, singlePool: true, action: lifecycle.DeleteAction}, {name: "single-pool-null", state: versioning.Enabled, singlePool: true, null: true, action: lifecycle.DeleteAction}, {name: "unversioned", null: true, action: lifecycle.DeleteAction}, {name: "current-null-enabled", state: versioning.Enabled, null: true, action: lifecycle.DeleteAction, wantScan: true}, {name: "current-null-suspended", state: versioning.Suspended, null: true, action: lifecycle.DeleteAction, wantScan: true}, {name: "transition", state: versioning.Enabled, transition: true, action: lifecycle.TransitionAction}, {name: "restored", state: versioning.Enabled, restored: true, action: lifecycle.DeleteRestoredAction}, } { t.Run(tc.name, func(t *testing.T) { s := newSplitILM(t, tc.state, splitILMRules+transitionRules) key := "exp-" + tc.name if tc.restored || tc.transition { key = "transition-" + tc.name } v := splitAt("v1", 5, 0) v.null, v.tiered = tc.null, tc.restored s.write(key, v) vid := s.ids[key+" v1"] if vid == "" { vid = nullVersionID } if tc.restored { globalTierConfigMgr.drivercache["WARM"] = &tierExpiryTestBackend{objects: map[string][]byte{"remote-v1": []byte(key + " v1")}} opts := ObjectOptions{VersionID: vid, Versioned: true} opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) { oi.UserDefined[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(-time.Hour)).String() return ReplicateDecision{}, nil } if _, err := s.z.PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil { t.Fatal(err) } } z := s.z if tc.singlePool { // Exercise the single-pool dispatch with the same storage fixture. z = &erasureServerPools{serverPools: s.z.serverPools[:1]} } router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"}) scans := lifecycleReadDisks(t, s, key, -1, nil) for _, method := range []string{http.MethodGet, http.MethodHead} { scans.Store(0) rec := consistencyRequest(t, router, method, s.bucket, key, vid) want := http.StatusOK if tc.action.Delete() && !tc.action.DeleteRestored() { want = http.StatusNotFound } if rec.Code != want || (scans.Load() > 0) != tc.wantScan { t.Errorf("%s status=%d want=%d scans=%d wantScan=%t body=%s", method, rec.Code, want, scans.Load(), tc.wantScan, rec.Body.String()) } if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" v1" { t.Errorf("%s lost object contents: %q", method, rec.Body.String()) } if tc.action.Delete() { select { case op := <-s.queue: task, ok := op.(expiryTask) if !ok || task.event.Action != tc.action || s.key(key, task.objInfo.VersionID) != s.key(key, vid) { t.Errorf("%s wrong expiry: %+v", method, task) } default: t.Errorf("%s missing expiry", method) } } if n := len(s.queue); n != 0 { t.Fatalf("%s queued %d unexpected expiries", method, n) } } }) } } func TestPoolsLifecycleReadProxy(t *testing.T) { s := newSplitILM(t, versioning.Enabled, splitILMRules) const key, body = "exp-proxied", "remote object remains readable" vid := mustGetUUID() target := newILMTestTarget(t) var remoteRequests atomic.Int64 registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) { remoteRequests.Add(1) w.Header().Set(xhttp.ContentLength, fmt.Sprint(len(body))) w.Header().Set(xhttp.LastModified, UTCNow().Add(-5*24*time.Hour).Format(http.TimeFormat)) w.Header().Set(xhttp.ETag, `"da39a3ee5e6b4b0d3255bfef95601890"`) w.Header().Set(xhttp.AmzVersionID, vid) if r.Method == http.MethodGet { _, _ = w.Write([]byte(body)) } }) setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket) // The delete-only target fixture omits credentials; proxy discovery clones // the target descriptor and requires its credentials field to be present. globalBucketTargetSys.Lock() globalBucketTargetSys.targetsMap[s.bucket][0].Credentials = &madmin.Credentials{AccessKey: "proxy-test"} globalBucketTargetSys.Unlock() stats := NewReplicationStats(t.Context(), nil) previous := globalReplicationStats.Swap(stats) t.Cleanup(func() { globalReplicationStats.Store(previous) }) scans := lifecycleReadDisks(t, s, key, -1, nil) router := initTestAPIEndPoints(s.z, nil) for _, method := range []string{http.MethodGet, http.MethodHead} { before := remoteRequests.Load() rec := consistencyRequest(t, router, method, s.bucket, key, vid) if rec.Code != http.StatusOK || remoteRequests.Load() == before || scans.Load() != 0 || len(s.queue) != 0 { t.Errorf("proxy %s: status=%d remote=%d scans=%d queued=%d body=%s", method, rec.Code, remoteRequests.Load()-before, scans.Load(), len(s.queue), rec.Body.String()) } if method == http.MethodGet && rec.Body.String() != body { t.Errorf("proxy body=%q", rec.Body.String()) } } }