// Copyright (c) 2026 PGSTY // // This file is part of Silo Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "slices" "testing" "time" "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/versioning" "github.com/minio/minio/internal/event" "github.com/minio/minio/internal/logger" loghttp "github.com/minio/minio/internal/logger/target/http" "github.com/minio/minio/internal/pubsub" xnet "github.com/pgsty/silo-pkg/v3/net" ) func TestILMExpiryNotDoneClassification(t *testing.T) { for _, tc := range []struct { err error reason string counted bool }{ {nil, "", false}, {ObjectNotFound{}, "", false}, {VersionNotFound{}, "", false}, {errLifecycleNotApplicable, ilmIncompleteNotApplicable, true}, {fmt.Errorf("wrapped: %w", errLifecycleNotApplicable), ilmIncompleteNotApplicable, true}, {ObjectLocked{}, ilmIncompleteLocked, true}, {InsufficientWriteQuorum{}, ilmIncompleteHistoryUnavailable, true}, {InsufficientReadQuorum{}, ilmIncompleteHistoryUnavailable, true}, {errFaultyDisk, ilmIncompleteError, true}, } { reason, counted := ilmExpiryNotDone(tc.err) if reason != tc.reason || counted != tc.counted { t.Errorf("%v: %q %v, want %q %v", tc.err, reason, counted, tc.reason, tc.counted) } } } // fixedDeleteResults is an object layer whose bulk deletes return errs. type fixedDeleteResults struct { ObjectLayer errs []error } func (o fixedDeleteResults) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { deleted := make([]DeletedObject, len(objects)) for i, obj := range objects { deleted[i] = DeletedObject{ObjectName: obj.ObjectName, VersionID: obj.VersionID} } return deleted, slices.Clone(o.errs) } // ilmTestAudit captures the ILMExpiry audit entries of bucket. func ilmTestAudit(t *testing.T, bucket string) <-chan map[string]any { t.Helper() if len(logger.AuditTargets()) != 0 { t.Fatal("unexpected pre-existing test audit targets") } entries := make(chan map[string]any, 16) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var entry struct { Error string `json:"error"` Tags map[string]any `json:"tags"` API struct{ Name, Bucket string } } if err := json.NewDecoder(r.Body).Decode(&entry); err == nil && entry.API.Name == "ILMExpiry" && entry.API.Bucket == bucket { tags := entry.Tags if tags == nil { tags = map[string]any{} } tags["error"] = entry.Error entries <- tags } w.WriteHeader(http.StatusOK) })) endpoint, err := xnet.ParseHTTPURL(server.URL) if err != nil { t.Fatal(err) } ctx := context.Background() if errs := logger.UpdateAuditWebhooks(ctx, map[string]loghttp.Config{"ilm": {Enabled: true, Name: "ilm", Endpoint: endpoint, BatchSize: 1, QueueSize: 128, MaxRetry: 1, RetryIntvl: time.Millisecond, HTTPTimeout: time.Second}}); len(errs) > 0 { t.Fatal(errs) } targets := logger.AuditTargets() t.Cleanup(func() { logger.UpdateAuditWebhooks(ctx, nil) for _, target := range targets { target.Cancel() } server.Close() }) return entries } // A refused or failed noncurrent purge is counted and traced with its reason, // a failure is audited, and only the version that was removed is announced. func TestDeleteObjectVersionsReportsRefusalAndFailure(t *testing.T) { s := newSplitILM(t, versioning.Enabled, splitILMRules) ctx := t.Context() audits := ilmTestAudit(t, s.bucket) done := make(chan struct{}) defer close(done) events := make(chan event.Event, 8) if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedAll), events, done, nil); err != nil { t.Fatal(err) } traces := make(chan madmin.TraceInfo, 8) if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil { t.Fatal(err) } before := globalILMIncomplete.snapshot() layer := fixedDeleteResults{ObjectLayer: s.z, errs: []error{errLifecycleNotApplicable, InsufficientWriteQuorum{}, nil}} toDel := []ObjectToDelete{ {ObjectV: ObjectV{ObjectName: "ncv-a", VersionID: "11111111-1111-1111-1111-111111111111"}}, {ObjectV: ObjectV{ObjectName: "ncv-b", VersionID: "22222222-2222-2222-2222-222222222222"}}, {ObjectV: ObjectV{ObjectName: "ncv-c", VersionID: "33333333-3333-3333-3333-333333333333"}}, } evs := []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}} deleteObjectVersions(ctx, layer, s.bucket, toDel, evs) after := globalILMIncomplete.snapshot() if d := after[ilmIncompleteNotApplicable] - before[ilmIncompleteNotApplicable]; d != 1 { t.Errorf("not_applicable moved by %d, want 1", d) } if d := after[ilmIncompleteHistoryUnavailable] - before[ilmIncompleteHistoryUnavailable]; d != 1 { t.Errorf("history_unavailable moved by %d, want 1", d) } var removed []string results := map[string]string{} for drained := false; !drained; { select { case ev := <-events: removed = append(removed, ev.S3.Object.Key) case tr := <-traces: if r := tr.Custom["result"]; r != "" { results[tr.Custom["version-id"]] = r } default: drained = true } } if !slices.Equal(removed, []string{"ncv-c"}) { t.Errorf("removal events for %v, want only ncv-c", removed) } if results[toDel[0].VersionID] != "refused" || results[toDel[1].VersionID] != "failed" { t.Errorf("traces %v, want refused and failed", results) } select { case tags := <-audits: if tags["ilm-incomplete-reason"] != ilmIncompleteHistoryUnavailable || tags["error"] == "" { t.Errorf("audit of the failure: %v", tags) } case <-time.After(5 * time.Second): t.Fatal("the failure was not audited") } select { case tags := <-audits: if _, failed := tags["ilm-incomplete-reason"]; failed { t.Errorf("a second incomplete expiry was audited: %v", tags) } case <-time.After(500 * time.Millisecond): } } // expiryEvent returns the removal events sent while run runs. func expiryEvents(t *testing.T, run func()) []event.Event { t.Helper() done := make(chan struct{}) defer close(done) events := make(chan event.Event, 8) if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedAll), events, done, nil); err != nil { t.Fatal(err) } run() var got []event.Event for { select { case ev := <-events: got = append(got, ev) default: return got } } } // The notification of a completed expiry names what its delete did: the // delete marker it created, or the version it removed. func TestExpiryEventsNameTheResult(t *testing.T) { type expiry struct { name string status versioning.State key string versions []splitVersion expire string // label of the version expired action lifecycle.Action wantEvent event.Name wantID func(s *splitILM, key string) []string // acceptable version IDs } marker := func(s *splitILM, key string) []string { oi, err := s.z.GetObjectInfo(s.t.Context(), s.bucket, key, ObjectOptions{}) if !oi.DeleteMarker { s.t.Fatalf("latest of %s is not a marker: %+v %v", key, oi, err) } return []string{oi.VersionID} } label := func(l string) func(s *splitILM, key string) []string { return func(s *splitILM, key string) []string { return []string{s.ids[key+" "+l]} } } null := func(*splitILM, string) []string { return []string{"", nullVersionID} } cases := []expiry{ {"latest-versioned", versioning.Enabled, "exp-ev-latest", []splitVersion{splitAt("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, marker}, {"noncurrent", versioning.Enabled, "ncv-ev-noncurrent", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 3, 0)}, "v1", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("v1")}, {"expired-marker", versioning.Enabled, "edm-ev-marker", []splitVersion{splitMarker("dm", 5, 0)}, "dm", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("dm")}, {"latest-suspended-null", versioning.Suspended, "exp-ev-null", []splitVersion{splitNull("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, null}, {"tiered-latest-versioned", versioning.Enabled, "exp-ev-tiered", []splitVersion{splitTiered("v1", 5, 0)}, "v1", lifecycle.DeleteAction, event.ObjectRemovedDeleteMarkerCreated, marker}, {"tiered-noncurrent", versioning.Enabled, "ncv-ev-tiered", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 3, 0)}, "v1", lifecycle.DeleteVersionAction, event.ObjectRemovedDelete, label("v1")}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := newSplitILM(t, tc.status, splitILMRules) installExpiryTestTier(t, map[string][]byte{"remote-v1": []byte("v1")}) for _, v := range tc.versions { s.write(tc.key, v) } vid := s.ids[tc.key+" "+tc.expire] if vid == "" { vid = nullVersionID } oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, tc.key, ObjectOptions{VersionID: vid}) if err != nil && !isErrMethodNotAllowed(err) { t.Fatal(err) } oi.Bucket, oi.Name = s.bucket, tc.key lcEvent := lifecycle.Event{Action: tc.action} got := expiryEvents(t, func() { if oi.TransitionedObject.Status != "" { if !applyExpiryOnTransitionedObject(t.Context(), s.z, oi, lcEvent, lcEventSrc_Scanner) { t.Fatal("transitioned expiry did not apply") } return } if !applyExpiryOnNonTransitionedObjects(t.Context(), s.z, oi, lcEvent, lcEventSrc_Scanner) { t.Fatal("expiry did not apply") } }) if len(got) != 1 { t.Fatalf("events %+v, want one", got) } if got[0].EventName != tc.wantEvent { t.Errorf("event %s, want %s", got[0].EventName, tc.wantEvent) } if want := tc.wantID(s, tc.key); !slices.Contains(want, got[0].S3.Object.VersionID) { t.Errorf("event names version %q, want one of %v", got[0].S3.Object.VersionID, want) } }) } } // expiry_incomplete_total is exported as a counter with a reason label by // both metrics versions. func TestILMExpiryIncompleteMetrics(t *testing.T) { // Metrics v3 exports a counter once it is above zero. for _, r := range ilmIncompleteReasons { globalILMIncomplete.add(r) } if md := ilmExpiryIncompleteTotalMD; md.Type != CounterMT || !slices.Equal(md.VariableLabels, []string{"reason"}) { t.Fatalf("v3 descriptor %+v", md) } d := map[MetricName]MetricDescriptor{} for _, md := range []MetricDescriptor{ilmExpiryPendingTasksMD, ilmTransitionActiveTasksMD, ilmTransitionPendingTasksMD, ilmTransitionMissedImmediateTasksMD, ilmVersionsScannedMD, ilmExpiryIncompleteTotalMD} { d[md.Name] = md } m := newMetricValues(d) if err := loadILMMetrics(t.Context(), m, nil); err != nil { t.Fatal(err) } reasons := map[string]bool{} for _, v := range m.values[expiryIncompleteTotal] { reasons[v.Labels["reason"]] = true } for _, r := range ilmIncompleteReasons { if !reasons[r] { t.Errorf("v3: reason %q not exported", r) } } reasons = map[string]bool{} for _, mv := range getILMNodeMetrics().Get() { if mv.Description.Name == expiryIncompleteTotal { if mv.Description.Type != counterMetric { t.Errorf("v2 type %v", mv.Description.Type) } reasons[mv.VariableLabels["reason"]] = true } } for _, r := range ilmIncompleteReasons { if !reasons[r] { t.Errorf("v2: reason %q not exported", r) } } }