// Copyright (c) 2015-2026 MinIO, Inc. // Copyright (c) 2026 PGSTY // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "github.com/minio/madmin-go/v3" "github.com/minio/minio-go/v7" "github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/replication" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/once" ) // ilmDeleteTarget models one replication destination for ILM expiry tests. type ilmDeleteTarget struct { arn, bucket string deletes atomic.Int32 } func newILMTestTarget(t *testing.T) *ilmDeleteTarget { return &ilmDeleteTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()} } func registerILMTestTarget(t *testing.T, obj ObjectLayer, bucket string, target *ilmDeleteTarget, handler http.HandlerFunc) { t.Helper() if err := obj.MakeBucket(t.Context(), target.bucket, MakeBucketOptions{}); err != nil { t.Fatal(err) } remote := httptest.NewServer(handler) t.Cleanup(remote.Close) client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1}) if err != nil { t.Fatal(err) } globalBucketTargetSys.Lock() globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()} globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket}) globalBucketTargetSys.Unlock() globalBucketTargetSys.hMutex.Lock() globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true} globalBucketTargetSys.hMutex.Unlock() } func setILMTestReplicationConfig(t *testing.T, bucket, arn, targetBucket string) { t.Helper() cfg := replication.Config{} rule := configs[0].Rules[0] rule.Priority = 1 rule.Destination = replication.Destination{ARN: arn, Bucket: targetBucket} cfg.Rules = append(cfg.Rules, rule) cfg.RoleArn = arn meta, err := globalBucketMetadataSys.Get(bucket) if err != nil { t.Fatal(err) } meta.replicationConfig = &cfg globalBucketMetadataSys.Set(bucket, meta) } func newILMTestReplicationPool(ctx context.Context, obj ObjectLayer, stats *ReplicationStats) (restore func(), receive func() DeletedObjectReplicationInfo) { oldPool := globalReplicationPool p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)} globalReplicationPool = once.NewSingleton[ReplicationPool]() globalReplicationPool.Set(p) return func() { globalReplicationPool = oldPool }, func() DeletedObjectReplicationInfo { select { case op := <-p.workers[0]: d, ok := op.(DeletedObjectReplicationInfo) if !ok { return DeletedObjectReplicationInfo{} } return d case <-time.After(3 * time.Second): return DeletedObjectReplicationInfo{} } } } // TestReplicateILMExpiryDeleteMarker verifies that a delete marker created by // ILM expiry carries delete replication state and is queued for replication, // exactly like a marker created through the S3 DELETE API. func TestReplicateILMExpiryDeleteMarker(t *testing.T) { defer DetectTestLeak(t)() ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ t: t, objAPITest: testReplicateILMExpiryDeleteMarker, }) } func testReplicateILMExpiryDeleteMarker(obj ObjectLayer, instanceType, bucket string, _ http.Handler, creds auth.Credentials, t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() stats := NewReplicationStats(ctx, nil) oldStats := globalReplicationStats.Swap(stats) defer globalReplicationStats.Store(oldStats) const name = "ilm-object" if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil { t.Fatal(err) } // DeleteObject refuses ILM expiry (Expiration.Expire) without lifecycle // rules; a date in the past makes every version eligible immediately. ilmXML := []byte(`ilm-expireEnabled2020-01-01T00:00:00Z`) if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, ilmXML); err != nil { t.Fatal(err) } oi, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true}) if err != nil { t.Fatal(err) } target := newILMTestTarget(t) registerILMTestTarget(t, obj, bucket, target, func(w http.ResponseWriter, r *http.Request) { topts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true} switch r.Method { case http.MethodHead: roi, rerr := obj.GetObjectInfo(r.Context(), target.bucket, name, topts) if roi.DeleteMarker { w.Header().Set(xhttp.AmzDeleteMarker, "true") w.Header().Set(xhttp.AmzVersionID, roi.VersionID) } if rerr != nil { writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), rerr)) return } w.WriteHeader(http.StatusOK) case http.MethodDelete: target.deletes.Add(1) if topts.VersionID == "" { t.Error("missing remote versionId") } topts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true" topts.ReplicationRequest = true topts.SetReplicaStatus(replication.Replica) if _, err := obj.DeleteObject(r.Context(), target.bucket, name, topts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) return } w.WriteHeader(http.StatusNoContent) default: t.Errorf("unexpected remote method %s", r.Method) } }) setILMTestReplicationConfig(t, bucket, target.arn, target.bucket) restorePool, receive := newILMTestReplicationPool(ctx, obj, stats) defer restorePool() if ok := applyExpiryOnNonTransitionedObjects(ctx, obj, oi, lifecycle.Event{Action: lifecycle.DeleteAction}, lcEventSrc_Scanner); !ok { t.Fatalf("%s: ILM expiry did not apply", instanceType) } deletion := receive() if deletion.ObjectName == "" { t.Fatalf("%s: ILM delete marker was not queued for replication", instanceType) } if !deletion.DeleteMarker || deletion.VersionID != "" || deletion.DeleteMarkerVersionID == "" { t.Fatalf("%s: unexpected queued delete: %+v", instanceType, deletion) } if deletion.EventType != ReplicateILMDelete { t.Fatalf("%s: queued event type = %q, want %q", instanceType, deletion.EventType, ReplicateILMDelete) } // The ILM-created marker must carry pending replication state. markerVID := deletion.DeleteMarkerVersionID moi, merr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: markerVID, Versioned: true}) if !isErrMethodNotAllowed(merr) || !moi.DeleteMarker { t.Fatalf("%s: ILM marker missing: %v %v", instanceType, moi, merr) } if moi.ReplicationStatus != replication.Pending { t.Fatalf("%s: ILM marker replication status = %q, want PENDING", instanceType, moi.ReplicationStatus) } result := replicateDelete(ctx, deletion, obj) if result.ReplicationStatus() != replication.Completed { t.Fatalf("%s: marker replication result: %+v", instanceType, result) } if target.deletes.Load() != 1 { t.Fatalf("%s: target saw %d deletes, want 1", instanceType, target.deletes.Load()) } // The marker must exist on the target. roi, rerr := obj.GetObjectInfo(ctx, target.bucket, name, ObjectOptions{VersionID: markerVID, Versioned: true}) if !isErrMethodNotAllowed(rerr) || !roi.DeleteMarker { t.Fatalf("%s: target marker missing: %v %v", instanceType, roi, rerr) } // The source marker records the completed replication per target. The // composite ReplicationStatus field is unreliable on read-back for // markers carrying a replication timestamp, so assert the raw state. moi, merr = obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: markerVID}) if !isErrMethodNotAllowed(merr) || !strings.Contains(string(moi.ReplicationStatusInternal), "=COMPLETED;") { t.Fatalf("%s: source marker state = %q (err %v), want COMPLETED", instanceType, moi.ReplicationStatusInternal, merr) } } // TestReplicateILMExpiryNoncurrentPurge verifies that a noncurrent version // removed by ILM carries a pending version purge state, is queued for delete // replication and converges on the target. func TestReplicateILMExpiryNoncurrentPurge(t *testing.T) { defer DetectTestLeak(t)() ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ t: t, objAPITest: testReplicateILMExpiryNoncurrentPurge, }) } func testReplicateILMExpiryNoncurrentPurge(obj ObjectLayer, instanceType, bucket string, _ http.Handler, creds auth.Credentials, t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() stats := NewReplicationStats(ctx, nil) oldStats := globalReplicationStats.Swap(stats) defer globalReplicationStats.Store(oldStats) const name = "ilm-object" if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil { t.Fatal(err) } if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("v1"), 2, "", ""), ObjectOptions{Versioned: true}); err != nil { t.Fatal(err) } v1Info, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("v2"), 2, "", ""), ObjectOptions{Versioned: true}) if err != nil { t.Fatal(err) } v1 := v1Info.VersionID if v1 == "" { t.Fatalf("%s: missing version ID on second version", instanceType) } target := newILMTestTarget(t) registerILMTestTarget(t, obj, bucket, target, func(w http.ResponseWriter, r *http.Request) { topts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true} switch r.Method { case http.MethodDelete: target.deletes.Add(1) if topts.VersionID == "" { t.Error("missing remote versionId") } topts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true" topts.ReplicationRequest = true topts.SetReplicaStatus(replication.Replica) if _, err := obj.DeleteObject(r.Context(), target.bucket, name, topts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) return } w.WriteHeader(http.StatusNoContent) default: t.Errorf("unexpected remote method %s", r.Method) } }) // Pre-create a marker version on the target with the same version ID as // the noncurrent version being purged, so the purge can be observed. if _, err := obj.DeleteObject(ctx, target.bucket, name, ObjectOptions{VersionID: v1, Versioned: true, DeleteMarker: true, ReplicationRequest: true}); err != nil { t.Fatal(err) } setILMTestReplicationConfig(t, bucket, target.arn, target.bucket) restorePool, receive := newILMTestReplicationPool(ctx, obj, stats) defer restorePool() // The scanner evaluates noncurrent version expiry per version; build the // delete entry exactly the way the scanner does for the oldest version. vi, verr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: v1, Versioned: true}) if verr != nil { t.Fatalf("%s: missing noncurrent version: %v", instanceType, verr) } otd := ilmPurgeToDelete(ctx, bucket, vi, true) if otd.VersionPurgeStatus != replication.VersionPurgePending || otd.VersionPurgeStatuses == "" || otd.ReplicateDecisionStr == "" { t.Fatalf("%s: ILM purge entry carries no replication state: %+v", instanceType, otd) } if plain := ilmPurgeToDelete(ctx, bucket, vi, false); plain.VersionPurgeStatus != "" || plain.ReplicateDecisionStr != "" { t.Fatalf("%s: non-replicating purge entry carries state: %+v", instanceType, plain) } deleteObjectVersions(ctx, obj, bucket, []ObjectToDelete{otd}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}}) // The version must remain locally as a tombstone with pending purge // state until replication completes. pi, perr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: v1, Versioned: true}) if !isErrMethodNotAllowed(perr) { t.Fatalf("%s: purged version metadata lost: %v", instanceType, perr) } if pi.VersionPurgeStatus != replication.VersionPurgePending { t.Fatalf("%s: purged version state = %q, want PENDING", instanceType, pi.VersionPurgeStatus) } deletion := receive() if deletion.ObjectName == "" { t.Fatalf("%s: ILM noncurrent purge was not queued for replication", instanceType) } if deletion.VersionID != v1 || deletion.DeleteMarkerVersionID != "" || deletion.DeleteMarker { t.Fatalf("%s: unexpected queued purge: %+v", instanceType, deletion) } result := replicateDelete(ctx, deletion, obj) if result.VersionPurgeStatus() != replication.VersionPurgeComplete { t.Fatalf("%s: purge replication result: %+v", instanceType, result) } if target.deletes.Load() != 1 { t.Fatalf("%s: target saw %d deletes, want 1", instanceType, target.deletes.Load()) } // The purge must have converged on the target. if _, terr := obj.GetObjectInfo(ctx, target.bucket, name, ObjectOptions{VersionID: v1, Versioned: true}); !isErrVersionNotFound(terr) && !isErrObjectNotFound(terr) { t.Fatalf("%s: target version still present: %v", instanceType, terr) } // Once the purge completes the local tombstone is freed and the version // is gone on both sides. if _, terr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: v1}); !isErrVersionNotFound(terr) && !isErrObjectNotFound(terr) { t.Fatalf("%s: purged version still present locally: %v", instanceType, terr) } }