// Copyright (c) 2015-2026 MinIO, Inc.
// Copyright (c) 2026 PGSTY
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
package cmd
import (
"context"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/bucket/versioning"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/once"
)
// ilmDeleteTarget models one replication destination for ILM expiry tests.
type ilmDeleteTarget struct {
arn, bucket string
deletes atomic.Int32
}
func newILMTestTarget(t *testing.T) *ilmDeleteTarget {
return &ilmDeleteTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()}
}
func registerILMTestTarget(t *testing.T, obj ObjectLayer, bucket string, target *ilmDeleteTarget, handler http.HandlerFunc) {
t.Helper()
if err := obj.MakeBucket(t.Context(), target.bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
remote := httptest.NewServer(handler)
t.Cleanup(remote.Close)
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
if err != nil {
t.Fatal(err)
}
globalBucketTargetSys.Lock()
globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()}
globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket})
globalBucketTargetSys.Unlock()
globalBucketTargetSys.hMutex.Lock()
globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true}
globalBucketTargetSys.hMutex.Unlock()
}
func setILMTestReplicationConfig(t *testing.T, bucket, arn, targetBucket string) {
t.Helper()
cfg := replication.Config{}
rule := configs[0].Rules[0]
rule.Priority = 1
rule.Destination = replication.Destination{ARN: arn, Bucket: targetBucket}
cfg.Rules = append(cfg.Rules, rule)
cfg.RoleArn = arn
meta, err := globalBucketMetadataSys.Get(bucket)
if err != nil {
t.Fatal(err)
}
meta.replicationConfig = &cfg
globalBucketMetadataSys.Set(bucket, meta)
}
func newILMTestReplicationPool(ctx context.Context, obj ObjectLayer, stats *ReplicationStats) (restore func(), receive func() DeletedObjectReplicationInfo) {
oldPool := globalReplicationPool
p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)}
globalReplicationPool = once.NewSingleton[ReplicationPool]()
globalReplicationPool.Set(p)
return func() { globalReplicationPool = oldPool }, func() DeletedObjectReplicationInfo {
select {
case op := <-p.workers[0]:
d, ok := op.(DeletedObjectReplicationInfo)
if !ok {
return DeletedObjectReplicationInfo{}
}
return d
case <-time.After(3 * time.Second):
return DeletedObjectReplicationInfo{}
}
}
}
// Lifecycle deletes are not replicated: a delete marker or a version purge
// that ILM makes stays on the source, as in S3. The replication heal still
// sends an empty-state marker when existing-object replication applies to
// it, as it always did; that path is pinned here too.
// ilmTestTarget registers a target that applies the deletes it receives and
// records the requests, and the replication configuration of bucket with
// existing-object replication set as given.
func ilmTestTarget(t *testing.T, obj ObjectLayer, bucket, name string, existing bool) (*ilmDeleteTarget, *atomic.Int32) {
t.Helper()
target := newILMTestTarget(t)
var requests atomic.Int32
registerILMTestTarget(t, obj, bucket, target, func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
topts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
switch r.Method {
case http.MethodHead:
roi, rerr := obj.GetObjectInfo(r.Context(), target.bucket, name, topts)
if roi.DeleteMarker {
w.Header().Set(xhttp.AmzDeleteMarker, "true")
w.Header().Set(xhttp.AmzVersionID, roi.VersionID)
}
if rerr != nil {
writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), rerr))
return
}
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
target.deletes.Add(1)
topts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
topts.ReplicationRequest = true
topts.SetReplicaStatus(replication.Replica)
if _, err := obj.DeleteObject(r.Context(), target.bucket, name, topts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
return
}
w.WriteHeader(http.StatusNoContent)
default:
t.Errorf("unexpected remote method %s", r.Method)
}
})
setILMTestReplicationConfig(t, bucket, target.arn, target.bucket)
if !existing {
meta, err := globalBucketMetadataSys.Get(bucket)
if err != nil {
t.Fatal(err)
}
cfg := *meta.replicationConfig
cfg.Rules = slices.Clone(cfg.Rules)
cfg.Rules[0].ExistingObjectReplication = replication.ExistingObjectReplication{Status: replication.Disabled}
meta.replicationConfig = &cfg
globalBucketMetadataSys.Set(bucket, meta)
}
return target, &requests
}
// ilmTestReplicationConfig is the replication configuration the scanner and
// the replication heal use for bucket.
func ilmTestReplicationConfig(t *testing.T, bucket string) replicationConfig {
t.Helper()
cfg, err := getReplicationConfig(t.Context(), bucket)
if err != nil || cfg == nil {
t.Fatalf("replication config: %v", err)
}
remotes, err := globalBucketTargetSys.ListBucketTargets(t.Context(), bucket)
if err != nil {
t.Fatalf("bucket targets: %v", err)
}
return replicationConfig{Config: cfg, remotes: remotes}
}
// An ILM delete marker carries no replication state and is not queued; with
// existing-object replication off, the replication heal leaves it too.
func TestILMExpiryDeleteMarkerNotReplicated(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testILMExpiryDeleteMarkerNotReplicated})
}
func testILMExpiryDeleteMarkerNotReplicated(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
ilmMarkerReplicationCase(obj, instanceType, bucket, t, false)
}
// The inherited exception: with existing-object replication on, the
// replication heal sends an empty-state ILM delete marker, as it did before
// lifecycle deletes were ever replication-aware.
func TestILMExpiryDeleteMarkerExistingObjectResync(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testILMExpiryDeleteMarkerExistingObjectResync})
}
func testILMExpiryDeleteMarkerExistingObjectResync(obj ObjectLayer, instanceType, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
ilmMarkerReplicationCase(obj, instanceType, bucket, t, true)
}
func ilmMarkerReplicationCase(obj ObjectLayer, instanceType, bucket string, t *testing.T, existing bool) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
stats := NewReplicationStats(ctx, nil)
oldStats := globalReplicationStats.Swap(stats)
defer globalReplicationStats.Store(oldStats)
const name = "ilm-object"
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatal(err)
}
ilmXML := []byte(`ilm-expireEnabled2020-01-01T00:00:00Z`)
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, ilmXML); err != nil {
t.Fatal(err)
}
oi, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true})
if err != nil {
t.Fatal(err)
}
target, requests := ilmTestTarget(t, obj, bucket, name, existing)
restorePool, _ := newILMTestReplicationPool(ctx, obj, stats)
defer restorePool()
queue := globalReplicationPool.Get().workers[0]
if ok := applyExpiryOnNonTransitionedObjects(ctx, obj, oi, lifecycle.Event{Action: lifecycle.DeleteAction}, lcEventSrc_Scanner); !ok {
t.Fatalf("%s: ILM expiry did not apply", instanceType)
}
if n := len(queue); n != 0 {
t.Fatalf("%s: ILM expiry queued %d replication tasks", instanceType, n)
}
marker, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{Versioned: true})
if !isErrMethodNotAllowed(err) && !isErrObjectNotFound(err) || !marker.DeleteMarker || marker.VersionID == "" {
t.Fatalf("%s: latest after expiry: %+v %v", instanceType, marker, err)
}
marker.Bucket, marker.Name = bucket, name
if !marker.ReplicationStatus.Empty() || marker.ReplicationStatusInternal != "" {
t.Fatalf("%s: ILM marker carries replication state %q / %q", instanceType, marker.ReplicationStatus, marker.ReplicationStatusInternal)
}
// The replication heal, as the scanner runs it for the marker.
queueReplicationHeal(ctx, bucket, marker, ilmTestReplicationConfig(t, bucket), 0)
if !existing {
if n := len(queue); n != 0 {
t.Fatalf("%s: replication heal queued %d tasks for an ILM marker", instanceType, n)
}
if n := requests.Load(); n != 0 {
t.Fatalf("%s: the target saw %d requests", instanceType, n)
}
return
}
// The test harness reuses the bucket name for each object layer type, so
// targets registered for earlier ones are still listed: expect a task for
// at least the target of this run. Existing-object replication may also
// resync the data version; only the marker's tasks matter here.
markerTasks := 0
for n := len(queue); n > 0; n-- {
op := <-queue
deletion, ok := op.(DeletedObjectReplicationInfo)
if !ok {
continue
}
if !deletion.DeleteMarker || deletion.DeleteMarkerVersionID != marker.VersionID {
t.Fatalf("%s: unexpected resync task %+v", instanceType, deletion)
}
markerTasks++
replicateDelete(ctx, deletion, obj)
}
if markerTasks == 0 {
t.Fatalf("%s: existing-object resync queued no task for the marker", instanceType)
}
if target.deletes.Load() == 0 {
t.Fatalf("%s: the target saw no delete", instanceType)
}
}
// An API delete of a version still carries a purge decision and is queued:
// only lifecycle deletes are left out of replication.
func TestAPIVersionDeleteStillReplicated(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIVersionDeleteStillReplicated})
}
func testAPIVersionDeleteStillReplicated(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
stats := NewReplicationStats(ctx, nil)
oldStats := globalReplicationStats.Swap(stats)
defer globalReplicationStats.Store(oldStats)
const name = "api-object"
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatal(err)
}
if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("v1"), 2, "", ""), ObjectOptions{Versioned: true}); err != nil {
t.Fatal(err)
}
v1, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("v2"), 2, "", ""), ObjectOptions{Versioned: true})
if err != nil {
t.Fatal(err)
}
ilmTestTarget(t, obj, bucket, name, false)
restorePool, receive := newILMTestReplicationPool(ctx, obj, stats)
defer restorePool()
req, err := newTestSignedRequestV4(http.MethodDelete, getDeleteObjectURL("", bucket, name)+"?versionId="+v1.VersionID, 0, nil, creds.AccessKey, creds.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("%s: DELETE: %d %s", instanceType, rec.Code, rec.Body.String())
}
deletion := receive()
if deletion.VersionID != v1.VersionID || deletion.VersionPurgeStatus() != replication.VersionPurgePending {
t.Fatalf("%s: API version delete not queued for replication: %+v", instanceType, deletion)
}
}
// The scanner's noncurrent expiry deletes the version at once: no pending
// purge tombstone, nothing queued, the target untouched.
func TestILMNoncurrentPurgeNotReplicatedThroughScanner(t *testing.T) {
s := newSplitILM(t, versioning.Enabled, splitILMRules)
ctx := t.Context()
stats := NewReplicationStats(ctx, nil)
oldStats := globalReplicationStats.Swap(stats)
t.Cleanup(func() { globalReplicationStats.Store(oldStats) })
const key = "ncv-scanner-replicated"
s.write(key, splitAt("old", 5, 0))
s.write(key, splitAt("new", 3, 0))
old := s.ids[key+" old"]
target, requests := ilmTestTarget(t, s.z, s.bucket, key, false)
putConsistencyObject(t, s.z, target.bucket, key, 0, "replica", ObjectOptions{Versioned: true, VersionID: old})
restorePool, _ := newILMTestReplicationPool(ctx, s.z, stats)
t.Cleanup(restorePool)
s.replication = ilmTestReplicationConfig(t, s.bucket)
s.scan(0)
if _, err := s.z.GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: old}); !isErrVersionNotFound(err) {
t.Fatalf("expired version: %v, want it gone (no pending purge tombstone)", err)
}
if n := len(globalReplicationPool.Get().workers[0]); n != 0 {
t.Fatalf("noncurrent expiry queued %d replication tasks", n)
}
if n := requests.Load(); n != 0 {
t.Fatalf("the target saw %d requests", n)
}
if _, err := s.z.GetObjectInfo(ctx, target.bucket, key, ObjectOptions{VersionID: old}); err != nil {
t.Fatalf("target copy changed: %v", err)
}
}