From 0c61128d23f05ce6b37e7ace713c3ffbfb68f4cb Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 00:25:12 +0800 Subject: [PATCH] fix(replication): retry marker purges through persisted MRF (cherry picked from commit cf381a7151ef25fc95ace5fedcd767fa19410de2) Signed-off-by: Feng Ruohang --- cmd/bucket-replication-utils.go | 3 + cmd/bucket-replication.go | 127 ++-- cmd/replication-delete-marker_test.go | 24 +- cmd/replication-delete-mrf_test.go | 614 ++++++++++++++++++ cmd/replication-delete-operation_test.go | 201 ++++++ docs/investigations/r6/README.md | 61 ++ docs/investigations/r6/consensus.md | 49 ++ docs/investigations/r6/decisions-v2.md | 18 + docs/investigations/r6/opus-prompt-v1.md | 5 + docs/investigations/r6/opus-prompt-v2.md | 3 + docs/investigations/r6/opus-prompt-v3.md | 5 + docs/investigations/r6/opus-v1-review.md | 78 +++ docs/investigations/r6/opus-v1.metadata.json | 23 + docs/investigations/r6/opus-v2-review.md | 86 +++ docs/investigations/r6/opus-v2.metadata.json | 23 + docs/investigations/r6/opus-v3-review.md | 77 +++ docs/investigations/r6/opus-v3.metadata.json | 25 + docs/investigations/r6/plan-v1.md | 54 ++ docs/investigations/r6/plan-v2.md | 59 ++ docs/investigations/r6/plan-v3.md | 69 ++ docs/investigations/r6/research.md | 52 ++ .../r6/review_integration_test.go.txt | 181 ++++++ .../r6/review_probe_test.go.txt | 68 ++ docs/investigations/r6/verification-notes.md | 39 ++ .../r6/verification/baseline-build.log | 0 .../r6/verification/baseline-lint.log | 1 + .../r6/verification/baseline-race.log | 288 ++++++++ .../r6/verification/baseline-scope.log | 444 +++++++++++++ .../verification/baseline-verification.json | 101 +++ .../r6/verification/baseline-vet.log | 0 30 files changed, 2716 insertions(+), 62 deletions(-) create mode 100644 cmd/replication-delete-mrf_test.go create mode 100644 cmd/replication-delete-operation_test.go create mode 100644 docs/investigations/r6/README.md create mode 100644 docs/investigations/r6/consensus.md create mode 100644 docs/investigations/r6/decisions-v2.md create mode 100644 docs/investigations/r6/opus-prompt-v1.md create mode 100644 docs/investigations/r6/opus-prompt-v2.md create mode 100644 docs/investigations/r6/opus-prompt-v3.md create mode 100644 docs/investigations/r6/opus-v1-review.md create mode 100644 docs/investigations/r6/opus-v1.metadata.json create mode 100644 docs/investigations/r6/opus-v2-review.md create mode 100644 docs/investigations/r6/opus-v2.metadata.json create mode 100644 docs/investigations/r6/opus-v3-review.md create mode 100644 docs/investigations/r6/opus-v3.metadata.json create mode 100644 docs/investigations/r6/plan-v1.md create mode 100644 docs/investigations/r6/plan-v2.md create mode 100644 docs/investigations/r6/plan-v3.md create mode 100644 docs/investigations/r6/research.md create mode 100644 docs/investigations/r6/review_integration_test.go.txt create mode 100644 docs/investigations/r6/review_probe_test.go.txt create mode 100644 docs/investigations/r6/verification-notes.md create mode 100644 docs/investigations/r6/verification/baseline-build.log create mode 100644 docs/investigations/r6/verification/baseline-lint.log create mode 100644 docs/investigations/r6/verification/baseline-race.log create mode 100644 docs/investigations/r6/verification/baseline-scope.log create mode 100644 docs/investigations/r6/verification/baseline-verification.json create mode 100644 docs/investigations/r6/verification/baseline-vet.log diff --git a/cmd/bucket-replication-utils.go b/cmd/bucket-replication-utils.go index 8c0ca7041..be8f8665f 100644 --- a/cmd/bucket-replication-utils.go +++ b/cmd/bucket-replication-utils.go @@ -418,6 +418,9 @@ func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID for _, rinfo := range rinfos.Targets { if rinfo.ResyncTimestamp != "" { + if rs.ResetStatusesMap == nil { + rs.ResetStatusesMap = make(map[string]string) + } rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp } } diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index bfba4741a..7c781c2a3 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -425,6 +425,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet // the mere presence or absence of the target version. func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos { var replicationStatus replication.StatusType + isPurge := dobj.isVersionPurge() bucket := dobj.Bucket versionID := dobj.DeleteMarkerVersionID if versionID == "" { @@ -484,6 +485,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName) lkctx, err := lk.GetLock(ctx, globalOperationTimeout) if err != nil { + dobj.RetryCount++ globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry()) sendEvent(eventArgs{ BucketName: bucket, @@ -546,25 +548,38 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj replicationStatus = rinfos.ReplicationStatus() prevStatus := dobj.DeleteMarkerReplicationStatus() - if dobj.VersionID != "" { - prevStatus = replication.StatusType(dobj.VersionPurgeStatus()) - replicationStatus = replication.StatusType(rinfos.VersionPurgeStatus()) + if isPurge { + prevStatus = purgeReplicationStatus(dobj.VersionPurgeStatus()) + replicationStatus = purgeReplicationStatus(rinfos.VersionPurgeStatus()) } // to decrement pending count later. for _, rinfo := range rinfos.Targets { - if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus { - globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus, - prevStatus) + status, previous := rinfo.ReplicationStatus, rinfo.PrevReplicationStatus + if isPurge { + status = purgeReplicationStatus(rinfo.VersionPurgeStatus) + previous = purgeReplicationStatus(dobj.ReplicationState.PurgeTargets[rinfo.Arn]) + } + if status != previous { + globalReplicationStats.Load().Update(dobj.Bucket, rinfo, status, previous) } } eventName := event.ObjectReplicationComplete if replicationStatus == replication.Failed { eventName = event.ObjectReplicationFailed + dobj.RetryCount++ globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry()) } drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID) + if isPurge { + // A purge must not rewrite the marker's creation/replica metadata. + // Multiple serialized empty target statuses can parse as nonempty, + // so explicitly send an empty creation update to the metadata writer. + drs.ReplicationStatusInternal = "" + drs.Targets = nil + drs.ReplicaStatus = "" + } if replicationStatus != prevStatus { drs.ReplicationTimeStamp = UTCNow() } @@ -606,6 +621,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj } func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) { + isPurge := dobj.isVersionPurge() versionID := dobj.DeleteMarkerVersionID if versionID == "" { versionID = dobj.VersionID @@ -615,42 +631,50 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI rinfo.OpType = dobj.OpType rinfo.endpoint = tgt.EndpointURL().Host rinfo.secure = tgt.EndpointURL().Scheme == "https" + // A purge leaves ReplicationStatus empty: the metadata writer interprets + // that as preserving the entire creation-status block, including targets + // outside this fan-out. Only VersionPurgeStatus records the purge outcome. defer func() { - if rinfo.ReplicationStatus == replication.Completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType { + completed := rinfo.ReplicationStatus == replication.Completed + if isPurge { + completed = rinfo.VersionPurgeStatus == replication.VersionPurgeComplete + } + if completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType { rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID) } }() - if dobj.VersionID == "" && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType { + if !isPurge && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType { rinfo.ReplicationStatus = rinfo.PrevReplicationStatus return rinfo } - if dobj.VersionID != "" && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete { + if isPurge && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete { return rinfo } if globalBucketTargetSys.isOffline(tgt.EndpointURL()) { - replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN) + rinfo.Err = fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN) + replLogOnceIf(ctx, rinfo.Err, "replication-target-offline-delete-"+tgt.ARN) sendEvent(eventArgs{ BucketName: dobj.Bucket, Object: ObjectInfo{ Bucket: dobj.Bucket, Name: dobj.ObjectName, - VersionID: dobj.VersionID, + VersionID: versionID, DeleteMarker: dobj.DeleteMarker, }, UserAgent: "Internal: [Replication]", Host: globalLocalNodeName, EventName: event.ObjectReplicationNotTracked, }) - if dobj.VersionID == "" { - rinfo.ReplicationStatus = replication.Failed - } else { + if isPurge { rinfo.VersionPurgeStatus = replication.VersionPurgeFailed + } else { + rinfo.ReplicationStatus = replication.Failed } return rinfo } // early return if already replicated delete marker for existing object replication/ healing delete markers - if dobj.DeleteMarkerVersionID != "" { + if !isPurge && dobj.DeleteMarkerVersionID != "" { toi, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, minio.StatObjectOptions{ VersionID: versionID, Internal: minio.AdvancedGetOptions{ @@ -662,16 +686,10 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI switch { case isErrMethodNotAllowed(serr): // delete marker already replicated - if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() { - rinfo.ReplicationStatus = replication.Completed - return rinfo - } + rinfo.ReplicationStatus = replication.Completed + return rinfo case isErrObjectNotFound(serr), isErrVersionNotFound(serr): - // version being purged is already not found on target. - if !rinfo.VersionPurgeStatus.Empty() { - rinfo.VersionPurgeStatus = replication.VersionPurgeComplete - return rinfo - } + // The marker still needs to be created on the target. case isErrReadQuorum(serr), isErrWriteQuorum(serr): // destination has some quorum issues, perform removeObject() anyways // to complete the operation. @@ -691,7 +709,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{ VersionID: versionID, Internal: minio.AdvancedRemoveOptions{ - ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "", + ReplicationDeleteMarker: !isPurge && dobj.DeleteMarkerVersionID != "", ReplicationMTime: dobj.DeleteMarkerMTime.Time, ReplicationStatus: minio.ReplicationStatusReplica, ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside @@ -699,20 +717,20 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI }) if rmErr != nil { rinfo.Err = rmErr - if dobj.VersionID == "" { - rinfo.ReplicationStatus = replication.Failed - } else { + if isPurge { rinfo.VersionPurgeStatus = replication.VersionPurgeFailed + } else { + rinfo.ReplicationStatus = replication.Failed } replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr)) if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) { globalBucketTargetSys.markOffline(tgt.EndpointURL()) } } else { - if dobj.VersionID == "" { - rinfo.ReplicationStatus = replication.Completed - } else { + if isPurge { rinfo.VersionPurgeStatus = replication.VersionPurgeComplete + } else { + rinfo.ReplicationStatus = replication.Completed } } return rinfo @@ -1923,11 +1941,26 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri // DeletedObjectReplicationInfo has info on deleted object type DeletedObjectReplicationInfo struct { DeletedObject - Bucket string - EventType string - OpType replication.Type - ResetID string - TargetArn string + Bucket string + EventType string + OpType replication.Type + ResetID string + TargetArn string + RetryCount int +} + +// isVersionPurge also recognizes the old marker-shaped purge task. Use the +// operation's state, rather than one target's possibly missing purge entry. +func (di DeletedObjectReplicationInfo) isVersionPurge() bool { + return di.VersionID != "" || di.DeleteMarkerVersionID != "" && !di.VersionPurgeStatus().Empty() +} + +// Purge metadata uses COMPLETE; operation statistics and audit use COMPLETED. +func purgeReplicationStatus(status VersionPurgeStatusType) replication.StatusType { + if replication.StatusType(status) == replication.CompletedLegacy { + return replication.Completed + } + return replication.StatusType(status) } // ToMRFEntry returns the relevant info needed by MRF @@ -1937,9 +1970,10 @@ func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry { versionID = di.VersionID } return MRFReplicateEntry{ - Bucket: di.Bucket, - Object: di.ObjectName, - versionID: versionID, + Bucket: di.Bucket, + Object: di.ObjectName, + versionID: versionID, + RetryCount: di.RetryCount, } } @@ -2431,6 +2465,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf case <-p.ctx.Done(): case ch <- doi: default: + doi.RetryCount++ p.queueMRFSave(doi.ToMRFEntry()) p.mu.RLock() prio := p.priority @@ -3794,9 +3829,10 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf DeleteMarkerMTime: DeleteMarkerMTime{roi.ModTime}, DeleteMarker: roi.DeleteMarker, }, - Bucket: roi.Bucket, - OpType: replication.HealReplicationType, - EventType: ReplicateHealDelete, + Bucket: roi.Bucket, + OpType: replication.HealReplicationType, + EventType: ReplicateHealDelete, + RetryCount: retryCount, } // heal delete marker replication failure or versioned delete replication failure if roi.ReplicationStatus == replication.Pending || @@ -4079,7 +4115,12 @@ func (p *ReplicationPool) queueMRFHeal() error { VersionID: vID, }) cancel() - if err != nil { + // A versioned marker lookup returns its metadata with a 405. Only + // accept that error with a real, matching marker identity. + validMarker := isErrMethodNotAllowed(err) && oi.DeleteMarker && + vID != "" && oi.VersionID == vID && !oi.ModTime.IsZero() && + oi.Bucket == e.Bucket && oi.Name != "" && oi.Name == decodeDirObject(e.Object) + if err != nil && !validMarker || oi.Name == "" { continue } diff --git a/cmd/replication-delete-marker_test.go b/cmd/replication-delete-marker_test.go index e1ac57e6f..874db6eb3 100644 --- a/cmd/replication-delete-marker_test.go +++ b/cmd/replication-delete-marker_test.go @@ -106,6 +106,7 @@ func TestReplicateDeleteMarkerTargetSemantics(t *testing.T) { func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T, legacy bool) { ctx := t.Context() + defer replicationTestCapacity(obj)() const arn = "arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket" const name = "marker" version := mustGetUUID() @@ -208,27 +209,12 @@ func testReplicateDeleteMarkerPurge(obj ObjectLayer, instanceType, bucket string t.Errorf("purge scheduled as marker creation: version=%q marker=%q", deletion.VersionID, deletion.DeleteMarkerVersionID) } if legacy { - // Reproduce the old producer's state and let the existing scanner/heal - // path recover it. Upgrades must also finish purges already left pending. + // Old task shapes must complete directly; scanner/MRF recovery is + // covered separately by TestReplicationMRFMarkerRecovery. deletion.VersionID, deletion.DeleteMarkerVersionID = "", version - replicateDelete(ctx, deletion, obj) - oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true}) - if oi.VersionPurgeStatus != replication.VersionPurgePending { - t.Fatalf("legacy source purge = %s, want PENDING", oi.VersionPurgeStatus) - } - targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) - if err != nil { - t.Fatal(err) - } - queueReplicationHeal(ctx, bucket, oi, replicationConfig{Config: &cfg, remotes: targets}, 0) - select { - case op := <-worker: - deletion = op.(DeletedObjectReplicationInfo) - case <-time.After(time.Second): - t.Fatal("legacy pending purge was not scheduled for healing") - } } - result := replicateDelete(context.Background(), deletion, obj) + + result := replicateDelete(context.Background(), deletion, markerPurgeUpdateLayer{ObjectLayer: obj, t: t}) if result.VersionPurgeStatus() != replication.VersionPurgeComplete { t.Errorf("remote purge result = %s, want COMPLETE", result.VersionPurgeStatus()) } diff --git a/cmd/replication-delete-mrf_test.go b/cmd/replication-delete-mrf_test.go new file mode 100644 index 000000000..c5d833c2d --- /dev/null +++ b/cmd/replication-delete-mrf_test.go @@ -0,0 +1,614 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-only + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + loghttp "github.com/minio/minio/internal/logger/target/http" + "github.com/minio/minio/internal/once" + xnet "github.com/pgsty/silo-pkg/v3/net" +) + +type markerRecoveryCase struct { + name string + legacy, creation, lostReply, partial, exhaust, lockFirst, invalidMRF, replicaSource, offline, unrecordedPurge bool + targets int +} + +func TestReplicationMRFMarkerRecovery(t *testing.T) { + for _, tc := range []markerRecoveryCase{ + {name: "canonical", targets: 1}, + {name: "lock-failure", lockFirst: true, targets: 1}, + {name: "invalid-MRF-metadata", invalidMRF: true, targets: 1}, + {name: "legacy", legacy: true, targets: 1}, + {name: "unrecorded-purge", unrecordedPurge: true, targets: 2}, + {name: "two-targets", legacy: true, targets: 2}, + {name: "two-targets-offline", offline: true, targets: 2}, + {name: "replica-source", replicaSource: true, targets: 1}, + {name: "lost-reply-canonical", lostReply: true, targets: 1}, + {name: "lost-reply-legacy", legacy: true, lostReply: true, targets: 1}, + {name: "creation", creation: true, targets: 1}, + {name: "partial-creation-block", partial: true, targets: 2}, + {name: "retry-budget-and-scanner", exhaust: true, targets: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, endpoints: []string{"DeleteObject"}, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) { + testReplicationMRFMarkerRecovery(t, obj, backend, bucket, router, creds, tc) + }}) + }) + } +} + +type markerRecoveryTarget struct { + arn, bucket string + client *minio.Client + reject, loseReply atomic.Bool + deletes atomic.Int32 +} + +// Only adapt host capacity accounting, using the existing real-disk adapter. +// Reads, object metadata, MRF files and writes all still use the fixture disks. +func replicationTestCapacity(obj ObjectLayer) func() { + var restore []func() + for _, pool := range obj.(*erasureServerPools).serverPools { + for _, set := range pool.sets { + original := set.getDisks + disks := append([]StorageAPI(nil), original()...) + for i, disk := range disks { + if disk != nil { + disks[i] = tagTestCapacityDisk{StorageAPI: disk} + } + } + set.getDisks = func() []StorageAPI { return disks } + restore = append(restore, func() { set.getDisks = original }) + } + } + return func() { + for _, fn := range restore { + fn() + } + } +} + +func testReplicationMRFMarkerRecovery(t *testing.T, obj ObjectLayer, backend, bucket string, router http.Handler, creds auth.Credentials, tc markerRecoveryCase) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + defer replicationTestCapacity(obj)() + stats := NewReplicationStats(ctx, nil) + oldStats := globalReplicationStats.Swap(stats) + defer globalReplicationStats.Store(oldStats) + oldPool := globalReplicationPool + defer func() { globalReplicationPool = oldPool }() + const name = "marker" + version := mustGetUUID() + creationTime := UTCNow().Add(-time.Hour).Truncate(time.Second) + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil { + t.Fatal(err) + } + if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil { + t.Fatal(err) + } + var targets []*markerRecoveryTarget + cfg := replication.Config{} + creationStates := make(map[string]replication.StatusType) + var creationInternal string + for i := 0; i < tc.targets; i++ { + target := &markerRecoveryTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()} + if err := obj.MakeBucket(ctx, target.bucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + if _, err := obj.PutObject(ctx, target.bucket, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil { + t.Fatal(err) + } + if !tc.creation { + opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, ReplicationRequest: true, MTime: creationTime} + opts.SetReplicaStatus(replication.Replica) + if _, err := obj.DeleteObject(ctx, target.bucket, name, opts); err != nil { + t.Fatal(err) + } + } + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true} + switch r.Method { + case http.MethodHead: + oi, err := obj.GetObjectInfo(r.Context(), target.bucket, name, opts) + if oi.DeleteMarker { + w.Header().Set(xhttp.AmzDeleteMarker, "true") + w.Header().Set(xhttp.AmzVersionID, oi.VersionID) + } + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err)) + return + } + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + target.deletes.Add(1) + if got := r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"; got != tc.creation { + t.Errorf("purge/creation wire flag=%v creation=%v", got, tc.creation) + } + if opts.VersionID == "" { + t.Error("missing remote versionId") + } + if target.reject.Load() { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDenied`) + return + } + opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true" + opts.ReplicationRequest = true + opts.SetReplicaStatus(replication.Replica) + _, err := obj.DeleteObject(r.Context(), target.bucket, name, opts) + if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) + return + } + if target.loseReply.Swap(false) { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() + return + } + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote method %s", r.Method) + } + })) + defer remote.Close() + client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + target.client = client + globalBucketTargetSys.Lock() + globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()} + globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket}) + globalBucketTargetSys.Unlock() + globalBucketTargetSys.hMutex.Lock() + globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true} + globalBucketTargetSys.hMutex.Unlock() + rule := configs[0].Rules[0] + rule.Priority = i + 1 + rule.Destination = replication.Destination{ARN: target.arn, Bucket: target.bucket} + cfg.Rules = append(cfg.Rules, rule) + creationStates[target.arn] = replication.Completed + creationInternal += target.arn + "=COMPLETED;" + targets = append(targets, target) + } + if tc.targets == 1 { + cfg.RoleArn = targets[0].arn + } + if !tc.creation { + opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, MTime: creationTime, DeleteReplication: ReplicationState{Targets: creationStates, ReplicationStatusInternal: creationInternal, ReplicationTimeStamp: creationTime}} + if tc.replicaSource { + opts.DeleteReplication = ReplicationState{} + opts.SetReplicaStatus(replication.Replica) + } + if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil { + t.Fatal(err) + } + } + meta, err := globalBucketMetadataSys.Get(bucket) + if err != nil { + t.Fatal(err) + } + meta.replicationConfig = &cfg + globalBucketMetadataSys.Set(bucket, meta) + newPool := func() *ReplicationPool { + p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)} + globalReplicationPool = once.NewSingleton[ReplicationPool]() + globalReplicationPool.Set(p) + return p + } + p := newPool() + receive := func() DeletedObjectReplicationInfo { + select { + case op := <-p.workers[0]: + d, ok := op.(DeletedObjectReplicationInfo) + if !ok { + t.Fatalf("wrong queued operation %T", op) + } + return d + case <-time.After(3 * time.Second): + t.Fatal("no marker task from actual MRF/handler/scanner queue") + return DeletedObjectReplicationInfo{} + } + } + uri := "/" + bucket + "/" + name + if !tc.creation { + uri += "?versionId=" + version + } + req, err := newTestSignedRequestV4(http.MethodDelete, uri, 0, nil, creds.AccessKey, creds.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("source DELETE status %d: %s", w.Code, w.Body) + } + deletion := receive() + if tc.creation { + version = deletion.DeleteMarkerVersionID + } else if deletion.VersionID != version || deletion.DeleteMarkerVersionID != "" { + t.Fatalf("current producer emitted noncanonical purge: %+v", deletion) + } + if tc.unrecordedPurge { + // Exercise a task carrying purge state while the disk marker still has + // only creation metadata. Recreate only this isolated fixture marker. + if _, err := obj.DeleteObject(ctx, bucket, name, ObjectOptions{VersionID: version, Versioned: true}); err != nil { + t.Fatal(err) + } + opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, MTime: creationTime, DeleteReplication: ReplicationState{Targets: creationStates, ReplicationStatusInternal: creationInternal, ReplicationTimeStamp: creationTime}} + if _, err := obj.DeleteObject(ctx, bucket, name, opts); err != nil { + t.Fatal(err) + } + } + if tc.legacy { + deletion.VersionID, deletion.DeleteMarkerVersionID = "", version + } + if tc.partial { + deletion.TargetArn = targets[len(targets)-1].arn + } + if tc.invalidMRF { + testReplicationMRFInvalidLookups(ctx, t, obj, p, newPool, deletion) + return + } + var auditStatuses <-chan string + if tc.name == "canonical" { + var stopAudit func() + auditStatuses, stopAudit = replicationTestAudit(ctx, t, bucket) + defer stopAudit() + } + assertAudit := func(want string) { + if auditStatuses == nil { + return + } + select { + case got := <-auditStatuses: + if got != want { + t.Fatalf("replication audit status=%q want %q", got, want) + } + case <-time.After(3 * time.Second): + t.Fatal("missing replication audit event") + } + } + deletion.OpType = replication.HealReplicationType // Exercise operation-status statistics too. + failing := targets[len(targets)-1] + failing.reject.Store(!tc.lostReply) + failing.loseReply.Store(tc.lostReply) + if tc.offline { + globalBucketTargetSys.hMutex.Lock() + globalBucketTargetSys.hc[failing.client.EndpointURL().Host] = epHealth{Online: false} + globalBucketTargetSys.hMutex.Unlock() + } + before, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version}) + beforeCreation := before.ReplicationStatusInternal + beforeStamp := before.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] + beforeReplica := before.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] + beforeReplicaStamp := before.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] + if !tc.creation && !tc.replicaSource && (len(replicationStatusesMap(beforeCreation)) != tc.targets || beforeStamp == "") { + t.Fatalf("missing seeded creation block: %+v", before) + } + if tc.replicaSource && (beforeReplica != "REPLICA" || beforeReplicaStamp == "") { + t.Fatalf("missing replica block: %+v", before) + } + updateObj := obj + if !tc.creation { + updateObj = markerPurgeUpdateLayer{ObjectLayer: obj, t: t} + } + rounds := 2 + if tc.lostReply { + rounds = 1 + } + if tc.exhaust { + rounds = mrfRetryLimit + 1 + } + for round := 1; round <= rounds; round++ { + callObj := updateObj + if tc.lockFirst && round == 1 { + callObj = markerLockFailureLayer{ObjectLayer: updateObj} + } + result := replicateDelete(ctx, deletion, callObj) + switch { + case tc.lockFirst && round == 1: + if len(result.Targets) != 0 { + t.Fatal("lock failure attempted a target") + } + case tc.creation: + if result.ReplicationStatus() != replication.Failed { + t.Fatalf("creation result: %+v", result) + } + case result.VersionPurgeStatus() != replication.VersionPurgeFailed: + t.Fatalf("purge failure result: %+v", result) + } + assertAudit("FAILED") + if round == 1 && !tc.lockFirst && !tc.creation { + stats.RLock() + failed := stats.Cache[bucket].Stats[failing.arn].FailStats.SinceUptime + stats.RUnlock() + if failed.Count != 1 || failed.Bytes != 0 { + t.Fatalf("purge failure stats=%+v, want count 1 and zero bytes", failed) + } + } + oi, err := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version}) + if !isErrMethodNotAllowed(err) || !oi.DeleteMarker { + t.Fatalf("source marker metadata/405 missing: %+v %v", oi, err) + } + if !tc.creation && (oi.ReplicationStatusInternal != beforeCreation || oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] != beforeStamp) { + t.Fatalf("purge rewrote creation block: before=%q/%q after=%q/%q", beforeCreation, beforeStamp, oi.ReplicationStatusInternal, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp]) + } + if !tc.creation && (oi.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] != beforeReplica || oi.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] != beforeReplicaStamp) { + t.Fatal("purge rewrote replica block") + } + if tc.targets == 2 && !tc.partial { + purges := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal) + if purges[targets[0].arn] != replication.VersionPurgeComplete || purges[failing.arn] != replication.VersionPurgeFailed { + t.Fatalf("incorrect persisted target states: %v", purges) + } + if targets[0].deletes.Load() != 1 { + t.Fatalf("successful target resent %d times", targets[0].deletes.Load()) + } + } + if tc.partial { + select { + case <-p.mrfSaveCh: + default: + t.Fatal("partial failure missing MRF") + } + dsc := deletion.ReplicationState.ReplicateDecisionStr + deletion.ReplicationState = oi.ReplicationState() + deletion.ReplicationState.ReplicateDecisionStr = dsc + continue + } + if tc.exhaust && round > mrfRetryLimit { + if len(p.mrfSaveCh) != 0 || atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount) != 1 { + t.Fatalf("retry budget not applied: queued=%d drops=%d", len(p.mrfSaveCh), stats.mrfStats.TotalDroppedCount) + } + break + } + var entry MRFReplicateEntry + select { + case entry = <-p.mrfSaveCh: + default: + t.Fatal("failure did not enter MRF") + } + if entry.RetryCount != round || entry.versionID != version { + t.Fatalf("MRF entry lost identity/budget: %+v, round=%d", entry, round) + } + p.saveMRFEntries(ctx, map[string]MRFReplicateEntry{entry.versionID: entry}) + record, err := p.loadMRF() + if err != nil || len(record.Entries) != 1 { + t.Fatalf("disk MRF missing: %+v %v", record, err) + } + if got := record.Entries[version]; got.RetryCount != round || got.Object != name || got.Bucket != bucket { + t.Fatalf("disk MRF mismatch: %+v", got) + } + p.saveMRFEntries(ctx, record.Entries) // loadMRF consumes the file; each replay uses a fresh disk record. + p = newPool() // no in-memory entries carried to the replacement pool. + if err := p.queueMRFHeal(); err != nil { + t.Fatal(err) + } + deletion = receive() + if deletion.RetryCount != round { + t.Fatalf("MRF retry count=%d want %d", deletion.RetryCount, round) + } + if !tc.creation && (deletion.VersionID != version || deletion.DeleteMarkerVersionID != "") { + t.Fatalf("MRF emitted wrong purge: %+v", deletion) + } + } + if tc.partial { + return + } + failing.reject.Store(false) + globalBucketTargetSys.hMutex.Lock() + globalBucketTargetSys.hc[failing.client.EndpointURL().Host] = epHealth{Online: true} + globalBucketTargetSys.hMutex.Unlock() + if tc.exhaust { + oi, _ := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: version}) + QueueReplicationHeal(ctx, bucket, oi, 0) // Separately prove the existing scanner fallback after MRF exhaustion. + deletion = receive() + if deletion.RetryCount != 0 { + t.Fatal("scanner did not start a fresh retry budget") + } + } + result := replicateDelete(ctx, deletion, updateObj) + assertAudit("COMPLETED") + if tc.creation { + if result.ReplicationStatus() != replication.Completed { + t.Fatalf("creation recovery failed: %+v", result) + } + } else if result.VersionPurgeStatus() != replication.VersionPurgeComplete { + t.Fatalf("purge recovery failed: %+v", result) + } + for _, b := range append([]string{bucket}, func() []string { + var b []string + for _, target := range targets { + b = append(b, target.bucket) + } + return b + }()...) { + oi, err := obj.GetObjectInfo(ctx, b, name, ObjectOptions{VersionID: version}) + if tc.creation { + if !oi.DeleteMarker || !isErrMethodNotAllowed(err) { + t.Fatalf("creation missing in %s: %+v %v", b, oi, err) + } + } else if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Fatalf("purge left marker in %s: %+v %v", b, oi, err) + } + } + if !tc.creation { + // Re-deliver an ambiguous purge directly. An absent marker must stay absent. + duplicate := deletion + duplicate.VersionID, duplicate.DeleteMarkerVersionID = "", version + duplicate.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{failing.arn: replication.VersionPurgePending} + duplicate.ReplicationState.VersionPurgeStatusInternal = "" + got := replicateDeleteToTarget(ctx, duplicate, &TargetClient{Client: failing.client, ARN: failing.arn, Bucket: failing.bucket}) + if got.VersionPurgeStatus != replication.VersionPurgeComplete { + t.Fatalf("duplicate purge: %+v", got) + } + if oi, err := obj.GetObjectInfo(ctx, failing.bucket, name, ObjectOptions{VersionID: version}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Fatalf("duplicate recreated marker: %+v %v", oi, err) + } + } + if len(p.mrfSaveCh) != 0 || len(p.workers[0]) != 0 { + t.Fatalf("completion left queued work: mrf=%d worker=%d", len(p.mrfSaveCh), len(p.workers[0])) + } + if !tc.creation && !tc.exhaust && !tc.lostReply { + stats.RLock() + got := stats.Cache[bucket].Stats[failing.arn].ReplicatedCount + size := stats.Cache[bucket].Stats[failing.arn].ReplicatedSize + stats.RUnlock() + if got != 1 || size != 0 { + t.Fatalf("purge completion statistic=%d want 1", got) + } + } + t.Logf("%s: %s recovered; %d target(s), persisted MRF, source/target metadata checked", backend, tc.name, tc.targets) +} + +func TestReplicationDeleteQueueFullRetryBudget(t *testing.T) { + stats := NewReplicationStats(t.Context(), nil) + p := &ReplicationPool{ctx: t.Context(), objLayer: &replicationMRFTestObjectLayer{}, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 1), priority: "slow"} + d := DeletedObjectReplicationInfo{Bucket: "bucket", DeletedObject: DeletedObject{ObjectName: "marker", VersionID: mustGetUUID()}, RetryCount: 2} + p.queueReplicaDeleteTask(d) + if got := <-p.mrfSaveCh; got.RetryCount != 3 { + t.Fatalf("queue-full retry count=%d", got.RetryCount) + } + d.RetryCount = mrfRetryLimit + p.queueReplicaDeleteTask(d) + if len(p.mrfSaveCh) != 0 || atomic.LoadUint64(&stats.mrfStats.TotalDroppedCount) != 1 { + t.Fatal("queue-full retry exceeded budget without a visible drop") + } +} + +type markerLockFailureLayer struct{ ObjectLayer } + +func (markerLockFailureLayer) NewNSLock(string, ...string) RWLocker { return markerFailedLock{} } + +type markerFailedLock struct{ RWLocker } + +func (markerFailedLock) GetLock(context.Context, *dynamicTimeout) (LockContext, error) { + return LockContext{}, context.DeadlineExceeded +} + +type markerLookupLayer struct { + ObjectLayer + mutate func(ObjectInfo, error) (ObjectInfo, error) + lookedUp chan struct{} +} + +func (l markerLookupLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts) + defer close(l.lookedUp) + return l.mutate(oi, err) +} + +func testReplicationMRFInvalidLookups(ctx context.Context, t *testing.T, obj ObjectLayer, p *ReplicationPool, newPool func() *ReplicationPool, d DeletedObjectReplicationInfo) { + for _, tc := range []struct { + name string + mutate func(ObjectInfo, error) (ObjectInfo, error) + }{ + {"empty-info", func(_ ObjectInfo, e error) (ObjectInfo, error) { return ObjectInfo{}, e }}, + {"not-a-marker", func(o ObjectInfo, e error) (ObjectInfo, error) { o.DeleteMarker = false; return o, e }}, + {"wrong-version", func(o ObjectInfo, e error) (ObjectInfo, error) { o.VersionID = mustGetUUID(); return o, e }}, + {"wrong-bucket", func(o ObjectInfo, e error) (ObjectInfo, error) { o.Bucket = "different-bucket"; return o, e }}, + {"wrong-object", func(o ObjectInfo, e error) (ObjectInfo, error) { o.Name = "different-object"; return o, e }}, + {"zero-modtime", func(o ObjectInfo, e error) (ObjectInfo, error) { o.ModTime = time.Time{}; return o, e }}, + {"missing", func(o ObjectInfo, _ error) (ObjectInfo, error) { + return o, ObjectNotFound{Bucket: o.Bucket, Object: o.Name} + }}, + {"read-error", func(o ObjectInfo, _ error) (ObjectInfo, error) { return o, InsufficientReadQuorum{} }}, + } { + t.Run(tc.name, func(t *testing.T) { + entry := d.ToMRFEntry() + p.saveMRFEntries(ctx, map[string]MRFReplicateEntry{entry.versionID: entry}) + fresh := newPool() + looked := make(chan struct{}) + fresh.objLayer = markerLookupLayer{ObjectLayer: obj, mutate: tc.mutate, lookedUp: looked} + if err := fresh.queueMRFHeal(); err != nil { + t.Fatal(err) + } + select { + case <-looked: + case <-time.After(3 * time.Second): + t.Fatal("disk MRF entry was not read") + } + select { + case op := <-fresh.workers[0]: + t.Fatalf("invalid lookup scheduled %T", op) + case <-time.After(100 * time.Millisecond): + } + }) + } +} + +// Capture the actual internal audit event through the supported webhook sink. +func replicationTestAudit(ctx context.Context, t *testing.T, bucket string) (<-chan string, func()) { + t.Helper() + if len(logger.AuditTargets()) != 0 { + t.Fatal("unexpected pre-existing test audit targets") + } + statuses := make(chan string, 16) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var entry struct { + API struct{ Name, Bucket, Status string } + } + if err := json.NewDecoder(r.Body).Decode(&entry); err != nil { + t.Error(err) + } else if entry.API.Name == ReplicateDeleteAPI && entry.API.Bucket == bucket { + statuses <- entry.API.Status + } + w.WriteHeader(http.StatusOK) + })) + endpoint, err := xnet.ParseHTTPURL(server.URL) + if err != nil { + t.Fatal(err) + } + if errs := logger.UpdateAuditWebhooks(ctx, map[string]loghttp.Config{"r6": {Enabled: true, Name: "r6", Endpoint: endpoint, BatchSize: 1, QueueSize: 128, MaxRetry: 1, RetryIntvl: time.Millisecond, HTTPTimeout: time.Second}}); len(errs) > 0 { + t.Fatal(errs) + } + targets := logger.AuditTargets() + return statuses, func() { + logger.UpdateAuditWebhooks(ctx, nil) + for _, target := range targets { + target.Cancel() + } + server.Close() + } +} + +type markerPurgeUpdateLayer struct { + ObjectLayer + t *testing.T +} + +func (l markerPurgeUpdateLayer) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + l.t.Helper() + rs := opts.DeleteReplication + if rs.ReplicationStatusInternal != "" || rs.Targets != nil || rs.ReplicaStatus != "" || !rs.CompositeReplicationStatus().Empty() { + l.t.Fatalf("purge has a nonempty creation update: %+v", rs) + } + if rs.CompositeVersionPurgeStatus().Empty() { + l.t.Fatal("purge update lost its purge status") + } + return l.ObjectLayer.DeleteObject(ctx, bucket, object, opts) +} diff --git a/cmd/replication-delete-operation_test.go b/cmd/replication-delete-operation_test.go new file mode 100644 index 000000000..3f4d759b0 --- /dev/null +++ b/cmd/replication-delete-operation_test.go @@ -0,0 +1,201 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-only + +package cmd + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" +) + +// Purge uses the version-delete wire form regardless of the in-memory shape. +// Creation status, purge status and successful resync markers are independent. +func TestReplicateDeleteOperationExits(t *testing.T) { + for _, shape := range []string{"marker-creation", "legacy-marker-purge", "marker-purge", "object-purge"} { + purge := shape != "marker-creation" + for _, tc := range []struct { + name string + creation replication.StatusType + priorPurge VersionPurgeStatusType + headCode, deleteCode int + headError string + offline, resync bool + }{ + {name: "pending-success", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 204}, + {name: "completed-creation", creation: replication.Completed, priorPurge: replication.VersionPurgePending, headCode: 405, deleteCode: 204}, + {name: "existing-marker", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 405, deleteCode: 204}, + // Use the quorum S3 code without 503, which is classified as backend-down first. + {name: "head-read-quorum", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 400, headError: "SlowDownRead", deleteCode: 204}, + {name: "replica-creation-status", creation: replication.Replica, priorPurge: replication.VersionPurgeFailed, headCode: 405, deleteCode: 204}, + {name: "head-unavailable", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 503, deleteCode: 204}, + {name: "head-forbidden", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 403, deleteCode: 204}, + {name: "delete-forbidden", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 403}, + {name: "delete-method-rejected", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 405}, + {name: "delete-unavailable", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 503}, + {name: "offline", creation: replication.Pending, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 204, offline: true}, + {name: "retry-failed", creation: replication.Completed, priorPurge: replication.VersionPurgeFailed, headCode: 404, deleteCode: 204}, + {name: "purge-complete", creation: replication.Pending, priorPurge: replication.VersionPurgeComplete, headCode: 404, deleteCode: 403}, + {name: "resync-success", creation: replication.Completed, priorPurge: replication.VersionPurgePending, headCode: 405, deleteCode: 204, resync: true}, + {name: "resync-already-purged", creation: replication.Pending, priorPurge: replication.VersionPurgeComplete, headCode: 404, deleteCode: 403, resync: true}, + {name: "resync-failure", creation: replication.Completed, priorPurge: replication.VersionPurgePending, headCode: 404, deleteCode: 403, resync: true}, + } { + t.Run(shape+"/"+tc.name, func(t *testing.T) { + version := mustGetUUID() + var heads, deletes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("versionId") != version { + t.Errorf("wrong request version: %s", r.URL) + } + switch r.Method { + case http.MethodHead: + heads.Add(1) + w.Header().Set(xhttp.AmzVersionID, version) + w.Header().Set(xhttp.LastModified, time.Now().UTC().Format(http.TimeFormat)) + if tc.headCode == 405 { + w.Header().Set(xhttp.AmzDeleteMarker, "true") + } + if tc.headError != "" { + w.Header().Set("x-minio-error-code", tc.headError) + } + w.WriteHeader(tc.headCode) + case http.MethodDelete: + deletes.Add(1) + if got := r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"; got == purge { + t.Errorf("source delete-marker header=%v, purge=%v", got, purge) + } + w.WriteHeader(tc.deleteCode) + if tc.deleteCode != 204 { + fmt.Fprintf(w, `%sinjected rejection`, map[int]string{403: "AccessDenied", 405: "MethodNotAllowed", 503: "ServiceUnavailable"}[tc.deleteCode]) + } + default: + t.Errorf("unexpected method %s", r.Method) + } + })) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + old := globalBucketTargetSys + globalBucketTargetSys = &BucketTargetSys{hc: map[string]epHealth{client.EndpointURL().Host: {Online: !tc.offline}}} + defer func() { globalBucketTargetSys = old }() + d := DeletedObjectReplicationInfo{Bucket: "source", DeletedObject: DeletedObject{ObjectName: "marker", DeleteMarker: shape != "object-purge"}} + if shape == "marker-creation" || shape == "legacy-marker-purge" { + d.DeleteMarkerVersionID = version + } else { + d.VersionID = version + } + d.ReplicationState.Targets = map[string]replication.StatusType{"arn1": tc.creation} + d.ReplicationState.ResetStatusesMap = map[string]string{"arn1": "previous-reset"} + if purge { + d.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{"arn1": tc.priorPurge} + } + if tc.resync { + d.OpType = replication.ExistingObjectReplicationType + } + got := replicateDeleteToTarget(t.Context(), d, &TargetClient{Client: client, ARN: "arn1", Bucket: "target", ResetID: "current-reset"}) + var wantCreation replication.StatusType + var wantPurge VersionPurgeStatusType + var wantHeads, wantDeletes int32 + success := false + if purge { + wantCreation = "" + wantPurge = replication.VersionPurgeComplete + switch { + case tc.priorPurge == replication.VersionPurgeComplete: + success = true + case tc.offline: + wantPurge = replication.VersionPurgeFailed + default: + wantDeletes = 1 + success = tc.deleteCode == 204 + if !success { + wantPurge = replication.VersionPurgeFailed + } + } + } else { + switch { + case tc.creation == replication.Completed && !tc.resync: + success = true + case tc.offline: + default: + wantHeads = 1 + switch tc.headCode { + case 405: + success = true + case 403, 503: + default: + wantDeletes = 1 + success = tc.deleteCode == 204 + } + } + if success { + wantCreation = replication.Completed + } else { + wantCreation = replication.Failed + } + } + if got.PrevReplicationStatus != tc.creation { + t.Error("previous creation state changed") + } + if got.ReplicationStatus != wantCreation || got.VersionPurgeStatus != wantPurge { + t.Errorf("status=%+v, want creation=%s purge=%s", got, wantCreation, wantPurge) + } + if heads.Load() != wantHeads || deletes.Load() != wantDeletes { + t.Errorf("HEAD/DELETE=%d/%d, want %d/%d", heads.Load(), deletes.Load(), wantHeads, wantDeletes) + } + if (got.Err != nil) == success { + t.Errorf("success=%v, error=%v", success, got.Err) + } + if tc.resync && success { + if !strings.HasSuffix(got.ResyncTimestamp, ";current-reset") { + t.Errorf("successful resync missing reset: %q", got.ResyncTimestamp) + } + } else if got.ResyncTimestamp != "previous-reset" { + t.Errorf("unexpected reset: %q", got.ResyncTimestamp) + } + }) + } + } +} + +func TestReplicateDeletePurgeMissingTargetState(t *testing.T) { + // Classification belongs to the operation, even if this target has no + // previous purge entry (another target supplies the tracked purge state). + var deletes atomic.Int32 + version := mustGetUUID() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Query().Get("versionId") != version || r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true" { + t.Errorf("wrong purge request: %s %s %v", r.Method, r.URL, r.Header) + } + deletes.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + old := globalBucketTargetSys + globalBucketTargetSys = &BucketTargetSys{hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}} + defer func() { globalBucketTargetSys = old }() + d := DeletedObjectReplicationInfo{Bucket: "source", DeletedObject: DeletedObject{ObjectName: "marker", DeleteMarker: true, DeleteMarkerVersionID: version, ReplicationState: ReplicationState{Targets: map[string]replication.StatusType{"arn1": replication.Completed}, PurgeTargets: map[string]VersionPurgeStatusType{"arn2": replication.VersionPurgePending}}}} + got := replicateDeleteToTarget(t.Context(), d, &TargetClient{Client: client, ARN: "arn1", Bucket: "target", ResetID: "reset"}) + if deletes.Load() != 1 || got.VersionPurgeStatus != replication.VersionPurgeComplete || got.ReplicationStatus != "" { + t.Fatalf("operation misclassified: %+v, deletes=%d", got, deletes.Load()) + } + got.ResyncTimestamp = "resync;reset" + state := getReplicationState(replicatedInfos{Targets: []replicatedTargetInfo{got}}, ReplicationState{}, "") + if state.ResetStatusesMap[targetResetHeader("arn1")] != got.ResyncTimestamp { + t.Fatal("resync timestamp not preserved with nil reset map") + } +} diff --git a/docs/investigations/r6/README.md b/docs/investigations/r6/README.md new file mode 100644 index 000000000..309f9c296 --- /dev/null +++ b/docs/investigations/r6/README.md @@ -0,0 +1,61 @@ +# R6：delete-marker purge 与 MRF 修复 + +## 当前交付状态 + +本地实现已完成，v3 已与真实 Opus 5.0 达成共识，原研究基线上的定向回归、race、完整构建、vet、lint 全部通过。正在将隔离分支同步到新主干快照 `af2b1794d38d9e70e1d2c3ee692426e4b6cab4bd` 并复验。 + +- 研究基线：`9ebe81c1b3611f9cc73e676b5b741c2be62c467a`。 +- 分支：`codex/r6-delete-marker-mrf`。 +- [PR #184](https://github.com/pgsty/silo/pull/184) 在最终核对时仍为 OPEN，head `6addf9eb916b5a4b837480cf534cd1efa5407d3c`。复用其按 purge 状态识别操作、接纳 marker 405 的方向，补齐实测遗漏；没有直接合并该 PR。 +- 新主干只新增 R4 的 SSE-KMS PUT 选项与对应材料，R6 涉及的生产文件、测试文件和依赖未发生交叉修改。 +- 本任务没有执行主干合并、远端推送、发布、部署或现网存量改写。 + +## 修复内容 + +| 操作 | 远端行为 | 结果与源端写回 | +|---|---|---| +| marker 创建 | 保留 HEAD 已存在/就绪检查及创建语义 | 更新创建状态；405 可以表示已创建 | +| 规范版本 purge | 发送指定 versionId 的永久删除 | 只更新 purge 状态，保留磁盘创建/replica 字段 | +| 旧 marker 形态 purge | 从任务级 purge 状态识别，沿用规范永久删除请求 | 不再被创建 COMPLETED 跳过；失败进入 MRF；已完成 purge 不重发 | + +- 离线、DELETE 拒绝、成功与 resync 出口使用操作自己的状态；失败 purge 不写成功 reset 标记。源端 purge 写回显式清空三个“创建更新”字段，避免多目标空状态被旧正则误解析后覆盖磁盘创建记录。 +- MRF 接受携带正确 marker、版本、对象、桶和非零时间的 405；其它错误或无效信息不调度删除。 +- 删除任务携带重试计数；锁失败、复制失败、工作队列满三个入 MRF 出口都递增。耗尽现有预算后继续保留 scanner 恢复路径。 +- purge 的内部 COMPLETE 保持不变；操作审计使用规范 COMPLETED，失败为 FAILED。按目标状态变化更新统计，成功 heal purge 的统计为次数增加、字节数为零。 +- 不改 wire 格式、MRF 磁盘格式、共享正则、复制状态合并框架或依赖版本。 + +## 真实 Opus 共识 + +使用本机 Claude Code 2.1.270，每轮显式指定 `claude-opus-5 --effort max`。全部记录到的 assistant 模型均为 `claude-opus-5`；实际调用成功，未用模拟评审或限流失败代替同意。 + +| 方案 | 结论 | 处置 | +|---|---|---| +| v1 | REVISE，1 个阻断 | 接受意见：不能用本次目标子集重写完整创建状态 | +| v2 | GO_WITH_NONBLOCKING_NOTES | 共识后实现；随后用真实存储发现多目标空状态正则反例 | +| v3 | GO_WITH_NONBLOCKING_NOTES，0 阻断 | reviewer 撤回过强的旧证明，确认三字段清空方案；共识后应用并验证 | + +最终不可变方案：[plan-v3.md](plan-v3.md)，SHA256 `dc9a67fc91b3113fa35218a2f903455807430cc4daf9c78a88d0be6b8fc27058`。 +详见 [完整共识及逐项处置](consensus.md)、[研究与 #184 审查](research.md)、[v3 原始评审正文](opus-v3-review.md)。模型只读权限不允许计算哈希；它核对了具体文件内容，本任务在评审前后计算并确认哈希不变。评审不替代执行验证。 + +## 验证范围 + +原基线证据：[机器记录](verification/baseline-verification.json)，以及同目录 scope/race/build/vet/lint 日志。 + +- 28 个顶层定向测试通过，包含 198 个通过条目（含子测试）。 +- 单盘、16 盘真实 erasure 存储；真实源端签名 DELETE、minio-go HTTP、源/目标 marker 元数据。 +- 失败 → MRF 文件持久化 → 新 ReplicationPool 读取 → 真实 marker+405 lookup → 工作队列 → 生产 replicateDelete → 恢复。 +- 覆盖创建/旧新 purge、部分目标重试、两目标一成一败/离线、远端已删除但响应丢失、重试预算耗尽和 scanner 接管；恢复后核对源和目标 marker 最终状态。 +- 两个目标空状态的实际存储反例已转绿；所有 purge 写回均通过断言确认创建字段为空，完整创建/replica 元数据和时间戳保留。 +- 验证实际审计 webhook 的 FAILED/COMPLETED，以及失败/成功统计变化；race 未报告数据竞争。 + +**边界：** 目标为受控 HTTP 适配器，调用真实 ObjectLayer；测试显式消费队列并执行生产复制函数，直接驱动 MRF 保存，没有启动后台定时器和完整 worker 循环。这是三端点 fan-out 与磁盘恢复验证，不是三台独立 SILO 进程的站点复制集群、接收端认证或进程崩溃验收。 + +6 项无关广义 DELETE 测试在种子数据写入时触发宿主机容量阈值，该次扩大测试未通过；R6 存储夹具使用仓库现有容量适配器，数据仍真实落盘。一次测试链接遇到磁盘空间耗尽，清理可确认属于本任务的旧 Go 缓存后复验。详情和中间失败记录：[verification-notes.md](verification-notes.md)。 + +## 仍然独立的事项 + +当前 DELETE/scanner/heal/resync 已产生规范 purge；旧任务形态不会序列化跨重启，不能宣称所有失败 purge 永久卡住。本修复主要恢复活跃的 marker MRF 路径，并完善旧形态兼容。 + +未覆盖或未修改：缺失客户端导致的目标状态遗漏、目标级 resync 的既有 purge 子集替换、复制跟踪已丢失、replica relay、purge 后延迟创建且无 tombstone、源端元数据写失败依赖 scanner、共享解析器的通用健壮性、额外 backoff/指标设计。完整多进程站点验收应另行安排。 + +原始大日志与临时复现：`/Users/vonng/tmp/silo-r6-20260915-aa3f/`。每轮 metadata 记录模型、命令、基线、方案及原始输出哈希。 diff --git a/docs/investigations/r6/consensus.md b/docs/investigations/r6/consensus.md new file mode 100644 index 000000000..65ace86af --- /dev/null +++ b/docs/investigations/r6/consensus.md @@ -0,0 +1,49 @@ +# R6 plan consensus — final v3 + +2026-09-15. Source baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`. + +Earlier v2 agreed immutable plan: [plan-v2.md](plan-v2.md), SHA256 `dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3`. + +Codex accepts this plan. Real Claude Code 2.1.270, explicitly `--model claude-opus-5 --effort max`, read the same file and returned **GO_WITH_NONBLOCKING_NOTES, no blockers**, with explicit consensus. Every recorded assistant model in both rounds is `claude-opus-5`. CLI auxiliary usage is listed separately in the metadata files. The reviewer had read-only tools, so verified content but did not independently compute the hash; Codex computed the hash before and after review and confirmed it unchanged. The plan is 59 lines as described by the reviewer. A `rate_limit_event` telemetry record is not a failure verdict: the actual final result is `subtype=success, is_error=false`, and includes the explicit review and consensus. + +| Round | Verdict | Outcome | +|---|---|---| +| v1 | REVISE, one blocker | Accepted B1: preserve the disk creation block using the existing empty-update signal. All seven notes resolved/scoped in decisions-v2.md. No production code changed. | +| v2 | GO_WITH_NONBLOCKING_NOTES | Same immutable plan accepted by both parties; implementation and tests now proceed. | + +Raw logs: `/Users/vonng/tmp/silo-r6-20260915-aa3f/opus-v1.jsonl`, `opus-v2.jsonl`, and matching stderr logs. Extracted reviews and SHA/model/command metadata are alongside this document. No simulated reviewer or fallback model was substituted. + +## v2 nonblocking dispositions + +1. Keep canonical success audit normalization COMPLETE → COMPLETED, using replication.CompletedLegacy for conversion; assert the actual audit outcome and document the visible string correction. +2. Use operation status for per-target change comparisons. Assert count-only successful heal purge deltas, zero bytes, and no pending operation outcome for failed purges. No new metrics policy. +3. Initialize ResetStatusesMap before an existing assignment when nil, unconditionally safe; no general merge change. +4. Already-COMPLETE purge also short-circuits under ExistingObjectReplicationType, matching canonical behavior; add a matrix row. +5. Offline errors populate Err for both creation and purge. This is explicit error reporting with the same failed outcome. +6. Preserve getReplicationState's existing unused third parameter and shape-agnostic behavior. +7. Validate real marker/nonempty version/nonzero ModTime plus bucket and decoded object identity. Use decodeDirObject for the name comparison so the stricter gate does not reject the internal directory-object encoding. +8. Use int for delete RetryCount, matching the persisted MRF field and QueueReplicationHeal input. No on-disk format changes. +9. Purge-status subset replacement is pre-existing and remains outside R6; creation-block preservation under partial fan-out is newly tested. +10. Tests drive saveMRFEntries directly, verify the real stored record, and create a fresh pool for each load/queue replay. Timer waiting and process-crash durability are not claimed. + +These are implementation refinements within the accepted plan; Opus explicitly stated they require no new review round/hash. Consensus is not test acceptance, merge, release or deployment. Production implementation begins only after this record was written. + + +## Final v3 consensus, 2026-09-16 CST + +Final immutable plan: [plan-v3.md](plan-v3.md), SHA256 `dc9a67fc91b3113fa35218a2f903455807430cc4daf9c78a88d0be6b8fc27058` (70 lines; locally recomputed unchanged after review). + +Codex accepts v3. Real `claude-opus-5 --effort max` returned GO_WITH_NONBLOCKING_NOTES, no blockers, and explicit consensus on the exact read content. As before, read-only reviewer tools could not compute the digest; the model verified the named content and Codex verified its hash. Original review/metadata are opus-v3-review.md and opus-v3.metadata.json. The review candidly withdraws the earlier multi-target regex proof. This is actual additional review, not a simulated amendment to the earlier output. + +The v2 implementation was completed only after v2 agreement. Codex then found and reproduced the two-empty-status parser counterexample on real storage; the incremental three-field v3 source change remained unapplied until this new consensus was recorded. + +| v3 note | Disposition | +|---|---| +| N1 payload invariant | Accepted as mandatory: wrap the real ObjectLayer update in purge tests and assert all three creation-update fields and their composite are empty. | +| N2 masking scope | Ordinary purges are protected through FileInfo.Deleted=false; ordinary object versions have another such guard. The unrecorded marker case corrupts creation metadata only on its first failed write. | +| N3 ReplicaStatus | Clearing it is defensive, not a repair of an observed producer population. | +| N4 shared parser | Remains unchanged; generic parser robustness is separate. No future arbitrary caller guarantee is claimed. | +| N5 timestamp assignment | Retained; empty creation-update payload makes it irrelevant to purge disk creation metadata. | +| N6 prior proof | v2's regex proof is correct only for a single target. The immutable raw reviews and executable counterexample are both retained. | + +Only v3's three field assignments and the required invariant remain to be applied after this record. All earlier compatible refinements and acceptance limits still stand. diff --git a/docs/investigations/r6/decisions-v2.md b/docs/investigations/r6/decisions-v2.md new file mode 100644 index 000000000..acccc6760 --- /dev/null +++ b/docs/investigations/r6/decisions-v2.md @@ -0,0 +1,18 @@ +# R6 v1 review disposition + +Actual reviewer: Claude Code 2.1.270, assistant model `claude-opus-5`, explicit effort max. Original verdict REVISE, one blocker; see opus-v1-review.md and opus-v1.metadata.json. No consensus or production implementation at this stage. + +| Item | Disposition | v2 change / evidence | +|---|---|---| +| B1 creation-status pin rewrites a partial target set | Accepted. The disk no-update semantics are preferable and smaller. | Purge results leave ReplicationStatus empty, only VersionPurgeStatus changes; assert full persisted creation state and timestamp across partial fan-out/repeated failure. No generic state merge rewrite. | +| N1 wire version fallback | Accepted. | Existing DeleteMarkerVersionID fallback retained; tests assert query version and false marker flag for every purge shape. | +| N2 queue-full retry budget | Accepted. | All three queueMRFSave sites for deletes increment RetryCount, including queueReplicaDeleteTask. | +| N3 completion statistics change | Accepted. | Check concrete Heal/ExistingObject counter deltas for COMPLETE-to-COMPLETED conversion. | +| N4 null/empty versions | Accepted as current boundary. | 405 recovery requires a real nonempty identity; empty/null special cases remain outside acceptance. | +| N5 live producer/old-shape scope | Accepted. | Old tasks are not serialized; robustness path distinguished from currently active MRF defect. | +| N6 detached MRF execution | Accepted. | Real disk persistence each round, new pool, synchronized queue receives with bounded timeout, no sleeping for presumed completion. | +| N7 fixture threshold | Resolved with actual execution. | Existing capacity adapter; baseline canonical and old scanner recovery pass. PR MRF canonical recovery completes on single/16-drive fixtures; old-shape failure still queues zero entries. | + +Correction to our research inference: a canonical purge's empty returned creation result causes replicatedInfos.ReplicationStatus() to report PENDING, but that does NOT show loss of the disk creation block. xlMetaV2 skips that block update when the composite creation status is empty. The temporary probe's in-memory assertion was too strong; do not promote it into a disk-state defect. The PR old-shape missing-MRF observation and all target-branch observations remain valid. + +The local-source metadata-write error path and missing-client/purge-subset merge behavior are explicitly documented acceptance limits. They are not new claims of convergence. v2 does not broaden the repair into those independent mechanisms. diff --git a/docs/investigations/r6/opus-prompt-v1.md b/docs/investigations/r6/opus-prompt-v1.md new file mode 100644 index 000000000..dfb54450d --- /dev/null +++ b/docs/investigations/r6/opus-prompt-v1.md @@ -0,0 +1,5 @@ +You are the independent Opus reviewer for SILO R6. Read AGENTS.md first. Review only; do not edit files. We need real independent technical scrutiny, not a ceremonial approval. + +Source commit: 9ebe81c1b3611f9cc73e676b5b741c2be62c467a. Plan file: docs/investigations/r6/plan-v1.md. Plan SHA256: bcd023e2b00ad3dc709baa738bb02551d47aeebb49879b0f76f2789f4c4dff8b. Read that exact file in full and inspect the referenced current source and tests. PR184 raw patch and fresh baseline probes are in /Users/vonng/tmp/silo-r6-20260915-aa3f/ (read pr184.diff, review_probe_test.go, baseline.log if available). Its code direction is useful but its claims are not accepted evidence. Focus on every exit of replicateDeleteToTarget and replicateDelete, persisted queueMRFHeal with MethodNotAllowed ObjectInfo, preserving statuses, resync accounting and actual wire behavior on retries. + +Critically verify whether the minimal proposed code could restore an already removed marker, whether retry budgets and multi-target status remain safe within the explicit bounded scope. Explain any disagreement with precise code evidence and a concrete minimal correction. Return a verdict GO, GO_WITH_NONBLOCKING_NOTES, or REVISE, explicitly for plan v1 hash bcd023e2b00ad3dc709baa738bb02551d47aeebb49879b0f76f2789f4c4dff8b. List blocking findings separately from nonblocking suggestions. If there are no blockers, explicitly state that consensus on this plan is technically acceptable, while implementation still requires tests. Do not claim to have run anything. diff --git a/docs/investigations/r6/opus-prompt-v2.md b/docs/investigations/r6/opus-prompt-v2.md new file mode 100644 index 000000000..2d95f2c7c --- /dev/null +++ b/docs/investigations/r6/opus-prompt-v2.md @@ -0,0 +1,3 @@ +Review the revised R6 plan, read-only. The exact source remains 9ebe81c1b3611f9cc73e676b5b741c2be62c467a. Read AGENTS.md, docs/investigations/r6/opus-v1-review.md, docs/investigations/r6/decisions-v2.md, and docs/investigations/r6/plan-v2.md (SHA256 dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3). Focus on resolving your B1 and coupled exits, and verify the amended retry queue-full site. We accepted your disk-preservation correction; the previous inference from an empty per-target return was too strong. No production implementation has started. Source and temporary probes remain available. Real storage fixtures now execute with the repository capacity adapter (only DiskInfo capacity changes); baseline-mrf.log and pr184-mrf.log have the evidence. + +Perform a targeted delta review of v2, inspect source as needed; do not re-review unrelated systems or attempt to write a file. Say GO, GO_WITH_NONBLOCKING_NOTES, or REVISE explicitly for plan-v2.md hash dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3, list any blockers, and if acceptable explicitly state consensus on the exact plan. Tests are future obligations, not completed acceptance. diff --git a/docs/investigations/r6/opus-prompt-v3.md b/docs/investigations/r6/opus-prompt-v3.md new file mode 100644 index 000000000..ddc20b023 --- /dev/null +++ b/docs/investigations/r6/opus-prompt-v3.md @@ -0,0 +1,5 @@ +Targeted R6 incremental review, read only, no file writes. Real v2 consensus was GO; v2 implementation was then authorized and implemented. We found an executable counterexample to part of both our empty-composite proofs. Read docs/investigations/r6/plan-v3.md (SHA256 dc9a67fc91b3113fa35218a2f903455807430cc4daf9c78a88d0be6b8fc27058) especially the final incremental section, and /Users/vonng/tmp/silo-r6-20260915-aa3f/v2-unrecorded-purge.log. Inspect the current implementation and the new TestReplicationMRFMarkerRecovery/unrecorded-purge test. Source identity: baseline 9ebe81c1 plus implementation-v2.diff in that evidence directory; v3's three-field correction is NOT applied yet. + +The regex requires a character but permits a semicolon: two empty entries parse into a nonempty fake status. Usual purge metadata sets FileInfo.Deleted=false and masks it, but a task ahead of source purge metadata reproduces real creation-block overwrite with two targets. Proposed minimal delta is ONLY in outer replicateDelete after getReplicationState: if isPurge then drs.ReplicationStatusInternal="", drs.Targets=nil, drs.ReplicaStatus="". This makes CompositeReplicationStatus truly empty, preserves disk creation/replica blocks, retains purge/reset metadata, and avoids changing shared parsing or merging. + +Please independently verify the counterexample, field selection, whether the proposal introduces any loss or leak, and whether it suffices. Focus this review on the delta and its interactions rather than restarting unrelated exploration. Return explicit GO/GO_WITH_NONBLOCKING_NOTES/REVISE for plan-v3.md hash dc9a67fc91b3113fa35218a2f903455807430cc4daf9c78a88d0be6b8fc27058; list blockers and state exact-plan consensus if acceptable. Do not claim tests ran or that the normal producer necessarily emits the special setup. The reviewers' earlier multi-target regex proof was too strong; revise the conclusion candidly. Other scope limits remain explicit. diff --git a/docs/investigations/r6/opus-v1-review.md b/docs/investigations/r6/opus-v1-review.md new file mode 100644 index 000000000..d7db521b4 --- /dev/null +++ b/docs/investigations/r6/opus-v1-review.md @@ -0,0 +1,78 @@ +I'll start by reading AGENTS.md and the plan file, then dig into the actual source. + +Let me locate the core source files and the evidence directory. + +I've verified the plan against the actual source. Writing up the review now. + +The `Write` tool is disabled in this session, so I can't create the plan file — the review is below in full. + +--- + +# Independent review — SILO R6 `plan-v1.md` + +**Plan hash reviewed:** `bcd023e2b00ad3dc709baa738bb02551d47aeebb49879b0f76f2789f4c4dff8b` +**Source:** `9ebe81c1b3611f9cc73e676b5b741c2be62c467a` +**I did not run anything.** All findings come from reading the source, the existing tests, PR #184's diff/prose, and the supplied `baseline.log` (read, not produced by me). + +## Verdict: **REVISE** + +One blocking finding. The classification rule, the canonical-wire choice, the MRF 405 work, and the retry-budget work are technically correct and better grounded than PR #184. One explicitly prescribed detail in §A causes a silent regression on the *canonical* purge shape — the shape every live producer emits — and §C would not catch it. The correction is small and stays inside the plan's bounded scope. + +--- + +## Blocking + +### B1 — Pinning `rinfo.ReplicationStatus = rinfo.PrevReplicationStatus` for purges makes the persisted marker creation-status block get rewritten from the fan-out subset + +§A prescribes this "on every exit" while also forbidding changes to `getReplicationState` merging. Together that is a regression, not preservation. + +1. `bucket-replication-utils.go:392-399` — `targetState()` never sets `ReplicationStatus`, so a canonical purge returns `""` today. +2. `bucket-replication-utils.go:92-101` — `ReplicationStatusInternal()` rebuilds `"arn=STATUS;"` **only from `rinfos.Targets`**, i.e. only fanned-out targets. +3. `bucket-replication.go:508-543` — fan-out skips `!Replicate`, non-matching `dobj.TargetArn`, and nil clients. `TargetArn` is really set by `queueReplicateDeletesWrapper` (`:2400-2409`). +4. `bucket-replication-utils.go:410-412` → `drs`, passed as `DeleteReplication` at `bucket-replication.go:572-582`. +5. `erasure-object.go:2163-2177` → `xl-storage-format-v2.go:1438-1446` — for a `DeleteType` version, **if `fi.DeleteMarkerReplicationStatus()` is non-empty**, `MetaSys[ReplicationStatus]` and `MetaSys[ReplicationTimestamp]` are overwritten. + +Today that guard never fires for a canonical purge: the string is `"arn1=;"`, `replicationStatusesMap` doesn't match it (`:428-439`), composite over the empty map is `""` (`:455-459`) — so the on-disk creation block is left **untouched**, which preserves it perfectly including non-attempted targets. Under the plan it becomes non-empty and is rewritten: + +- **Status loss:** `"arn1=COMPLETED;arn2=COMPLETED;"` with a fan-out covering only `arn2` persists `"arn2=COMPLETED;"` — `arn1` silently dropped. That is the exact failure the plan exists to fix, newly introduced on the main shape. +- **Zero timestamp:** `rs.ReplicationTimeStamp = rinfos.ReplicationTimeStamp` (`:413`) is never assigned in `replicateDelete`; it's only rescued by `bucket-replication.go:568-570` when the composite *changes*. A repeated FAILED→FAILED purge writes `0001-01-01T00:00:00Z`. +- **Empty ReplicaStatus:** under a legacy `RoleArn` config the composite can be `REPLICA` (`bucket-replication-utils.go:497-503`), taking `xl-storage-format-v2.go:1398-1400`, which writes `ReplicaStatus` — never populated by `ObjectInfo.ReplicationState()` (`:569-587`). + +Only delete-**marker** versions are affected; `ObjectType` versions only touch `VersionPurgeStatusKey` (`xl-storage-format-v2.go:1465-1476`). + +**Minimal correction (strictly smaller than the plan):** keep the classification and every exit fix, but for purges **leave `rinfo.ReplicationStatus` at its zero value** rather than pinning it. That is what the canonical path already does, and it preserves the on-disk block byte-for-byte including non-attempted targets, with no change to `getReplicationState`. Consequences that then become mandatory, not optional: + +- purge exits write only `VersionPurgeStatus` — never `Failed`, `Completed`, or `PrevReplicationStatus`; +- the resync defer (`:618-622`) must be gated on the operation's own success. Under the plan as written, pinning `Completed` would stamp the reset marker for a **failed** purge on any marker whose creation was COMPLETED — the two changes are coupled and cannot land separately; +- the stats gate at `:556` must be replaced by the per-target operation-status comparison §A already calls for, otherwise purges stop being reported at all. + +Add a partial-fan-out test: two ARNs persisted, one excluded from fan-out, failed purge → marker metadata unchanged. §C item 5 fans out to *both* targets and cannot detect this. + +--- + +## Nonblocking + +1. **The wire version ID is load-bearing.** For the legacy shape `dobj.VersionID` is empty, so the implementation must keep the existing `versionID` fallback (`bucket-replication.go:609-612`). A `RemoveObject` with empty `VersionID` against a versioned target **creates a new marker** (`erasure-object.go:2126-2149`) — the exact resurrection being fixed. Make §C1 assert the outgoing `versionId` and `x-minio-source-deletemarker` explicitly. +2. **Retry increment must cover all three delete-path MRF sites:** `:487` (lock), `:565` (aggregate failure), `:2427` (queue full). §B names only the first two; missing `:2427` leaves an unbounded loop once marker MRF is live. Mirror `ri.RetryCount++` at `:1310`. +3. **Stats will move.** `COMPLETE`→`COMPLETED` makes `ReplicationStats.Update` reach its `Completed` case for Heal/ExistingObject deletes (`bucket-replication-stats.go:184-201`, `replication.go:139-144`). Today `"COMPLETE" != "COMPLETED"` so nothing is recorded. Assert expected counter deltas rather than discovering them. +4. **§B's identity gate excludes null-version markers by construction** — `GetObjectInfo` returns `ObjectNotFound`, not 405, when `VersionID == ""` (`erasure-object.go:996-999`), and `ToObjectInfo` leaves `VersionID` empty when `versioned` is false (`erasure-metadata.go:120-123`). No regression, but document it instead of implying MRF healing is complete. +5. **Legacy-shape scope.** I found no producer of that shape on this baseline: `object-handlers.go:3225-3228`, `bucket-handlers.go:565-570`, `bucket-replication.go:3323-3327`/`:3775-3779` are mutually exclusive, and `erasure-object.go:1752-1766` only sets `DeleteMarkerVersionID` when `VersionID == ""`. `DeletedObjectReplicationInfo` isn't serialized, so it can't survive a restart. §A's legacy handling is upgrade/robustness work; the live value of R6 is mostly §B. Say so in the PR text so the fork doesn't inherit #184's overclaiming. +6. **§C item 3 is testable but fiddly.** `globalLocalDrivesMap` is populated by `newErasureServerPools` (`erasure-server-pool.go:174-181`), so save/load works — but `loadMRF` **deletes the file after reading** (`:4013-4015`) and `queueMRFHeal` dispatches a detached goroutine with a 1s per-entry context (`:4067-4081`). Re-persist between rounds and synchronize, don't sleep. +7. **Confirm the fixtures run.** The supplied `baseline.log` shows both `TestReplicateDeleteMarkerPurge` subtests aborting at `replication-delete-marker_test.go:121` with "Storage reached its minimum free drive threshold" — an environment failure. §C items 2, 3, 5 all depend on those fixtures. + +--- + +## Independently confirmed as correct in the plan + +- **Observation 1 holds at every exit:** `:624` (early-return without sending), `:645-649`/`:702-706`/`:712-716` (field selected on `dobj.VersionID`), `:684-688` (HEAD-not-ready overwrites creation status unconditionally), `:628` (completed-purge early-out only for non-empty `VersionID`), `:618-622` (resync stamp keyed on `ReplicationStatus`). The supplied probe log agrees. +- **Observation 2 is a real regression in PR #184.** `replicateDelete` selects on `dobj.VersionID != ""` (`:549-552`), so with only the target-side fix a failed legacy purge aggregates to `COMPLETED`, emits `ObjectReplicationComplete`, and **skips `queueMRFSave`** (`:563-566`) — worse than baseline. +- **Task-level classification is required; #184's per-target `isDMPurge` is wrong.** `VersionPurgeStatus()` needs `completed == len(ri.Targets)` (`bucket-replication-utils.go:122-139`); a target classified as a creation never sets the purge field, so the composite can never reach COMPLETE. The task-level rule is also safe here — no current producer emits a creation with a non-empty composite purge status. +- **The canonical-wire choice really does fix resurrection; #184 does not.** With `ReplicationDeleteMarker=true` and an absent version the receiver re-creates the marker (`xl-storage.go:1346-1349`; `xl-storage-format-v2.go:1517-1520`). With `false`, the receiver returns VersionNotFound (`erasure-object.go:2013-2027`) and the handler answers 204 (`object-handlers.go:3171-3192`). Scope it honestly: this only changes *legacy-shaped* purges (canonical already sends `false`), and does nothing for a delayed creation after a purge — correctly filed as separate. +- **§B's 405 premise is exact:** `erasure-object.go:996-1002` returns a populated `ObjectInfo` with `toObjectErr(errMethodNotAllowed,…)` → `MethodNotAllowed{}` (`object-api-errors.go:96-102`), preserved multi-pool too (`erasure-server-pool.go:1063-1077`). +- **§B's retry gap is real:** `ToMRFEntry()` (`:1927-1937`) never sets `RetryCount`, so the `> mrfRetryLimit` drop (`:3883-3887`) can never fire for deletes. No msgp code exists for the type, so the field is genuinely schema-free. +- **Resync accounting already handles both shapes** — `resyncTargetSucceeded` (`:3154-3162`) keys on `roi.VersionPurgeStatus`. The plan correctly leaves it alone. +- **The pushback on #184's permanence/topology prose is correct.** Canonical purge retries are not short-circuited on this baseline (`:624` requires `VersionID == ""`; `:628` exits only on `VersionPurgeComplete`). #184's Bugs 1 and 2 as narrated are legacy-shape-only. + +--- + +Because B1 is a blocker, I am not stating consensus on this exact hash. Resolve B1 (and the coupled resync-defer/stats-gate items it forces) and the plan becomes technically acceptable in my judgement — implementation would still require the tests in §C plus the partial-fan-out regression above. diff --git a/docs/investigations/r6/opus-v1.metadata.json b/docs/investigations/r6/opus-v1.metadata.json new file mode 100644 index 000000000..f08cd80f1 --- /dev/null +++ b/docs/investigations/r6/opus-v1.metadata.json @@ -0,0 +1,23 @@ +{ + "source_commit": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a", + "requested_model": "claude-opus-5", + "requested_effort": "max", + "cli_version": "2.1.270", + "plan": "plan-v1.md", + "plan_sha256": "bcd023e2b00ad3dc709baa738bb02551d47aeebb49879b0f76f2789f4c4dff8b", + "assistant_models": [ + "claude-opus-5" + ], + "raw_path": "/Users/vonng/tmp/silo-r6-20260915-aa3f/opus-v1.jsonl", + "raw_sha256": "16f1954b213b5813a72a8db6d960d7933e1065c63e9ae2d6f139d0cff881011f", + "command": "claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --add-dir /Users/vonng/tmp/silo-r6-20260915-aa3f --output-format stream-json --verbose", + "auxiliary_usage_models": [ + "claude-haiku-4-5-20251001", + "claude-opus-5" + ], + "subtype": "success", + "is_error": false, + "session_id": "927a163e-c14f-4366-b1bc-dbb363fbe793", + "duration_ms": 357292, + "num_turns": 61 +} diff --git a/docs/investigations/r6/opus-v2-review.md b/docs/investigations/r6/opus-v2-review.md new file mode 100644 index 000000000..0b9453cf1 --- /dev/null +++ b/docs/investigations/r6/opus-v2-review.md @@ -0,0 +1,86 @@ +I'll start by reading the required documents and verifying the plan hash. + +Now let me verify the plan hash and inspect the source claims. + +I read AGENTS.md, opus-v1-review.md, decisions-v2.md, and plan-v2.md, then re-checked every claim v2 changed against the source at `9ebe81c1b`. I ran nothing; findings are source reading plus the supplied logs. + +One honesty note up front: **no shell tool is available in this session, so I could not compute the SHA256 myself.** My verdict is on the exact content I read at `docs/investigations/r6/plan-v2.md` — 59 lines, opening `# R6 plan v2 — marker purge operation and MRF recovery`, dated 2026-09-15, closing with the "v2 changes from Opus v1" section that lists B1/N1–N7 dispositions. If that is `dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3`, the verdict below attaches to that hash. + +# Verdict: GO_WITH_NONBLOCKING_NOTES + +**Blockers: none.** B1 is resolved, and resolved by the correct mechanism rather than by wording. + +--- + +## B1 and the coupled exits — resolved, mechanism verified end-to-end + +§A now says purges leave `rinfo.ReplicationStatus` empty on every exit. I traced the preservation chain to make sure that is actually load-bearing and not just an absence: + +1. `replicatedInfos.ReplicationStatusInternal()` (`cmd/bucket-replication-utils.go:92-101`) emits `"arn1=;"` — non-empty, so it does reach `ReplicationState`. +2. `replicationStatusesMap` (`:428-439`) uses `replStatusRegex` (`:168`), whose second group `([^,].*?)` requires ≥1 char before `;`. `"arn1=;"` does not match → **empty `Targets` map**. +3. `CompositeReplicationStatus()` (`:356-372`) therefore takes the `default:` branch → `getCompositeReplicationStatus(empty)` → `""`; the `ReplicaTimeStamp` fall-through at `:366-371` also returns `""` because `replStatus == Completed` is false. So the composite is `""` under every sub-case. +4. `xl-storage-format-v2.go:1438` gates the `DeleteType` rewrite on `!fi.DeleteMarkerReplicationStatus().Empty()` → **guard never fires** → `MetaSys[ReplicationStatus]`, `MetaSys[ReplicationTimestamp]` and the Replica pair at `:1441-1442` are left byte-for-byte intact, including ARNs excluded from fan-out. `updateVersion` is still true via `:1390`, so `MetaSys[VersionPurgeStatusKey]` at `:1448-1449` is written as intended. + +All three consequences I said were mandatory and non-separable are present: + +| Coupled item | v2 | +|---|---| +| Purge exits write only `VersionPurgeStatus` | §A, explicit, "on every exit"; `PrevReplicationStatus` retained for inspection only — harmless, `targetState()` sets it and nothing persists it | +| Resync defer gated on the operation's own success | §A "Do not stamp the current resync reset for a failed purge"; §C1 asserts the reset marker | +| `:556` stats gate replaced by per-target operation-status comparison | §A "Feed per-target old/new operation status into stats rather than selecting changes from the unrelated creation status" | +| Partial-fan-out regression test | §C5, with two creation ARNs, a non-zero timestamp, and fan-out restricted to one ARN | + +The zero-timestamp and `ReplicaStatus` sub-findings from v1 are dissolved rather than patched: with the composite empty, `:1444-1445` is never reached, so `drs.ReplicationTimeStamp` (set at `bucket-replication.go:568-570`) cannot land on disk for a purge at all. + +I also checked the downstream consumer the change could have silently flipped: `resyncTargetSucceeded` (`bucket-replication.go:3154-3162`) keys the purge branch on `roi.VersionPurgeStatus` and reads `t.VersionPurgeStatus`, never `t.ReplicationStatus`. Leaving `ReplicationStatus` empty makes resync accounting strictly more correct for the old shape (today a resynced old-shape purge sets `ReplicationStatus = Completed` at `:713`). §A's "leave it alone" is right. + +## Amended retry queue-full site — correct + +`cmd/bucket-replication.go:2411-2448`, `queueReplicaDeleteTask`, `default:` branch of the select at `:2426-2427` → `p.queueMRFSave(doi.ToMRFEntry())`. §B names it exactly ("queueReplicaDeleteTask queue-full fallback"). I grepped every `queueMRFSave(` call: the delete-path sites are precisely `:487` (lock), `:565` (aggregate failure), `:2427` (queue-full) — three, no more. `:1206`, `:1311`, `:2339`, `:2370` are object-path. + +The supporting claims hold too: `MRFReplicateEntry.RetryCount` already exists with msgp tag `rc` (`bucket-replication-utils.go:792`), so no format change; `DeletedObjectReplicationInfo.ToMRFEntry()` (`:1927-1937`) sets only `Bucket`/`Object`/`versionID`; `versionID` is unexported but survives as the map key (`persistMRF` `:3870`, read back at `:4068`); the `> mrfRetryLimit` drop at `:3883-3887` is therefore currently unreachable for deletes; and `DeletedObjectReplicationInfo` has no generated msgp code, so the new field is genuinely schema-free. The gap's exact location is `queueReplicationHeal:3762` setting `roi.RetryCount` while `dv` at `:3781-3793` drops it — which is what §B closes. + +## N7 evidence — checks out + +`baseline-mrf.log` shows `TestReplicateDeleteMarkerPurge` both subtests **PASS**; the "Storage reached its minimum free drive threshold" abort from v1 is gone, so §C items 2/3/5/6 have a working fixture. The logs also independently corroborate two plan premises on real storage: `err=Method not allowed` on the source marker lookup (§B's 405 premise), and `legacy=true … MRF=0` on both baseline and PR #184 (observation 2, and #184's residual gap). Your retraction is right, and I'd add that the mechanism is visible: `replicatedInfos.ReplicationStatus()` (`bucket-replication-utils.go:103-119`) counts *every* target including empty-status ones, so `creation=PENDING` for a canonical purge is an artifact of that aggregate, not disk state. The retained logs still carry the old "failure stored in incorrect status field" assertion text on the canonical rows — worth a pointer to decisions-v2's retraction beside them so a later reader doesn't re-derive the wrong conclusion. + +--- + +## Non-blocking notes + +1. **Audit `Status` string changes on the live canonical path.** §A folds audit into the COMPLETE→COMPLETED mapping. The audit defer at `bucket-replication.go:434-444` logs `Status: string(replicationStatus)`, so every successful canonical versioned-delete replication goes from `COMPLETE` to `COMPLETED`. That is a user-visible change on the *live* path, not the legacy one. Keep it — `CompletedLegacy` is documented as an error at `internal/bucket/replication/datatypes.go:35-36` — but assert the audit string in §C, reuse `replication.CompletedLegacy` rather than a `"COMPLETE"` literal, and mention it in the PR text. +2. **The stats delta is wider than §C5's Heal/ExistingObject framing.** The `Completed` case gate is right (`bucket-replication-stats.go:189-192` requires `IsDataReplication()`, which excludes the unset OpType on handler-originated deletes — `replication.go:139-145`). But replacing the `:556` gate also changes *which* purges reach `Update`: today `""` vs `COMPLETED` makes that gate fire for nearly every purge of a previously-replicated version, and old-shape purges currently record a spurious `Pending` (aggregate `Pending`, prev `COMPLETED`). Assert that spurious `Pending` disappears too. Note `ri.Size` is never set in `replicateDeleteToTarget`, so `Completed` deltas are count-only, zero bytes. +3. **Make the `ResetStatusesMap` nil-guard unconditional** instead of contingent on tests exposing it (plan line 51). `getReplicationState:419-422` writes into `prevState.ResetStatusesMap` unguarded; `ObjectToDelete.ReplicationState()` (`:590-600`) leaves it nil, unlike `ObjectInfo.ReplicationState()` (`:576`). Today it is unreachable only because the resync defer requires `ReplicationStatus == Completed`, which purges never reach — and §A re-gates exactly that defer. I traced the live producers: the sole `ExistingObjectReplicationType` delete is `:3329-3342`, fed by `getHealReplicateObjectInfo` → `oi.ReplicationState()` → non-nil, so this is robustness and test-fixture safety, not a live panic. Two lines; just do it. +4. **Pin the already-COMPLETE purge under ExistingObject resync in §C1.** The purge early-out at `:628` has no `OpType != ExistingObjectReplicationType` exclusion, unlike the creation early-out at `:624`. Routing old-shape purges through it means a resync of an already-COMPLETE old-shape purge now short-circuits where today it re-sends. That matches canonical behaviour and is probably intended, but §C1's "resync success/failure" row currently leaves the implementer free to pick either. +5. **Scope "Preserve actual target failure in Err, including offline error where appropriate."** The offline exit (`:631-651`) sets no `Err` today, and `Err` flows into `replStat.set(...)` → `srUpdate` → site-replication stats. Say whether creations also start carrying an offline `Err`, or restrict it to purges; otherwise §C6's offline row has no fixed expectation. +6. **Leave `getReplicationState`'s third parameter alone.** `vID` (`bucket-replication-utils.go:402`) is entirely unused in the body. Since §A re-plumbs shape classification, someone will be tempted to wire it up; the empty-composite preservation depends on that function staying shape-agnostic. +7. **§B identity gate: key on `versionID` + `DeleteMarker` + non-zero `ModTime`.** `queueMRFHeal` calls `GetObjectInfo` with `ObjectOptions{VersionID: vID}` and no `Versioned`, so `ToObjectInfo` (`erasure-metadata.go:118-123`) returns `fi.VersionID` verbatim — the gate works. Bucket/object equality is trivially satisfied (they are the request arguments); the one component that can be perturbed is a strict `oi.Name == e.Object` against `decodeDirObject` (`cmd/utils.go:899-904`) for directory objects. `erasure-server-pool.go:1063-1077` does return the populated `oi` with the error upward, so the gate has real data to inspect. +8. **RetryCount type.** `ReplicateObjectInfo.RetryCount` is `uint32` (`:3762`), `MRFReplicateEntry.RetryCount` is `int` (`:792`), `QueueReplicationHeal` takes `int`. Pick one for the new delete-task field and keep conversions in one place. +9. **State the purge-subset loss as pre-existing, not merely excluded.** The asymmetry §C5 carves out is structural: creation status survives partial fan-out *because* the composite is empty (`:1438`), while `VersionPurgeStatusInternal()` is non-empty and unconditionally rewrites `MetaSys[VersionPurgeStatusKey]` from the fan-out subset (`:1448-1449`). Canonical purges already lose excluded-ARN purge status today, so R6 neither causes nor fixes it. Say that, or a reader will assume both were repaired. +10. **§C3 flush mechanics.** `persistMRF` (`:3866-3874`) only writes to disk on the 5-minute timer, ctx cancel, or 1M entries; the re-queued entry lands in an in-memory map first. N6's "persist each failed entry again" covers it, but make explicit that the test drives `saveMRFEntries` directly rather than waiting on the timer — and that `loadMRF` deletes the file after reading (`:4013-4015`), so each round needs a fresh write. + +--- + +## Consensus + +I state consensus on **`docs/investigations/r6/plan-v2.md` exactly as reviewed here** (hash `dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3` as supplied; content verified, digest not independently computed for lack of a shell tool). The classification rule, the all-exits treatment, the canonical-wire choice, the empty-creation-status preservation mechanism, the valid-405 gate, the three-site retry budget, and the §C matrix are technically correct against `9ebe81c1b`, and the scope limits in the "Existing state, compatibility, and delivery" section are accurate rather than overclaimed. The ten notes above are refinements and assertions to add, not conditions on this agreement — none of them requires another review round or a new hash. + +Tests in §C remain future obligations. Nothing here asserts they pass; the only executed evidence I relied on is the supplied baseline/PR logs, which establish fixture viability and the two defect premises, not acceptance. + +The review above is complete and stands as delivered. Summarizing the required outputs: + +**Verdict for `docs/investigations/r6/plan-v2.md`, hash `dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3`: GO_WITH_NONBLOCKING_NOTES** + +**Blockers: none.** + +**Consensus:** I state consensus on the exact plan as reviewed. The one caveat is procedural, not substantive: no shell tool is available in this session, so I verified the plan's *content* (59 lines, opening `# R6 plan v2 — marker purge operation and MRF recovery`, closing with the B1/N1–N7 disposition list) but could not independently compute its digest. Consensus attaches to that content. + +What I resolved this round: +- **B1** — accepted resolution, and I confirmed the mechanism is real rather than merely absent: `"arn1=;"` fails `replStatusRegex`, so `Targets` is empty, so `CompositeReplicationStatus()` returns `""` under every sub-case including the `ReplicaTimeStamp` fall-through, so the `xl-storage-format-v2.go:1438` guard never fires and the on-disk creation block survives partial fan-out intact — while `updateVersion` still writes `VersionPurgeStatusKey`. +- **Coupled exits** — all three present (purge-only status writes; resync defer gated on the operation's own success; `:556` gate replaced by per-target operation status), plus the §C5 partial-fan-out test. I also checked the one consumer that could have flipped silently, `resyncTargetSucceeded:3154-3162`, and it keys on `VersionPurgeStatus` — so the change makes resync accounting strictly more correct. +- **Queue-full retry site** — `queueReplicaDeleteTask`, `default:` branch, `bucket-replication.go:2427`; the delete-path `queueMRFSave` sites are exactly `:487`, `:565`, `:2427`, and the `RetryCount` disk field already exists with msgp tag `rc`. +- **N7** — the fixture-threshold failure is gone; `TestReplicateDeleteMarkerPurge` passes on baseline, and the logs corroborate the 405 premise and the `legacy=true … MRF=0` gap on both baseline and #184. + +The ten non-blocking notes are in my response; the three worth acting on before you branch are the audit-string change on the live canonical path (note 1), the wider-than-stated stats delta (note 2), and making the `ResetStatusesMap` nil-guard unconditional (note 3). None requires another review round or a new hash. + +Tests in §C remain future obligations — nothing here asserts they pass. I made no edits and wrote no files. diff --git a/docs/investigations/r6/opus-v2.metadata.json b/docs/investigations/r6/opus-v2.metadata.json new file mode 100644 index 000000000..ee21bdaaf --- /dev/null +++ b/docs/investigations/r6/opus-v2.metadata.json @@ -0,0 +1,23 @@ +{ + "source_commit": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a", + "requested_model": "claude-opus-5", + "requested_effort": "max", + "cli_version": "2.1.270", + "plan": "plan-v2.md", + "plan_sha256": "dae51753a3ab4b2ea98b85e720144338fdf52541ca6daf968c7dac5ce564d8c3", + "assistant_models": [ + "claude-opus-5" + ], + "raw_path": "/Users/vonng/tmp/silo-r6-20260915-aa3f/opus-v2.jsonl", + "raw_sha256": "aa4d04931875e82ae850a8fa2b875c094f36625784c8c090781f9a0273d20871", + "command": "claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --add-dir /Users/vonng/tmp/silo-r6-20260915-aa3f --output-format stream-json --verbose", + "auxiliary_usage_models": [ + "claude-haiku-4-5-20251001", + "claude-opus-5" + ], + "subtype": "success", + "is_error": false, + "session_id": "c6dfdf82-d542-47a1-8e69-a48753d61b12", + "duration_ms": 219946, + "num_turns": 41 +} diff --git a/docs/investigations/r6/opus-v3-review.md b/docs/investigations/r6/opus-v3-review.md new file mode 100644 index 000000000..bfdc66364 --- /dev/null +++ b/docs/investigations/r6/opus-v3-review.md @@ -0,0 +1,77 @@ +I'll start by reading the key evidence documents and the current implementation state. + +I have everything I need. Let me verify one last interaction — whether the delta is pinned by any existing test — and then write the review. + +Write is disabled in this session (consistent with your read-only constraint), so the review is delivered here in full. + +# Verdict: **GO_WITH_NONBLOCKING_NOTES** — `plan-v3.md` + +**Blockers: none.** + +Digest caveat: no shell tool in this session, so I could not compute SHA256 and cannot attest `dc9a67fc91b3113fa35218a2f903455807430cc4daf9c78a88d0be6b8fc27058`. Consensus attaches to the content I read — 70 lines, opening `# R6 plan v3 — marker purge operation and MRF recovery`, closing `...including the proof now corrected by the executable counterexample.` — the same procedural caveat the v2 reviewer recorded. I ran no tests; `v2-unrecorded-purge.log` is treated as reported evidence, not as my own execution. + +Source reviewed: worktree = `9ebe81c1` + `implementation-v2.diff`, v3 delta **not** applied. + +--- + +## 1. Counterexample verified — and I am withdrawing the general form of the earlier proof + +The v2 consensus rested on: empty per-target status ⇒ `"arn=;"` ⇒ no regex match ⇒ empty `Targets` ⇒ empty composite ⇒ guard at `cmd/xl-storage-format-v2.go:1438` never fires. **That holds only for |targets| ≤ 1. It is false for ≥ 2.** The reviewers' multi-target regex proof was too strong; the executable counterexample is correct. + +Trace against `replStatusRegex = ([^=].*?)=([^,].*?);` (`cmd/bucket-replication-utils.go:168`), input `arn1=;arn2=;`: +- group 1 lazily reaches the first `=` → `arn1` +- group 2's `[^,]` **accepts `;`** (the class excludes comma, not semicolon), then `.*?` runs to the next `;` → `;arn2=` +- one match spans the whole string → `{arn1: ";arn2="}` + +Then `CompositeReplicationStatus` (`:356-379`): non-empty internal, not a legacy literal → `default` → `getCompositeReplicationStatus` sees one bogus entry → **`Pending`**. The `ReplicaTimeStamp` fall-through at `:366-371` cannot rescue it (it only returns `ReplicaStatus` when `replStatus == Completed`). Generalizes: N=1 → `""`; every N ≥ 2 → `Pending`, always via a `;`-prefixed value, so it can never accidentally land on a real status. + +**Why the ordinary multi-target tests passed** — plan §v3 is right, with one mechanism refinement: the masking is not in the guard, it is in `FileInfo.DeleteMarkerReplicationStatus()` (`cmd/erasure-metadata.go:670-675`), which returns `""` whenever `!fi.Deleted`, regardless of the composite. `erasureObjects.DeleteObject` sets `deleteMarker=false` when `versionFound && !goi.VersionPurgeStatus.Empty()` (`cmd/erasure-object.go:2100-2101`) — the normal state after the handler's PENDING write. A **second independent masking condition** sits at `:2102` (`else if !goi.DeleteMarker`), so purges of ordinary object versions are never exposed: the hole is confined to delete-marker versions whose on-disk purge status is absent. + +Unrecorded path: disk marker has creation metadata only → `deleteMarker` stays `opts.Versioned=true` → `fi.Deleted=true` → `DeleteMarkerReplicationStatus()` = bogus `Pending` → guard at `:1438` fires → `default` branch rewrites `ReplicationStatus` = `arn1=;arn2=;` and `ReplicationTimestamp` = `UTCNow()` (supplied by `:576`, since FAILED ≠ PENDING). Exactly the recorded log: statuses emptied, stamp jumping `15:06:15Z` → `16:06:15.612037Z`. + +The fixture is sound: the recreate at `replication-delete-mrf_test.go:258` genuinely removes the marker (`updateVersion=false` → removal branch at `:1457`), the reseed at `:261-264` writes a creation-only block, and `deletion` — captured from the real signed handler DELETE at `:249` — still carries PENDING purge state. `before` is sampled at `:304`, after the reseed. + +**Scoping, unweakened:** the normal producer does not demonstrably race. `DeleteObjectHandler` enqueues only after a successful write and derives the task from the returned `objInfo` (`cmd/object-handlers.go:3164, 3222-3242`); `queueReplicationHeal` derives it from current disk state (`cmd/bucket-replication.go:3808-3821`). Reaching the unrecorded shape needs disk/task divergence — e.g. a partial-quorum PENDING write later healed back from a stale shard while the in-memory task survives. Narrow, not impossible, unproven. Two facts bound the blast radius: the damage is **one-shot** (the same failed write records the purge status, masking every later round), and it is erased if the purge ever completes. It remains real metadata corruption while a marker is stuck — garbage per-target creation statuses plus a lost creation timestamp. + +## 2. Field selection — correct, with one candid downgrade + +| Field | Assessment | +|---|---| +| `ReplicationStatusInternal = ""` | **Load-bearing.** Direct cause; forces the composite to case 3. | +| `Targets = nil` | **Consistency hygiene.** The composite switches on the string, so it changes nothing today; it prevents an internally inconsistent state (empty string, populated map) misleading a future reader. Keep. | +| `ReplicaStatus = ""` | **Defensive, not load-bearing today.** The fallback at `:374-375` is real, but no production producer populates it: `ObjectInfo.ReplicationState()` (`:572-590`) and `ObjectToDelete.ReplicationState()` (`:593-602`) both omit the replica fields, and every delete-task construction site (`bucket-replication.go:2678, 3361, 3813`, `object-handlers.go:3237`, `erasure-object.go:1715, 1758, 1764`) routes through one of them. Keep it — free, and it closes the documented fallback — but it does not repair anything observed. | + +`isPurge` is the right gate: one value computed pre-fan-out at `:428`, identical to the one `replicateDeleteToTarget` uses at `:616`. Creations untouched. + +## 3. Loss / leak — none found + +- **Creation + replica blocks preserved.** Both write sites (`:1396` ventry, `:1438` in-place) sit behind the same now-dead guard, and `:1430-1453` mutates the existing `ver.DeleteMarker.MetaSys` by key, never clearing it. A no-update payload, exactly as the plan says. +- **Purge block / reset map still written** (`:1448-1453`); `updateVersion` unchanged (`:1380` diverts on the non-empty purge status, `:1390` sets it for any non-COMPLETE purge). +- **Successful purge:** composite COMPLETE → `updateVersion=false` → version removed at `:1457`; `:1458`/`:1460` both stay false. Unchanged. +- **ventry path (`:1395-1411`, added at `:1506`) unreachable for purges:** reaching `:1506` requires an `ObjectType` version, and for those `fi.Deleted` is always false via `:2100-2103`. +- **Zero fan-out (nil clients):** two of three assignments are already no-ops; I traced the `ReplicaStatus` case through `:2081/:2084` and `:1380` for replica and non-replica sources — identical outcome with and without the delta. Pre-existing behavior, already an explicit exclusion. +- **No re-derivation risk:** `SetDeleteReplicationState` runs only under `opts.EvalMetadataFn != nil` (`erasure-object.go:2030-2038`, `erasure-server-pool-consistency.go:342-351`); `replicateDelete` never sets it, so emptied fields are not refilled with a PENDING decision. +- **`ReplicationState.Equal`** is used only by `FileInfo.ReplicationInfoEquals` comparing two on-disk infos. No interaction. +- **Only observable change:** `dobjInfo` handed to `sendEvent` at `:604-610` carries empty `ReplicationStatusInternal`/`ReplicationStatus` instead of bogus `PENDING` for multi-target purges (`erasure-metadata.go:160-162`). Strict improvement; that payload never reflected disk state. Worth one line in the PR description. +- **Stats / audit / MRF** are computed from `rinfos` before `drs` (`:548-573`). Untouched. + +## 4. Sufficiency — yes, for the class + +- `getReplicationState` has exactly **one** production caller (`bucket-replication.go:574`). No second write path to patch. +- With the composite forced empty, the guard is dead for all purges under every (`fi.Deleted`, `updateVersion`) combination — not just the fixture's. +- `versionPurgeStatusesMap` is not exposed to the same misparse: `VersionPurgeStatusInternal()` skips empty statuses (`:147-149`), so `arn=;` never appears there. +- After the delta no known producer emits an empty entry at all: purges write `""`; every `!isPurge` exit of `replicateDeleteToTarget` assigns a status (`:640, 664, 681, 695, 715, 725`); the object path pre-initializes `ReplicationStatus: replication.Failed` (`:1338`). Shared parser untouched, as required. +- No existing test asserts a purge writes a non-empty creation status, so no expected regression. `partial-creation-block` restricts fan-out to one ARN → single entry → composite already `""` → unchanged. + +## 5. Non-blocking notes + +- **N1 (strongest — promote from optional to required).** The plan's "small invariant over the source update payload **if useful**" is the only thing that can pin `Targets` and `ReplicaStatus`; the disk assertion at `replication-delete-mrf_test.go:353` pins `ReplicationStatusInternal` alone. Make it mandatory: wrap the `ObjectLayer` passed to `replicateDelete` and assert, for every purge subtest, that `opts.DeleteReplication` has all three fields empty and `CompositeReplicationStatus() == ""`. A `replicaSource + unrecordedPurge, targets: 2` case is optional — it may pass either way because the replica timestamp round-trips byte-identically, which is exactly why the payload invariant is the better instrument. +- **N2.** Tighten §v3's mechanism sentence: `fi.Deleted=false` protects via `DeleteMarkerReplicationStatus()` returning `""` (`erasure-metadata.go:670-675`), not via the guard; record the second masking condition (`erasure-object.go:2102`) confining exposure to delete-marker versions, and the one-shot nature of the corruption. +- **N3.** State `ReplicaStatus = ""` as defensive (§2), not as a fix for an observed failure. +- **N4.** Residual, explicitly out of R6: `replStatusRegex` still misparses any future two-empty-entry string. No producer remains after this delta; record as a follow-up issue, do not fix here. +- **N5.** `drs.ReplicationTimeStamp = UTCNow()` at `:576` is now provably never written for purges. Harmless — leave it rather than widen a three-line delta that has been reviewed field by field. +- **N6.** For the record: the v2 "empty composite" argument was sound only for a single target; the earlier reviews in this lineage, including the one I am extending, overstated it. + +## 6. Consensus + +I state consensus on the exact plan content as read, including §A's three-field instruction (line 23) and the §v3 restatement (line 67), subject to the digest caveat. All v2 dispositions and scope limits carry forward unchanged: no merge/publish/deploy, no live storage rewrite, no parser or `getReplicationState` merge change, nil-client and already-erased-tracking states remain excluded, and the 3-endpoint fixture is not a production three-daemon SR mesh. Proceed with the three-field delta plus the N1 payload invariant. diff --git a/docs/investigations/r6/opus-v3.metadata.json b/docs/investigations/r6/opus-v3.metadata.json new file mode 100644 index 000000000..6dce8683d --- /dev/null +++ b/docs/investigations/r6/opus-v3.metadata.json @@ -0,0 +1,25 @@ +{ + "source_commit": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a", + "requested_model": "claude-opus-5", + "requested_effort": "max", + "cli_version": "2.1.270", + "plan": "plan-v3.md", + "plan_sha256": "dc9a67fc91b3113fa35218a2f903455807430cc4daf9c78a88d0be6b8fc27058", + "assistant_models": [ + "claude-opus-5" + ], + "raw_path": "/Users/vonng/tmp/silo-r6-20260915-aa3f/opus-v3.jsonl", + "raw_sha256": "7e63d6d522919206d5833b7d908d028123e3a403e9c34a26a07934af79e870c1", + "command": "claude --print --model claude-opus-5 --effort max --safe-mode --permission-mode plan --tools Read,Grep,Glob --strict-mcp-config --no-session-persistence --add-dir /Users/vonng/tmp/silo-r6-20260915-aa3f --output-format stream-json --verbose", + "auxiliary_usage_models": [ + "claude-haiku-4-5-20251001", + "claude-opus-5" + ], + "subtype": "success", + "is_error": false, + "session_id": "07e73b10-dab0-4992-bcad-a71ddac270a3", + "duration_ms": 282053, + "num_turns": 40, + "source_diff": "/Users/vonng/tmp/silo-r6-20260915-aa3f/implementation-v2.diff", + "source_diff_sha256": "30f154da13626b5b38b48fbd48755b21a2377ff7b1a5534dc64feda2cc208488" +} diff --git a/docs/investigations/r6/plan-v1.md b/docs/investigations/r6/plan-v1.md new file mode 100644 index 000000000..75739bb74 --- /dev/null +++ b/docs/investigations/r6/plan-v1.md @@ -0,0 +1,54 @@ +# R6 plan v1 — marker purge operation and MRF recovery + +Date: 2026-09-15. Implementation has NOT started; this is the review candidate. +Baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a` (clean detached worktree and freshly fetched origin/main agree). +PR #184: OPEN; head `6addf9eb916b5a4b837480cf534cd1efa5407d3c`, code commit `96b21557a85cd4a615ba8a797bfc4c556e413db4`. +Evidence directory: `/Users/vonng/tmp/silo-r6-20260915-aa3f/` (PR JSON/diff, baseline overlays/probes/logs). +Current PGSTY support policy is read from local AGENTS.md, copied verbatim from the main checkout's ignored AGENTS.md. No dependency or supported-stack change. + +## Observed mechanisms and scope + +1. `replicateDeleteToTarget`: an old-shaped task has VersionID empty, DeleteMarkerVersionID set, creation target COMPLETED, purge target PENDING/FAILED. The creation early-return suppresses its DELETE. Offline/RemoveObject error/success select the wrong field for this shape. HEAD-not-ready always overwrites creation status, including purges. The resync defer uses creation COMPLETED even when purge fails; purge COMPLETE early-out only recognizes nonempty VersionID. +2. `replicateDelete`: aggregate audit/event/MRF status and the stats change check still select on VersionID/creation status. Fixing only the target routine leaves a failed old-shaped purge reporting Completed and not entering MRF. +3. `queueMRFHeal`: disk entries are consumed, GetObjectInfo(marker version) returns real metadata plus MethodNotAllowed, and every error is discarded. Direct queueReplicationHeal is NOT an MRF test. +4. Current DELETE producer, scanner/heal, and resync already construct the canonical VersionID purge shape. Existing `TestReplicateDeleteMarkerPurge/recover_legacy_true` proves scanner/heal can recover old state. This is NOT evidence that every failed purge is permanently stuck, nor a defect exclusive to >2 sites. +5. PR #184 has the right state-based classification and 405 recovery direction, but misses the outer status, HEAD failure, resync defer, already-complete guard, bounded delete retry propagation, and real MRF coverage. Its existing-marker test confirms one attempt through a helper HTTP endpoint, not a three-site deployment. Its prose makes stronger permanence/topology claims than the current baseline proves. + +## Proposed minimal implementation + +### A. One operation classification, all exits + +Add a small `DeletedObjectReplicationInfo.isVersionPurge()` helper: true when VersionID is nonempty OR DeleteMarkerVersionID is nonempty and the task's composite VersionPurgeStatus is nonempty. Use the task-level decision in both outer and target functions; do not classify a multi-target task differently merely because one target lacks a map entry. + +For purges preserve `rinfo.ReplicationStatus = rinfo.PrevReplicationStatus` on every exit. Only creation changes this field; only purge changes VersionPurgeStatus. Use this classification for completed early-outs, offline/error/success and resync success. Do not stamp the current resync reset for a failed purge. Existing creation semantics (HEAD 405 means already created, readiness gate, quorum fall-through) remain. + +Route ALL purges through the canonical permanent-delete request already produced by today's handlers: explicit version ID, `ReplicationDeleteMarker=false`. Perform marker HEAD/readiness probes only for creations. Purge authorization/failure is determined by the DELETE itself. This removes the obsolete old-shape HEAD error path, prevents a lost-response retry from re-creating an absent marker, and needs no new wire header or receiver change. Keep current RemoveObject 404/idempotency semantics. Preserve actual target failure in Err, including offline error where appropriate. + +In the outer routine use purge outcomes for audit/event/MRF and for per-target change detection. Map internal purge COMPLETE to operation COMPLETED only when passing an operation status to the existing statistics/event/audit logic; stored purge metadata remains COMPLETE. Feed per-target old/new operation status into stats rather than selecting changes from the unrelated creation status. Keep existing stats policy, no new metrics framework. + +For pre-fan-out exits: config/decision failures remain not-tracked; lock failure retains MRF scheduling; missing configured clients remain logged/skipped and are a documented separate state-preservation limitation. Do not rewrite general `getReplicationState` target merging in this issue. Do not claim global convergence for nil clients or resync narrowed to one of many targets. + +### B. Real MRF healing and bounded retries + +Accept GetObjectInfo MethodNotAllowed only when returned ObjectInfo is a delete marker with nonempty matching bucket/object/version identity. QueueReplicationHeal still rejects zero ModTime. Other errors and invalid/empty ObjectInfo are not queued. Reuse the existing disk persistence/load/queue path; no timer/backoff redesign or format change. + +Carry a RetryCount in the in-memory delete task, pass it from queueReplicationHeal, increment it when a failure/lock error is submitted to MRF, and include it in ToMRFEntry (whose disk format already has RetryCount). Respect the existing mrfRetryLimit and drop accounting; after budget exhaustion the scanner can still start a fresh heal. This closes the retry-count omission exposed by re-enabling marker MRF. No additional persistence schema. + +### C. Regression and acceptance matrix + +1. Target operation table: create (new/HEAD405/completed/readiness failure/quorum), canonical object purge, canonical marker purge, old marker purge. Pending/failed/completed purge, creation pending/completed/replica; success, DELETE403/405/503, offline, absent version, response lost after real removal, resync success/failure. Verify HTTP method/version/header, exact status fields and reset marker. No HEAD for purges. +2. Full outer call: old/new purge shapes, source/target real erasure metadata, first failure yields purge FAILED while creation state survives; MRF queue entry exists; correct operation status accounting. +3. Persist that MRF entry with saveMRFEntries, create a fresh ReplicationPool with no in-memory entry, then queueMRFHeal -> loadMRF -> real GetObjectInfo(405) -> QueueReplicationHeal -> delete queue -> replicateDelete. Initially still failing, entry reappears with increased retry count. Recover target, reload/replay and prove source AND target marker versions removed; repeated successful purge remains absent. Repeat in single-drive and 16-drive fixtures. No direct queueReplicationHeal substitute for MRF proof. +4. Creation MRF: failed marker creation metadata also travels the real disk MRF path and reaches the target. Negative MRF lookups (missing/corrupt/nonmarker 405/empty identity) do not schedule mutations. Verify retry budget still drops with counters, scanner fallback works. +5. Two independent HTTP targets plus source in local erasure fixtures: one target succeeds, the other fails/offline; source remains with per-target COMPLETE/FAILED purge states and unchanged creation states. Restore failed target via persisted MRF; successful target is not resent, both targets and source are absent at completion. This is 3-endpoint fan-out/metadata evidence, not a production three-daemon SR mesh, process-crash/power-loss or cross-region test. Report that distinction. +6. Run focused replication/delete/MRF/resync tests and race tests, then build/vet appropriate to changed code. Do not run other tasks' whole-repository suites concurrently. Tests must fail on baseline for the repaired paths; existing canonical purge and scanner recovery remain covered. + +## Existing state, compatibility, and delivery + +Pending/failed old marker metadata is consumed using the normal heal/MRF machinery; no bulk migration or live object rewrite. Disk entries whose marker lookup was previously dropped can be rediscovered by the scanner. Already missing/erased replication tracking or nil target clients cannot be recovered by this patch alone. A durable receiver tombstone for a delayed *creation* after a purge, replica relay behavior, full arbitrary-mesh convergence, and new backoff/observability are separate issues. + +Reuse PR #184's operation-state separation and valid-405 intent, and adapt its useful target/legacy convergence tests; do not copy its unproven permanence claims or adopt the entire PR blindly. Expected work: 1 production file plus focused tests and investigation records (helper placement may touch bucket-replication-utils.go if clearer). Branch only after exact-version Opus consensus. Normal implementation/tests authorized; merge, publish, deploy, and live storage rewrites excluded. + +## Review request + +Verify the classification and every exit against the exact source; challenge the canonical-wire choice, status conversion, MRF validity/retry budget, test sufficiency and scope. Respond with explicit GO/GO_WITH_NONBLOCKING_NOTES/REVISE for plan-v1.md and list blockers separately. Agreement must be on this exact plan hash. No implementation before resolving blockers. diff --git a/docs/investigations/r6/plan-v2.md b/docs/investigations/r6/plan-v2.md new file mode 100644 index 000000000..cbb73bc5c --- /dev/null +++ b/docs/investigations/r6/plan-v2.md @@ -0,0 +1,59 @@ +# R6 plan v2 — marker purge operation and MRF recovery + +Date: 2026-09-15. Implementation has NOT started; this is the review candidate. +Baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a` (clean detached worktree and freshly fetched origin/main agree). +PR #184: OPEN; head `6addf9eb916b5a4b837480cf534cd1efa5407d3c`, code commit `96b21557a85cd4a615ba8a797bfc4c556e413db4`. +Evidence directory: `/Users/vonng/tmp/silo-r6-20260915-aa3f/` (PR JSON/diff, baseline overlays/probes/logs). +Current PGSTY support policy is read from local AGENTS.md, copied verbatim from the main checkout's ignored AGENTS.md. No dependency or supported-stack change. + +## Observed mechanisms and scope + +1. `replicateDeleteToTarget`: an old-shaped task has VersionID empty, DeleteMarkerVersionID set, creation target COMPLETED, purge target PENDING/FAILED. The creation early-return suppresses its DELETE. Offline/RemoveObject error/success select the wrong field for this shape. HEAD-not-ready always overwrites creation status, including purges. The resync defer uses creation COMPLETED even when purge fails; purge COMPLETE early-out only recognizes nonempty VersionID. +2. `replicateDelete`: aggregate audit/event/MRF status and the stats change check still select on VersionID/creation status. Fixing only the target routine leaves a failed old-shaped purge reporting Completed and not entering MRF. +3. `queueMRFHeal`: disk entries are consumed, GetObjectInfo(marker version) returns real metadata plus MethodNotAllowed, and every error is discarded. Direct queueReplicationHeal is NOT an MRF test. +4. Current DELETE producer, scanner/heal, and resync already construct the canonical VersionID purge shape. Existing `TestReplicateDeleteMarkerPurge/recover_legacy_true` proves scanner/heal can recover old state. This is NOT evidence that every failed purge is permanently stuck, nor a defect exclusive to >2 sites. +5. PR #184 has the right state-based classification and 405 recovery direction, but misses the outer status, HEAD failure, resync defer, already-complete guard, bounded delete retry propagation, and real MRF coverage. Its existing-marker test confirms one attempt through a helper HTTP endpoint, not a three-site deployment. Its prose makes stronger permanence/topology claims than the current baseline proves. + +## Proposed minimal implementation + +### A. One operation classification, all exits + +Add a small `DeletedObjectReplicationInfo.isVersionPurge()` helper: true when VersionID is nonempty OR DeleteMarkerVersionID is nonempty and the task's composite VersionPurgeStatus is nonempty. Use the task-level decision in both outer and target functions; do not classify a multi-target task differently merely because one target lacks a map entry. + +For purges leave `rinfo.ReplicationStatus` EMPTY on every exit; `PrevReplicationStatus` still carries the previous value for inspection. This is the existing canonical-purge no-update signal: the disk marker metadata layer does not overwrite creation status or its timestamp when the composite creation status is empty. Pinning previous status here would instead rebuild creation state from the fan-out subset and is explicitly rejected. Only creation sets ReplicationStatus; only purge sets VersionPurgeStatus. Preserve the full on-disk creation/replica status and timestamp block without rewriting it, including excluded targets. Use this classification for completed early-outs, offline/error/success and resync success. Do not stamp the current resync reset for a failed purge. Existing creation semantics (HEAD 405 means already created, readiness gate, quorum fall-through) remain. + +Route ALL purges through the canonical permanent-delete request already produced by today's handlers: explicit version ID, `ReplicationDeleteMarker=false`. Perform marker HEAD/readiness probes only for creations. Purge authorization/failure is determined by the DELETE itself. This removes the obsolete old-shape HEAD error path, prevents a lost-response retry from re-creating an absent marker, and needs no new wire header or receiver change. Keep current RemoveObject 404/idempotency semantics. Preserve actual target failure in Err, including offline error where appropriate. + +In the outer routine use purge outcomes for audit/event/MRF and for per-target change detection. Map internal purge COMPLETE to operation COMPLETED only when passing an operation status to the existing statistics/event/audit logic; stored purge metadata remains COMPLETE. Feed per-target old/new operation status into stats rather than selecting changes from the unrelated creation status. Keep existing stats policy, no new metrics framework. + +For pre-fan-out exits: config/decision failures remain not-tracked; lock failure retains MRF scheduling; missing configured clients remain logged/skipped and are a documented separate state-preservation limitation. Do not rewrite general `getReplicationState` target merging in this issue. Do not claim global convergence for nil clients or resync narrowed to one of many targets. + +### B. Real MRF healing and bounded retries + +Accept GetObjectInfo MethodNotAllowed only when returned ObjectInfo is a delete marker with nonempty matching bucket/object/version identity. QueueReplicationHeal still rejects zero ModTime. Other errors and invalid/empty ObjectInfo are not queued. Reuse the existing disk persistence/load/queue path; no timer/backoff redesign or format change. + +Carry a RetryCount in the in-memory delete task, pass it from queueReplicationHeal, increment it at ALL THREE submission sites: aggregate failure, lock failure, and queueReplicaDeleteTask queue-full fallback, and include it in ToMRFEntry (whose disk format already has RetryCount). Respect the existing mrfRetryLimit and drop accounting; after budget exhaustion the scanner can still start a fresh heal. This closes the retry-count omission exposed by re-enabling marker MRF. No additional persistence schema. + +### C. Regression and acceptance matrix + +1. Target operation table: create (new/HEAD405/completed/readiness failure/quorum), canonical object purge, canonical marker purge, old marker purge. Pending/failed/completed purge, creation pending/completed/replica; success, DELETE403/405/503, offline, absent version, response lost after real removal, resync success/failure. Verify HTTP method/version/header, exact status fields and reset marker. No HEAD for purges. +2. Full outer call: old/new purge shapes, source/target real erasure metadata, first failure yields purge FAILED while the complete on-disk creation/replica status and timestamp survive (empty creation field in the per-target result is intentional); MRF queue entry exists; correct operation status accounting. +3. Persist that MRF entry with saveMRFEntries, create a fresh ReplicationPool with no in-memory entry, then queueMRFHeal -> loadMRF -> real GetObjectInfo(405) -> QueueReplicationHeal -> delete queue -> replicateDelete. Initially still failing, entry reappears with increased retry count. Recover target, reload/replay and prove source AND target marker versions removed; repeated successful purge remains absent. Repeat in single-drive and 16-drive fixtures. No direct queueReplicationHeal substitute for MRF proof. +4. Creation MRF: failed marker creation metadata also travels the real disk MRF path and reaches the target. Negative MRF lookups (missing/corrupt/nonmarker 405/empty identity) do not schedule mutations. Verify retry budget still drops with counters, including worker-queue saturation, and scanner fallback works. +5. Also persist two creation ARNs and a nonzero creation timestamp, restrict fan-out to one ARN, fail/repeat the purge and verify the full creation block/timestamp is unchanged. This checks preservation only; the separate purge-target subset-merging limitation remains excluded. Verify COMPLETE-to-COMPLETED statistics deltas for Heal/ExistingObject operations. +6. Two independent HTTP targets plus source in local erasure fixtures: one target succeeds, the other fails/offline; source remains with per-target COMPLETE/FAILED purge states and unchanged creation states. Restore failed target via persisted MRF; successful target is not resent, both targets and source are absent at completion. This is 3-endpoint fan-out/metadata evidence, not a production three-daemon SR mesh, process-crash/power-loss or cross-region test. Report that distinction. +7. Run focused replication/delete/MRF/resync tests and race tests, then build/vet appropriate to changed code. Do not run other tasks' whole-repository suites concurrently. Tests must fail on baseline for the repaired paths; existing canonical purge and scanner recovery remain covered. + +## Existing state, compatibility, and delivery + +No current producer emits the old in-memory shape, and DeletedObjectReplicationInfo is not serialized across restart. Old-shape handling is robustness/upgrade compatibility; the directly active repair is marker MRF. Pending/failed marker metadata is consumed using the normal heal/MRF machinery; no bulk migration or live object rewrite. Disk entries whose marker lookup was previously dropped can be rediscovered by the scanner. The valid-405 gate covers nonempty version identities; null/empty-version markers that return ObjectNotFound remain outside it. Local source DeleteObject metadata-write errors retain the existing scanner recovery behavior; this issue does not change that independent failure path. These are explicit acceptance limits, not claims of full failure-mode coverage. Already missing/erased replication tracking or nil target clients cannot be recovered by this patch alone. A durable receiver tombstone for a delayed *creation* after a purge, replica relay behavior, full arbitrary-mesh convergence, and new backoff/observability are separate issues. + +Reuse PR #184's operation-state separation and valid-405 intent, and adapt its useful target/legacy convergence tests; do not copy its unproven permanence claims or adopt the entire PR blindly. Expected work: 1 production file plus focused tests and investigation records (helper placement may touch bucket-replication-utils.go if clearer). No getReplicationState merge change is proposed: the empty creation-update signal is the minimal preservation mechanism. If tests expose a nil ResetStatusesMap on a newly successful purge resync, initialize that map before its existing assignment without changing its merge semantics. Branch only after exact-version Opus consensus. Normal implementation/tests authorized; merge, publish, deploy, and live storage rewrites excluded. + +## Review request + +Verify the classification and every exit against the exact source; challenge the canonical-wire choice, status conversion, MRF validity/retry budget, test sufficiency and scope. Respond with explicit GO/GO_WITH_NONBLOCKING_NOTES/REVISE for plan-v2.md and list blockers separately. Agreement must be on this exact plan hash. No implementation before resolving blockers. + +## v2 changes from Opus v1 + +B1 accepted: remove the proposed creation-status pin, require disk-block preservation and partial-fan-out regression. N1 accepted: outgoing versionId/header assertions remain mandatory. N2 accepted: include queue-full RetryCount increment. N3 accepted: assert status-conversion counter deltas. N4/N5 accepted: scope empty/null identities and unsaved old task shape accurately. N6 accepted: persist each failed entry again and synchronize actual queue delivery between rounds. N7 resolved: capacity-adapted real-storage baseline/PR probes completed; see research.md and baseline-mrf.log/pr184-mrf.log. A prior probe incorrectly treated the empty returned creation result as evidence of a disk-state loss; the reviewer corrected that inference. Raw logs are retained. diff --git a/docs/investigations/r6/plan-v3.md b/docs/investigations/r6/plan-v3.md new file mode 100644 index 000000000..3afcd2216 --- /dev/null +++ b/docs/investigations/r6/plan-v3.md @@ -0,0 +1,69 @@ +# R6 plan v3 — marker purge operation and MRF recovery + +Date: 2026-09-15. The v2 implementation exists locally after v2 consensus; the v3 incremental correction below has NOT been implemented pending review. +Baseline: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a` (clean detached worktree and freshly fetched origin/main agree). +PR #184: OPEN; head `6addf9eb916b5a4b837480cf534cd1efa5407d3c`, code commit `96b21557a85cd4a615ba8a797bfc4c556e413db4`. +Evidence directory: `/Users/vonng/tmp/silo-r6-20260915-aa3f/` (PR JSON/diff, baseline overlays/probes/logs). +Current PGSTY support policy is read from local AGENTS.md, copied verbatim from the main checkout's ignored AGENTS.md. No dependency or supported-stack change. + +## Observed mechanisms and scope + +1. `replicateDeleteToTarget`: an old-shaped task has VersionID empty, DeleteMarkerVersionID set, creation target COMPLETED, purge target PENDING/FAILED. The creation early-return suppresses its DELETE. Offline/RemoveObject error/success select the wrong field for this shape. HEAD-not-ready always overwrites creation status, including purges. The resync defer uses creation COMPLETED even when purge fails; purge COMPLETE early-out only recognizes nonempty VersionID. +2. `replicateDelete`: aggregate audit/event/MRF status and the stats change check still select on VersionID/creation status. Fixing only the target routine leaves a failed old-shaped purge reporting Completed and not entering MRF. +3. `queueMRFHeal`: disk entries are consumed, GetObjectInfo(marker version) returns real metadata plus MethodNotAllowed, and every error is discarded. Direct queueReplicationHeal is NOT an MRF test. +4. Current DELETE producer, scanner/heal, and resync already construct the canonical VersionID purge shape. Existing `TestReplicateDeleteMarkerPurge/recover_legacy_true` proves scanner/heal can recover old state. This is NOT evidence that every failed purge is permanently stuck, nor a defect exclusive to >2 sites. +5. PR #184 has the right state-based classification and 405 recovery direction, but misses the outer status, HEAD failure, resync defer, already-complete guard, bounded delete retry propagation, and real MRF coverage. Its existing-marker test confirms one attempt through a helper HTTP endpoint, not a three-site deployment. Its prose makes stronger permanence/topology claims than the current baseline proves. + +## Proposed minimal implementation + +### A. One operation classification, all exits + +Add a small `DeletedObjectReplicationInfo.isVersionPurge()` helper: true when VersionID is nonempty OR DeleteMarkerVersionID is nonempty and the task's composite VersionPurgeStatus is nonempty. Use the task-level decision in both outer and target functions; do not classify a multi-target task differently merely because one target lacks a map entry. + +For purges leave `rinfo.ReplicationStatus` EMPTY on every exit; `PrevReplicationStatus` still carries the previous value for inspection. This is the per-target no-update intent. Before the outer source DeleteObject write, explicitly enforce the no-update signal for purge: after getReplicationState, set drs.ReplicationStatusInternal="", drs.Targets=nil, drs.ReplicaStatus="". Leave purge fields, reset map, replica timestamp and other fields unchanged. This is necessary because two serialized empty statuses (arn1=;arn2=;) are misparsed by the existing regexp into a nonempty composite. Do not change that shared parser/serializer or getReplicationState merging in R6. Pinning previous status here would instead rebuild creation state from the fan-out subset and is explicitly rejected. Only creation sets ReplicationStatus; only purge sets VersionPurgeStatus. Preserve the full on-disk creation/replica status and timestamp block without rewriting it, including excluded targets. Use this classification for completed early-outs, offline/error/success and resync success. Do not stamp the current resync reset for a failed purge. Existing creation semantics (HEAD 405 means already created, readiness gate, quorum fall-through) remain. + +Route ALL purges through the canonical permanent-delete request already produced by today's handlers: explicit version ID, `ReplicationDeleteMarker=false`. Perform marker HEAD/readiness probes only for creations. Purge authorization/failure is determined by the DELETE itself. This removes the obsolete old-shape HEAD error path, prevents a lost-response retry from re-creating an absent marker, and needs no new wire header or receiver change. Keep current RemoveObject 404/idempotency semantics. Preserve actual target failure in Err, including offline error where appropriate. + +In the outer routine use purge outcomes for audit/event/MRF and for per-target change detection. Map internal purge COMPLETE to operation COMPLETED only when passing an operation status to the existing statistics/event/audit logic; stored purge metadata remains COMPLETE. Feed per-target old/new operation status into stats rather than selecting changes from the unrelated creation status. Keep existing stats policy, no new metrics framework. + +For pre-fan-out exits: config/decision failures remain not-tracked; lock failure retains MRF scheduling; missing configured clients remain logged/skipped and are a documented separate state-preservation limitation. Do not rewrite general `getReplicationState` target merging in this issue. Do not claim global convergence for nil clients or resync narrowed to one of many targets. + +### B. Real MRF healing and bounded retries + +Accept GetObjectInfo MethodNotAllowed only when returned ObjectInfo is a delete marker with nonempty matching bucket/object/version identity. QueueReplicationHeal still rejects zero ModTime. Other errors and invalid/empty ObjectInfo are not queued. Reuse the existing disk persistence/load/queue path; no timer/backoff redesign or format change. + +Carry a RetryCount in the in-memory delete task, pass it from queueReplicationHeal, increment it at ALL THREE submission sites: aggregate failure, lock failure, and queueReplicaDeleteTask queue-full fallback, and include it in ToMRFEntry (whose disk format already has RetryCount). Respect the existing mrfRetryLimit and drop accounting; after budget exhaustion the scanner can still start a fresh heal. This closes the retry-count omission exposed by re-enabling marker MRF. No additional persistence schema. + +### C. Regression and acceptance matrix + +1. Target operation table: create (new/HEAD405/completed/readiness failure/quorum), canonical object purge, canonical marker purge, old marker purge. Pending/failed/completed purge, creation pending/completed/replica; success, DELETE403/405/503, offline, absent version, response lost after real removal, resync success/failure. Verify HTTP method/version/header, exact status fields and reset marker. No HEAD for purges. +2. Full outer call: old/new purge shapes, source/target real erasure metadata, first failure yields purge FAILED while the complete on-disk creation/replica status and timestamp survive (empty creation field in the per-target result is intentional); MRF queue entry exists; correct operation status accounting. +3. Persist that MRF entry with saveMRFEntries, create a fresh ReplicationPool with no in-memory entry, then queueMRFHeal -> loadMRF -> real GetObjectInfo(405) -> QueueReplicationHeal -> delete queue -> replicateDelete. Initially still failing, entry reappears with increased retry count. Recover target, reload/replay and prove source AND target marker versions removed; repeated successful purge remains absent. Repeat in single-drive and 16-drive fixtures. No direct queueReplicationHeal substitute for MRF proof. +4. Creation MRF: failed marker creation metadata also travels the real disk MRF path and reaches the target. Negative MRF lookups (missing/corrupt/nonmarker 405/empty identity) do not schedule mutations. Verify retry budget still drops with counters, including worker-queue saturation, and scanner fallback works. +5. Also persist two creation ARNs and a nonzero creation timestamp, restrict fan-out to one ARN, fail/repeat the purge and verify the full creation block/timestamp is unchanged. This checks preservation only; the separate purge-target subset-merging limitation remains excluded. Verify COMPLETE-to-COMPLETED statistics deltas for Heal/ExistingObject operations. +6. Two independent HTTP targets plus source in local erasure fixtures: one target succeeds, the other fails/offline; source remains with per-target COMPLETE/FAILED purge states and unchanged creation states. Restore failed target via persisted MRF; successful target is not resent, both targets and source are absent at completion. This is 3-endpoint fan-out/metadata evidence, not a production three-daemon SR mesh, process-crash/power-loss or cross-region test. Report that distinction. +7. Run focused replication/delete/MRF/resync tests and race tests, then build/vet appropriate to changed code. Do not run other tasks' whole-repository suites concurrently. Tests must fail on baseline for the repaired paths; existing canonical purge and scanner recovery remain covered. + +## Existing state, compatibility, and delivery + +No current producer emits the old in-memory shape, and DeletedObjectReplicationInfo is not serialized across restart. Old-shape handling is robustness/upgrade compatibility; the directly active repair is marker MRF. Pending/failed marker metadata is consumed using the normal heal/MRF machinery; no bulk migration or live object rewrite. Disk entries whose marker lookup was previously dropped can be rediscovered by the scanner. The valid-405 gate covers nonempty version identities; null/empty-version markers that return ObjectNotFound remain outside it. Local source DeleteObject metadata-write errors retain the existing scanner recovery behavior; this issue does not change that independent failure path. These are explicit acceptance limits, not claims of full failure-mode coverage. Already missing/erased replication tracking or nil target clients cannot be recovered by this patch alone. A durable receiver tombstone for a delayed *creation* after a purge, replica relay behavior, full arbitrary-mesh convergence, and new backoff/observability are separate issues. + +Reuse PR #184's operation-state separation and valid-405 intent, and adapt its useful target/legacy convergence tests; do not copy its unproven permanence claims or adopt the entire PR blindly. Expected work: 1 production file plus focused tests and investigation records (helper placement may touch bucket-replication-utils.go if clearer). No getReplicationState merge change is proposed: explicitly empty the creation/replica update fields only at the outer purge write; this enforces the intended no-update signal even for multiple empty target statuses. If tests expose a nil ResetStatusesMap on a newly successful purge resync, initialize that map before its existing assignment without changing its merge semantics. Branch only after exact-version Opus consensus. Normal implementation/tests authorized; merge, publish, deploy, and live storage rewrites excluded. + +## Review request + +Verify the classification and every exit against the exact source; challenge the canonical-wire choice, status conversion, MRF validity/retry budget, test sufficiency and scope. Respond with explicit GO/GO_WITH_NONBLOCKING_NOTES/REVISE for plan-v3.md and list blockers separately. Agreement must be on this exact plan hash. No implementation before resolving blockers. + +## v2 changes from Opus v1 + +B1 accepted: remove the proposed creation-status pin, require disk-block preservation and partial-fan-out regression. N1 accepted: outgoing versionId/header assertions remain mandatory. N2 accepted: include queue-full RetryCount increment. N3 accepted: assert status-conversion counter deltas. N4/N5 accepted: scope empty/null identities and unsaved old task shape accurately. N6 accepted: persist each failed entry again and synchronize actual queue delivery between rounds. N7 resolved: capacity-adapted real-storage baseline/PR probes completed; see research.md and baseline-mrf.log/pr184-mrf.log. A prior probe incorrectly treated the empty returned creation result as evidence of a disk-state loss; the reviewer corrected that inference. Raw logs are retained. + +## v3 incremental finding and correction + +The v2 proof that empty rinfo.ReplicationStatus necessarily leads to an empty composite was incomplete for multiple targets: regexp `([^=].*?)=([^,].*?);` matches `arn1=;arn2=;` with status `;arn2=`. The usual source purge already has VersionPurgeStatus on disk, which makes erasureObjects.DeleteObject set FileInfo.Deleted=false; that independently protects the disk creation block and explains why the ordinary v2 multi-target tests passed. A task carrying purge state while the disk marker still has only creation metadata can instead set FileInfo.Deleted=true; a failed two-target purge then rewrites the full creation block with empty entries and a new timestamp. + +Real erasure proof: `TestReplicationMRFMarkerRecovery/unrecorded-purge` in cmd/replication-delete-mrf_test.go; raw `v2-unrecorded-purge.log` in the evidence directory. Test fixture creates a normal signed purge task, then locally restores only its test source marker's creation metadata while retaining that task's purge state. With one target succeeding and another failing, v2 rewrites creation fields (test fails). This is a boundary/robustness scenario, not a claim that today's normal producer necessarily races its persisted purge state. It also reveals an inherited weakness; no unrelated parser fix is proposed. + +Minimal correction: the outer `replicateDelete`, immediately after `drs := getReplicationState(...)`, executes the three field assignments listed in §A inside `if isPurge`. `ReplicaStatus` must be empty too because CompositeReplicationStatus otherwise falls back to it. This is a no-update payload, not a request to delete disk creation/replica fields; the xlMetaV2 guard does not run for empty composite. Purge maps and reset timestamps retain their current behavior. Keep per-target status empty as agreed in v2. Add the real unrecorded-purge scenario, plus a small invariant over the source update payload if useful, and rerun ordinary multi-target/partial/replica preservation, MRF, race and lint. + +Production source under review is baseline 9ebe81c1 plus `/Users/vonng/tmp/silo-r6-20260915-aa3f/implementation-v2.diff` (v2 has been implemented after its recorded consensus). Only this additional correction awaits v3 agreement. All other v2 dispositions and scope limits remain. The earlier consensus/raw reviews remain immutable evidence, including the proof now corrected by the executable counterexample. diff --git a/docs/investigations/r6/research.md b/docs/investigations/r6/research.md new file mode 100644 index 000000000..af6e17797 --- /dev/null +++ b/docs/investigations/r6/research.md @@ -0,0 +1,52 @@ +# R6 research and PR #184 review + +## Source identity + +- Worktree baseline / freshly fetched origin/main: `9ebe81c1b3611f9cc73e676b5b741c2be62c467a`. +- PR URL: https://github.com/pgsty/silo/pull/184 +- PR head at this review: `6addf9eb916b5a4b837480cf534cd1efa5407d3c`; OPEN, no reviews returned by GitHub API. Its recorded PR base OID `89637554d60c27cfc51d2281d0a4fe15e415f06d` is older than the fetched branch; current worktree source wins. +- Production diff is exactly 24 insertions/11 removals in bucket-replication.go relative to this baseline. The PR additionally changes tests and adds a convergence write-up. +- No production edits were made during research. All test overlays and raw logs: `/Users/vonng/tmp/silo-r6-20260915-aa3f/`. + +## Reproduced target-path findings + +`review_probe_test.go` is a temporary in-package probe using a real minio-go HTTP client and counted HEAD/DELETE requests. `base-overlay.json` adds the probe; `pr-overlay.json` replaces only bucket-replication.go with the exact PR version for comparison. + +| Probe | Baseline | PR #184 | Required behavior | +|---|---|---|---| +| old shape, creation COMPLETED, purge PENDING, successful target | DELETE 0; creation COMPLETED; purge PENDING | DELETE 1; creation COMPLETED; purge COMPLETE | send purge and finish purge state | +| old shape, creation PENDING, HEAD 403 | creation FAILED; purge PENDING | same | purge must not overwrite creation | +| old shape, ExistingObjectReplicationType, DELETE 403 | purge still PENDING | purge FAILED, but current reset timestamp recorded | failed operation must not mark successful resync | +| old shape, purge already COMPLETE, creation PENDING | two HTTP calls, creation FAILED | two HTTP calls, purge becomes FAILED | skip completed purge, preserve states | + +Raw logs: `baseline.log`, `pr184-probes.log`. The first baseline run also attempted the existing storage tests, which failed at seed PutObject due to the host's free-space percentage threshold. This is a fixture/environment failure, not replication evidence. Follow-up overlays use the existing `tagTestCapacityDisk` adapter: only DiskInfo total/used capacity is adapted; actual storage writes remain on test disks. Host df showed about 10 GiB available but 100% used by filesystem percentage rounding. + +## Additional code-review findings + +- Outer `replicateDelete` chooses operation outcome using VersionID, so PR's preserved creation COMPLETED can conceal old-shape purge FAILED from ObjectReplicationFailed and queueMRFSave. Per-target stats comparisons also use creation fields. +- `VersionPurgeComplete` is `COMPLETE`, while ordinary replication completion is `COMPLETED`. A raw string cast does not take the completion branch of ReplicationStats.Update. +- The old marker-shape request sends ReplicationDeleteMarker=true for purges. Today's canonical purge tasks use false and explicit VersionID. Reuse that existing wire meaning to avoid creating a marker at the absent-version fallback after an ambiguous successful DELETE. +- MRF persists entries on actual local drives, removes the loaded file, then asynchronously reads each object. MethodNotAllowed accompanies real marker ObjectInfo in erasureObjects.getObjectInfo. Tests must cover save/load/fresh pool and subsequent queueing, not call queueReplicationHeal directly. +- DeletedObjectReplicationInfo currently lacks RetryCount; its ToMRFEntry always serializes zero. Restoring marker MRF makes the existing budget omission reachable on the fast retry path. + +## Claims excluded from acceptance + +The current DELETE, scanner/heal, and resync producers already build canonical purges. The inherited old-shape test intentionally repairs a legacy PENDING marker with scanner/heal. It therefore contradicts an unqualified claim that all failed purges persist forever or only manual resync repairs them. R6 does not establish the frequency of failures for any site count, nor prove the cited production 405 traffic is entirely caused by these paths. + +PR documentation describes missing-client status loss, replica relaying, tombstones and arbitrary-mesh recovery. They are separate mechanisms. This patch's acceptance concerns tracked source fan-out with configured reachable/recoverable targets; it does not establish global multi-site convergence under lost state, nil clients, delayed creation after purge, or process/storage failure at every possible point. + +## Real erasure and disk-MRF probes + +`review_integration_test.go` reuses the existing signed DELETE/storage fixture, injects a target 403, preserves the intended creation state explicitly, and persists the actual failed entry. A fresh ReplicationPool then loads the disk record and attempts queueMRFHeal. It validates source and target absence after recovery. + +- Baseline canonical failure: creation result PENDING (lost prior COMPLETED), purge FAILED, one MRF entry. Real source lookup yields matching marker ObjectInfo and MethodNotAllowed. Disk-MRF replay schedules no task (probe fails as expected). +- Baseline old-shaped failure: creation COMPLETED, purge still PENDING, zero MRF entries (probe fails as expected). +- PR canonical failure: creation result is still empty (disk preservation clarified below). Disk-MRF replay now runs and source/target removal succeeds on both single-drive and 16-drive fixtures. +- PR old-shaped failure: creation COMPLETED and purge FAILED, but zero MRF entries. This directly confirms the outer-function omission. +- The existing baseline old-shape recovery test PASSES using its explicit scanner/heal fallback, on both storage fixtures. Therefore scanner recovery is retained as observed evidence, not only a code inference. + +Raw logs: `baseline-mrf.log`, `pr184-mrf.log`. Temporary probe development first hit an unused import and then an uninitialized statistics fixture; those were corrected without changing production code. They do not count as defect evidence. Final logs above contain actual assertion outcomes. + +## Correction after independent Opus review + +Opus v1 B1 disproved the interpretation that a canonical purge's empty creation result means the disk creation state was lost. It is an intentional no-update signal in xlMetaV2.DeleteVersion: the disk block is retained if the composite creation state is empty. `baseline-mrf.log` and `pr184-mrf.log` include a temporary assertion that was too strong; their empty per-target result is not a disk-loss finding. v2 keeps that existing no-update signal, tests the actual persisted block/timestamp, and avoids introducing the partial-fan-out overwrite that a creation-status pin would cause. See decisions-v2.md for all dispositions. diff --git a/docs/investigations/r6/review_integration_test.go.txt b/docs/investigations/r6/review_integration_test.go.txt new file mode 100644 index 000000000..53ef5d8c3 --- /dev/null +++ b/docs/investigations/r6/review_integration_test.go.txt @@ -0,0 +1,181 @@ +// Copyright (c) 2026 PGSTY +// SPDX-License-Identifier: AGPL-3.0-only + +package cmd + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/once" +) + +// Exercise the actual single-object DELETE handler and erasure metadata. A +// marker purge must delete the target version and finish the source purge; +// merely seeing a 405 on HEAD is not evidence of a completed permanent delete. +func TestReviewR6FailureMRF(t *testing.T) { + defer DetectTestLeak(t)() + for _, legacy := range []bool{false, true} { + t.Run(fmt.Sprintf("recover_legacy_%v", legacy), func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, endpoints: []string{"DeleteObject"}, + objAPITest: func(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) { + testReviewR6FailureMRF(obj, instanceType, bucket, router, creds, t, legacy) + }, + }) + }) + } +} + +func testReviewR6FailureMRF(obj ObjectLayer, instanceType, bucket string, router http.Handler, creds auth.Credentials, t *testing.T, legacy bool) { + ctx := t.Context() + oldStats := globalReplicationStats.Swap(NewReplicationStats(ctx, nil)) + defer globalReplicationStats.Store(oldStats) + if pools, ok := obj.(*erasureServerPools); ok { + for _, pool := range pools.serverPools { for _, set := range pool.sets { + disks := set.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i,d := range disks { if d!=nil {wrapped[i]=tagTestCapacityDisk{StorageAPI:d}} } + set.getDisks = func() []StorageAPI {return wrapped} + }} + } + + const arn = "arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket" + const name = "marker" + version := mustGetUUID() + remoteBucket := getRandomBucketName() + if err := obj.MakeBucket(ctx, remoteBucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil { + t.Fatal(err) + } + for _, b := range []string{bucket, remoteBucket} { + if _, err := obj.PutObject(ctx, b, name, mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), ObjectOptions{Versioned: true}); err != nil { + t.Fatal(err) + } + opts := ObjectOptions{VersionID: version, Versioned: true, DeleteMarker: true, ReplicationRequest: true, MTime: UTCNow()} + opts.SetReplicaStatus(replication.Replica) + if _, err := obj.DeleteObject(ctx, b, name, opts); err != nil { + t.Fatalf("%s: seed marker in %s: %v", instanceType, b, err) + } + } + var reject atomic.Bool + reject.Store(true) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true} + switch r.Method { + case http.MethodHead: + oi, err := obj.GetObjectInfo(r.Context(), remoteBucket, name, opts) + if oi.DeleteMarker { + w.Header().Set(xhttp.AmzDeleteMarker, "true") + w.Header().Set(xhttp.AmzVersionID, oi.VersionID) + } + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err)) + return + } + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + if reject.Load() {w.WriteHeader(403); fmt.Fprint(w, `AccessDenied`);return} + opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true" + opts.SetReplicaStatus(replication.Replica) + _, err := obj.DeleteObject(r.Context(), remoteBucket, name, opts) + if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) + return + } + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote method %s", r.Method) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer remote.Close() + client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1"}) + if err != nil { + t.Fatal(err) + } + target := &TargetClient{Client: client, ARN: arn, Bucket: remoteBucket} + globalBucketTargetSys.Lock() + globalBucketTargetSys.arnRemotesMap[arn] = arnTarget{Client: target, lastRefresh: UTCNow()} + globalBucketTargetSys.targetsMap[bucket] = []madmin.BucketTarget{{Arn: arn, TargetBucket: remoteBucket}} + globalBucketTargetSys.Unlock() + globalBucketTargetSys.hMutex.Lock() + globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true} + globalBucketTargetSys.hMutex.Unlock() + meta, err := globalBucketMetadataSys.Get(bucket) + if err != nil { + t.Fatal(err) + } + cfg := configs[0] + cfg.RoleArn = arn + meta.replicationConfig = &cfg + globalBucketMetadataSys.Set(bucket, meta) + worker := make(chan ReplicationWorkerOperation, 1) + p := &ReplicationPool{ + ctx: ctx, + objLayer: obj, + workers: []chan ReplicationWorkerOperation{worker}, + stats: globalReplicationStats.Load(), + mrfSaveCh: make(chan MRFReplicateEntry, 1), + } + oldPool := globalReplicationPool + globalReplicationPool = once.NewSingleton[ReplicationPool]() + globalReplicationPool.Set(p) + defer func() { globalReplicationPool = oldPool }() + + req, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+name+"?versionId="+version, 0, nil, creds.AccessKey, creds.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("DELETE status %d: %s", w.Code, w.Body.String()) + } + var deletion DeletedObjectReplicationInfo + select { + case op := <-worker: + deletion = op.(DeletedObjectReplicationInfo) + case <-time.After(time.Second): + t.Fatal("DELETE did not schedule replication") + } + if deletion.VersionID != version || deletion.DeleteMarkerVersionID != "" { + t.Errorf("purge scheduled as marker creation: version=%q marker=%q", deletion.VersionID, deletion.DeleteMarkerVersionID) + } + deletion.ReplicationState.Targets = map[string]replication.StatusType{arn: replication.Completed} + deletion.ReplicationState.ReplicationStatusInternal = arn+"=COMPLETED;" + if legacy {deletion.VersionID,deletion.DeleteMarkerVersionID="",version} + result:=replicateDelete(ctx,deletion,obj) + t.Logf("legacy=%v result creation=%s purge=%s MRF=%d",legacy,result.ReplicationStatus(),result.VersionPurgeStatus(),len(p.mrfSaveCh)) + if result.VersionPurgeStatus()!=replication.VersionPurgeFailed || result.ReplicationStatus()!=replication.Completed {t.Error("failure stored in incorrect status field")} + var entry MRFReplicateEntry + select {case entry= <-p.mrfSaveCh:default:t.Fatal("failed purge did not enter MRF")} + oi,lookupErr:=obj.GetObjectInfo(ctx,bucket,name,ObjectOptions{VersionID:version}) + t.Logf("real source lookup: name=%s version=%s marker=%v purge=%s err=%v",oi.Name,oi.VersionID,oi.DeleteMarker,oi.VersionPurgeStatus,lookupErr) + if !isErrMethodNotAllowed(lookupErr)||!oi.DeleteMarker {t.Fatal("expected real marker plus 405")} + p.saveMRFEntries(ctx,map[string]MRFReplicateEntry{entry.versionID:entry}) + fresh:= &ReplicationPool{ctx:ctx,objLayer:obj,workers:[]chan ReplicationWorkerOperation{worker},stats:globalReplicationStats.Load(),mrfSaveCh:make(chan MRFReplicateEntry,1)} + globalReplicationPool=once.NewSingleton[ReplicationPool]();globalReplicationPool.Set(fresh) + reject.Store(false) + if err:=fresh.queueMRFHeal();err!=nil{t.Fatal(err)} + select {case op:= <-worker:deletion=op.(DeletedObjectReplicationInfo);case <-time.After(time.Second):t.Fatal("persisted MRF marker lookup was skipped")} + result=replicateDelete(ctx,deletion,obj) + if result.VersionPurgeStatus()!=replication.VersionPurgeComplete {t.Fatalf("MRF retry result: %+v",result)} + for _,b:=range []string{bucket,remoteBucket}{ + oi,err:=obj.GetObjectInfo(ctx,b,name,ObjectOptions{VersionID:version}) + if !isErrVersionNotFound(err)&&!isErrObjectNotFound(err){t.Errorf("marker remains in %s: %+v %v",b,oi,err)} + } +} diff --git a/docs/investigations/r6/review_probe_test.go.txt b/docs/investigations/r6/review_probe_test.go.txt new file mode 100644 index 000000000..4d678e100 --- /dev/null +++ b/docs/investigations/r6/review_probe_test.go.txt @@ -0,0 +1,68 @@ +package cmd +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "github.com/minio/minio-go/v7" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" +) +func TestReviewR6LegacyPurgeRetries(t *testing.T) { + var deletes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + w.Header().Set(xhttp.AmzDeleteMarker, "true") + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + deletes.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + old := globalBucketTargetSys + globalBucketTargetSys = &BucketTargetSys{hc: map[string]epHealth{client.EndpointURL().Host: {Online: true}}} + defer func() { globalBucketTargetSys = old }() + d := DeletedObjectReplicationInfo{Bucket: "source", DeletedObject: DeletedObject{ObjectName: "marker", DeleteMarker: true, DeleteMarkerVersionID: mustGetUUID()}} + d.ReplicationState.Targets = map[string]replication.StatusType{"arn1": replication.Completed} + d.ReplicationState.PurgeTargets = map[string]VersionPurgeStatusType{"arn1": replication.VersionPurgePending} + result := replicateDeleteToTarget(t.Context(), d, &TargetClient{Client: client, ARN: "arn1", Bucket: "target"}) + t.Logf("remote DELETE calls=%d, creation=%s, purge=%s", deletes.Load(), result.ReplicationStatus, result.VersionPurgeStatus) + if deletes.Load() != 1 || result.VersionPurgeStatus != replication.VersionPurgeComplete { + t.Error("pending legacy purge was not delivered") + } +} + + +func TestReviewR6PurgeExits(t *testing.T) { + for _, tc := range []struct{name string; creation replication.StatusType; purge VersionPurgeStatusType; head int; existing bool; want VersionPurgeStatusType; wantCalls int; wantReset bool}{ + {"head_forbidden", replication.Pending, replication.VersionPurgePending, 403, false, replication.VersionPurgeFailed, 1, false}, + {"failed_resync", replication.Completed, replication.VersionPurgePending, 405, true, replication.VersionPurgeFailed, 2, false}, + {"already_purged", replication.Pending, replication.VersionPurgeComplete, 405, false, replication.VersionPurgeComplete, 0, false}, + } { + t.Run(tc.name, func(t *testing.T){ + var calls atomic.Int32 + server:=httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter,r *http.Request){ + calls.Add(1) + if r.Method==http.MethodHead {w.Header().Set(xhttp.AmzDeleteMarker,"true"); w.WriteHeader(tc.head);return} + w.WriteHeader(403);fmt.Fprint(w,`AccessDenied`) + })) + defer server.Close() + client,err:=minio.New(strings.TrimPrefix(server.URL,"http://"),&minio.Options{Region:"us-east-1",MaxRetries:1});if err!=nil{t.Fatal(err)} + old:=globalBucketTargetSys;globalBucketTargetSys=&BucketTargetSys{hc:map[string]epHealth{client.EndpointURL().Host:{Online:true}}};defer func(){globalBucketTargetSys=old}() + d:=DeletedObjectReplicationInfo{Bucket:"source",DeletedObject:DeletedObject{ObjectName:"marker",DeleteMarker:true,DeleteMarkerVersionID:mustGetUUID()}} + d.ReplicationState.Targets=map[string]replication.StatusType{"arn1":tc.creation} + d.ReplicationState.PurgeTargets=map[string]VersionPurgeStatusType{"arn1":tc.purge} + if tc.existing {d.OpType=replication.ExistingObjectReplicationType} + got:=replicateDeleteToTarget(t.Context(),d,&TargetClient{Client:client,ARN:"arn1",Bucket:"target",ResetID:"new-reset"}) + t.Logf("calls=%d, creation=%s, purge=%s, reset=%q",calls.Load(),got.ReplicationStatus,got.VersionPurgeStatus,got.ResyncTimestamp) + if got.ReplicationStatus!=tc.creation || got.VersionPurgeStatus!=tc.want || int(calls.Load())!=tc.wantCalls || (got.ResyncTimestamp!="")!=tc.wantReset {t.Errorf("operation classified inconsistently: %+v",got)} + }) + } +} diff --git a/docs/investigations/r6/verification-notes.md b/docs/investigations/r6/verification-notes.md new file mode 100644 index 000000000..9032e4690 --- /dev/null +++ b/docs/investigations/r6/verification-notes.md @@ -0,0 +1,39 @@ +# R6 verification notes and evidence boundaries + +## Reproduction evidence before acceptance + +Raw evidence directory: `/Users/vonng/tmp/silo-r6-20260915-aa3f/`. + +| Log | What it establishes | +|---|---| +| baseline.log | Old task shape skips DELETE; wrong HEAD/outcome fields and already-complete behavior. Initial storage fixture attempts hit the host capacity threshold. | +| baseline-mrf.log | After adapting fixture capacity, existing scanner/heal recovery passes. Actual disk-MRF replay drops a real marker+405. Old-shaped task queues no MRF. The canonical per-target creation-result assertion was later retracted as a disk-loss inference; see decisions-v2.md. | +| pr184-probes.log | Exact PR #184 fixes old successful delivery but leaves HEAD failure, failed resync stamp and already-complete retry errors. | +| pr184-mrf.log | PR canonical disk-MRF recovery works; failed old-shaped purge still queues no MRF. | +| v2-unrecorded-purge.log | V2 can rewrite creation metadata with two empty target statuses when disk purge state is absent. Real source/target storage, no simulated metadata layer. This led to v3 review and the explicit empty update payload. | + +The temporary baseline probes were developed with two fixture corrections (unused import, then missing ReplicationStats initialization). Those development failures are not treated as product evidence. Likewise the initial in-memory `creation=PENDING` observation does not establish a disk-state loss; Opus v1 corrected that inference. Opus v3 then corrected the earlier multi-target regex proof after the real storage counterexample. + +## What the local storage tests exercise + +The suite uses the existing single-drive and 16-drive erasure fixtures, the real signed source DELETE handler, real source/target marker metadata and a real minio-go client over HTTP. Target HTTP adapters call the real ObjectLayer, and can reject a request, be marked offline, or close the connection **after** removing the marker. They are controlled replication target adapters, not three independently booted SILO site-replication daemons. The target adapters do not exercise receiver authentication or the complete target HTTP router; those were not changed by this patch. + +`saveMRFEntries` writes real MRF files to registered fixture drives. Tests read and check the encoded record, re-persist it because loadMRF consumes the file, construct a new empty ReplicationPool, and call queueMRFHeal. The real GetObjectInfo(VersionID) returns marker metadata with 405 and the queued task goes through the actual replication worker channel. The test explicitly receives that task and invokes production replicateDelete to control each failure/recovery round; it does not start the long-running background worker loop. Tests drive persistence directly, rather than waiting for the five-minute timer. They prove pool replacement/disk reload, not process crash or power-loss durability. + +Negative MRF lookups wait for the actual lookup and assert no task arrives within a bounded observation window. Missing, read-error, nonmarker, wrong bucket/object/version, empty info and zero timestamp responses are covered. Null/empty marker version identities that return ObjectNotFound are explicitly outside the new valid-405 gate. + +The multi-target fixture checks one target complete while the other fails/offline, persisted per-target purge states, no extra delivery to the successful target, then recovery and marker removal on source and both targets. The separate partial-fan-out test preserves the complete creation block and its timestamp. It does not claim to fix the pre-existing purge-status subset replacement. + +## Environment failures and fixes to test inputs + +- The host reports a high used-space percentage. Six unrelated broad DELETE tests abort at seed PutObject with `Storage reached its minimum free drive threshold`: TestDeleteObjectConditional, TestDeleteObjectConditionalWithReadQuorumFailure, TestDeleteObjectConditionalVersioned, TestDeleteObjectsVersioned, TestDeleteObject, TestDeleteObjectVersionMarker. See replication-suite.log. That exploratory broad suite was not a pass. No unrelated production capacity policy or those tests were changed. +- R6 storage fixtures reuse the repository's tagTestCapacityDisk adapter: Total/Used are adapted to the actual free space; object/MRF metadata and data still go to real disks. This isolates host occupancy from replication semantics. +- A test link later failed with `no space left on device` (head-exits.log). Thirteen old Go cache artifacts containing this exact worktree path, totaling 1495 MiB, were removed after identification. No repository data or other tasks' cache entries were selected. The manifest is owned-cache-cleanup.json in the raw evidence directory. Available space also changed due to unrelated host activity; we do not attribute the whole increase to this cleanup. +- The HEAD quorum test initially used HTTP 503 and expected a quorum-code fall-through. Existing ErrorRespToObjectError classifies 503 as backend-down before its S3 code conversion. Final tests separately cover 503 not-ready failure and a non-503 SlowDownRead code reaching the existing quorum branch. No production change to that classification was made. +- Initial lint reported five test-style issues, subsequently corrected. v2-scope-regression.log is an intermediate failed run, including the still-unfixed v3 counterexample and the initial quorum test expectation. It is not final acceptance. + +## Limits retained after R6 + +No three-daemon/full-mesh SR deployment, process restart/crash, cross-region test, production repair, merge or release is claimed. The following pre-existing mechanisms remain separate: absent configured clients and omitted purge target state, target-scoped resync replacing the purge subset, loss of replication tracking, replica relay behavior, delayed marker **creation** after a purge without tombstones, local source metadata-write failures relying on scanner recovery, generic status parser robustness, and new MRF timer/backoff/observability design. + +Current normal producers already emit canonical version purges. The old in-memory task shape is not serialized through restart. Its support is robustness compatibility; marker MRF is the directly active retry-path repair. Do not reuse PR #184's unqualified permanence, request-rate or site-count claims as acceptance conclusions. diff --git a/docs/investigations/r6/verification/baseline-build.log b/docs/investigations/r6/verification/baseline-build.log new file mode 100644 index 000000000..e69de29bb diff --git a/docs/investigations/r6/verification/baseline-lint.log b/docs/investigations/r6/verification/baseline-lint.log new file mode 100644 index 000000000..6a3ebaa7e --- /dev/null +++ b/docs/investigations/r6/verification/baseline-lint.log @@ -0,0 +1 @@ +0 issues. diff --git a/docs/investigations/r6/verification/baseline-race.log b/docs/investigations/r6/verification/baseline-race.log new file mode 100644 index 000000000..320c759d0 --- /dev/null +++ b/docs/investigations/r6/verification/baseline-race.log @@ -0,0 +1,288 @@ +=== RUN TestReplicateDeleteMarkerPurge +=== RUN TestReplicateDeleteMarkerPurge/recover_legacy_false +=== RUN TestReplicateDeleteMarkerPurge/recover_legacy_true +--- PASS: TestReplicateDeleteMarkerPurge (0.84s) + --- PASS: TestReplicateDeleteMarkerPurge/recover_legacy_false (0.39s) + --- PASS: TestReplicateDeleteMarkerPurge/recover_legacy_true (0.44s) +=== RUN TestReplicateDeleteMarkerTargetSemantics +=== RUN TestReplicateDeleteMarkerTargetSemantics/existing_marker_is_idempotent +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_removes_an_existing_marker +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_forbidden +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_method_rejected +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_unavailable +--- PASS: TestReplicateDeleteMarkerTargetSemantics (0.01s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/existing_marker_is_idempotent (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_removes_an_existing_marker (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_forbidden (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_method_rejected (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_unavailable (0.01s) +=== RUN TestReplicationMRFMarkerRecovery +=== RUN TestReplicationMRFMarkerRecovery/canonical + replication-delete-mrf_test.go:485: ErasureSD: canonical recovered; 1 target(s), persisted MRF, source/target metadata checked +goroutine 5773 [running]: +runtime/debug.Stack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:26 +0x6c +runtime/debug.PrintStack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:18 +0x24 +github.com/minio/minio/internal/ioutil.SafeClose[...](...) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/ioutil/ioutil.go:470 +github.com/minio/minio/internal/logger/target/http.(*Target).Cancel(0xc002fd4680) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/logger/target/http/http.go:630 +0x98 +github.com/minio/minio/cmd.replicationTestAudit.func2() + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:593 +0xa4 +github.com/minio/minio/cmd.testReplicationMRFMarkerRecovery(0xc000f31d48, {0x1099ca8b0, _}, {_, _}, {_, _}, {_, _}, {{0x105dfce44, ...}, ...}, ...) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:486 +0x5cb8 +github.com/minio/minio/cmd.TestReplicationMRFMarkerRecovery.func1.1({0x1099ca8b0, 0xc002432410}, {0x105df9251, 0x9}, {0xc0024cc180, 0x3c}, {0x109974420, _}, {{0x105dfce44, 0xa}, ...}, ...) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:53 +0x124 +github.com/minio/minio/cmd.ExecObjectLayerAPITest({0xc000f31d48, 0xc002f98030, {0xc002e26720, 0x1, 0x1}, 0x0, {0x0, 0x0, 0x0, {0x0, ...}, ...}}) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/test-utils_test.go:1789 +0x32c +github.com/minio/minio/cmd.TestReplicationMRFMarkerRecovery.func1(0xc000f31d48) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:52 +0x15c +testing.tRunner(0xc000f31d48, 0xc002777e00) + /opt/homebrew/Cellar/go/1.27.1/libexec/src/testing/testing.go:2193 +0x168 +created by testing.(*T).Run in goroutine 5772 + /opt/homebrew/Cellar/go/1.27.1/libexec/src/testing/testing.go:2258 +0x7c0 + replication-delete-mrf_test.go:485: Erasure: canonical recovered; 1 target(s), persisted MRF, source/target metadata checked +goroutine 5773 [running]: +runtime/debug.Stack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:26 +0x6c +runtime/debug.PrintStack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:18 +0x24 +github.com/minio/minio/internal/ioutil.SafeClose[...](...) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/ioutil/ioutil.go:470 +github.com/minio/minio/internal/logger/target/http.(*Target).Cancel(0xc000fed6c0) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/logger/target/http/http.go:630 +0x98 +github.com/minio/minio/cmd.replicationTestAudit.func2() + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:593 +0xa4 +github.com/minio/minio/cmd.testReplicationMRFMarkerRecovery(0xc000f31d48, {0x1099ca8b0, _}, {_, _}, {_, _}, {_, _}, {{0x105dfce44, ...}, ...}, ...) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:486 +0x5cb8 +github.com/minio/minio/cmd.TestReplicationMRFMarkerRecovery.func1.1({0x1099ca8b0, 0xc00251aea0}, {0x105df2e8e, 0x7}, {0xc0024cc700, 0x3c}, {0x109974420, _}, {{0x105dfce44, 0xa}, ...}, ...) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:53 +0x124 +github.com/minio/minio/cmd.ExecObjectLayerAPITest({0xc000f31d48, 0xc002f98030, {0xc002e26720, 0x1, 0x1}, 0x0, {0x0, 0x0, 0x0, {0x0, ...}, ...}}) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/test-utils_test.go:1817 +0x5f0 +github.com/minio/minio/cmd.TestReplicationMRFMarkerRecovery.func1(0xc000f31d48) + /Users/vonng/.codex/worktrees/aa3f/silo/cmd/replication-delete-mrf_test.go:52 +0x15c +testing.tRunner(0xc000f31d48, 0xc002777e00) + /opt/homebrew/Cellar/go/1.27.1/libexec/src/testing/testing.go:2193 +0x168 +created by testing.(*T).Run in goroutine 5772 + /opt/homebrew/Cellar/go/1.27.1/libexec/src/testing/testing.go:2258 +0x7c0 +=== RUN TestReplicationMRFMarkerRecovery/lock-failure + replication-delete-mrf_test.go:485: ErasureSD: lock-failure recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: lock-failure recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error#01 +=== RUN TestReplicationMRFMarkerRecovery/legacy + replication-delete-mrf_test.go:485: ErasureSD: legacy recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: legacy recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/unrecorded-purge + replication-delete-mrf_test.go:485: ErasureSD: unrecorded-purge recovered; 2 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: unrecorded-purge recovered; 2 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/two-targets + replication-delete-mrf_test.go:485: ErasureSD: two-targets recovered; 2 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: two-targets recovered; 2 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/two-targets-offline + replication-delete-mrf_test.go:485: ErasureSD: two-targets-offline recovered; 2 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: two-targets-offline recovered; 2 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/replica-source + replication-delete-mrf_test.go:485: ErasureSD: replica-source recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: replica-source recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/lost-reply-canonical + replication-delete-mrf_test.go:485: ErasureSD: lost-reply-canonical recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: lost-reply-canonical recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/lost-reply-legacy + replication-delete-mrf_test.go:485: ErasureSD: lost-reply-legacy recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: lost-reply-legacy recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/creation + replication-delete-mrf_test.go:485: ErasureSD: creation recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: creation recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/partial-creation-block +=== RUN TestReplicationMRFMarkerRecovery/retry-budget-and-scanner + replication-delete-mrf_test.go:485: ErasureSD: retry-budget-and-scanner recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: retry-budget-and-scanner recovered; 1 target(s), persisted MRF, source/target metadata checked +--- PASS: TestReplicationMRFMarkerRecovery (8.55s) + --- PASS: TestReplicationMRFMarkerRecovery/canonical (0.51s) + --- PASS: TestReplicationMRFMarkerRecovery/lock-failure (0.56s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata (2.02s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error#01 (0.11s) + --- PASS: TestReplicationMRFMarkerRecovery/legacy (0.48s) + --- PASS: TestReplicationMRFMarkerRecovery/unrecorded-purge (0.59s) + --- PASS: TestReplicationMRFMarkerRecovery/two-targets (0.47s) + --- PASS: TestReplicationMRFMarkerRecovery/two-targets-offline (0.53s) + --- PASS: TestReplicationMRFMarkerRecovery/replica-source (0.43s) + --- PASS: TestReplicationMRFMarkerRecovery/lost-reply-canonical (0.68s) + --- PASS: TestReplicationMRFMarkerRecovery/lost-reply-legacy (0.52s) + --- PASS: TestReplicationMRFMarkerRecovery/creation (0.43s) + --- PASS: TestReplicationMRFMarkerRecovery/partial-creation-block (0.78s) + --- PASS: TestReplicationMRFMarkerRecovery/retry-budget-and-scanner (0.55s) +=== RUN TestReplicationDeleteQueueFullRetryBudget +--- PASS: TestReplicationDeleteQueueFullRetryBudget (0.00s) +=== RUN TestReplicateDeleteOperationExits +=== RUN TestReplicateDeleteOperationExits/marker-creation/pending-success +=== RUN TestReplicateDeleteOperationExits/marker-creation/completed-creation +=== RUN TestReplicateDeleteOperationExits/marker-creation/existing-marker +=== RUN TestReplicateDeleteOperationExits/marker-creation/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/marker-creation/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/marker-creation/head-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-creation/head-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-creation/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-creation/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/marker-creation/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-creation/offline +=== RUN TestReplicateDeleteOperationExits/marker-creation/retry-failed +=== RUN TestReplicateDeleteOperationExits/marker-creation/purge-complete +=== RUN TestReplicateDeleteOperationExits/marker-creation/resync-success +=== RUN TestReplicateDeleteOperationExits/marker-creation/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/marker-creation/resync-failure +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/pending-success +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/completed-creation +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/existing-marker +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/head-unavailable +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/head-forbidden +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/offline +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/retry-failed +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/purge-complete +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/resync-success +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/resync-failure +=== RUN TestReplicateDeleteOperationExits/marker-purge/pending-success +=== RUN TestReplicateDeleteOperationExits/marker-purge/completed-creation +=== RUN TestReplicateDeleteOperationExits/marker-purge/existing-marker +=== RUN TestReplicateDeleteOperationExits/marker-purge/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/marker-purge/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/marker-purge/head-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-purge/head-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-purge/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-purge/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/marker-purge/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-purge/offline +=== RUN TestReplicateDeleteOperationExits/marker-purge/retry-failed +=== RUN TestReplicateDeleteOperationExits/marker-purge/purge-complete +=== RUN TestReplicateDeleteOperationExits/marker-purge/resync-success +=== RUN TestReplicateDeleteOperationExits/marker-purge/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/marker-purge/resync-failure +=== RUN TestReplicateDeleteOperationExits/object-purge/pending-success +=== RUN TestReplicateDeleteOperationExits/object-purge/completed-creation +=== RUN TestReplicateDeleteOperationExits/object-purge/existing-marker +=== RUN TestReplicateDeleteOperationExits/object-purge/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/object-purge/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/object-purge/head-unavailable +=== RUN TestReplicateDeleteOperationExits/object-purge/head-forbidden +=== RUN TestReplicateDeleteOperationExits/object-purge/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/object-purge/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/object-purge/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/object-purge/offline +=== RUN TestReplicateDeleteOperationExits/object-purge/retry-failed +=== RUN TestReplicateDeleteOperationExits/object-purge/purge-complete +=== RUN TestReplicateDeleteOperationExits/object-purge/resync-success +=== RUN TestReplicateDeleteOperationExits/object-purge/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/object-purge/resync-failure +--- PASS: TestReplicateDeleteOperationExits (0.58s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/head-read-quorum (0.07s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/head-unavailable (0.11s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/delete-unavailable (0.10s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/resync-failure (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/head-read-quorum (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/head-unavailable (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/delete-unavailable (0.06s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/resync-failure (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/head-read-quorum (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/head-unavailable (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/delete-unavailable (0.03s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/resync-failure (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/head-read-quorum (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/head-unavailable (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/delete-unavailable (0.16s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/resync-failure (0.00s) +=== RUN TestReplicateDeletePurgeMissingTargetState +--- PASS: TestReplicateDeletePurgeMissingTargetState (0.00s) +=== RUN TestReplicationMRFDropsVisible +=== RUN TestReplicationMRFDropsVisible/bucket +=== RUN TestReplicationMRFDropsVisible/node +--- PASS: TestReplicationMRFDropsVisible (0.01s) + --- PASS: TestReplicationMRFDropsVisible/bucket (0.00s) + --- PASS: TestReplicationMRFDropsVisible/node (0.00s) +PASS +ok github.com/minio/minio/cmd 14.031s diff --git a/docs/investigations/r6/verification/baseline-scope.log b/docs/investigations/r6/verification/baseline-scope.log new file mode 100644 index 000000000..489b98293 --- /dev/null +++ b/docs/investigations/r6/verification/baseline-scope.log @@ -0,0 +1,444 @@ +=== RUN TestReplicatedInfos +--- PASS: TestReplicatedInfos (0.00s) +=== RUN TestReplicationResync +--- PASS: TestReplicationResync (0.00s) +=== RUN TestReplicationResyncwrapper +--- PASS: TestReplicationResyncwrapper (0.00s) +=== RUN TestReplicationValidationObjectUsesRulePrefix +=== RUN TestReplicationValidationObjectUsesRulePrefix/empty_prefix +=== RUN TestReplicationValidationObjectUsesRulePrefix/filter_prefix +=== RUN TestReplicationValidationObjectUsesRulePrefix/and_prefix +--- PASS: TestReplicationValidationObjectUsesRulePrefix (0.00s) + --- PASS: TestReplicationValidationObjectUsesRulePrefix/empty_prefix (0.00s) + --- PASS: TestReplicationValidationObjectUsesRulePrefix/filter_prefix (0.00s) + --- PASS: TestReplicationValidationObjectUsesRulePrefix/and_prefix (0.00s) +=== RUN TestResyncBucketFinalize +=== RUN TestResyncBucketFinalize/persists_complete_counts +=== RUN TestResyncBucketFinalize/parent_cancel_during_drain_downgrades_to_failed +=== RUN TestResyncBucketFinalize/user_cancel_persists_canceled +--- PASS: TestResyncBucketFinalize (0.11s) + --- PASS: TestResyncBucketFinalize/persists_complete_counts (0.01s) + --- PASS: TestResyncBucketFinalize/parent_cancel_during_drain_downgrades_to_failed (0.01s) + --- PASS: TestResyncBucketFinalize/user_cancel_persists_canceled (0.01s) +=== RUN TestResyncFinishDrainsResults +--- PASS: TestResyncFinishDrainsResults (0.00s) +=== RUN TestResyncFinishWaitsForInflightWorker +--- PASS: TestResyncFinishWaitsForInflightWorker (0.00s) +=== RUN TestResyncResultFor +=== RUN TestResyncResultFor/completed_update +=== RUN TestResyncResultFor/failed_update_over_existing_version +=== RUN TestResyncResultFor/completed_but_errored_is_a_failure +=== RUN TestResyncResultFor/delete_failed +=== RUN TestResyncResultFor/delete_marker_replicated_counts_zero_bytes +=== RUN TestResyncResultFor/arn_not_attempted_is_a_failure +=== RUN TestResyncResultFor/completed_with_zero_size_falls_back_to_object_size +=== RUN TestResyncResultFor/version_purge_complete_is_a_success +=== RUN TestResyncResultFor/version_purge_failed_is_a_failure +=== RUN TestResyncResultFor/benign_duplicate_412_is_a_success +--- PASS: TestResyncResultFor (0.00s) + --- PASS: TestResyncResultFor/completed_update (0.00s) + --- PASS: TestResyncResultFor/failed_update_over_existing_version (0.00s) + --- PASS: TestResyncResultFor/completed_but_errored_is_a_failure (0.00s) + --- PASS: TestResyncResultFor/delete_failed (0.00s) + --- PASS: TestResyncResultFor/delete_marker_replicated_counts_zero_bytes (0.00s) + --- PASS: TestResyncResultFor/arn_not_attempted_is_a_failure (0.00s) + --- PASS: TestResyncResultFor/completed_with_zero_size_falls_back_to_object_size (0.00s) + --- PASS: TestResyncResultFor/version_purge_complete_is_a_success (0.00s) + --- PASS: TestResyncResultFor/version_purge_failed_is_a_failure (0.00s) + --- PASS: TestResyncResultFor/benign_duplicate_412_is_a_success (0.00s) +=== RUN TestReplicationActionForTargetRetentionRemoval +=== RUN TestReplicationActionForTargetRetentionRemoval/removal_confirmed_by_destination +=== RUN TestReplicationActionForTargetRetentionRemoval/destination_still_holds_the_retention_hidden_from_HEAD +=== RUN TestReplicationActionForTargetRetentionRemoval/retention_hidden_from_HEAD_by_permissions +=== RUN TestReplicationActionForTargetRetentionRemoval/destination_reports_no_object_lock_configuration +=== RUN TestReplicationActionForTargetRetentionRemoval/version_never_had_retention_is_not_confirmed +--- PASS: TestReplicationActionForTargetRetentionRemoval (0.00s) + --- PASS: TestReplicationActionForTargetRetentionRemoval/removal_confirmed_by_destination (0.00s) + --- PASS: TestReplicationActionForTargetRetentionRemoval/destination_still_holds_the_retention_hidden_from_HEAD (0.00s) + --- PASS: TestReplicationActionForTargetRetentionRemoval/retention_hidden_from_HEAD_by_permissions (0.00s) + --- PASS: TestReplicationActionForTargetRetentionRemoval/destination_reports_no_object_lock_configuration (0.00s) + --- PASS: TestReplicationActionForTargetRetentionRemoval/version_never_had_retention_is_not_confirmed (0.00s) +=== RUN TestReplicationActionForTargetNullVersionResync +=== RUN TestReplicationActionForTargetNullVersionResync/destination_holds_retention +=== RUN TestReplicationActionForTargetNullVersionResync/retention_read_denied +--- PASS: TestReplicationActionForTargetNullVersionResync (0.00s) + --- PASS: TestReplicationActionForTargetNullVersionResync/destination_holds_retention (0.00s) + --- PASS: TestReplicationActionForTargetNullVersionResync/retention_read_denied (0.00s) +=== RUN TestReplicationActionForTargetTimestampOnlyRemoval +=== RUN TestReplicationActionForTargetTimestampOnlyRemoval/retention_hidden_from_HEAD_by_permissions +=== RUN TestReplicationActionForTargetTimestampOnlyRemoval/destination_still_holds_the_retention +=== RUN TestReplicationActionForTargetTimestampOnlyRemoval/removal_confirmed_by_destination +--- PASS: TestReplicationActionForTargetTimestampOnlyRemoval (0.00s) + --- PASS: TestReplicationActionForTargetTimestampOnlyRemoval/retention_hidden_from_HEAD_by_permissions (0.00s) + --- PASS: TestReplicationActionForTargetTimestampOnlyRemoval/destination_still_holds_the_retention (0.00s) + --- PASS: TestReplicationActionForTargetTimestampOnlyRemoval/removal_confirmed_by_destination (0.00s) +=== RUN TestReplicateDeleteMarkerPurge +=== RUN TestReplicateDeleteMarkerPurge/recover_legacy_false +=== RUN TestReplicateDeleteMarkerPurge/recover_legacy_true +--- PASS: TestReplicateDeleteMarkerPurge (0.47s) + --- PASS: TestReplicateDeleteMarkerPurge/recover_legacy_false (0.26s) + --- PASS: TestReplicateDeleteMarkerPurge/recover_legacy_true (0.21s) +=== RUN TestReplicateDeleteMarkerTargetSemantics +=== RUN TestReplicateDeleteMarkerTargetSemantics/existing_marker_is_idempotent +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_removes_an_existing_marker +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_forbidden +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_method_rejected +=== RUN TestReplicateDeleteMarkerTargetSemantics/purge_unavailable +--- PASS: TestReplicateDeleteMarkerTargetSemantics (0.17s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/existing_marker_is_idempotent (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_removes_an_existing_marker (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_forbidden (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_method_rejected (0.00s) + --- PASS: TestReplicateDeleteMarkerTargetSemantics/purge_unavailable (0.17s) +=== RUN TestReplicationMRFMarkerRecovery +=== RUN TestReplicationMRFMarkerRecovery/canonical + replication-delete-mrf_test.go:485: ErasureSD: canonical recovered; 1 target(s), persisted MRF, source/target metadata checked +goroutine 6684 [running]: +runtime/debug.Stack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:26 +0x64 +runtime/debug.PrintStack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:18 +0x1c +github.com/minio/minio/internal/ioutil.SafeClose[...](...) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/ioutil/ioutil.go:470 +github.com/minio/minio/internal/logger/target/http.(*Target).Cancel(0x4add905116c0) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/logger/target/http/http.go:630 +0x74 +created by github.com/minio/minio/internal/logger.cancelTargets in goroutine 6745 + /Users/vonng/.codex/worktrees/aa3f/silo/internal/logger/targets.go:213 +0x28 + replication-delete-mrf_test.go:485: Erasure: canonical recovered; 1 target(s), persisted MRF, source/target metadata checked +goroutine 10189 [running]: +runtime/debug.Stack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:26 +0x64 +runtime/debug.PrintStack() + /opt/homebrew/Cellar/go/1.27.1/libexec/src/runtime/debug/stack.go:18 +0x1c +github.com/minio/minio/internal/ioutil.SafeClose[...](...) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/ioutil/ioutil.go:470 +github.com/minio/minio/internal/logger/target/http.(*Target).Cancel(0x4add9bf7d380) + /Users/vonng/.codex/worktrees/aa3f/silo/internal/logger/target/http/http.go:630 +0x74 +created by github.com/minio/minio/internal/logger.cancelTargets in goroutine 6745 + /Users/vonng/.codex/worktrees/aa3f/silo/internal/logger/targets.go:213 +0x28 +=== RUN TestReplicationMRFMarkerRecovery/lock-failure + replication-delete-mrf_test.go:485: ErasureSD: lock-failure recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: lock-failure recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing#01 +=== RUN TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error#01 +=== RUN TestReplicationMRFMarkerRecovery/legacy + replication-delete-mrf_test.go:485: ErasureSD: legacy recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: legacy recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/unrecorded-purge + replication-delete-mrf_test.go:485: ErasureSD: unrecorded-purge recovered; 2 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: unrecorded-purge recovered; 2 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/two-targets + replication-delete-mrf_test.go:485: ErasureSD: two-targets recovered; 2 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: two-targets recovered; 2 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/two-targets-offline + replication-delete-mrf_test.go:485: ErasureSD: two-targets-offline recovered; 2 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: two-targets-offline recovered; 2 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/replica-source + replication-delete-mrf_test.go:485: ErasureSD: replica-source recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: replica-source recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/lost-reply-canonical + replication-delete-mrf_test.go:485: ErasureSD: lost-reply-canonical recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: lost-reply-canonical recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/lost-reply-legacy + replication-delete-mrf_test.go:485: ErasureSD: lost-reply-legacy recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: lost-reply-legacy recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/creation + replication-delete-mrf_test.go:485: ErasureSD: creation recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: creation recovered; 1 target(s), persisted MRF, source/target metadata checked +=== RUN TestReplicationMRFMarkerRecovery/partial-creation-block +=== RUN TestReplicationMRFMarkerRecovery/retry-budget-and-scanner + replication-delete-mrf_test.go:485: ErasureSD: retry-budget-and-scanner recovered; 1 target(s), persisted MRF, source/target metadata checked + replication-delete-mrf_test.go:485: Erasure: retry-budget-and-scanner recovered; 1 target(s), persisted MRF, source/target metadata checked +--- PASS: TestReplicationMRFMarkerRecovery (6.13s) + --- PASS: TestReplicationMRFMarkerRecovery/canonical (0.22s) + --- PASS: TestReplicationMRFMarkerRecovery/lock-failure (0.24s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata (1.90s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/empty-info#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/not-a-marker#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-version#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-bucket#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/wrong-object#01 (0.12s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/zero-modtime#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/missing#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/invalid-MRF-metadata/read-error#01 (0.10s) + --- PASS: TestReplicationMRFMarkerRecovery/legacy (0.26s) + --- PASS: TestReplicationMRFMarkerRecovery/unrecorded-purge (0.27s) + --- PASS: TestReplicationMRFMarkerRecovery/two-targets (0.28s) + --- PASS: TestReplicationMRFMarkerRecovery/two-targets-offline (0.29s) + --- PASS: TestReplicationMRFMarkerRecovery/replica-source (0.28s) + --- PASS: TestReplicationMRFMarkerRecovery/lost-reply-canonical (0.56s) + --- PASS: TestReplicationMRFMarkerRecovery/lost-reply-legacy (0.56s) + --- PASS: TestReplicationMRFMarkerRecovery/creation (0.34s) + --- PASS: TestReplicationMRFMarkerRecovery/partial-creation-block (0.47s) + --- PASS: TestReplicationMRFMarkerRecovery/retry-budget-and-scanner (0.45s) +=== RUN TestReplicationDeleteQueueFullRetryBudget +--- PASS: TestReplicationDeleteQueueFullRetryBudget (0.00s) +=== RUN TestReplicateDeleteOperationExits +=== RUN TestReplicateDeleteOperationExits/marker-creation/pending-success +=== RUN TestReplicateDeleteOperationExits/marker-creation/completed-creation +=== RUN TestReplicateDeleteOperationExits/marker-creation/existing-marker +=== RUN TestReplicateDeleteOperationExits/marker-creation/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/marker-creation/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/marker-creation/head-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-creation/head-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-creation/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-creation/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/marker-creation/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-creation/offline +=== RUN TestReplicateDeleteOperationExits/marker-creation/retry-failed +=== RUN TestReplicateDeleteOperationExits/marker-creation/purge-complete +=== RUN TestReplicateDeleteOperationExits/marker-creation/resync-success +=== RUN TestReplicateDeleteOperationExits/marker-creation/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/marker-creation/resync-failure +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/pending-success +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/completed-creation +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/existing-marker +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/head-unavailable +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/head-forbidden +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/offline +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/retry-failed +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/purge-complete +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/resync-success +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/legacy-marker-purge/resync-failure +=== RUN TestReplicateDeleteOperationExits/marker-purge/pending-success +=== RUN TestReplicateDeleteOperationExits/marker-purge/completed-creation +=== RUN TestReplicateDeleteOperationExits/marker-purge/existing-marker +=== RUN TestReplicateDeleteOperationExits/marker-purge/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/marker-purge/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/marker-purge/head-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-purge/head-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-purge/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/marker-purge/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/marker-purge/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/marker-purge/offline +=== RUN TestReplicateDeleteOperationExits/marker-purge/retry-failed +=== RUN TestReplicateDeleteOperationExits/marker-purge/purge-complete +=== RUN TestReplicateDeleteOperationExits/marker-purge/resync-success +=== RUN TestReplicateDeleteOperationExits/marker-purge/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/marker-purge/resync-failure +=== RUN TestReplicateDeleteOperationExits/object-purge/pending-success +=== RUN TestReplicateDeleteOperationExits/object-purge/completed-creation +=== RUN TestReplicateDeleteOperationExits/object-purge/existing-marker +=== RUN TestReplicateDeleteOperationExits/object-purge/head-read-quorum +=== RUN TestReplicateDeleteOperationExits/object-purge/replica-creation-status +=== RUN TestReplicateDeleteOperationExits/object-purge/head-unavailable +=== RUN TestReplicateDeleteOperationExits/object-purge/head-forbidden +=== RUN TestReplicateDeleteOperationExits/object-purge/delete-forbidden +=== RUN TestReplicateDeleteOperationExits/object-purge/delete-method-rejected +=== RUN TestReplicateDeleteOperationExits/object-purge/delete-unavailable +=== RUN TestReplicateDeleteOperationExits/object-purge/offline +=== RUN TestReplicateDeleteOperationExits/object-purge/retry-failed +=== RUN TestReplicateDeleteOperationExits/object-purge/purge-complete +=== RUN TestReplicateDeleteOperationExits/object-purge/resync-success +=== RUN TestReplicateDeleteOperationExits/object-purge/resync-already-purged +=== RUN TestReplicateDeleteOperationExits/object-purge/resync-failure +--- PASS: TestReplicateDeleteOperationExits (0.79s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/head-read-quorum (0.06s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/head-unavailable (0.15s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/delete-unavailable (0.20s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-creation/resync-failure (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/head-read-quorum (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/head-unavailable (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/delete-unavailable (0.05s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/legacy-marker-purge/resync-failure (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/head-read-quorum (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/head-unavailable (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/delete-unavailable (0.14s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/marker-purge/resync-failure (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/pending-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/completed-creation (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/existing-marker (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/head-read-quorum (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/replica-creation-status (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/head-unavailable (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/head-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/delete-forbidden (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/delete-method-rejected (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/delete-unavailable (0.16s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/offline (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/retry-failed (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/purge-complete (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/resync-success (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/resync-already-purged (0.00s) + --- PASS: TestReplicateDeleteOperationExits/object-purge/resync-failure (0.00s) +=== RUN TestReplicateDeletePurgeMissingTargetState +--- PASS: TestReplicateDeletePurgeMissingTargetState (0.00s) +=== RUN TestReplicationMRFDropsVisible +=== RUN TestReplicationMRFDropsVisible/bucket +=== RUN TestReplicationMRFDropsVisible/node +--- PASS: TestReplicationMRFDropsVisible (0.00s) + --- PASS: TestReplicationMRFDropsVisible/bucket (0.00s) + --- PASS: TestReplicationMRFDropsVisible/node (0.00s) +=== RUN TestReplicationObjectDeleteWorkerAffinity +--- PASS: TestReplicationObjectDeleteWorkerAffinity (0.00s) +=== RUN TestSiteResyncCancelState +--- PASS: TestSiteResyncCancelState (0.00s) +=== RUN TestResyncRecoveryOwnsLeaderContext +=== RUN TestResyncRecoveryOwnsLeaderContext/lose_leader_false +=== RUN TestResyncRecoveryOwnsLeaderContext/lose_leader_true +--- PASS: TestResyncRecoveryOwnsLeaderContext (0.00s) + --- PASS: TestResyncRecoveryOwnsLeaderContext/lose_leader_false (0.00s) + --- PASS: TestResyncRecoveryOwnsLeaderContext/lose_leader_true (0.00s) +=== RUN TestResyncCancelRouting +--- PASS: TestResyncCancelRouting (0.00s) +=== RUN TestResyncCancellationWinsFinalization +--- PASS: TestResyncCancellationWinsFinalization (0.00s) +=== RUN TestResyncCancelFullWorkerQueue +--- PASS: TestResyncCancelFullWorkerQueue (0.00s) +=== RUN TestResyncCancelBlockedWalkReceive +--- PASS: TestResyncCancelBlockedWalkReceive (0.00s) +=== RUN TestResyncCancelsOwnedWalkOnError +--- PASS: TestResyncCancelsOwnedWalkOnError (0.00s) +=== RUN TestReplicationTrustControlsInternalOptionsAndEvents +--- PASS: TestReplicationTrustControlsInternalOptionsAndEvents (0.00s) +PASS +ok github.com/minio/minio/cmd 10.118s +=== RUN TestReplicate +=== RUN TestReplicate/#00 +=== RUN TestReplicate/c1test +=== RUN TestReplicate/c1test#01 +=== RUN TestReplicate/c1test#02 +=== RUN TestReplicate/c1test#03 +=== RUN TestReplicate/c1test#04 +=== RUN TestReplicate/c1test#05 +=== RUN TestReplicate/c1test#06 +=== RUN TestReplicate/c1test#07 +=== RUN TestReplicate/c2test +=== RUN TestReplicate/c2test#01 +=== RUN TestReplicate/c2test#02 +=== RUN TestReplicate/c2test#03 +=== RUN TestReplicate/c2test#04 +=== RUN TestReplicate/c2test#05 +=== RUN TestReplicate/c2test#06 +=== RUN TestReplicate/xy/c3test +=== RUN TestReplicate/xyz/c3test +=== RUN TestReplicate/xyz/c3test#01 +=== RUN TestReplicate/xyz/c3test#02 +=== RUN TestReplicate/xyz/c3test#03 +=== RUN TestReplicate/xyz/c3test#04 +=== RUN TestReplicate/xy/c3test#01 +=== RUN TestReplicate/xyz/c3test#05 +=== RUN TestReplicate/xyz/c3test#06 +=== RUN TestReplicate/xyz/c3test#07 +=== RUN TestReplicate/abc/c3test +=== RUN TestReplicate/xy/c4test +=== RUN TestReplicate/xa/c4test +=== RUN TestReplicate/xyz/c4test +=== RUN TestReplicate/xyz/c4test#01 +=== RUN TestReplicate/xyz/c4test#02 +=== RUN TestReplicate/xyz/c4test#03 +=== RUN TestReplicate/abc/c4test +=== RUN TestReplicate/abc/c4test#01 +=== RUN TestReplicate/abc/c4test#02 +=== RUN TestReplicate/abc/c4test#03 +=== RUN TestReplicate/abc/c4test#04 +=== RUN TestReplicate/xy/c5test +=== RUN TestReplicate/xa/c5test +--- PASS: TestReplicate (0.00s) + --- PASS: TestReplicate/#00 (0.00s) + --- PASS: TestReplicate/c1test (0.00s) + --- PASS: TestReplicate/c1test#01 (0.00s) + --- PASS: TestReplicate/c1test#02 (0.00s) + --- PASS: TestReplicate/c1test#03 (0.00s) + --- PASS: TestReplicate/c1test#04 (0.00s) + --- PASS: TestReplicate/c1test#05 (0.00s) + --- PASS: TestReplicate/c1test#06 (0.00s) + --- PASS: TestReplicate/c1test#07 (0.00s) + --- PASS: TestReplicate/c2test (0.00s) + --- PASS: TestReplicate/c2test#01 (0.00s) + --- PASS: TestReplicate/c2test#02 (0.00s) + --- PASS: TestReplicate/c2test#03 (0.00s) + --- PASS: TestReplicate/c2test#04 (0.00s) + --- PASS: TestReplicate/c2test#05 (0.00s) + --- PASS: TestReplicate/c2test#06 (0.00s) + --- PASS: TestReplicate/xy/c3test (0.00s) + --- PASS: TestReplicate/xyz/c3test (0.00s) + --- PASS: TestReplicate/xyz/c3test#01 (0.00s) + --- PASS: TestReplicate/xyz/c3test#02 (0.00s) + --- PASS: TestReplicate/xyz/c3test#03 (0.00s) + --- PASS: TestReplicate/xyz/c3test#04 (0.00s) + --- PASS: TestReplicate/xy/c3test#01 (0.00s) + --- PASS: TestReplicate/xyz/c3test#05 (0.00s) + --- PASS: TestReplicate/xyz/c3test#06 (0.00s) + --- PASS: TestReplicate/xyz/c3test#07 (0.00s) + --- PASS: TestReplicate/abc/c3test (0.00s) + --- PASS: TestReplicate/xy/c4test (0.00s) + --- PASS: TestReplicate/xa/c4test (0.00s) + --- PASS: TestReplicate/xyz/c4test (0.00s) + --- PASS: TestReplicate/xyz/c4test#01 (0.00s) + --- PASS: TestReplicate/xyz/c4test#02 (0.00s) + --- PASS: TestReplicate/xyz/c4test#03 (0.00s) + --- PASS: TestReplicate/abc/c4test (0.00s) + --- PASS: TestReplicate/abc/c4test#01 (0.00s) + --- PASS: TestReplicate/abc/c4test#02 (0.00s) + --- PASS: TestReplicate/abc/c4test#03 (0.00s) + --- PASS: TestReplicate/abc/c4test#04 (0.00s) + --- PASS: TestReplicate/xy/c5test (0.00s) + --- PASS: TestReplicate/xa/c5test (0.00s) +PASS +ok github.com/minio/minio/internal/bucket/replication 0.624s diff --git a/docs/investigations/r6/verification/baseline-verification.json b/docs/investigations/r6/verification/baseline-verification.json new file mode 100644 index 000000000..bd58e9b87 --- /dev/null +++ b/docs/investigations/r6/verification/baseline-verification.json @@ -0,0 +1,101 @@ +{ + "baseline": "9ebe81c1b3611f9cc73e676b5b741c2be62c467a", + "source_sha256": { + "cmd/bucket-replication.go": "999c2818a8980cbeb55cfcbc244840069fb44e042b4b4d6ef7668c2660ce31e0", + "cmd/bucket-replication-utils.go": "365641c760901641e8320cc5123697ab92a46f1add612048b991ed2d1cfad43b", + "cmd/replication-delete-marker_test.go": "d967787804d558ac6266b113228fdf4a4f9fcb7cab39138a4fb07558814ccca4", + "cmd/replication-delete-operation_test.go": "2888a04a543324776041316de2821f388d28c3c1a6f5d0031e5ac9d34f58d504", + "cmd/replication-delete-mrf_test.go": "5e160f7e19cbbb8cd5fa4e7ffd9cff9e09361b3fc4c5ee5ae61458f99c777781" + }, + "checks": [ + { + "name": "scope", + "command": [ + "go", + "test", + "-p", + "2", + "./cmd", + "./internal/bucket/replication", + "-run", + "TestReplication|TestReplicate|TestMRF|TestResync|TestSiteResync", + "-count=1", + "-v" + ], + "exit_code": 0, + "seconds": 38.664, + "log": "/Users/vonng/tmp/silo-r6-20260915-aa3f/final-scope.log", + "log_sha256": "c0f286b61b0a8e668dc92f4d62800f010970331e257ea49db504373d39574370", + "finished_at_utc": "2026-09-15T16:16:22.027596+00:00" + }, + { + "name": "race", + "command": [ + "go", + "test", + "-race", + "-p", + "2", + "./cmd", + "-run", + "TestReplicateDelete|TestReplicationMRF|TestReplicationDeleteQueueFull", + "-count=1", + "-v" + ], + "exit_code": 0, + "seconds": 72.883, + "log": "/Users/vonng/tmp/silo-r6-20260915-aa3f/final-race.log", + "log_sha256": "826c3de1632f026f806474328807408c8e197726a30a70ffff4376004ec51e53", + "finished_at_utc": "2026-09-15T16:17:34.912761+00:00" + }, + { + "name": "build", + "command": [ + "go", + "build", + "-p", + "2", + "./..." + ], + "exit_code": 0, + "seconds": 39.464, + "log": "/Users/vonng/tmp/silo-r6-20260915-aa3f/final-build.log", + "log_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "finished_at_utc": "2026-09-15T16:18:14.378361+00:00" + }, + { + "name": "vet", + "command": [ + "go", + "vet", + "-p", + "2", + "./cmd", + "./internal/bucket/replication" + ], + "exit_code": 0, + "seconds": 10.278, + "log": "/Users/vonng/tmp/silo-r6-20260915-aa3f/final-vet.log", + "log_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "finished_at_utc": "2026-09-15T16:18:24.657311+00:00" + }, + { + "name": "lint", + "command": [ + "/Users/vonng/pgsty/silo/.bin/golangci/v2.13.1/golangci-lint", + "run", + "--build-tags", + "kqueue", + "--timeout=10m", + "--config", + "./.golangci.yml" + ], + "exit_code": 0, + "seconds": 126.658, + "log": "/Users/vonng/tmp/silo-r6-20260915-aa3f/final-lint.log", + "log_sha256": "e92606b0bf483111dff0a120c315ea165821348f31365020e2468a0059095c47", + "finished_at_utc": "2026-09-15T16:20:31.317818+00:00" + } + ], + "source_unchanged": true +} diff --git a/docs/investigations/r6/verification/baseline-vet.log b/docs/investigations/r6/verification/baseline-vet.log new file mode 100644 index 000000000..e69de29bb