fix(restore): queue the heal of a partial commit under the normalized ID

A native restore commit that reaches a write quorum but not every drive
queues an MRF heal of the version. It passed the raw version ID, which is
empty for the null version, while the convergence path passes the
normalized "null". The heal boundary already reads an empty ID as the
null version, so the old entry healed the right version; this makes the
two paths queue the same identity.

The header of restore-capability.go still described the multipart upload
of the first native-restore design. It now describes what the capability
gate protects in the two-phase design: restore ownership through the
restore lock and the attempt marker, the staged commit under the object
lock, and the attempt fence. It also says why an unreachable server
blocks native restore: it is not proven stopped.

Test: TestRestorePartialCommitQueuesNullVersionHeal fails one drive's
commit on a noncurrent null version, checks the queued entry names
"null", and heals it back onto that drive. It fails on the base at the
identity check only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-06 19:40:24 +08:00
parent 94d91703b5
commit 3268fbea7e
3 changed files with 76 additions and 7 deletions
+61
View File
@@ -856,3 +856,64 @@ func TestRestorePublicationBelowWriteQuorumOnWideSet(t *testing.T) {
}
}
}
// A commit that reaches a write quorum but not every drive queues a heal of
// the version it restored: for the null version, the normalized null ID that
// the convergence path queues too. That heal repairs the null version on the
// drive that missed the commit, also where the null version is noncurrent.
func TestRestorePartialCommitQueuesNullVersionHeal(t *testing.T) {
saved := globalStorageClass
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 2}})
t.Cleanup(func() { globalStorageClass.Update(saved) })
r := newRestoreSafetyTest(t, 1, versioning.Suspended)
oi, body := nativeRestoreSeed(t, r, 0, true, false)
// A version written while versioning was enabled makes the null version
// noncurrent.
newer := []byte("newer")
if _, err := r.z.PutObject(t.Context(), r.bucket, r.object,
mustGetPutObjReader(t, bytes.NewReader(newer), int64(len(newer)), "", ""), ObjectOptions{Versioned: true}); err != nil {
t.Fatal(err)
}
failing := &restoreTwoPhaseFaults{}
failing.rename.Store("fail")
disks := restoreTwoPhaseFaulty(t, r, nil, nil, nil, failing)
// The test owns the MRF queue while the commit runs: the heal routines of
// the test servers would otherwise take the entry first.
queue := make(chan PartialOperation, mrfOpsQueueSize)
previous := globalMRFState.opCh
globalMRFState.opCh = queue
err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
globalMRFState.opCh = previous
if err != nil {
t.Fatalf("restore with one drive failing its commit: %v", err)
}
var queued []PartialOperation
for len(queue) > 0 {
if op := <-queue; op.Bucket == r.bucket && op.Object == r.object {
queued = append(queued, op)
}
}
if len(queued) != 1 || queued[0].VersionID != nullVersionID {
t.Fatalf("queued heals %+v, want one of version %q", queued, nullVersionID)
}
if restoreTwoPhaseRestored(restoreTwoPhaseEntries(t, r, disks, oi.VersionID)[3]) {
t.Fatal("the drive that failed the commit holds the restored copy")
}
failing.rename.Store("")
if _, err := r.z.HealObject(t.Context(), r.bucket, r.object, queued[0].VersionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan}); err != nil {
t.Fatalf("heal of the queued version: %v", err)
}
for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
if !restoreTwoPhaseRestored(fi) {
t.Errorf("drive %d: not restored after the heal: %q", i, fi.Metadata[xhttp.AmzRestore])
}
}
if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
t.Fatalf("restored contents: %v", err)
}
if latest, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); err != nil || latest.VersionID == "" || latest.VersionID == nullVersionID {
t.Fatalf("latest = %q, %v; want the newer version", latest.VersionID, err)
}
}