mirror of
https://github.com/pgsty/minio.git
synced 2026-11-10 01:31:52 +03:00
fix(restore): queue the heal of a partial commit under the normalized ID
A native restore commit that reaches a write quorum but not every drive queues an MRF heal of the version. It passed the raw version ID, which is empty for the null version, while the convergence path passes the normalized "null". The heal boundary already reads an empty ID as the null version, so the old entry healed the right version; this makes the two paths queue the same identity. The header of restore-capability.go still described the multipart upload of the first native-restore design. It now describes what the capability gate protects in the two-phase design: restore ownership through the restore lock and the attempt marker, the staged commit under the object lock, and the attempt fence. It also says why an unreachable server blocks native restore: it is not proven stopped. Test: TestRestorePartialCommitQueuesNullVersionHeal fails one drive's commit on a noncurrent null version, checks the queued entry names "null", and heals it back onto that drive. It fails on the base at the identity check only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
@@ -856,3 +856,64 @@ func TestRestorePublicationBelowWriteQuorumOnWideSet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A commit that reaches a write quorum but not every drive queues a heal of
|
||||
// the version it restored: for the null version, the normalized null ID that
|
||||
// the convergence path queues too. That heal repairs the null version on the
|
||||
// drive that missed the commit, also where the null version is noncurrent.
|
||||
func TestRestorePartialCommitQueuesNullVersionHeal(t *testing.T) {
|
||||
saved := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 2}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(saved) })
|
||||
r := newRestoreSafetyTest(t, 1, versioning.Suspended)
|
||||
oi, body := nativeRestoreSeed(t, r, 0, true, false)
|
||||
// A version written while versioning was enabled makes the null version
|
||||
// noncurrent.
|
||||
newer := []byte("newer")
|
||||
if _, err := r.z.PutObject(t.Context(), r.bucket, r.object,
|
||||
mustGetPutObjReader(t, bytes.NewReader(newer), int64(len(newer)), "", ""), ObjectOptions{Versioned: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
failing := &restoreTwoPhaseFaults{}
|
||||
failing.rename.Store("fail")
|
||||
disks := restoreTwoPhaseFaulty(t, r, nil, nil, nil, failing)
|
||||
// The test owns the MRF queue while the commit runs: the heal routines of
|
||||
// the test servers would otherwise take the entry first.
|
||||
queue := make(chan PartialOperation, mrfOpsQueueSize)
|
||||
previous := globalMRFState.opCh
|
||||
globalMRFState.opCh = queue
|
||||
err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
|
||||
globalMRFState.opCh = previous
|
||||
if err != nil {
|
||||
t.Fatalf("restore with one drive failing its commit: %v", err)
|
||||
}
|
||||
var queued []PartialOperation
|
||||
for len(queue) > 0 {
|
||||
if op := <-queue; op.Bucket == r.bucket && op.Object == r.object {
|
||||
queued = append(queued, op)
|
||||
}
|
||||
}
|
||||
if len(queued) != 1 || queued[0].VersionID != nullVersionID {
|
||||
t.Fatalf("queued heals %+v, want one of version %q", queued, nullVersionID)
|
||||
}
|
||||
if restoreTwoPhaseRestored(restoreTwoPhaseEntries(t, r, disks, oi.VersionID)[3]) {
|
||||
t.Fatal("the drive that failed the commit holds the restored copy")
|
||||
}
|
||||
|
||||
failing.rename.Store("")
|
||||
if _, err := r.z.HealObject(t.Context(), r.bucket, r.object, queued[0].VersionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan}); err != nil {
|
||||
t.Fatalf("heal of the queued version: %v", err)
|
||||
}
|
||||
for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
|
||||
if !restoreTwoPhaseRestored(fi) {
|
||||
t.Errorf("drive %d: not restored after the heal: %q", i, fi.Metadata[xhttp.AmzRestore])
|
||||
}
|
||||
}
|
||||
if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("restored contents: %v", err)
|
||||
}
|
||||
if latest, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); err != nil || latest.VersionID == "" || latest.VersionID == nullVersionID {
|
||||
t.Fatalf("latest = %q, %v; want the newer version", latest.VersionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user