astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-11-10 17:45:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

test(restore): give native-restore liveness waits one minute

Browse Source 
The native-restore tests waited 10 seconds for steps that do real drive
work: metadata writes with synchronous persistence and renames on every
drive. A full suite run on a machine whose SSD was busy with another
decommission run failed TestRestoreUploadStagingFailsClosed/staged once
with "background restore did not finish"; about 430 reruns, under CPU and
fsync stress as well, passed.

The test uses local locks, and after its release the completion takes no
new namespace lock, so the bound is a liveness bound, not a latency
contract. All eight such waits in the native-restore tests now use one
restoreTestWait of one minute, and the background-restore helper prints
the goroutine stacks when it times out. Assertions that something must not
happen keep their own bounds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-05 11:34:34 +08:00
parent 7fac646923
commit 67a6532dec
2 changed files with 17 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/restore-native-generation_test.go
+5 -5
View File
@@ -369,7 +369,7 @@ func TestRestoreUploadUnreadableFailsClosed(t *testing.T) {
if mode == "stale" {
select {
case <-gate.entered:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("restore never reached completion")
}
} else {
@@ -701,7 +701,7 @@ func testRestoreUploadStaging(t *testing.T, mode string) {
}
select {
case <-entered:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("completion never staged its commit")
}
stored := restoreIsolationUploads(t, r, 0)
@@ -784,7 +784,7 @@ func TestRestoreSplitKeepsRunningUpload(t *testing.T) {
}
select {
case <-gate.entered:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("restore never reached completion")
}
stored := restoreIsolationUploads(t, r, 0)
@@ -1277,7 +1277,7 @@ func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) {
readOnce.Do(func() { close(readRelease) })
select {
case <-cleanupDone:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("stale cleanup never returned")
}
stageOnce.Do(func() { close(stageRelease) })
@@ -1403,7 +1403,7 @@ func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) {
paused.Do(func() { close(pause.release) })
select {
case <-stageEntered:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("completion never staged after the rename ended")
}
stageOnce.Do(func() { close(stageRelease) })
cmd/restore-native-isolation_test.go
+12 -4
View File
@@ -28,6 +28,7 @@ import (
"net/http/httptest"
"net/url"
"reflect"
"runtime"
"strings"
"sync"
"sync/atomic"
@@ -82,14 +83,21 @@ func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, b
return err
}
// restoreTestWait bounds how long a restore test waits for a step that does
// real drive work: metadata writes with synchronous persistence and renames
// on every drive. It is a liveness bound, not a latency contract; a loaded
// machine has stalled such a step past ten seconds.
const restoreTestWait = time.Minute
// wait returns the result of the next background restore.
func (l *restoreIsolationLayer) wait(t *testing.T) error {
t.Helper()
select {
case err := <-l.done:
return err
case <-time.After(10 * time.Second):
t.Fatal("background restore did not finish")
case <-time.After(restoreTestWait):
buf := make([]byte, 1<<20)
t.Fatalf("background restore did not finish in %v; goroutines:\n%s", restoreTestWait, buf[:runtime.Stack(buf, true)])
return nil
}
}
@@ -255,7 +263,7 @@ func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) {
t.Helper()
select {
case <-tier.entered:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("restore never reached its download gate")
}
}
@@ -1062,7 +1070,7 @@ func TestRestoreUploadOutlivesStaleAge(t *testing.T) {
if gated {
select {
case <-gate.entered:
case <-time.After(10 * time.Second):
case <-time.After(restoreTestWait):
t.Fatal("restore never reached completion")
}
} else {