diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go index d79bc1030..48b5d19ea 100644 --- a/cmd/restore-native-generation_test.go +++ b/cmd/restore-native-generation_test.go @@ -369,7 +369,7 @@ func TestRestoreUploadUnreadableFailsClosed(t *testing.T) { if mode == "stale" { select { case <-gate.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached completion") } } else { @@ -701,7 +701,7 @@ func testRestoreUploadStaging(t *testing.T, mode string) { } select { case <-entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("completion never staged its commit") } stored := restoreIsolationUploads(t, r, 0) @@ -784,7 +784,7 @@ func TestRestoreSplitKeepsRunningUpload(t *testing.T) { } select { case <-gate.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached completion") } stored := restoreIsolationUploads(t, r, 0) @@ -1277,7 +1277,7 @@ func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) { readOnce.Do(func() { close(readRelease) }) select { case <-cleanupDone: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("stale cleanup never returned") } stageOnce.Do(func() { close(stageRelease) }) @@ -1403,7 +1403,7 @@ func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) { paused.Do(func() { close(pause.release) }) select { case <-stageEntered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("completion never staged after the rename ended") } stageOnce.Do(func() { close(stageRelease) }) diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go index 232eb4323..6e5e412da 100644 --- a/cmd/restore-native-isolation_test.go +++ b/cmd/restore-native-isolation_test.go @@ -28,6 +28,7 @@ import ( "net/http/httptest" "net/url" "reflect" + "runtime" "strings" "sync" "sync/atomic" @@ -82,14 +83,21 @@ func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, b return err } +// restoreTestWait bounds how long a restore test waits for a step that does +// real drive work: metadata writes with synchronous persistence and renames +// on every drive. It is a liveness bound, not a latency contract; a loaded +// machine has stalled such a step past ten seconds. +const restoreTestWait = time.Minute + // wait returns the result of the next background restore. func (l *restoreIsolationLayer) wait(t *testing.T) error { t.Helper() select { case err := <-l.done: return err - case <-time.After(10 * time.Second): - t.Fatal("background restore did not finish") + case <-time.After(restoreTestWait): + buf := make([]byte, 1<<20) + t.Fatalf("background restore did not finish in %v; goroutines:\n%s", restoreTestWait, buf[:runtime.Stack(buf, true)]) return nil } } @@ -255,7 +263,7 @@ func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) { t.Helper() select { case <-tier.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached its download gate") } } @@ -1062,7 +1070,7 @@ func TestRestoreUploadOutlivesStaleAge(t *testing.T) { if gated { select { case <-gate.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached completion") } } else {