diff --git a/cmd/erasure-server-pool-recovery_test.go b/cmd/erasure-server-pool-recovery_test.go new file mode 100644 index 000000000..d1f1fc6ca --- /dev/null +++ b/cmd/erasure-server-pool-recovery_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Feng Ruohang +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// An acknowledged recovery overwrite must resolve the unreadable pool, even +// when another pool already has a readable copy of the same object. +// Real inline metadata is combined in the same 15-old/1-new state used by +// that regression. Capacity adapters only make destination choice deterministic. +func TestPoolsMinorityOverwriteRecovery(t *testing.T) { + for _, mode := range []string{"single-pool", "dual-pool-only-inconsistent-copy", "dual-pool-readable-other-copy"} { + t.Run(mode, func(t *testing.T) { + var z *erasureServerPools + if mode == "single-pool" { + z, _ = mutationPools(t) + } else { + z, _ = consistencyPools(t) + } + const object = "review-retry" + for _, pool := range z.serverPools { + restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} }) + defer restore() + } + bucket, router := conditionalPutBucket(t, z, "unversioned") + oldTime := time.Now().UTC().Add(-time.Hour) + oldBody := bytes.Repeat([]byte{'a'}, 8192) + pendingBody := bytes.Repeat([]byte{'b'}, 8192) + recoveryBody := bytes.Repeat([]byte{'c'}, 8192) + seed := func(pool int, body []byte, when time.Time) { + t.Helper() + putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{MTime: when}) + } + seed(0, oldBody, oldTime) + disks := z.serverPools[0].getHashedSet(object).getDisks() + oldMeta := make([][]byte, len(disks)) + for i, disk := range disks { + oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object) + } + seed(0, pendingBody, oldTime.Add(time.Minute)) + for i := 0; i < len(disks)-1; i++ { + if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil { + t.Fatal(err) + } + } + if mode == "dual-pool-readable-other-copy" { + seed(1, oldBody, oldTime.Add(-time.Minute)) + // Ordinary allocation can select this healthy other pool. + // Force that permitted choice without altering metadata reads. + restore := conditionalPutPool(t, z, object, 1) + defer restore() + } + before := consistencyRequest(t, router, http.MethodGet, bucket, object, "") + if before.Code != http.StatusServiceUnavailable { + t.Fatalf("fixture must refuse ambiguous read; got %d", before.Code) + } + req, err := newTestSignedRequestV4(http.MethodPut, getGetObjectURL("", bucket, object), int64(len(recoveryBody)), bytes.NewReader(recoveryBody), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + result := httptest.NewRecorder() + router.ServeHTTP(result, req.WithContext(t.Context())) + t.Logf("recovery PUT HTTP=%d response=%s", result.Code, result.Body.String()) + if result.Code != http.StatusOK { + t.Fatalf("unconditional recovery PUT: %d %s", result.Code, result.Body.String()) + } + after := consistencyRequest(t, router, http.MethodGet, bucket, object, "") + t.Logf("after acknowledged recovery PUT: GET HTTP=%d matches_new=%t", after.Code, bytes.Equal(after.Body.Bytes(), recoveryBody)) + for i, pool := range z.serverPools { + oi, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}) + t.Logf("pool=%d mtime=%s err=%v", i, oi.ModTime, err) + } + if after.Code != http.StatusOK || !bytes.Equal(after.Body.Bytes(), recoveryBody) { + t.Fatalf("acknowledged recovery overwrite did not restore reads: HTTP=%d body=%s", after.Code, after.Body.String()) + } + }) + } +}