mirror of
https://github.com/pgsty/minio.git
synced 2026-10-31 21:02:06 +03:00
test: reject recovery ACK when another pool still wins reads
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -117,7 +118,7 @@ func (d recoveryReadFaultDisk) RenameData(ctx context.Context, sv, sp string, fi
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
||||||
for _, mode := range []string{"two-inconsistent", "suspended", "rebalancing", "offline", "read-fails-after-commit"} {
|
for _, mode := range []string{"two-inconsistent", "suspended", "rebalancing", "offline", "read-fails-after-commit", "newer-readable-other-copy"} {
|
||||||
t.Run(mode, func(t *testing.T) {
|
t.Run(mode, func(t *testing.T) {
|
||||||
z, _ := consistencyPools(t)
|
z, _ := consistencyPools(t)
|
||||||
const object = "refused-recovery"
|
const object = "refused-recovery"
|
||||||
@@ -146,12 +147,17 @@ func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
inconsistent(0)
|
inconsistent(0)
|
||||||
putConsistencyObject(t, z, bucket, object, 1, string(old), ObjectOptions{MTime: base.Add(-time.Minute)})
|
otherTime := base.Add(-time.Minute)
|
||||||
|
if mode == "newer-readable-other-copy" {
|
||||||
|
otherTime = base.Add(2 * time.Hour)
|
||||||
|
}
|
||||||
|
putConsistencyObject(t, z, bucket, object, 1, string(old), ObjectOptions{MTime: otherTime})
|
||||||
if mode == "two-inconsistent" {
|
if mode == "two-inconsistent" {
|
||||||
inconsistent(1)
|
inconsistent(1)
|
||||||
}
|
}
|
||||||
// Select a healthy destination if the resolver incorrectly ignores pool0.
|
// Select a healthy destination if the resolver incorrectly ignores pool0.
|
||||||
if mode != "read-fails-after-commit" {
|
postCommitFailure := mode == "read-fails-after-commit" || mode == "newer-readable-other-copy"
|
||||||
|
if !postCommitFailure {
|
||||||
restore := conditionalPutPool(t, z, object, 1)
|
restore := conditionalPutPool(t, z, object, 1)
|
||||||
defer restore()
|
defer restore()
|
||||||
}
|
}
|
||||||
@@ -196,19 +202,31 @@ func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
|||||||
if result.Code != http.StatusServiceUnavailable {
|
if result.Code != http.StatusServiceUnavailable {
|
||||||
t.Errorf("unresolved recovery returned HTTP=%d: %s", result.Code, result.Body.String())
|
t.Errorf("unresolved recovery returned HTTP=%d: %s", result.Code, result.Body.String())
|
||||||
}
|
}
|
||||||
if mode != "read-fails-after-commit" {
|
if !postCommitFailure {
|
||||||
after := snapshot()
|
after := snapshot()
|
||||||
for i := range before {
|
for i := range before {
|
||||||
if before[i] != after[i] {
|
if before[i] != after[i] {
|
||||||
t.Errorf("refused preflight modified shard %d", i)
|
t.Errorf("refused preflight modified shard %d", i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else if mode == "read-fails-after-commit" {
|
||||||
fail.Store(false)
|
fail.Store(false)
|
||||||
got := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
got := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||||
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), replacement) {
|
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), replacement) {
|
||||||
t.Errorf("read validation failure must not undo committed recovery: HTTP=%d", got.Code)
|
t.Errorf("read validation failure must not undo committed recovery: HTTP=%d", got.Code)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// A readable but different latest copy cannot confirm recovery.
|
||||||
|
// The committed replacement must still remain in its target pool.
|
||||||
|
r, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(r)
|
||||||
|
r.Close()
|
||||||
|
if err != nil || !bytes.Equal(body, replacement) {
|
||||||
|
t.Fatalf("failed recovery validation must preserve committed data: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user