mirror of
https://github.com/pgsty/minio.git
synced 2026-10-17 15:16:00 +03:00
fix(listing): confirm current groups and bound recovery lock waits
Distinguish all-drive native absence from read uncertainty and confirm current pool targets before omitting a recovered name. Cover both existing single-pool writer lock domains, release before encoding or output, and preserve cache terminal errors without blocking the HTTP error response on remote publication. Keep ordinary entries free of recovery I/O and locks. Bound the selective short lock-conflict wait, preserve existing retry intervals and quorum, and make retries cancelable with late-grant cleanup coverage. Restore the original raw listing callback and unchanged scanner/healing/migration callers now that recovery errors originate in the merger. Add deterministic deletion, pool ordering, cache, lock-routing and cancellation regressions to race CI. Focused cmd/dsync/lsync race suites and verifiers pass; complete current-main integration and bare-binary performance validation are recorded in PR #239. Refs: https://github.com/pgsty/silo/issues/218 Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,218 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package dsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
// retryTestLocker deliberately leaves unexpected methods unimplemented.
|
||||
// These tests exercise the real DRWMutex retry and cleanup code without HTTP.
|
||||
type retryTestLocker struct {
|
||||
NetLocker
|
||||
acquire func(context.Context, LockArgs) (bool, error)
|
||||
release func(context.Context, LockArgs) (bool, error)
|
||||
}
|
||||
|
||||
func (l retryTestLocker) Lock(ctx context.Context, args LockArgs) (bool, error) {
|
||||
return l.acquire(ctx, args)
|
||||
}
|
||||
|
||||
func (l retryTestLocker) RLock(ctx context.Context, args LockArgs) (bool, error) {
|
||||
return l.acquire(ctx, args)
|
||||
}
|
||||
|
||||
func (l retryTestLocker) Unlock(ctx context.Context, args LockArgs) (bool, error) {
|
||||
return l.release(ctx, args)
|
||||
}
|
||||
|
||||
func (l retryTestLocker) RUnlock(ctx context.Context, args LockArgs) (bool, error) {
|
||||
return l.release(ctx, args)
|
||||
}
|
||||
|
||||
func (l retryTestLocker) String() string { return "retry-test-locker" }
|
||||
|
||||
func newRetryTestMutex(lockers []NetLocker) *DRWMutex {
|
||||
dm := NewDRWMutex(&Dsync{
|
||||
GetLockers: func() ([]NetLocker, string) { return lockers, "retry-test-owner" },
|
||||
Timeouts: DefaultTimeouts,
|
||||
}, "retry-test-resource")
|
||||
dm.rng = rand.New(&lockedRandSource{src: rand.NewSource(1)})
|
||||
return dm
|
||||
}
|
||||
|
||||
func retryTestAcquire(ctx context.Context, dm *DRWMutex, read bool, opts Options) bool {
|
||||
if read {
|
||||
return dm.GetRLock(ctx, nil, "retry-test-uid", "retry-test", opts)
|
||||
}
|
||||
return dm.GetLock(ctx, nil, "retry-test-uid", "retry-test", opts)
|
||||
}
|
||||
|
||||
func TestDRWMutexRetryCancellation(t *testing.T) {
|
||||
for _, read := range []bool{false, true} {
|
||||
for _, opts := range []Options{
|
||||
{Timeout: time.Hour},
|
||||
{Timeout: time.Hour, RetryInterval: time.Hour},
|
||||
} {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
lockers := make([]NetLocker, 4)
|
||||
for i := range lockers {
|
||||
lockers[i] = retryTestLocker{acquire: func(context.Context, LockArgs) (bool, error) {
|
||||
calls.Add(1)
|
||||
return false, nil
|
||||
}}
|
||||
}
|
||||
dm := newRetryTestMutex(lockers)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan bool, 1)
|
||||
go func() {
|
||||
done <- retryTestAcquire(ctx, dm, read, opts)
|
||||
}()
|
||||
synctest.Wait()
|
||||
if got := calls.Load(); got != 4 {
|
||||
t.Fatalf("first attempt made %d RPCs, want 4", got)
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case locked := <-done:
|
||||
if locked {
|
||||
t.Fatal("canceled waiter acquired lock")
|
||||
}
|
||||
default:
|
||||
t.Errorf("read=%v opts=%+v: cancellation did not interrupt retry wait", read, opts)
|
||||
<-done
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
synctest.Wait()
|
||||
if got := calls.Load(); got != 4 {
|
||||
t.Fatalf("canceled waiter issued a new RPC attempt: %d calls", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDRWMutexRetryDeadline(t *testing.T) {
|
||||
for _, interval := range []time.Duration{0, time.Hour} {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
locker := retryTestLocker{acquire: func(context.Context, LockArgs) (bool, error) {
|
||||
return false, nil
|
||||
}}
|
||||
dm := newRetryTestMutex([]NetLocker{locker})
|
||||
start := time.Now()
|
||||
if retryTestAcquire(t.Context(), dm, false, Options{Timeout: 10 * time.Millisecond, RetryInterval: interval}) {
|
||||
t.Fatal("unavailable lock was acquired")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 10*time.Millisecond {
|
||||
t.Fatalf("retry wait overran deadline: %v", elapsed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDRWMutexCanceledPartialAndLateGrants(t *testing.T) {
|
||||
for _, read := range []bool{false, true} {
|
||||
for _, slowRelease := range []bool{false, true} {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
lateGrant := make(chan struct{})
|
||||
releaseGate := make(chan struct{})
|
||||
if !slowRelease {
|
||||
close(releaseGate)
|
||||
}
|
||||
var calls, releases atomic.Int64
|
||||
var held [4]atomic.Bool
|
||||
lockers := make([]NetLocker, len(held))
|
||||
for i := range lockers {
|
||||
lockers[i] = retryTestLocker{
|
||||
acquire: func(ctx context.Context, _ LockArgs) (bool, error) {
|
||||
calls.Add(1)
|
||||
if i > 0 {
|
||||
<-lateGrant
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Error("NetLocker acquisition unexpectedly inherits caller cancellation")
|
||||
}
|
||||
held[i].Store(true)
|
||||
return true, nil
|
||||
},
|
||||
release: func(context.Context, LockArgs) (bool, error) {
|
||||
if i == 0 {
|
||||
<-releaseGate
|
||||
}
|
||||
held[i].Store(false)
|
||||
releases.Add(1)
|
||||
return true, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
dm := newRetryTestMutex(lockers)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan bool, 1)
|
||||
go func() { done <- retryTestAcquire(ctx, dm, read, Options{Timeout: time.Hour}) }()
|
||||
synctest.Wait()
|
||||
if !held[0].Load() || calls.Load() != 4 {
|
||||
t.Fatal("partial acquisition was not established")
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
if slowRelease {
|
||||
// Cancellation of retry sleep is not a whole-call bound:
|
||||
// synchronous cleanup of a partial grant can still block.
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("acquisition returned before partial-grant cleanup completed")
|
||||
default:
|
||||
}
|
||||
close(releaseGate)
|
||||
synctest.Wait()
|
||||
}
|
||||
select {
|
||||
case locked := <-done:
|
||||
if locked {
|
||||
t.Fatal("canceled partial acquisition returned success")
|
||||
}
|
||||
default:
|
||||
t.Error("acquisition did not return after partial-grant cleanup")
|
||||
<-done
|
||||
}
|
||||
if held[0].Load() || releases.Load() != 1 {
|
||||
t.Fatal("partial grant was not released")
|
||||
}
|
||||
close(lateGrant)
|
||||
synctest.Wait()
|
||||
for i := range held {
|
||||
if held[i].Load() {
|
||||
t.Errorf("late grant on node %d was not released", i)
|
||||
}
|
||||
}
|
||||
if calls.Load() != 4 || releases.Load() != 4 {
|
||||
t.Fatalf("calls=%d releases=%d, want 4/4", calls.Load(), releases.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user