// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
package cmd
import (
"context"
"fmt"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/minio/minio/internal/dsync"
)
func listingLockerTestWrite(t *testing.T, l *localLocker, args dsync.LockArgs) {
t.Helper()
if ok, err := l.Lock(t.Context(), args); !ok || err != nil {
t.Fatalf("Lock = %v, %v", ok, err)
}
}
func listingLockerTestUnlock(t *testing.T, l *localLocker, args dsync.LockArgs) {
t.Helper()
if ok, err := l.Unlock(t.Context(), args); !ok || err != nil {
t.Fatalf("Unlock = %v, %v", ok, err)
}
}
func TestLocalLockerListingReverseWaitRechecks(t *testing.T) {
for _, tc := range []struct {
name string
change func(*testing.T, *localLocker, dsync.LockArgs)
want bool
}{
{"writer-released", func(t *testing.T, l *localLocker, w dsync.LockArgs) { listingLockerTestUnlock(t, l, w) }, true},
{"writer-still-held", func(*testing.T, *localLocker, dsync.LockArgs) {}, false},
{"replacement-writer", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
listingLockerTestUnlock(t, l, w)
listingLockerTestWrite(t, l, listingLockerTestArgs("new-writer", "key", "writer"))
}, false},
{"ordinary-reader-arrives", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
listingLockerTestUnlock(t, l, w)
listingLockerTestRead(t, l, listingLockerTestArgs("ordinary-reader", "key", "ordinary-reader"))
}, true},
{"listing-reader-arrives", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
listingLockerTestUnlock(t, l, w)
listingLockerTestRead(t, l, listingLockerTestArgs("other-listing", "key", listingLockerTestSource))
}, true},
{"force-unlock", func(t *testing.T, l *localLocker, w dsync.LockArgs) {
if ok, err := l.ForceUnlock(t.Context(), w); !ok || err != nil {
t.Fatalf("ForceUnlock = %v, %v", ok, err)
}
}, true},
{"expiry", func(_ *testing.T, l *localLocker, _ dsync.LockArgs) { l.expireOldLocks(-time.Minute) }, true},
} {
t.Run(tc.name, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
l := newLocker()
writer := listingLockerTestArgs("writer", "key", "writer")
reader := listingLockerTestArgs("listing", "key", listingLockerTestSource)
listingLockerTestWrite(t, l, writer)
var got bool
var err error
start := time.Now()
go func() { got, err = l.RLock(t.Context(), reader) }()
synctest.Wait()
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
t.Fatalf("waiting = %d, want 1", state.waiting)
}
listingLockerTestWrite(t, l, listingLockerTestArgs("unrelated", "other-key", "writer"))
if elapsed := time.Since(start); elapsed != 0 {
t.Fatalf("unrelated key waited %v", elapsed)
}
time.Sleep(time.Millisecond)
tc.change(t, l, writer)
time.Sleep(time.Millisecond)
synctest.Wait()
if state := listingLockerTestSnapshot(l); got != tc.want || err != nil || state.waiting != 0 {
t.Fatalf("RLock = %v, %v, waiting = %d; want %v, nil, 0", got, err, state.waiting, tc.want)
}
if elapsed := time.Since(start); elapsed != 2*time.Millisecond {
t.Fatalf("one attempt elapsed %v, want 2ms", elapsed)
}
found := false
for _, holder := range l.DupLockMap()["key"] {
if holder.UID == reader.UID {
found = true
if holder.Writer || holder.Timestamp != UTCNow().UnixNano() || holder.TimeLastRefresh != holder.Timestamp {
t.Fatalf("invalid granted reader: %+v", holder)
}
}
}
if found != got {
t.Fatalf("reader present = %v, granted = %v", found, got)
}
})
})
}
}
func TestLocalLockerListingReverseWaitCancellation(t *testing.T) {
for _, when := range []string{"before-call", "during-wait", "deadline"} {
t.Run(when, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
l := newLocker()
writer := listingLockerTestArgs("writer", "key", "writer")
listingLockerTestWrite(t, l, writer)
ctx, cancel := context.WithCancel(t.Context())
if when == "deadline" {
cancel()
ctx, cancel = context.WithTimeout(t.Context(), time.Millisecond)
}
defer cancel()
if when == "before-call" {
cancel()
}
var got bool
var err error
start := time.Now()
go func() { got, err = l.RLock(ctx, listingLockerTestArgs("listing", "key", listingLockerTestSource)) }()
synctest.Wait()
if when != "before-call" {
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
t.Fatalf("waiting = %d, want 1", state.waiting)
}
time.Sleep(time.Millisecond)
// Removal must not turn the canceled request into a grant.
listingLockerTestUnlock(t, l, writer)
cancel()
} else {
listingLockerTestUnlock(t, l, writer)
}
synctest.Wait()
state := listingLockerTestSnapshot(l)
if got || err == nil || err != ctx.Err() || state != (listingLockerTestState{}) {
t.Fatalf("canceled RLock = %v, %v; context %v; state %+v", got, err, ctx.Err(), state)
}
if elapsed := time.Since(start); elapsed >= 2*time.Millisecond {
t.Fatalf("canceled request waited for full timer: %v", elapsed)
}
})
})
}
}
func TestLocalLockerListingReverseWaitExcluded(t *testing.T) {
for _, source := range []string{"ordinary-reader", "", "[:0:()]", "listingRLock", "[metacache-server-pool.go:0:(*erasureServerPools).listingRLock()]"} {
t.Run(source, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
l := newLocker()
listingLockerTestWrite(t, l, listingLockerTestArgs("writer", "key", "writer"))
start := time.Now()
if ok, err := l.RLock(t.Context(), listingLockerTestArgs("reader", "key", source)); ok || err != nil {
t.Fatalf("excluded RLock = %v, %v", ok, err)
}
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != 0 || state.uids != 1 {
t.Fatalf("excluded RLock changed timing or state: %+v", state)
}
})
})
}
}
func TestLocalLockerListingReverseWaitFastPaths(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
l := newLocker()
start := time.Now()
for i, source := range []string{listingLockerTestSource, "ordinary-reader", ""} {
listingLockerTestRead(t, l, listingLockerTestArgs(fmt.Sprintf("shared-%d", i), "shared", source))
listingLockerTestRead(t, l, listingLockerTestArgs(fmt.Sprintf("free-%d", i), fmt.Sprintf("free-%d", i), source))
}
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != 0 || state.uids != 6 {
t.Fatalf("uncontended readers changed timing or state: %+v", state)
}
})
}
func TestLocalLockerListingReverseWaitOverloaded(t *testing.T) {
for _, when := range []string{"before-call", "during-wait"} {
t.Run(when, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
l := newLocker()
writer := listingLockerTestArgs("writer", "key", "writer")
listingLockerTestWrite(t, l, writer)
defer l.waitMutex.Store(0)
if when == "before-call" {
l.waitMutex.Store(lockMutexWaitLimit + 1)
}
var got bool
var err error
go func() {
got, err = l.RLock(t.Context(), listingLockerTestArgs("listing", "key", listingLockerTestSource))
}()
synctest.Wait()
if when == "during-wait" {
if state := listingLockerTestSnapshot(l); state.waiting != 1 {
t.Fatalf("waiting = %d, want 1", state.waiting)
}
listingLockerTestUnlock(t, l, writer)
l.waitMutex.Store(lockMutexWaitLimit + 1)
time.Sleep(2 * time.Millisecond)
synctest.Wait()
} else {
listingLockerTestUnlock(t, l, writer)
}
if state := listingLockerTestSnapshot(l); got || err != nil || state != (listingLockerTestState{}) || l.locksOverloaded.Load() != 1 {
t.Fatalf("overloaded RLock = %v, %v; state %+v; overloads %d", got, err, state, l.locksOverloaded.Load())
}
})
})
}
}
func TestLocalLockerListingWaitSharedAdmission(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
l := newLocker()
listingLockerTestRead(t, l, listingLockerTestArgs("reader", "read-key", listingLockerTestSource))
listingLockerTestWrite(t, l, listingLockerTestArgs("writer", "write-key", "writer"))
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
results := make(chan error, listingLockWaitLimit)
for i := range listingLockWaitLimit {
go func() {
var ok bool
var err error
if i%2 == 0 {
ok, err = l.Lock(ctx, listingLockerTestArgs(fmt.Sprintf("writer-%d", i), "read-key", "writer"))
} else {
ok, err = l.RLock(ctx, listingLockerTestArgs(fmt.Sprintf("listing-%d", i), "write-key", listingLockerTestSource))
}
if ok {
err = fmt.Errorf("granted over a conflicting holder")
}
results <- err
}()
}
synctest.Wait()
if state := listingLockerTestSnapshot(l); state.waiting != listingLockWaitLimit {
t.Fatalf("combined waiting = %d, want %d", state.waiting, listingLockWaitLimit)
}
start := time.Now()
if ok, err := l.Lock(ctx, listingLockerTestArgs("excess-writer", "read-key", "writer")); ok || err != nil {
t.Fatalf("excess Lock = %v, %v", ok, err)
}
if ok, err := l.RLock(ctx, listingLockerTestArgs("excess-listing", "write-key", listingLockerTestSource)); ok || err != nil {
t.Fatalf("excess RLock = %v, %v", ok, err)
}
listingLockerTestWrite(t, l, listingLockerTestArgs("unrelated-writer", "unrelated-write", "writer"))
listingLockerTestRead(t, l, listingLockerTestArgs("unrelated-listing", "unrelated-read", listingLockerTestSource))
if state := listingLockerTestSnapshot(l); time.Since(start) != 0 || state.waiting != listingLockWaitLimit || state.uids != 4 {
t.Fatalf("combined cap changed immediate paths: %+v", state)
}
cancel()
synctest.Wait()
for range listingLockWaitLimit {
if err := <-results; err != context.Canceled {
t.Fatalf("canceled waiter: %v", err)
}
}
if state := listingLockerTestSnapshot(l); state.waiting != 0 || state.uids != 4 {
t.Fatalf("combined wait leaked: %+v", state)
}
})
}
type listingReverseCountLocker struct {
*localLocker
calls atomic.Int32
grants atomic.Int32
reject bool
}
func (l *listingReverseCountLocker) RLock(ctx context.Context, args dsync.LockArgs) (bool, error) {
l.calls.Add(1)
if l.reject {
return false, nil
}
ok, err := l.localLocker.RLock(ctx, args)
if ok {
l.grants.Add(1)
}
return ok, err
}
func TestDistributedListingReverseWait(t *testing.T) {
for _, tc := range []struct {
name string
free, held int
immediateRejects int
cancel bool
want bool
}{
{name: "all-writers-release", want: true},
{name: "two-writers-remain", held: 2, want: true},
{name: "three-writers-remain", held: 3},
{name: "partial-grant-then-quorum", free: 1, want: true},
{name: "partial-grant-without-quorum", free: 1, held: 3},
{name: "late-grant-after-rejection", immediateRejects: 3},
{name: "partial-and-late-grants-after-cancel", free: 1, cancel: true},
} {
t.Run(tc.name, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
clients := make([]dsync.NetLocker, 4)
writer := listingLockerTestArgs("writer", "key", "writer")
for i := range clients {
l := &listingReverseCountLocker{localLocker: newLocker(), reject: i < tc.immediateRejects}
clients[i] = l
if i >= tc.free {
listingLockerTestWrite(t, l.localLocker, writer)
}
}
ds := &dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, writer.Owner }, Timeouts: dsync.DefaultTimeouts}
dm := dsync.NewDRWMutex(ds, "key")
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
result := make(chan bool, 1)
go func() {
result <- dm.GetRLock(ctx, nil, "listing", listingLockerTestSource, dsync.Options{Timeout: time.Second, RetryInterval: -1})
}()
synctest.Wait()
for i, c := range clients {
l := c.(*listingReverseCountLocker)
if i < tc.free && l.grants.Load() != 1 {
t.Fatal("uncontended partial grant did not arrive immediately")
}
if i >= tc.free && !l.reject && listingLockerTestSnapshot(l.localLocker).waiting != 1 {
t.Fatal("conflicting reader did not wait in its first RPC")
}
}
earlyResult := tc.immediateRejects == 3 || tc.cancel
if tc.cancel {
cancel()
synctest.Wait()
}
if earlyResult {
select {
case got := <-result:
if got {
t.Fatal("attempt succeeded before delayed replies")
}
default:
t.Fatal("attempt did not return before delayed replies")
}
for i := range tc.free {
if state := listingLockerTestSnapshot(clients[i].(*listingReverseCountLocker).localLocker); state.uids != 0 {
t.Fatal("early partial grant was not released")
}
}
}
time.Sleep(time.Millisecond)
for i, c := range clients {
if i >= tc.free+tc.held {
listingLockerTestUnlock(t, c.(*listingReverseCountLocker).localLocker, writer)
}
}
time.Sleep(time.Millisecond)
synctest.Wait()
if !earlyResult {
if got := <-result; got != tc.want {
t.Fatalf("GetRLock = %v, want %v", got, tc.want)
}
}
for i, c := range clients {
l := c.(*listingReverseCountLocker)
if l.calls.Load() != 1 || listingLockerTestSnapshot(l.localLocker).waiting != 0 {
t.Fatalf("peer %d calls = %d or leaked admission", i, l.calls.Load())
}
wantGrant := !l.reject && (i < tc.free || i >= tc.free+tc.held)
if (l.grants.Load() == 1) != wantGrant {
t.Fatalf("peer %d grants = %d, want granted = %v", i, l.grants.Load(), wantGrant)
}
if !tc.want {
for _, holder := range l.DupLockMap()["key"] {
if holder.UID == "listing" {
t.Fatalf("peer %d leaked partial or late read grant", i)
}
}
}
}
if tc.want {
dm.RUnlock(t.Context())
synctest.Wait()
}
for i, c := range clients {
l := c.(*listingReverseCountLocker)
if i >= tc.free && i < tc.free+tc.held {
listingLockerTestUnlock(t, l.localLocker, writer)
}
if state := listingLockerTestSnapshot(l.localLocker); state != (listingLockerTestState{}) {
t.Fatalf("peer %d final state is not empty: %+v", i, state)
}
}
})
})
}
}