Files
minio/cmd/metacache-listing-lock-routes_test.go
T
Feng Ruohang 87c1c6b3dc fix(listing): confirm current groups and bound recovery lock waits
Distinguish all-drive native absence from read uncertainty and confirm current pool targets before omitting a recovered name. Cover both existing single-pool writer lock domains, release before encoding or output, and preserve cache terminal errors without blocking the HTTP error response on remote publication.

Keep ordinary entries free of recovery I/O and locks. Bound the selective short lock-conflict wait, preserve existing retry intervals and quorum, and make retries cancelable with late-grant cleanup coverage. Restore the original raw listing callback and unchanged scanner/healing/migration callers now that recovery errors originate in the merger.

Add deterministic deletion, pool ordering, cache, lock-routing and cancellation regressions to race CI. Focused cmd/dsync/lsync race suites and verifiers pass; complete current-main integration and bare-binary performance validation are recorded in PR #239.

Refs: https://github.com/pgsty/silo/issues/218
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-17 11:18:08 +08:00

132 lines
4.4 KiB
Go

// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"testing"
"time"
"github.com/minio/minio/internal/dsync"
)
// Use the real distributed-lock algorithm with in-process lock servers.
// This is a protocol-level check, not a network or rolling-restart test.
func TestListFallbackDistributedLockRoute(t *testing.T) {
for _, tc := range []struct {
name string
multiPool, sharedLockers bool
}{
{"multi-pool-shared-lockers", true, true},
{"multi-pool-disjoint-lockers", true, false},
{"single-pool-target-set-lockers", false, false},
{"single-pool-first-set-lockers", false, false},
{"single-pool-shared-ns-distinct-lockers", false, false},
} {
t.Run(tc.name, func(t *testing.T) {
a := []dsync.NetLocker{newLocker(), newLocker()}
b := []dsync.NetLocker{newLocker(), newLocker()}
if tc.sharedLockers {
b = a
}
set0 := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return a, "listing-test" }}
target := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return b, "listing-test" }}
if tc.name == "single-pool-shared-ns-distinct-lockers" {
target.nsMutex = set0.nsMutex
}
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{set0, target}}}}
if tc.multiPool {
z.serverPools = []*erasureSets{{sets: []*erasureObjects{set0}}, {sets: []*erasureObjects{target}}}
}
writer := target.NewNSLock("bucket", "object")
if tc.multiPool || tc.name == "single-pool-first-set-lockers" {
writer = z.NewNSLock("bucket", "object")
}
wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
if err != nil {
t.Fatal(err)
}
locked := true
defer func() {
if locked {
writer.Unlock(wctx)
}
}()
ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
defer cancel()
_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
if err == nil {
unlock()
t.Fatal("fallback read lock acquired while ordinary writer lock is held")
}
writer.Unlock(wctx)
locked = false
ctx, cancel = context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
_, unlock, err = z.listingRLock(ctx, "bucket", "object", target)
if err != nil {
t.Fatalf("fallback read lock failed after writer unlock: %v", err)
}
unlock()
})
}
}
// Cancellation while waiting for the second domain must release the first.
func TestListingSecondLockCancellationReleasesFirst(t *testing.T) {
first := &erasureObjects{nsMutex: newNSLock(false)}
target := &erasureObjects{nsMutex: newNSLock(false)}
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{first, target}}}}
writer := target.NewNSLock("bucket", "object")
wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
if err != nil {
t.Fatal(err)
}
defer writer.Unlock(wctx)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan struct{})
var lockErr error
go func() {
defer close(done)
_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
lockErr = err
if err == nil {
unlock()
}
}()
waitForListingReadLock(t, target.nsMutex, "bucket", "object", done)
cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("second lock did not cancel")
}
if lockErr == nil {
t.Fatal("canceled second lock succeeded")
}
poolWriter := z.NewNSLock("bucket", "object")
ctx, cancel = context.WithTimeout(t.Context(), time.Second)
defer cancel()
pctx, err := poolWriter.GetLock(ctx, globalOperationTimeout)
if err != nil {
t.Fatalf("first lock leaked after second failed: %v", err)
}
poolWriter.Unlock(pctx)
}