mirror of
https://github.com/pgsty/minio.git
synced 2026-10-17 15:16:00 +03:00
87c1c6b3dc
Distinguish all-drive native absence from read uncertainty and confirm current pool targets before omitting a recovered name. Cover both existing single-pool writer lock domains, release before encoding or output, and preserve cache terminal errors without blocking the HTTP error response on remote publication. Keep ordinary entries free of recovery I/O and locks. Bound the selective short lock-conflict wait, preserve existing retry intervals and quorum, and make retries cancelable with late-grant cleanup coverage. Restore the original raw listing callback and unchanged scanner/healing/migration callers now that recovery errors originate in the merger. Add deterministic deletion, pool ordering, cache, lock-routing and cancellation regressions to race CI. Focused cmd/dsync/lsync race suites and verifiers pass; complete current-main integration and bare-binary performance validation are recorded in PR #239. Refs: https://github.com/pgsty/silo/issues/218 Signed-off-by: Feng Ruohang <rh@vonng.com>
132 lines
4.4 KiB
Go
132 lines
4.4 KiB
Go
// Copyright (c) 2026 Feng Ruohang
|
|
//
|
|
// This file is part of Silo Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/minio/internal/dsync"
|
|
)
|
|
|
|
// Use the real distributed-lock algorithm with in-process lock servers.
|
|
// This is a protocol-level check, not a network or rolling-restart test.
|
|
func TestListFallbackDistributedLockRoute(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
multiPool, sharedLockers bool
|
|
}{
|
|
{"multi-pool-shared-lockers", true, true},
|
|
{"multi-pool-disjoint-lockers", true, false},
|
|
{"single-pool-target-set-lockers", false, false},
|
|
{"single-pool-first-set-lockers", false, false},
|
|
{"single-pool-shared-ns-distinct-lockers", false, false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
a := []dsync.NetLocker{newLocker(), newLocker()}
|
|
b := []dsync.NetLocker{newLocker(), newLocker()}
|
|
if tc.sharedLockers {
|
|
b = a
|
|
}
|
|
set0 := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return a, "listing-test" }}
|
|
target := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return b, "listing-test" }}
|
|
if tc.name == "single-pool-shared-ns-distinct-lockers" {
|
|
target.nsMutex = set0.nsMutex
|
|
}
|
|
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{set0, target}}}}
|
|
if tc.multiPool {
|
|
z.serverPools = []*erasureSets{{sets: []*erasureObjects{set0}}, {sets: []*erasureObjects{target}}}
|
|
}
|
|
writer := target.NewNSLock("bucket", "object")
|
|
if tc.multiPool || tc.name == "single-pool-first-set-lockers" {
|
|
writer = z.NewNSLock("bucket", "object")
|
|
}
|
|
wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
locked := true
|
|
defer func() {
|
|
if locked {
|
|
writer.Unlock(wctx)
|
|
}
|
|
}()
|
|
ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
|
|
defer cancel()
|
|
_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
|
|
if err == nil {
|
|
unlock()
|
|
t.Fatal("fallback read lock acquired while ordinary writer lock is held")
|
|
}
|
|
writer.Unlock(wctx)
|
|
locked = false
|
|
ctx, cancel = context.WithTimeout(t.Context(), 5*time.Second)
|
|
defer cancel()
|
|
_, unlock, err = z.listingRLock(ctx, "bucket", "object", target)
|
|
if err != nil {
|
|
t.Fatalf("fallback read lock failed after writer unlock: %v", err)
|
|
}
|
|
unlock()
|
|
})
|
|
}
|
|
}
|
|
|
|
// Cancellation while waiting for the second domain must release the first.
|
|
func TestListingSecondLockCancellationReleasesFirst(t *testing.T) {
|
|
first := &erasureObjects{nsMutex: newNSLock(false)}
|
|
target := &erasureObjects{nsMutex: newNSLock(false)}
|
|
z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{first, target}}}}
|
|
writer := target.NewNSLock("bucket", "object")
|
|
wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer writer.Unlock(wctx)
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
done := make(chan struct{})
|
|
var lockErr error
|
|
go func() {
|
|
defer close(done)
|
|
_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
|
|
lockErr = err
|
|
if err == nil {
|
|
unlock()
|
|
}
|
|
}()
|
|
waitForListingReadLock(t, target.nsMutex, "bucket", "object", done)
|
|
cancel()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("second lock did not cancel")
|
|
}
|
|
if lockErr == nil {
|
|
t.Fatal("canceled second lock succeeded")
|
|
}
|
|
poolWriter := z.NewNSLock("bucket", "object")
|
|
ctx, cancel = context.WithTimeout(t.Context(), time.Second)
|
|
defer cancel()
|
|
pctx, err := poolWriter.GetLock(ctx, globalOperationTimeout)
|
|
if err != nil {
|
|
t.Fatalf("first lock leaked after second failed: %v", err)
|
|
}
|
|
poolWriter.Unlock(pctx)
|
|
}
|