mirror of
https://github.com/pgsty/minio.git
synced 2026-11-10 17:45:32 +03:00
test(pools): make the copy and bulk lock tests race-free and stricter
The implementation review of the lock fixes found test flaws, which are corrected here; the production behavior is unchanged. - The alias and merge tests replaced globalOperationTimeout while background healing reads it, a data race under -race. They now bound each request with its own context instead. - The alias test now holds the cleaned name's write lock while the copy runs, so it proves the names are checked before anything is locked; it checks both objects before any later write, and covers a/./x as a destination. - The merge test picks aliases that hash to the clean name's set and, where sets have their own lockers, ones that do not, and requires the alias's own table to hold a read lock: readers admitted, writers excluded. - Release polling creates a fresh lock instance for every probe. - The getNSLocks event hook and the bulk delete group limit are atomic, so tests can set them while other goroutines read them. Without the name check and the resource merge, both tests now fail in every lock layout. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
@@ -44,38 +44,42 @@ func ldCopyAPI(t *testing.T, mode string) (*erasureServerPools, string, http.Han
|
||||
// A lock is taken on the cleaned name, so an invalid name that aliases
|
||||
// another one locks the same resource. A copy whose source or destination is
|
||||
// such an alias is refused before it locks anything, as the release refused
|
||||
// it when it read the source.
|
||||
// it when it read the source: the refusal comes at once while another writer
|
||||
// holds the cleaned name.
|
||||
func TestLockDomainCopyRejectsAliasNames(t *testing.T) {
|
||||
for _, mode := range append(slices.Clone(ldModes), "pools") {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
z, bucket, router := ldCopyAPI(t, mode)
|
||||
prev := globalOperationTimeout
|
||||
globalOperationTimeout = newDynamicTimeout(5*time.Second, 5*time.Second)
|
||||
t.Cleanup(func() { globalOperationTimeout = prev })
|
||||
ldMustPut(t, z, bucket, "alias/x", "victim")
|
||||
ldMustPut(t, z, bucket, "alias/src", "source")
|
||||
|
||||
for _, tc := range []struct{ src, dst string }{
|
||||
{"alias//x", "alias/x"},
|
||||
{"alias/./x", "alias/x"},
|
||||
{"alias/x", "alias//x"},
|
||||
{"alias/src", "alias//x"},
|
||||
{"alias/src", "alias/./x"},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s->%s", tc.src, tc.dst), func(t *testing.T) {
|
||||
// Another writer holds the cleaned name of the alias.
|
||||
release := ldHoldLock(t, z.NewObjectNSLock(bucket, "alias/x"))
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
start := time.Now()
|
||||
rec := ldCopyRequest(t.Context(), t, router, bucket, tc.src, tc.dst, nil)
|
||||
if took := time.Since(start); took > time.Second {
|
||||
rec := ldCopyRequest(ctx, t, router, bucket, tc.src, tc.dst, nil)
|
||||
took := time.Since(start)
|
||||
cancel()
|
||||
release()
|
||||
if took > time.Second {
|
||||
t.Errorf("refused after %v", took)
|
||||
}
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("status %d, want 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
if err := ldPut(ctx, z, bucket, "alias/x", "victim"); err != nil {
|
||||
t.Errorf("a PUT of alias/x after the copy: %v", err)
|
||||
}
|
||||
if body, _ := ldBody(t, z, bucket, "alias/x"); body != "victim" {
|
||||
t.Errorf("alias/x = %q", body)
|
||||
}
|
||||
if body, _ := ldBody(t, z, bucket, "alias/src"); body != "source" {
|
||||
t.Errorf("alias/src = %q", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -109,37 +113,59 @@ func TestLockDomainCopyErrorPrecedence(t *testing.T) {
|
||||
}
|
||||
|
||||
// lockObjects merges the requests of one resource in one lock table, also for
|
||||
// names that differ before cleaning; the same resource in two distributed
|
||||
// lock domains stays two locks.
|
||||
// names that differ before cleaning, into one lock that is a write lock when
|
||||
// either asks for one. The same resource in two distributed lock domains
|
||||
// stays two locks: a read lock in the alias's domain and a write lock in the
|
||||
// clean name's.
|
||||
func TestLockObjectsMergesByResource(t *testing.T) {
|
||||
for _, mode := range ldModes {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
z, bucket := ldPool(t, mode)
|
||||
pool := z.serverPools[0]
|
||||
prev := globalOperationTimeout
|
||||
globalOperationTimeout = newDynamicTimeout(3*time.Second, 3*time.Second)
|
||||
t.Cleanup(func() { globalOperationTimeout = prev })
|
||||
for i := range 16 {
|
||||
clean := fmt.Sprintf("merge-%d/x", i)
|
||||
alias := fmt.Sprintf("merge-%d//x", i)
|
||||
split := !pool.sharesLocalNSLock()
|
||||
// Names whose alias hashes to the clean name's set, and, where the
|
||||
// sets have their own lockers, names whose alias does not.
|
||||
pick := func(sameSet bool) (string, string) {
|
||||
for i := range 1000 {
|
||||
clean, alias := fmt.Sprintf("merge-%d/x", i), fmt.Sprintf("merge-%d//x", i)
|
||||
if (pool.getHashedSetIndex(alias) == pool.getHashedSetIndex(clean)) == sameSet {
|
||||
return clean, alias
|
||||
}
|
||||
}
|
||||
t.Fatalf("no names with sameSet=%v", sameSet)
|
||||
return "", ""
|
||||
}
|
||||
cases := []bool{true}
|
||||
if split {
|
||||
cases = append(cases, false)
|
||||
}
|
||||
for _, sameSet := range cases {
|
||||
clean, alias := pick(sameSet)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
|
||||
start := time.Now()
|
||||
_, unlock, err := z.lockObjects(t.Context(),
|
||||
_, unlock, err := z.lockObjects(ctx,
|
||||
objectLockRequest{bucket: bucket, object: alias},
|
||||
objectLockRequest{bucket: bucket, object: clean, write: true})
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatalf("%s + %s: %v after %v", alias, clean, err, time.Since(start))
|
||||
}
|
||||
if took := time.Since(start); took > time.Second {
|
||||
t.Errorf("%s + %s took %v", alias, clean, took)
|
||||
}
|
||||
sameTable := pool.sharesLocalNSLock() || pool.getHashedSetIndex(alias) == pool.getHashedSetIndex(clean)
|
||||
// The resource is write locked in the clean name's table.
|
||||
// The clean name's table holds a write lock.
|
||||
if ldTry(t, pool.getHashedSet(clean).NewNSLock(bucket, clean), false) {
|
||||
t.Errorf("%s: not write locked", clean)
|
||||
t.Errorf("%s: readers admitted in its own table", clean)
|
||||
}
|
||||
if !sameTable && !ldTry(t, pool.getHashedSet(alias).NewNSLock(bucket, clean), false) {
|
||||
// The alias holds only a read lock in its own domain.
|
||||
t.Errorf("%s: alias domain write locked", alias)
|
||||
if split && !sameSet {
|
||||
// The alias's table holds the alias's read lock.
|
||||
aliasSet := pool.getHashedSet(alias)
|
||||
if !ldTry(t, aliasSet.NewNSLock(bucket, alias), false) {
|
||||
t.Errorf("%s: readers excluded in its own table", alias)
|
||||
}
|
||||
if ldTry(t, aliasSet.NewNSLock(bucket, alias), true) {
|
||||
t.Errorf("%s: writers admitted in its own table, the alias lock is missing", alias)
|
||||
}
|
||||
}
|
||||
unlock()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user