fix(listing): confirm current groups and bound recovery lock waits

Distinguish all-drive native absence from read uncertainty and confirm current pool targets before omitting a recovered name. Cover both existing single-pool writer lock domains, release before encoding or output, and preserve cache terminal errors without blocking the HTTP error response on remote publication.

Keep ordinary entries free of recovery I/O and locks. Bound the selective short lock-conflict wait, preserve existing retry intervals and quorum, and make retries cancelable with late-grant cleanup coverage. Restore the original raw listing callback and unchanged scanner/healing/migration callers now that recovery errors originate in the merger.

Add deterministic deletion, pool ordering, cache, lock-routing and cancellation regressions to race CI. Focused cmd/dsync/lsync race suites and verifiers pass; complete current-main integration and bare-binary performance validation are recorded in PR #239.

Refs: https://github.com/pgsty/silo/issues/218
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/metacache-listing-lock-routes_test.go

@@ -35,6 +35,8 @@ func TestListFallbackDistributedLockRoute(t *testing.T) {
 		{"multi-pool-shared-lockers", true, true},
 		{"multi-pool-disjoint-lockers", true, false},
 		{"single-pool-target-set-lockers", false, false},
+		{"single-pool-first-set-lockers", false, false},
+		{"single-pool-shared-ns-distinct-lockers", false, false},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			a := []dsync.NetLocker{newLocker(), newLocker()}
@@ -44,12 +46,15 @@ func TestListFallbackDistributedLockRoute(t *testing.T) {
 			}
 			set0 := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return a, "listing-test" }}
 			target := &erasureObjects{nsMutex: newNSLock(true), getLockers: func() ([]dsync.NetLocker, string) { return b, "listing-test" }}
+			if tc.name == "single-pool-shared-ns-distinct-lockers" {
+				target.nsMutex = set0.nsMutex
+			}
 			z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{set0, target}}}}
 			if tc.multiPool {
 				z.serverPools = []*erasureSets{{sets: []*erasureObjects{set0}}, {sets: []*erasureObjects{target}}}
 			}
 			writer := target.NewNSLock("bucket", "object")
-			if tc.multiPool {
+			if tc.multiPool || tc.name == "single-pool-first-set-lockers" {
 				writer = z.NewNSLock("bucket", "object")
 			}
 			wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
@@ -62,23 +67,65 @@ func TestListFallbackDistributedLockRoute(t *testing.T) {
 					writer.Unlock(wctx)
 				}
 			}()
-			reader := z.listingNSLock(target)("bucket", "object")
 			ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
 			defer cancel()
-			rctx, err := reader.GetRLock(ctx, globalOperationTimeout)
+			_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
 			if err == nil {
-				reader.RUnlock(rctx)
+				unlock()
 				t.Fatal("fallback read lock acquired while ordinary writer lock is held")
 			}
 			writer.Unlock(wctx)
 			locked = false
 			ctx, cancel = context.WithTimeout(t.Context(), 5*time.Second)
 			defer cancel()
-			rctx, err = reader.GetRLock(ctx, globalOperationTimeout)
+			_, unlock, err = z.listingRLock(ctx, "bucket", "object", target)
 			if err != nil {
 				t.Fatalf("fallback read lock failed after writer unlock: %v", err)
 			}
-			reader.RUnlock(rctx)
+			unlock()
 		})
 	}
 }
+
+// Cancellation while waiting for the second domain must release the first.
+func TestListingSecondLockCancellationReleasesFirst(t *testing.T) {
+	first := &erasureObjects{nsMutex: newNSLock(false)}
+	target := &erasureObjects{nsMutex: newNSLock(false)}
+	z := &erasureServerPools{serverPools: []*erasureSets{{sets: []*erasureObjects{first, target}}}}
+	writer := target.NewNSLock("bucket", "object")
+	wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer writer.Unlock(wctx)
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	done := make(chan struct{})
+	var lockErr error
+	go func() {
+		defer close(done)
+		_, unlock, err := z.listingRLock(ctx, "bucket", "object", target)
+		lockErr = err
+		if err == nil {
+			unlock()
+		}
+	}()
+	waitForListingReadLock(t, target.nsMutex, "bucket", "object", done)
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("second lock did not cancel")
+	}
+	if lockErr == nil {
+		t.Fatal("canceled second lock succeeded")
+	}
+	poolWriter := z.NewNSLock("bucket", "object")
+	ctx, cancel = context.WithTimeout(t.Context(), time.Second)
+	defer cancel()
+	pctx, err := poolWriter.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		t.Fatalf("first lock leaked after second failed: %v", err)
+	}
+	poolWriter.Unlock(pctx)
+}