fix(listing): confirm current groups and bound recovery lock waits

Distinguish all-drive native absence from read uncertainty and confirm current pool targets before omitting a recovered name. Cover both existing single-pool writer lock domains, release before encoding or output, and preserve cache terminal errors without blocking the HTTP error response on remote publication.

Keep ordinary entries free of recovery I/O and locks. Bound the selective short lock-conflict wait, preserve existing retry intervals and quorum, and make retries cancelable with late-grant cleanup coverage. Restore the original raw listing callback and unchanged scanner/healing/migration callers now that recovery errors originate in the merger.

Add deterministic deletion, pool ordering, cache, lock-routing and cancellation regressions to race CI. Focused cmd/dsync/lsync race suites and verifiers pass; complete current-main integration and bare-binary performance validation are recorded in PR #239.

Refs: https://github.com/pgsty/silo/issues/218
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/metacache-listing-cache_test.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/xml"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -324,3 +325,79 @@ func TestListingCacheFullPageDoesNotWaitForTail(t *testing.T) {
 		t.Fatalf("cached remainder lost keys: %v", keys)
 	}
 }
+
+// Keep the error update pending at the actual peer RPC boundary. Other cache
+// operations, including readers and abandoned-cache cleanup, remain live.
+func listingCacheErrorPeer(t *testing.T, fail bool) (*peerRESTClient, <-chan struct{}, func()) {
+	t.Helper()
+	tg, err := grid.SetupTestGrid(2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ready, resume := make(chan struct{}), make(chan struct{})
+	release := sync.OnceFunc(func() { close(resume) })
+	t.Cleanup(func() {
+		release()
+		tg.Cleanup()
+	})
+	var entered sync.Once
+	server := &peerRESTServer{}
+	if err := getMetacacheListingRPC.Register(tg.Managers[1], server.GetMetacacheListingHandler); err != nil {
+		t.Fatal(err)
+	}
+	if err := updateMetacacheListingRPC.Register(tg.Managers[1], func(m *metacache) (*metacache, *grid.RemoteErr) {
+		if m.error != "" && m.error != "no longer used" {
+			entered.Do(func() { close(ready) })
+			select {
+			case <-resume:
+			case <-t.Context().Done():
+				return nil, grid.NewRemoteErr(t.Context().Err())
+			}
+			if fail {
+				return nil, grid.NewRemoteErr(errors.New("cache error update failed"))
+			}
+		}
+		return server.UpdateMetacacheListingHandler(m)
+	}); err != nil {
+		t.Fatal(err)
+	}
+	return &peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}, ready, release
+}
+
+func TestListingCacheRetainsErrorAfterPeerFailure(t *testing.T) {
+	client, ready, release := listingCacheErrorPeer(t, true)
+	o := listPathOptions{ID: mustGetUUID(), Bucket: "bucket"}
+	m := o.newMetacache()
+	mc := metaCacheRPC{meta: &m, rpc: client, o: o}
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		mc.setErr("terminal listing failure")
+	}()
+	t.Cleanup(func() {
+		release()
+		select {
+		case <-done:
+		case <-time.After(5 * time.Second):
+			t.Error("setErr worker did not finish before grid cleanup")
+		}
+	})
+	select {
+	case <-ready:
+	case <-time.After(5 * time.Second):
+		t.Fatal("error update did not reach peer")
+	}
+	release()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("failed peer update did not return")
+	}
+	if m.id != o.ID || m.bucket != o.Bucket || m.status != scanStateError || m.error != "terminal listing failure" || mc.getErr() == nil {
+		t.Fatalf("peer failure erased local failure: %+v", m)
+	}
+	mc.setErr("later failure")
+	if m.error != "terminal listing failure" {
+		t.Fatalf("later failure replaced the terminal error: %+v", m)
+	}
+}