fix(restore): name the server that keeps native restore unavailable

The native restore gate refused a POST restore and a queued restore
with one message, "unavailable during a rolling upgrade: not every
server of the cluster supports it yet", whether a server answered
another protocol or was simply not connected. A cluster with one server
down thus reported a rolling upgrade, and the operator looked for an
older binary instead of the server.

The gate now returns an error that wraps errNativeRestoreUnavailable
and says why: errNativeRestorePeerUnreachable names the server that is
not connected ("server host:port is not reachable, so it cannot be
shown to have stopped an older restore"), errNativeRestoreUnsupported
keeps the rolling-upgrade text for a server that reports another
protocol or reconnected while it was asked, and a check that its
deadline ended carries that error. The POST handler maps the error to
503 ServiceUnavailable with the message as before, the worker still
clears its marker and stops, and errors.Is(err,
errNativeRestoreUnavailable) holds for every refusal. The gate remains
closed in every case it was closed before.

TestNativeRestoreRefusalReasons posts a restore with a fake server that
is not connected and one that answers protocol 1; before this change
the first answered 503 with the rolling-upgrade text and without "not
reachable".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/restore-capability_test.go

@@ -74,21 +74,26 @@ func TestNativeRestoreCapabilityGrid(t *testing.T) {
 	}
 }
 
-// restoreCapabilityFake is a server whose connection and answer a test sets.
+// restoreCapabilityFake is a server whose connection and answer a test sets:
+// capable answers the question, protocol, when set, with another protocol.
 type restoreCapabilityFake struct {
+	server    string
 	capable   atomic.Bool
 	connected atomic.Bool
 	conn      atomic.Int64
 	asked     atomic.Int64
+	protocol  atomic.Pointer[string]
 }
 
 func newRestoreCapabilityFake() *restoreCapabilityFake {
-	p := &restoreCapabilityFake{}
+	p := &restoreCapabilityFake{server: "server-2:9000"}
 	p.capable.Store(true)
 	p.connected.Store(true)
 	return p
 }
 
+func (p *restoreCapabilityFake) restoreServer() string { return p.server }
+
 func (p *restoreCapabilityFake) restoreConnection() (string, bool) {
 	return strings.Repeat("c", int(p.conn.Load())), p.connected.Load()
 }
@@ -98,6 +103,9 @@ func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, erro
 	if !p.capable.Load() {
 		return "", errors.New("Invalid Handler for type: RestoreCapability")
 	}
+	if protocol := p.protocol.Load(); protocol != nil {
+		return *protocol, nil
+	}
 	return nativeRestoreCapability, nil
 }
 
@@ -186,6 +194,55 @@ func TestNativeRestoreRollingUpgradeGate(t *testing.T) {
 	}
 }
 
+// A refused POST restore says why. A server that is not connected is named,
+// as not reachable: that is not a rolling upgrade, and the operator looks for
+// the server. A server that speaks another protocol is one. Neither refusal
+// writes anything.
+func TestNativeRestoreRefusalReasons(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+	p := newRestoreCapabilityFake()
+	previous := globalRestoreCapability
+	globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+	t.Cleanup(func() { globalRestoreCapability = previous })
+	before := r.fileInfo(oi.VersionID)
+	layer := newRestoreIsolationLayer(r)
+
+	p.connected.Store(false)
+	rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+	if body := rec.Body.String(); rec.Code != http.StatusServiceUnavailable || !strings.Contains(body, "not reachable") ||
+		!strings.Contains(body, p.server) || strings.Contains(body, "rolling upgrade") {
+		t.Errorf("POST with a server not connected: %d %s", rec.Code, body)
+	}
+	if err := nativeRestoreAvailable(t.Context()); !errors.Is(err, errNativeRestorePeerUnreachable) ||
+		!errors.Is(err, errNativeRestoreUnavailable) || errors.Is(err, errNativeRestoreUnsupported) {
+		t.Errorf("gate with a server not connected: %v", err)
+	}
+
+	p.connected.Store(true)
+	older := "1"
+	p.protocol.Store(&older)
+	rec = restoreIsolationPOST(t, r, layer, oi.VersionID)
+	if body := rec.Body.String(); rec.Code != http.StatusServiceUnavailable || !strings.Contains(body, "rolling upgrade") ||
+		strings.Contains(body, "not reachable") {
+		t.Errorf("POST with a server of protocol %s: %d %s", older, rec.Code, body)
+	}
+	if err := nativeRestoreAvailable(t.Context()); !errors.Is(err, errNativeRestoreUnsupported) ||
+		!errors.Is(err, errNativeRestoreUnavailable) || errors.Is(err, errNativeRestorePeerUnreachable) {
+		t.Errorf("gate with a server of protocol %s: %v", older, err)
+	}
+
+	if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) {
+		t.Error("a refused POST changed the version")
+	}
+	select {
+	case err := <-layer.done:
+		t.Fatalf("a refused POST started a restore: %v", err)
+	default:
+	}
+}
+
 // restoreCapabilityHeld is a fake server whose answer a test holds.
 type restoreCapabilityHeld struct {
 	restoreCapabilityFake