fix(restore): name the server that keeps native restore unavailable

The native restore gate refused a POST restore and a queued restore
with one message, "unavailable during a rolling upgrade: not every
server of the cluster supports it yet", whether a server answered
another protocol or was simply not connected. A cluster with one server
down thus reported a rolling upgrade, and the operator looked for an
older binary instead of the server.

The gate now returns an error that wraps errNativeRestoreUnavailable
and says why: errNativeRestorePeerUnreachable names the server that is
not connected ("server host:port is not reachable, so it cannot be
shown to have stopped an older restore"), errNativeRestoreUnsupported
keeps the rolling-upgrade text for a server that reports another
protocol or reconnected while it was asked, and a check that its
deadline ended carries that error. The POST handler maps the error to
503 ServiceUnavailable with the message as before, the worker still
clears its marker and stops, and errors.Is(err,
errNativeRestoreUnavailable) holds for every refusal. The gate remains
closed in every case it was closed before.

TestNativeRestoreRefusalReasons posts a restore with a fake server that
is not connected and one that answers protocol 1; before this change
the first answered 503 with the rolling-upgrade text and without "not
reachable".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/restore-capability.go

@@ -55,7 +55,34 @@ const (
 	restoreCapabilityTimeout = 5 * time.Second
 )
 
-var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet")
+// errNativeRestoreUnavailable is the refusal of a native restore. The gate
+// returns an error that wraps it and names the reason, so that an operator
+// looks for a server that is not connected, not for an older binary.
+var errNativeRestoreUnavailable = errors.New("native restore is unavailable")
+
+var (
+	// errNativeRestoreUnsupported: a server of the cluster does not report
+	// this protocol, as an older server does not, or reconnected while it
+	// was asked, so its answer does not count.
+	errNativeRestoreUnsupported = fmt.Errorf("%w during a rolling upgrade: not every server of the cluster supports it yet", errNativeRestoreUnavailable)
+	// errNativeRestorePeerUnreachable: a server of the cluster is not
+	// connected. nativeRestoreUnreachableError names it.
+	errNativeRestorePeerUnreachable = fmt.Errorf("%w: a server of the cluster is not reachable", errNativeRestoreUnavailable)
+)
+
+// nativeRestoreUnreachableError is errNativeRestorePeerUnreachable naming
+// the server.
+type nativeRestoreUnreachableError struct{ server string }
+
+func (e nativeRestoreUnreachableError) Error() string {
+	server := "a server of the cluster"
+	if e.server != "" {
+		server = "server " + e.server
+	}
+	return fmt.Sprintf("native restore is unavailable: %s is not reachable, so it cannot be shown to have stopped an older restore", server)
+}
+
+func (nativeRestoreUnreachableError) Unwrap() error { return errNativeRestorePeerUnreachable }
 
 // restoreCapabilityHandler answers the question on this server.
 func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
@@ -64,6 +91,8 @@ func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
 
 // restoreCapabilityPeer is another server of the cluster.
 type restoreCapabilityPeer interface {
+	// restoreServer names the server, for messages; empty when unknown.
+	restoreServer() string
 	// restoreConnection identifies the current connection to the server; it
 	// changes when the server reconnects, as a restarted one does. ok is
 	// false when the server is not connected.
@@ -73,6 +102,13 @@ type restoreCapabilityPeer interface {
 
 type restorePeerClient struct{ *peerRESTClient }
 
+func (c restorePeerClient) restoreServer() string {
+	if c.peerRESTClient == nil || c.host == nil {
+		return ""
+	}
+	return c.host.String()
+}
+
 func (c restorePeerClient) restoreConnection() (string, bool) {
 	if c.peerRESTClient == nil || c.gridConn == nil {
 		return "", false
@@ -126,8 +162,9 @@ func notificationRestorePeers() []restoreCapabilityPeer {
 	return peers
 }
 
-// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every
-// other server of the cluster supports native restore.
+// nativeRestoreAvailable returns an error that wraps
+// errNativeRestoreUnavailable, and says why, unless every other server of
+// the cluster supports native restore.
 func nativeRestoreAvailable(ctx context.Context) error {
 	return globalRestoreCapability.check(ctx)
 }
@@ -137,34 +174,35 @@ func (g *restoreCapabilityGate) check(ctx context.Context) error {
 	// One deadline covers the wait for another check and the questions.
 	ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout)
 	defer cancel()
+	ended := func() error { return fmt.Errorf("%w: %w", errNativeRestoreUnavailable, ctx.Err()) }
 	select {
 	case g.turn <- struct{}{}:
 	case <-ctx.Done():
-		return errNativeRestoreUnavailable
+		return ended()
 	}
 	defer func() { <-g.turn }()
 
 	// Read the connections only now: one may have changed while this check
-	// waited for another.
+	// waited for another. A server that is not connected is named.
 	peers := g.peers()
-	connections := func() ([]string, bool) {
+	connections := func() ([]string, error) {
 		keys := make([]string, len(peers))
 		for i, p := range peers {
 			key, ok := p.restoreConnection()
 			if !ok {
-				return nil, false
+				return nil, nativeRestoreUnreachableError{server: p.restoreServer()}
 			}
 			keys[i] = key
 		}
-		return keys, true
+		return keys, nil
 	}
-	keys, ok := connections()
-	if !ok {
+	keys, err := connections()
+	if err != nil {
 		g.checked, g.keys = time.Time{}, nil
-		return errNativeRestoreUnavailable
+		return err
 	}
 	if ctx.Err() != nil {
-		return errNativeRestoreUnavailable
+		return ended()
 	}
 	if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) {
 		return nil
@@ -183,9 +221,14 @@ func (g *restoreCapabilityGate) check(ctx context.Context) error {
 	wg.Wait()
 	// An answer counts only for the connection it was asked on: a server
 	// that reconnected meanwhile may be another binary.
-	after, ok := connections()
-	if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil {
-		return errNativeRestoreUnavailable
+	after, err := connections()
+	switch {
+	case err != nil:
+		return err
+	case ctx.Err() != nil:
+		return ended()
+	case !slices.Equal(keys, after), slices.Contains(answers, false):
+		return errNativeRestoreUnsupported
 	}
 	g.checked, g.keys = time.Now(), keys
 	return nil