feat: add the native silo healthcheck subcommand

Add 'silo healthcheck [live|ready|cluster|cluster-read]', a thin anonymous HTTP client for the server's own /minio/health/* endpoints, so containers without a shell, curl, or mc can still run health checks. Design: silo.pgsty.com/compatibility/feature/healthcheck/ The check vocabulary maps 1:1 onto the health API paths; the probe target is derived from the server's own --address/MINIO_ADDRESS contract with HTTPS auto-detected from the certs directory, and can be overridden with --url. Exit codes are 0/1 only (Docker reserves 2); diagnostics (x-minio-server-status, quorum headers) go into a single output line for docker inspect. The request is strictly anonymous (a credentialed request would be rejected by the reserved-path guard), the transport bypasses HTTP_PROXY, and certificate verification is skipped to match kubelet HTTPS probe behavior. Cluster checks default to a 15s deadline so the server's 10s cluster_deadline can elapse. Compatibility notes: the preserved /minio/health/* path literals and the MINIO_ADDRESS env var are upstream wire/config surface, reused on purpose; the rebrand-guard baseline is regenerated for the new route literals (tests included) with zero new exported symbols. The docker entrypoint argv translation learns the new command name. Verified: unit tests, entrypoint tests, go vet, plus an end-to-end run against a live server covering all four checks, --maintenance (412), --json, usage errors, unreachable and timeout paths. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-09 15:53:28 +03:00 · 2026-08-06 16:32:16 +08:00
parent 219670d317
commit 2ff594f4bb
7 changed files with 536 additions and 1 deletions
@@ -0,0 +1,301 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/minio/cli"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// Default probe deadlines. Cluster checks are evaluated server-side under the
+// (default 10s) cluster_deadline, so their client deadline must be longer or
+// an unhealthy cluster answer would never be received.
+const (
+	healthcheckLocalTimeout   = 5 * time.Second
+	healthcheckClusterTimeout = 15 * time.Second
+)
+
+// healthcheckChecks maps the CLI check vocabulary 1:1 onto the server's
+// /minio/health/<path> endpoints. The path literals are shared with the
+// health router; the semantics live server-side only.
+var healthcheckChecks = map[string]string{
+	"live":         healthCheckLivenessPath,
+	"ready":        healthCheckReadinessPath,
+	"cluster":      healthCheckClusterPath,
+	"cluster-read": healthCheckClusterReadPath,
+}
+
+var healthcheckFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:   "address",
+		Value:  ":" + GlobalMinioDefaultPort,
+		Usage:  "probe the server bound to a specific ADDRESS:PORT, an empty ADDRESS is probed as 127.0.0.1",
+		EnvVar: "MINIO_ADDRESS",
+	},
+	cli.StringFlag{
+		Name:  "url",
+		Usage: "probe this base URL (http[s]://HOST:PORT) instead of deriving one from --address and the certs directory",
+	},
+	cli.BoolFlag{
+		Name:  "maintenance",
+		Usage: "with the cluster check only: ask whether taking this node down would lose quorum (HTTP 412 means it would)",
+	},
+	cli.DurationFlag{
+		Name:  "timeout",
+		Usage: "overall probe deadline (default: 5s for live/ready, 15s for cluster checks)",
+	},
+}
+
+var healthcheckCmd = cli.Command{
+	Name:   "healthcheck",
+	Usage:  "Probe the health of a Silo server and report it as the exit code",
+	Flags:  append(healthcheckFlags, GlobalFlags...),
+	Action: healthcheckMain,
+	CustomHelpTemplate: `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}[CHECK]
+
+CHECK:
+  live          the process is serving requests (default); touches no external system
+  ready         live, plus KMS and etcd reachability when they are configured
+  cluster       cluster-wide write quorum across every erasure set
+  cluster-read  cluster-wide read quorum across every erasure set
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+EXIT CODE:
+  0 - healthy (with --maintenance: safe to take the node down)
+  1 - anything else
+
+EXAMPLES:
+  1. Probe local liveness, e.g. as a container HEALTHCHECK:
+     {{.Prompt}} {{.HelpName}}
+  2. Probe readiness of a server on a non-default port:
+     {{.Prompt}} {{.HelpName}} --address :9010 ready
+  3. Ask whether this node can be taken down without losing HA:
+     {{.Prompt}} {{.HelpName}} --maintenance cluster
+`,
+}
+
+// healthcheckResult is the outcome of a single probe. It doubles as the
+// --json output schema, so field changes are compatibility-relevant.
+type healthcheckResult struct {
+	Check         string `json:"check"`
+	Healthy       bool   `json:"healthy"`
+	StatusCode    int    `json:"status,omitempty"`
+	DurationMS    int64  `json:"durationMs,omitempty"`
+	ServerStatus  string `json:"serverStatus,omitempty"`
+	WriteQuorum   string `json:"writeQuorum,omitempty"`
+	ReadQuorum    string `json:"readQuorum,omitempty"`
+	HealingDrives string `json:"healingDrives,omitempty"`
+	Err           string `json:"error,omitempty"`
+}
+
+// line renders the single human-readable output line. Container runtimes
+// store only the first 4096 bytes of probe output, so it stays short.
+func (r healthcheckResult) line() string {
+	if r.Err != "" {
+		return fmt.Sprintf("%s: unreachable (%s)", r.Check, r.Err)
+	}
+	if r.Healthy {
+		return fmt.Sprintf("%s: ok (%d, %dms)", r.Check, r.StatusCode, r.DurationMS)
+	}
+	label := "unhealthy"
+	if r.StatusCode == http.StatusPreconditionFailed {
+		label = "not safe for maintenance"
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, "%s: %s (%d)", r.Check, label, r.StatusCode)
+	for _, kv := range []struct{ k, v string }{
+		{"server-status", r.ServerStatus},
+		{"write-quorum", r.WriteQuorum},
+		{"read-quorum", r.ReadQuorum},
+		{"healing-drives", r.HealingDrives},
+	} {
+		if kv.v != "" {
+			fmt.Fprintf(&b, " %s=%s", kv.k, kv.v)
+		}
+	}
+	return b.String()
+}
+
+// healthcheckTarget derives the base URL to probe. An explicit rawURL wins;
+// otherwise the address' host:port is used, with the scheme decided by the
+// same certificate presence check the server performs at startup.
+func healthcheckTarget(rawURL, address, certsDir string) (string, error) {
+	if rawURL != "" {
+		u, err := url.Parse(rawURL)
+		if err != nil {
+			return "", fmt.Errorf("invalid --url %q: %w", rawURL, err)
+		}
+		if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+			return "", fmt.Errorf("invalid --url %q: expected http[s]://HOST:PORT", rawURL)
+		}
+		return u.Scheme + "://" + u.Host, nil
+	}
+
+	host, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return "", fmt.Errorf("invalid --address %q: %w", address, err)
+	}
+	if host == "" {
+		host = "127.0.0.1"
+	}
+	scheme := "http"
+	if isFile(filepath.Join(certsDir, publicCertFile)) && isFile(filepath.Join(certsDir, privateKeyFile)) {
+		scheme = "https"
+	}
+	return scheme + "://" + net.JoinHostPort(host, port), nil
+}
+
+// probeHealth performs one bounded, strictly anonymous GET against the
+// health endpoint for check. Anonymity is load-bearing: a credentialed
+// request is rejected by the reserved-path guard instead of answered.
+func probeHealth(baseURL, check string, maintenance bool, timeout time.Duration) healthcheckResult {
+	res := healthcheckResult{Check: check}
+
+	probeURL := baseURL + healthCheckPathPrefix + healthcheckChecks[check]
+	if maintenance {
+		probeURL += "?maintenance=true"
+	}
+
+	// Proxy is nil on purpose: a loopback probe must never be routed through
+	// an HTTP_PROXY inherited from the container environment. Certificate
+	// verification is skipped to match the kubelet's HTTPS probe behavior.
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy:             nil,
+			TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
+			DisableKeepAlives: true,
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil)
+	if err != nil {
+		res.Err = err.Error()
+		return res
+	}
+	req.Header.Set("User-Agent", "silo-healthcheck/"+ReleaseTag)
+
+	started := time.Now()
+	resp, err := client.Do(req)
+	res.DurationMS = time.Since(started).Milliseconds()
+	if err != nil {
+		res.Err = err.Error()
+		return res
+	}
+	defer resp.Body.Close()
+	io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
+
+	res.StatusCode = resp.StatusCode
+	res.Healthy = resp.StatusCode == http.StatusOK
+	res.ServerStatus = resp.Header.Get(xhttp.MinIOServerStatus)
+	res.WriteQuorum = resp.Header.Get(xhttp.MinIOWriteQuorum)
+	res.ReadQuorum = resp.Header.Get(xhttp.MinIOReadQuorum)
+	res.HealingDrives = resp.Header.Get(xhttp.MinIOHealingDrives)
+	return res
+}
+
+// healthcheckCertsDir mirrors the server's certs-dir resolution without its
+// side effects: an explicit --certs-dir wins, an explicit --config-dir
+// implies <config-dir>/certs, and the shared default applies otherwise.
+func healthcheckCertsDir(ctx *cli.Context) string {
+	switch {
+	case ctx.IsSet("certs-dir"):
+		return ctx.String("certs-dir")
+	case ctx.GlobalIsSet("certs-dir"):
+		return ctx.GlobalString("certs-dir")
+	case ctx.IsSet("config-dir"):
+		return filepath.Join(ctx.String("config-dir"), certsDir)
+	case ctx.GlobalIsSet("config-dir"):
+		return filepath.Join(ctx.GlobalString("config-dir"), certsDir)
+	}
+	return defaultCertsDir.Get()
+}
+
+func healthcheckMain(ctx *cli.Context) {
+	fail := func(format string, args ...any) {
+		fmt.Fprintf(os.Stderr, "healthcheck: "+format+"\n", args...)
+		os.Exit(1)
+	}
+
+	if len(ctx.Args()) > 1 {
+		fail("too many arguments, expected at most one CHECK")
+	}
+	check := "live"
+	if arg := ctx.Args().First(); arg != "" {
+		check = arg
+	}
+	if _, ok := healthcheckChecks[check]; !ok {
+		fail("unknown check %q, expected one of: live, ready, cluster, cluster-read", check)
+	}
+	if ctx.Bool("maintenance") && check != "cluster" {
+		fail("--maintenance applies to the cluster check only")
+	}
+
+	timeout := ctx.Duration("timeout")
+	if !ctx.IsSet("timeout") {
+		timeout = healthcheckLocalTimeout
+		if strings.HasPrefix(check, "cluster") {
+			timeout = healthcheckClusterTimeout
+		}
+	}
+
+	baseURL, err := healthcheckTarget(ctx.String("url"), ctx.String("address"), healthcheckCertsDir(ctx))
+	if err != nil {
+		fail("%v", err)
+	}
+
+	res := probeHealth(baseURL, check, ctx.Bool("maintenance"), timeout)
+
+	quiet := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
+	if ctx.IsSet("json") || ctx.GlobalIsSet("json") {
+		buf, jerr := json.Marshal(res)
+		if jerr != nil {
+			fail("%v", jerr)
+		}
+		fmt.Println(string(buf))
+	} else if !res.Healthy {
+		fmt.Fprintln(os.Stderr, res.line())
+	} else if !quiet {
+		fmt.Println(res.line())
+	}
+
+	if !res.Healthy {
+		os.Exit(1)
+	}
+}
@@ -0,0 +1,199 @@
+// Copyright (c) 2015-2026 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func TestHealthcheckTarget(t *testing.T) {
+	plainDir := t.TempDir()
+
+	tlsDir := t.TempDir()
+	for _, name := range []string{publicCertFile, privateKeyFile} {
+		if err := os.WriteFile(filepath.Join(tlsDir, name), []byte("test"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// A lone public.crt without its key must not flip the scheme.
+	halfDir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(halfDir, publicCertFile), []byte("test"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name     string
+		rawURL   string
+		address  string
+		certsDir string
+		want     string
+		wantErr  bool
+	}{
+		{name: "default address", address: ":9000", certsDir: plainDir, want: "http://127.0.0.1:9000"},
+		{name: "explicit host", address: "10.0.0.7:9010", certsDir: plainDir, want: "http://10.0.0.7:9010"},
+		{name: "tls certs present", address: ":9000", certsDir: tlsDir, want: "https://127.0.0.1:9000"},
+		{name: "cert without key stays http", address: ":9000", certsDir: halfDir, want: "http://127.0.0.1:9000"},
+		{name: "url override wins", rawURL: "https://silo.internal:9000", address: ":9000", certsDir: plainDir, want: "https://silo.internal:9000"},
+		{name: "url path is dropped", rawURL: "http://silo.internal:9000/minio/health/live", address: ":9000", certsDir: plainDir, want: "http://silo.internal:9000"},
+		{name: "address without port", address: "localhost", certsDir: plainDir, wantErr: true},
+		{name: "url without scheme", rawURL: "silo.internal:9000", certsDir: plainDir, wantErr: true},
+		{name: "url with bad scheme", rawURL: "ftp://silo.internal:9000", certsDir: plainDir, wantErr: true},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := healthcheckTarget(test.rawURL, test.address, test.certsDir)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("healthcheckTarget() error = %v, wantErr = %v", err, test.wantErr)
+			}
+			if err == nil && got != test.want {
+				t.Fatalf("healthcheckTarget() = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestProbeHealthChecksAndVerdicts(t *testing.T) {
+	var gotPath, gotQuery, gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		gotQuery = r.URL.RawQuery
+		gotAuth = r.Header.Get("Authorization")
+		switch r.URL.Path {
+		case "/minio/health/live", "/minio/health/ready":
+			w.WriteHeader(http.StatusOK)
+		case "/minio/health/cluster":
+			if r.URL.Query().Get("maintenance") == "true" {
+				w.Header().Set(xhttp.MinIOWriteQuorum, "3")
+				w.Header().Set(xhttp.MinIOHealingDrives, "2")
+				w.WriteHeader(http.StatusPreconditionFailed)
+				return
+			}
+			w.Header().Set(xhttp.MinIOServerStatus, "iam-offline")
+			w.Header().Set(xhttp.MinIOWriteQuorum, "3")
+			w.WriteHeader(http.StatusServiceUnavailable)
+		case "/minio/health/cluster/read":
+			w.Header().Set(xhttp.MinIOReadQuorum, "2")
+			w.WriteHeader(http.StatusOK)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer srv.Close()
+
+	res := probeHealth(srv.URL, "live", false, time.Second)
+	if !res.Healthy || res.StatusCode != http.StatusOK {
+		t.Fatalf("live: expected healthy 200, got %+v", res)
+	}
+	if gotPath != "/minio/health/live" {
+		t.Fatalf("live: probed %q", gotPath)
+	}
+	if gotAuth != "" {
+		t.Fatalf("probe must be anonymous, sent Authorization %q", gotAuth)
+	}
+
+	res = probeHealth(srv.URL, "cluster", false, time.Second)
+	if res.Healthy || res.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("cluster: expected unhealthy 503, got %+v", res)
+	}
+	if res.ServerStatus != "iam-offline" || res.WriteQuorum != "3" {
+		t.Fatalf("cluster: headers not decoded, got %+v", res)
+	}
+	if gotQuery != "" {
+		t.Fatalf("cluster without --maintenance sent query %q", gotQuery)
+	}
+
+	res = probeHealth(srv.URL, "cluster", true, time.Second)
+	if res.Healthy || res.StatusCode != http.StatusPreconditionFailed {
+		t.Fatalf("cluster maintenance: expected 412, got %+v", res)
+	}
+	if res.HealingDrives != "2" {
+		t.Fatalf("cluster maintenance: headers not decoded, got %+v", res)
+	}
+	if gotQuery != "maintenance=true" {
+		t.Fatalf("cluster --maintenance sent query %q", gotQuery)
+	}
+
+	res = probeHealth(srv.URL, "cluster-read", false, time.Second)
+	if !res.Healthy || res.ReadQuorum != "2" {
+		t.Fatalf("cluster-read: expected healthy with read quorum, got %+v", res)
+	}
+	if gotPath != "/minio/health/cluster/read" {
+		t.Fatalf("cluster-read: probed %q", gotPath)
+	}
+}
+
+func TestProbeHealthTLSSkipsVerification(t *testing.T) {
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	res := probeHealth(srv.URL, "live", false, time.Second)
+	if !res.Healthy {
+		t.Fatalf("self-signed TLS probe must succeed, got %+v", res)
+	}
+}
+
+func TestProbeHealthUnreachableAndTimeout(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	deadURL := srv.URL
+	srv.Close()
+
+	res := probeHealth(deadURL, "live", false, time.Second)
+	if res.Healthy || res.Err == "" {
+		t.Fatalf("probe of a closed server must report unreachable, got %+v", res)
+	}
+
+	slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(500 * time.Millisecond)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer slow.Close()
+
+	res = probeHealth(slow.URL, "live", false, 50*time.Millisecond)
+	if res.Healthy || res.Err == "" {
+		t.Fatalf("probe past its deadline must fail, got %+v", res)
+	}
+}
+
+func TestHealthcheckResultLine(t *testing.T) {
+	tests := []struct {
+		res  healthcheckResult
+		want string
+	}{
+		{healthcheckResult{Check: "live", Healthy: true, StatusCode: 200, DurationMS: 2}, "live: ok (200, 2ms)"},
+		{healthcheckResult{Check: "cluster", StatusCode: 503, ServerStatus: "iam-offline", WriteQuorum: "3", HealingDrives: "2"}, "cluster: unhealthy (503) server-status=iam-offline write-quorum=3 healing-drives=2"},
+		{healthcheckResult{Check: "cluster", StatusCode: 412, WriteQuorum: "3"}, "cluster: not safe for maintenance (412) write-quorum=3"},
+		{healthcheckResult{Check: "ready", Err: "connection refused"}, "ready: unreachable (connection refused)"},
+	}
+	for _, test := range tests {
+		if got := test.res.line(); got != test.want {
+			t.Fatalf("line() = %q, want %q", got, test.want)
+		}
+	}
+}
@@ -138,6 +138,7 @@ func newApp(name string) *cli.App {
 	// Register all commands.
 	registerCommand(serverCmd)
 	registerCommand(fmtGenCmd)
+	registerCommand(healthcheckCmd)

 	// Set up app.
 	cli.HelpFlag = cli.BoolFlag{