diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index 5657b75d8..70037e633 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -852,6 +852,10 @@ "/metrics/v3", "/minio/grid/", "/minio/grid/lock/", + "/minio/health/cluster", + "/minio/health/cluster/read", + "/minio/health/live", + "/minio/health/ready", "/myobject*", "/netperf", "/newfolder", diff --git a/cmd/healthcheck-main.go b/cmd/healthcheck-main.go new file mode 100644 index 000000000..0137e819a --- /dev/null +++ b/cmd/healthcheck-main.go @@ -0,0 +1,301 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/cli" + xhttp "github.com/minio/minio/internal/http" +) + +// Default probe deadlines. Cluster checks are evaluated server-side under the +// (default 10s) cluster_deadline, so their client deadline must be longer or +// an unhealthy cluster answer would never be received. +const ( + healthcheckLocalTimeout = 5 * time.Second + healthcheckClusterTimeout = 15 * time.Second +) + +// healthcheckChecks maps the CLI check vocabulary 1:1 onto the server's +// /minio/health/ endpoints. The path literals are shared with the +// health router; the semantics live server-side only. +var healthcheckChecks = map[string]string{ + "live": healthCheckLivenessPath, + "ready": healthCheckReadinessPath, + "cluster": healthCheckClusterPath, + "cluster-read": healthCheckClusterReadPath, +} + +var healthcheckFlags = []cli.Flag{ + cli.StringFlag{ + Name: "address", + Value: ":" + GlobalMinioDefaultPort, + Usage: "probe the server bound to a specific ADDRESS:PORT, an empty ADDRESS is probed as 127.0.0.1", + EnvVar: "MINIO_ADDRESS", + }, + cli.StringFlag{ + Name: "url", + Usage: "probe this base URL (http[s]://HOST:PORT) instead of deriving one from --address and the certs directory", + }, + cli.BoolFlag{ + Name: "maintenance", + Usage: "with the cluster check only: ask whether taking this node down would lose quorum (HTTP 412 means it would)", + }, + cli.DurationFlag{ + Name: "timeout", + Usage: "overall probe deadline (default: 5s for live/ready, 15s for cluster checks)", + }, +} + +var healthcheckCmd = cli.Command{ + Name: "healthcheck", + Usage: "Probe the health of a Silo server and report it as the exit code", + Flags: append(healthcheckFlags, GlobalFlags...), + Action: healthcheckMain, + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}[CHECK] + +CHECK: + live the process is serving requests (default); touches no external system + ready live, plus KMS and etcd reachability when they are configured + cluster cluster-wide write quorum across every erasure set + cluster-read cluster-wide read quorum across every erasure set +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +EXIT CODE: + 0 - healthy (with --maintenance: safe to take the node down) + 1 - anything else + +EXAMPLES: + 1. Probe local liveness, e.g. as a container HEALTHCHECK: + {{.Prompt}} {{.HelpName}} + 2. Probe readiness of a server on a non-default port: + {{.Prompt}} {{.HelpName}} --address :9010 ready + 3. Ask whether this node can be taken down without losing HA: + {{.Prompt}} {{.HelpName}} --maintenance cluster +`, +} + +// healthcheckResult is the outcome of a single probe. It doubles as the +// --json output schema, so field changes are compatibility-relevant. +type healthcheckResult struct { + Check string `json:"check"` + Healthy bool `json:"healthy"` + StatusCode int `json:"status,omitempty"` + DurationMS int64 `json:"durationMs,omitempty"` + ServerStatus string `json:"serverStatus,omitempty"` + WriteQuorum string `json:"writeQuorum,omitempty"` + ReadQuorum string `json:"readQuorum,omitempty"` + HealingDrives string `json:"healingDrives,omitempty"` + Err string `json:"error,omitempty"` +} + +// line renders the single human-readable output line. Container runtimes +// store only the first 4096 bytes of probe output, so it stays short. +func (r healthcheckResult) line() string { + if r.Err != "" { + return fmt.Sprintf("%s: unreachable (%s)", r.Check, r.Err) + } + if r.Healthy { + return fmt.Sprintf("%s: ok (%d, %dms)", r.Check, r.StatusCode, r.DurationMS) + } + label := "unhealthy" + if r.StatusCode == http.StatusPreconditionFailed { + label = "not safe for maintenance" + } + var b strings.Builder + fmt.Fprintf(&b, "%s: %s (%d)", r.Check, label, r.StatusCode) + for _, kv := range []struct{ k, v string }{ + {"server-status", r.ServerStatus}, + {"write-quorum", r.WriteQuorum}, + {"read-quorum", r.ReadQuorum}, + {"healing-drives", r.HealingDrives}, + } { + if kv.v != "" { + fmt.Fprintf(&b, " %s=%s", kv.k, kv.v) + } + } + return b.String() +} + +// healthcheckTarget derives the base URL to probe. An explicit rawURL wins; +// otherwise the address' host:port is used, with the scheme decided by the +// same certificate presence check the server performs at startup. +func healthcheckTarget(rawURL, address, certsDir string) (string, error) { + if rawURL != "" { + u, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("invalid --url %q: %w", rawURL, err) + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", fmt.Errorf("invalid --url %q: expected http[s]://HOST:PORT", rawURL) + } + return u.Scheme + "://" + u.Host, nil + } + + host, port, err := net.SplitHostPort(address) + if err != nil { + return "", fmt.Errorf("invalid --address %q: %w", address, err) + } + if host == "" { + host = "127.0.0.1" + } + scheme := "http" + if isFile(filepath.Join(certsDir, publicCertFile)) && isFile(filepath.Join(certsDir, privateKeyFile)) { + scheme = "https" + } + return scheme + "://" + net.JoinHostPort(host, port), nil +} + +// probeHealth performs one bounded, strictly anonymous GET against the +// health endpoint for check. Anonymity is load-bearing: a credentialed +// request is rejected by the reserved-path guard instead of answered. +func probeHealth(baseURL, check string, maintenance bool, timeout time.Duration) healthcheckResult { + res := healthcheckResult{Check: check} + + probeURL := baseURL + healthCheckPathPrefix + healthcheckChecks[check] + if maintenance { + probeURL += "?maintenance=true" + } + + // Proxy is nil on purpose: a loopback probe must never be routed through + // an HTTP_PROXY inherited from the container environment. Certificate + // verification is skipped to match the kubelet's HTTPS probe behavior. + client := &http.Client{ + Transport: &http.Transport{ + Proxy: nil, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil) + if err != nil { + res.Err = err.Error() + return res + } + req.Header.Set("User-Agent", "silo-healthcheck/"+ReleaseTag) + + started := time.Now() + resp, err := client.Do(req) + res.DurationMS = time.Since(started).Milliseconds() + if err != nil { + res.Err = err.Error() + return res + } + defer resp.Body.Close() + io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + + res.StatusCode = resp.StatusCode + res.Healthy = resp.StatusCode == http.StatusOK + res.ServerStatus = resp.Header.Get(xhttp.MinIOServerStatus) + res.WriteQuorum = resp.Header.Get(xhttp.MinIOWriteQuorum) + res.ReadQuorum = resp.Header.Get(xhttp.MinIOReadQuorum) + res.HealingDrives = resp.Header.Get(xhttp.MinIOHealingDrives) + return res +} + +// healthcheckCertsDir mirrors the server's certs-dir resolution without its +// side effects: an explicit --certs-dir wins, an explicit --config-dir +// implies /certs, and the shared default applies otherwise. +func healthcheckCertsDir(ctx *cli.Context) string { + switch { + case ctx.IsSet("certs-dir"): + return ctx.String("certs-dir") + case ctx.GlobalIsSet("certs-dir"): + return ctx.GlobalString("certs-dir") + case ctx.IsSet("config-dir"): + return filepath.Join(ctx.String("config-dir"), certsDir) + case ctx.GlobalIsSet("config-dir"): + return filepath.Join(ctx.GlobalString("config-dir"), certsDir) + } + return defaultCertsDir.Get() +} + +func healthcheckMain(ctx *cli.Context) { + fail := func(format string, args ...any) { + fmt.Fprintf(os.Stderr, "healthcheck: "+format+"\n", args...) + os.Exit(1) + } + + if len(ctx.Args()) > 1 { + fail("too many arguments, expected at most one CHECK") + } + check := "live" + if arg := ctx.Args().First(); arg != "" { + check = arg + } + if _, ok := healthcheckChecks[check]; !ok { + fail("unknown check %q, expected one of: live, ready, cluster, cluster-read", check) + } + if ctx.Bool("maintenance") && check != "cluster" { + fail("--maintenance applies to the cluster check only") + } + + timeout := ctx.Duration("timeout") + if !ctx.IsSet("timeout") { + timeout = healthcheckLocalTimeout + if strings.HasPrefix(check, "cluster") { + timeout = healthcheckClusterTimeout + } + } + + baseURL, err := healthcheckTarget(ctx.String("url"), ctx.String("address"), healthcheckCertsDir(ctx)) + if err != nil { + fail("%v", err) + } + + res := probeHealth(baseURL, check, ctx.Bool("maintenance"), timeout) + + quiet := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet") + if ctx.IsSet("json") || ctx.GlobalIsSet("json") { + buf, jerr := json.Marshal(res) + if jerr != nil { + fail("%v", jerr) + } + fmt.Println(string(buf)) + } else if !res.Healthy { + fmt.Fprintln(os.Stderr, res.line()) + } else if !quiet { + fmt.Println(res.line()) + } + + if !res.Healthy { + os.Exit(1) + } +} diff --git a/cmd/healthcheck-main_test.go b/cmd/healthcheck-main_test.go new file mode 100644 index 000000000..50f61acfc --- /dev/null +++ b/cmd/healthcheck-main_test.go @@ -0,0 +1,199 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + xhttp "github.com/minio/minio/internal/http" +) + +func TestHealthcheckTarget(t *testing.T) { + plainDir := t.TempDir() + + tlsDir := t.TempDir() + for _, name := range []string{publicCertFile, privateKeyFile} { + if err := os.WriteFile(filepath.Join(tlsDir, name), []byte("test"), 0o600); err != nil { + t.Fatal(err) + } + } + + // A lone public.crt without its key must not flip the scheme. + halfDir := t.TempDir() + if err := os.WriteFile(filepath.Join(halfDir, publicCertFile), []byte("test"), 0o600); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + rawURL string + address string + certsDir string + want string + wantErr bool + }{ + {name: "default address", address: ":9000", certsDir: plainDir, want: "http://127.0.0.1:9000"}, + {name: "explicit host", address: "10.0.0.7:9010", certsDir: plainDir, want: "http://10.0.0.7:9010"}, + {name: "tls certs present", address: ":9000", certsDir: tlsDir, want: "https://127.0.0.1:9000"}, + {name: "cert without key stays http", address: ":9000", certsDir: halfDir, want: "http://127.0.0.1:9000"}, + {name: "url override wins", rawURL: "https://silo.internal:9000", address: ":9000", certsDir: plainDir, want: "https://silo.internal:9000"}, + {name: "url path is dropped", rawURL: "http://silo.internal:9000/minio/health/live", address: ":9000", certsDir: plainDir, want: "http://silo.internal:9000"}, + {name: "address without port", address: "localhost", certsDir: plainDir, wantErr: true}, + {name: "url without scheme", rawURL: "silo.internal:9000", certsDir: plainDir, wantErr: true}, + {name: "url with bad scheme", rawURL: "ftp://silo.internal:9000", certsDir: plainDir, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := healthcheckTarget(test.rawURL, test.address, test.certsDir) + if (err != nil) != test.wantErr { + t.Fatalf("healthcheckTarget() error = %v, wantErr = %v", err, test.wantErr) + } + if err == nil && got != test.want { + t.Fatalf("healthcheckTarget() = %q, want %q", got, test.want) + } + }) + } +} + +func TestProbeHealthChecksAndVerdicts(t *testing.T) { + var gotPath, gotQuery, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotQuery = r.URL.RawQuery + gotAuth = r.Header.Get("Authorization") + switch r.URL.Path { + case "/minio/health/live", "/minio/health/ready": + w.WriteHeader(http.StatusOK) + case "/minio/health/cluster": + if r.URL.Query().Get("maintenance") == "true" { + w.Header().Set(xhttp.MinIOWriteQuorum, "3") + w.Header().Set(xhttp.MinIOHealingDrives, "2") + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set(xhttp.MinIOServerStatus, "iam-offline") + w.Header().Set(xhttp.MinIOWriteQuorum, "3") + w.WriteHeader(http.StatusServiceUnavailable) + case "/minio/health/cluster/read": + w.Header().Set(xhttp.MinIOReadQuorum, "2") + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + res := probeHealth(srv.URL, "live", false, time.Second) + if !res.Healthy || res.StatusCode != http.StatusOK { + t.Fatalf("live: expected healthy 200, got %+v", res) + } + if gotPath != "/minio/health/live" { + t.Fatalf("live: probed %q", gotPath) + } + if gotAuth != "" { + t.Fatalf("probe must be anonymous, sent Authorization %q", gotAuth) + } + + res = probeHealth(srv.URL, "cluster", false, time.Second) + if res.Healthy || res.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("cluster: expected unhealthy 503, got %+v", res) + } + if res.ServerStatus != "iam-offline" || res.WriteQuorum != "3" { + t.Fatalf("cluster: headers not decoded, got %+v", res) + } + if gotQuery != "" { + t.Fatalf("cluster without --maintenance sent query %q", gotQuery) + } + + res = probeHealth(srv.URL, "cluster", true, time.Second) + if res.Healthy || res.StatusCode != http.StatusPreconditionFailed { + t.Fatalf("cluster maintenance: expected 412, got %+v", res) + } + if res.HealingDrives != "2" { + t.Fatalf("cluster maintenance: headers not decoded, got %+v", res) + } + if gotQuery != "maintenance=true" { + t.Fatalf("cluster --maintenance sent query %q", gotQuery) + } + + res = probeHealth(srv.URL, "cluster-read", false, time.Second) + if !res.Healthy || res.ReadQuorum != "2" { + t.Fatalf("cluster-read: expected healthy with read quorum, got %+v", res) + } + if gotPath != "/minio/health/cluster/read" { + t.Fatalf("cluster-read: probed %q", gotPath) + } +} + +func TestProbeHealthTLSSkipsVerification(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + res := probeHealth(srv.URL, "live", false, time.Second) + if !res.Healthy { + t.Fatalf("self-signed TLS probe must succeed, got %+v", res) + } +} + +func TestProbeHealthUnreachableAndTimeout(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + deadURL := srv.URL + srv.Close() + + res := probeHealth(deadURL, "live", false, time.Second) + if res.Healthy || res.Err == "" { + t.Fatalf("probe of a closed server must report unreachable, got %+v", res) + } + + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(500 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer slow.Close() + + res = probeHealth(slow.URL, "live", false, 50*time.Millisecond) + if res.Healthy || res.Err == "" { + t.Fatalf("probe past its deadline must fail, got %+v", res) + } +} + +func TestHealthcheckResultLine(t *testing.T) { + tests := []struct { + res healthcheckResult + want string + }{ + {healthcheckResult{Check: "live", Healthy: true, StatusCode: 200, DurationMS: 2}, "live: ok (200, 2ms)"}, + {healthcheckResult{Check: "cluster", StatusCode: 503, ServerStatus: "iam-offline", WriteQuorum: "3", HealingDrives: "2"}, "cluster: unhealthy (503) server-status=iam-offline write-quorum=3 healing-drives=2"}, + {healthcheckResult{Check: "cluster", StatusCode: 412, WriteQuorum: "3"}, "cluster: not safe for maintenance (412) write-quorum=3"}, + {healthcheckResult{Check: "ready", Err: "connection refused"}, "ready: unreachable (connection refused)"}, + } + for _, test := range tests { + if got := test.res.line(); got != test.want { + t.Fatalf("line() = %q, want %q", got, test.want) + } + } +} diff --git a/cmd/main.go b/cmd/main.go index 7e0f6cd1d..fad6c8483 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -138,6 +138,7 @@ func newApp(name string) *cli.App { // Register all commands. registerCommand(serverCmd) registerCommand(fmtGenCmd) + registerCommand(healthcheckCmd) // Set up app. cli.HelpFlag = cli.BoolFlag{ diff --git a/dockerscripts/docker-entrypoint.sh b/dockerscripts/docker-entrypoint.sh index 88f0c3ac4..ad9f46fe0 100755 --- a/dockerscripts/docker-entrypoint.sh +++ b/dockerscripts/docker-entrypoint.sh @@ -13,7 +13,7 @@ case "${1:-}" in ;; silo) ;; - -*|server|fmt-gen) + -*|server|fmt-gen|healthcheck) set -- silo "$@" ;; esac diff --git a/dockerscripts/docker-entrypoint_test.sh b/dockerscripts/docker-entrypoint_test.sh index bfb7ffa8c..e08892a61 100755 --- a/dockerscripts/docker-entrypoint_test.sh +++ b/dockerscripts/docker-entrypoint_test.sh @@ -40,6 +40,7 @@ run_case() { run_case default $'silo\n' run_case server $'silo\nserver\n/data\n' server /data run_case option $'silo\n--version\n' --version +run_case healthcheck $'silo\nhealthcheck\nready\n' healthcheck ready run_case explicit-silo $'silo\nserver\n/data\n' silo server /data run_case legacy-minio $'silo\nserver\n/data\n' minio server /data diff --git a/docs/metrics/healthcheck/README.md b/docs/metrics/healthcheck/README.md index 6946f6f6d..8343ac155 100644 --- a/docs/metrics/healthcheck/README.md +++ b/docs/metrics/healthcheck/README.md @@ -2,6 +2,35 @@ Silo server exposes three un-authenticated, healthcheck endpoints liveness probe and a cluster probe at `/minio/health/live` and `/minio/health/cluster` respectively. +## Native CLI probe + +The `silo` binary can probe those endpoints itself, which makes health checking possible in containers that ship no shell, `curl`, or `mc`: + +``` +silo healthcheck [FLAGS] [live|ready|cluster|cluster-read] +``` + +The check name maps 1:1 onto `/minio/health/`; `live` is the default. The exit code is `0` when healthy and `1` otherwise, and one diagnostic line (including the `x-minio-server-status` and quorum headers on failure) is printed for `docker inspect` to capture. The probe target is derived the same way the server derives its own listen address — `--address` / `MINIO_ADDRESS`, with HTTPS auto-detected from `public.crt` and `private.key` in `--certs-dir` — or overridden wholesale with `--url`. Certificate verification is skipped, matching the kubelet's behavior for HTTPS probes. + +Use it as an image `HEALTHCHECK` (exec form, since there may be no shell): + +``` +HEALTHCHECK --interval=30s --timeout=5s --start-period=2m --start-interval=2s --retries=3 \ + CMD ["/usr/bin/silo", "healthcheck", "ready"] +``` + +or as a Docker Compose healthcheck: + +``` +healthcheck: + test: ["CMD", "/usr/bin/silo", "healthcheck", "ready"] + interval: 5s + timeout: 5s + retries: 5 +``` + +`silo healthcheck --maintenance cluster` answers the pre-drain question documented below: exit `0` when the node can be taken down safely, exit `1` (HTTP 412) when doing so would lose HA. Keep the `cluster` checks out of per-container liveness probes — they reflect cluster-wide quorum, not this process. + ## Liveness probe This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When liveness probe fails, Kubernetes like platforms restart the container.