mirror of
https://github.com/pgsty/minio.git
synced 2026-08-10 00:03:29 +03:00
feat: add the native silo healthcheck subcommand
Add 'silo healthcheck [live|ready|cluster|cluster-read]', a thin anonymous HTTP client for the server's own /minio/health/* endpoints, so containers without a shell, curl, or mc can still run health checks. Design: silo.pgsty.com/compatibility/feature/healthcheck/ The check vocabulary maps 1:1 onto the health API paths; the probe target is derived from the server's own --address/MINIO_ADDRESS contract with HTTPS auto-detected from the certs directory, and can be overridden with --url. Exit codes are 0/1 only (Docker reserves 2); diagnostics (x-minio-server-status, quorum headers) go into a single output line for docker inspect. The request is strictly anonymous (a credentialed request would be rejected by the reserved-path guard), the transport bypasses HTTP_PROXY, and certificate verification is skipped to match kubelet HTTPS probe behavior. Cluster checks default to a 15s deadline so the server's 10s cluster_deadline can elapse. Compatibility notes: the preserved /minio/health/* path literals and the MINIO_ADDRESS env var are upstream wire/config surface, reused on purpose; the rebrand-guard baseline is regenerated for the new route literals (tests included) with zero new exported symbols. The docker entrypoint argv translation learns the new command name. Verified: unit tests, entrypoint tests, go vet, plus an end-to-end run against a live server covering all four checks, --maintenance (412), --json, usage errors, unreachable and timeout paths. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -852,6 +852,10 @@
|
|||||||
"/metrics/v3",
|
"/metrics/v3",
|
||||||
"/minio/grid/",
|
"/minio/grid/",
|
||||||
"/minio/grid/lock/",
|
"/minio/grid/lock/",
|
||||||
|
"/minio/health/cluster",
|
||||||
|
"/minio/health/cluster/read",
|
||||||
|
"/minio/health/live",
|
||||||
|
"/minio/health/ready",
|
||||||
"/myobject*",
|
"/myobject*",
|
||||||
"/netperf",
|
"/netperf",
|
||||||
"/newfolder",
|
"/newfolder",
|
||||||
|
|||||||
@@ -0,0 +1,301 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/cli"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Default probe deadlines. Cluster checks are evaluated server-side under the
|
||||||
|
// (default 10s) cluster_deadline, so their client deadline must be longer or
|
||||||
|
// an unhealthy cluster answer would never be received.
|
||||||
|
const (
|
||||||
|
healthcheckLocalTimeout = 5 * time.Second
|
||||||
|
healthcheckClusterTimeout = 15 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// healthcheckChecks maps the CLI check vocabulary 1:1 onto the server's
|
||||||
|
// /minio/health/<path> endpoints. The path literals are shared with the
|
||||||
|
// health router; the semantics live server-side only.
|
||||||
|
var healthcheckChecks = map[string]string{
|
||||||
|
"live": healthCheckLivenessPath,
|
||||||
|
"ready": healthCheckReadinessPath,
|
||||||
|
"cluster": healthCheckClusterPath,
|
||||||
|
"cluster-read": healthCheckClusterReadPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
var healthcheckFlags = []cli.Flag{
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "address",
|
||||||
|
Value: ":" + GlobalMinioDefaultPort,
|
||||||
|
Usage: "probe the server bound to a specific ADDRESS:PORT, an empty ADDRESS is probed as 127.0.0.1",
|
||||||
|
EnvVar: "MINIO_ADDRESS",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
Name: "url",
|
||||||
|
Usage: "probe this base URL (http[s]://HOST:PORT) instead of deriving one from --address and the certs directory",
|
||||||
|
},
|
||||||
|
cli.BoolFlag{
|
||||||
|
Name: "maintenance",
|
||||||
|
Usage: "with the cluster check only: ask whether taking this node down would lose quorum (HTTP 412 means it would)",
|
||||||
|
},
|
||||||
|
cli.DurationFlag{
|
||||||
|
Name: "timeout",
|
||||||
|
Usage: "overall probe deadline (default: 5s for live/ready, 15s for cluster checks)",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var healthcheckCmd = cli.Command{
|
||||||
|
Name: "healthcheck",
|
||||||
|
Usage: "Probe the health of a Silo server and report it as the exit code",
|
||||||
|
Flags: append(healthcheckFlags, GlobalFlags...),
|
||||||
|
Action: healthcheckMain,
|
||||||
|
CustomHelpTemplate: `NAME:
|
||||||
|
{{.HelpName}} - {{.Usage}}
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
{{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}[CHECK]
|
||||||
|
|
||||||
|
CHECK:
|
||||||
|
live the process is serving requests (default); touches no external system
|
||||||
|
ready live, plus KMS and etcd reachability when they are configured
|
||||||
|
cluster cluster-wide write quorum across every erasure set
|
||||||
|
cluster-read cluster-wide read quorum across every erasure set
|
||||||
|
{{if .VisibleFlags}}
|
||||||
|
FLAGS:
|
||||||
|
{{range .VisibleFlags}}{{.}}
|
||||||
|
{{end}}{{end}}
|
||||||
|
EXIT CODE:
|
||||||
|
0 - healthy (with --maintenance: safe to take the node down)
|
||||||
|
1 - anything else
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
1. Probe local liveness, e.g. as a container HEALTHCHECK:
|
||||||
|
{{.Prompt}} {{.HelpName}}
|
||||||
|
2. Probe readiness of a server on a non-default port:
|
||||||
|
{{.Prompt}} {{.HelpName}} --address :9010 ready
|
||||||
|
3. Ask whether this node can be taken down without losing HA:
|
||||||
|
{{.Prompt}} {{.HelpName}} --maintenance cluster
|
||||||
|
`,
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthcheckResult is the outcome of a single probe. It doubles as the
|
||||||
|
// --json output schema, so field changes are compatibility-relevant.
|
||||||
|
type healthcheckResult struct {
|
||||||
|
Check string `json:"check"`
|
||||||
|
Healthy bool `json:"healthy"`
|
||||||
|
StatusCode int `json:"status,omitempty"`
|
||||||
|
DurationMS int64 `json:"durationMs,omitempty"`
|
||||||
|
ServerStatus string `json:"serverStatus,omitempty"`
|
||||||
|
WriteQuorum string `json:"writeQuorum,omitempty"`
|
||||||
|
ReadQuorum string `json:"readQuorum,omitempty"`
|
||||||
|
HealingDrives string `json:"healingDrives,omitempty"`
|
||||||
|
Err string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// line renders the single human-readable output line. Container runtimes
|
||||||
|
// store only the first 4096 bytes of probe output, so it stays short.
|
||||||
|
func (r healthcheckResult) line() string {
|
||||||
|
if r.Err != "" {
|
||||||
|
return fmt.Sprintf("%s: unreachable (%s)", r.Check, r.Err)
|
||||||
|
}
|
||||||
|
if r.Healthy {
|
||||||
|
return fmt.Sprintf("%s: ok (%d, %dms)", r.Check, r.StatusCode, r.DurationMS)
|
||||||
|
}
|
||||||
|
label := "unhealthy"
|
||||||
|
if r.StatusCode == http.StatusPreconditionFailed {
|
||||||
|
label = "not safe for maintenance"
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, "%s: %s (%d)", r.Check, label, r.StatusCode)
|
||||||
|
for _, kv := range []struct{ k, v string }{
|
||||||
|
{"server-status", r.ServerStatus},
|
||||||
|
{"write-quorum", r.WriteQuorum},
|
||||||
|
{"read-quorum", r.ReadQuorum},
|
||||||
|
{"healing-drives", r.HealingDrives},
|
||||||
|
} {
|
||||||
|
if kv.v != "" {
|
||||||
|
fmt.Fprintf(&b, " %s=%s", kv.k, kv.v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthcheckTarget derives the base URL to probe. An explicit rawURL wins;
|
||||||
|
// otherwise the address' host:port is used, with the scheme decided by the
|
||||||
|
// same certificate presence check the server performs at startup.
|
||||||
|
func healthcheckTarget(rawURL, address, certsDir string) (string, error) {
|
||||||
|
if rawURL != "" {
|
||||||
|
u, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("invalid --url %q: %w", rawURL, err)
|
||||||
|
}
|
||||||
|
if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||||
|
return "", fmt.Errorf("invalid --url %q: expected http[s]://HOST:PORT", rawURL)
|
||||||
|
}
|
||||||
|
return u.Scheme + "://" + u.Host, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
host, port, err := net.SplitHostPort(address)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("invalid --address %q: %w", address, err)
|
||||||
|
}
|
||||||
|
if host == "" {
|
||||||
|
host = "127.0.0.1"
|
||||||
|
}
|
||||||
|
scheme := "http"
|
||||||
|
if isFile(filepath.Join(certsDir, publicCertFile)) && isFile(filepath.Join(certsDir, privateKeyFile)) {
|
||||||
|
scheme = "https"
|
||||||
|
}
|
||||||
|
return scheme + "://" + net.JoinHostPort(host, port), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeHealth performs one bounded, strictly anonymous GET against the
|
||||||
|
// health endpoint for check. Anonymity is load-bearing: a credentialed
|
||||||
|
// request is rejected by the reserved-path guard instead of answered.
|
||||||
|
func probeHealth(baseURL, check string, maintenance bool, timeout time.Duration) healthcheckResult {
|
||||||
|
res := healthcheckResult{Check: check}
|
||||||
|
|
||||||
|
probeURL := baseURL + healthCheckPathPrefix + healthcheckChecks[check]
|
||||||
|
if maintenance {
|
||||||
|
probeURL += "?maintenance=true"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proxy is nil on purpose: a loopback probe must never be routed through
|
||||||
|
// an HTTP_PROXY inherited from the container environment. Certificate
|
||||||
|
// verification is skipped to match the kubelet's HTTPS probe behavior.
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
Proxy: nil,
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||||
|
DisableKeepAlives: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
res.Err = err.Error()
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "silo-healthcheck/"+ReleaseTag)
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
res.DurationMS = time.Since(started).Milliseconds()
|
||||||
|
if err != nil {
|
||||||
|
res.Err = err.Error()
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
||||||
|
|
||||||
|
res.StatusCode = resp.StatusCode
|
||||||
|
res.Healthy = resp.StatusCode == http.StatusOK
|
||||||
|
res.ServerStatus = resp.Header.Get(xhttp.MinIOServerStatus)
|
||||||
|
res.WriteQuorum = resp.Header.Get(xhttp.MinIOWriteQuorum)
|
||||||
|
res.ReadQuorum = resp.Header.Get(xhttp.MinIOReadQuorum)
|
||||||
|
res.HealingDrives = resp.Header.Get(xhttp.MinIOHealingDrives)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthcheckCertsDir mirrors the server's certs-dir resolution without its
|
||||||
|
// side effects: an explicit --certs-dir wins, an explicit --config-dir
|
||||||
|
// implies <config-dir>/certs, and the shared default applies otherwise.
|
||||||
|
func healthcheckCertsDir(ctx *cli.Context) string {
|
||||||
|
switch {
|
||||||
|
case ctx.IsSet("certs-dir"):
|
||||||
|
return ctx.String("certs-dir")
|
||||||
|
case ctx.GlobalIsSet("certs-dir"):
|
||||||
|
return ctx.GlobalString("certs-dir")
|
||||||
|
case ctx.IsSet("config-dir"):
|
||||||
|
return filepath.Join(ctx.String("config-dir"), certsDir)
|
||||||
|
case ctx.GlobalIsSet("config-dir"):
|
||||||
|
return filepath.Join(ctx.GlobalString("config-dir"), certsDir)
|
||||||
|
}
|
||||||
|
return defaultCertsDir.Get()
|
||||||
|
}
|
||||||
|
|
||||||
|
func healthcheckMain(ctx *cli.Context) {
|
||||||
|
fail := func(format string, args ...any) {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck: "+format+"\n", args...)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ctx.Args()) > 1 {
|
||||||
|
fail("too many arguments, expected at most one CHECK")
|
||||||
|
}
|
||||||
|
check := "live"
|
||||||
|
if arg := ctx.Args().First(); arg != "" {
|
||||||
|
check = arg
|
||||||
|
}
|
||||||
|
if _, ok := healthcheckChecks[check]; !ok {
|
||||||
|
fail("unknown check %q, expected one of: live, ready, cluster, cluster-read", check)
|
||||||
|
}
|
||||||
|
if ctx.Bool("maintenance") && check != "cluster" {
|
||||||
|
fail("--maintenance applies to the cluster check only")
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := ctx.Duration("timeout")
|
||||||
|
if !ctx.IsSet("timeout") {
|
||||||
|
timeout = healthcheckLocalTimeout
|
||||||
|
if strings.HasPrefix(check, "cluster") {
|
||||||
|
timeout = healthcheckClusterTimeout
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
baseURL, err := healthcheckTarget(ctx.String("url"), ctx.String("address"), healthcheckCertsDir(ctx))
|
||||||
|
if err != nil {
|
||||||
|
fail("%v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res := probeHealth(baseURL, check, ctx.Bool("maintenance"), timeout)
|
||||||
|
|
||||||
|
quiet := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||||
|
if ctx.IsSet("json") || ctx.GlobalIsSet("json") {
|
||||||
|
buf, jerr := json.Marshal(res)
|
||||||
|
if jerr != nil {
|
||||||
|
fail("%v", jerr)
|
||||||
|
}
|
||||||
|
fmt.Println(string(buf))
|
||||||
|
} else if !res.Healthy {
|
||||||
|
fmt.Fprintln(os.Stderr, res.line())
|
||||||
|
} else if !quiet {
|
||||||
|
fmt.Println(res.line())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !res.Healthy {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHealthcheckTarget(t *testing.T) {
|
||||||
|
plainDir := t.TempDir()
|
||||||
|
|
||||||
|
tlsDir := t.TempDir()
|
||||||
|
for _, name := range []string{publicCertFile, privateKeyFile} {
|
||||||
|
if err := os.WriteFile(filepath.Join(tlsDir, name), []byte("test"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A lone public.crt without its key must not flip the scheme.
|
||||||
|
halfDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(halfDir, publicCertFile), []byte("test"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
rawURL string
|
||||||
|
address string
|
||||||
|
certsDir string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "default address", address: ":9000", certsDir: plainDir, want: "http://127.0.0.1:9000"},
|
||||||
|
{name: "explicit host", address: "10.0.0.7:9010", certsDir: plainDir, want: "http://10.0.0.7:9010"},
|
||||||
|
{name: "tls certs present", address: ":9000", certsDir: tlsDir, want: "https://127.0.0.1:9000"},
|
||||||
|
{name: "cert without key stays http", address: ":9000", certsDir: halfDir, want: "http://127.0.0.1:9000"},
|
||||||
|
{name: "url override wins", rawURL: "https://silo.internal:9000", address: ":9000", certsDir: plainDir, want: "https://silo.internal:9000"},
|
||||||
|
{name: "url path is dropped", rawURL: "http://silo.internal:9000/minio/health/live", address: ":9000", certsDir: plainDir, want: "http://silo.internal:9000"},
|
||||||
|
{name: "address without port", address: "localhost", certsDir: plainDir, wantErr: true},
|
||||||
|
{name: "url without scheme", rawURL: "silo.internal:9000", certsDir: plainDir, wantErr: true},
|
||||||
|
{name: "url with bad scheme", rawURL: "ftp://silo.internal:9000", certsDir: plainDir, wantErr: true},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := healthcheckTarget(test.rawURL, test.address, test.certsDir)
|
||||||
|
if (err != nil) != test.wantErr {
|
||||||
|
t.Fatalf("healthcheckTarget() error = %v, wantErr = %v", err, test.wantErr)
|
||||||
|
}
|
||||||
|
if err == nil && got != test.want {
|
||||||
|
t.Fatalf("healthcheckTarget() = %q, want %q", got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeHealthChecksAndVerdicts(t *testing.T) {
|
||||||
|
var gotPath, gotQuery, gotAuth string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotPath = r.URL.Path
|
||||||
|
gotQuery = r.URL.RawQuery
|
||||||
|
gotAuth = r.Header.Get("Authorization")
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/minio/health/live", "/minio/health/ready":
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case "/minio/health/cluster":
|
||||||
|
if r.URL.Query().Get("maintenance") == "true" {
|
||||||
|
w.Header().Set(xhttp.MinIOWriteQuorum, "3")
|
||||||
|
w.Header().Set(xhttp.MinIOHealingDrives, "2")
|
||||||
|
w.WriteHeader(http.StatusPreconditionFailed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set(xhttp.MinIOServerStatus, "iam-offline")
|
||||||
|
w.Header().Set(xhttp.MinIOWriteQuorum, "3")
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
case "/minio/health/cluster/read":
|
||||||
|
w.Header().Set(xhttp.MinIOReadQuorum, "2")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res := probeHealth(srv.URL, "live", false, time.Second)
|
||||||
|
if !res.Healthy || res.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("live: expected healthy 200, got %+v", res)
|
||||||
|
}
|
||||||
|
if gotPath != "/minio/health/live" {
|
||||||
|
t.Fatalf("live: probed %q", gotPath)
|
||||||
|
}
|
||||||
|
if gotAuth != "" {
|
||||||
|
t.Fatalf("probe must be anonymous, sent Authorization %q", gotAuth)
|
||||||
|
}
|
||||||
|
|
||||||
|
res = probeHealth(srv.URL, "cluster", false, time.Second)
|
||||||
|
if res.Healthy || res.StatusCode != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("cluster: expected unhealthy 503, got %+v", res)
|
||||||
|
}
|
||||||
|
if res.ServerStatus != "iam-offline" || res.WriteQuorum != "3" {
|
||||||
|
t.Fatalf("cluster: headers not decoded, got %+v", res)
|
||||||
|
}
|
||||||
|
if gotQuery != "" {
|
||||||
|
t.Fatalf("cluster without --maintenance sent query %q", gotQuery)
|
||||||
|
}
|
||||||
|
|
||||||
|
res = probeHealth(srv.URL, "cluster", true, time.Second)
|
||||||
|
if res.Healthy || res.StatusCode != http.StatusPreconditionFailed {
|
||||||
|
t.Fatalf("cluster maintenance: expected 412, got %+v", res)
|
||||||
|
}
|
||||||
|
if res.HealingDrives != "2" {
|
||||||
|
t.Fatalf("cluster maintenance: headers not decoded, got %+v", res)
|
||||||
|
}
|
||||||
|
if gotQuery != "maintenance=true" {
|
||||||
|
t.Fatalf("cluster --maintenance sent query %q", gotQuery)
|
||||||
|
}
|
||||||
|
|
||||||
|
res = probeHealth(srv.URL, "cluster-read", false, time.Second)
|
||||||
|
if !res.Healthy || res.ReadQuorum != "2" {
|
||||||
|
t.Fatalf("cluster-read: expected healthy with read quorum, got %+v", res)
|
||||||
|
}
|
||||||
|
if gotPath != "/minio/health/cluster/read" {
|
||||||
|
t.Fatalf("cluster-read: probed %q", gotPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeHealthTLSSkipsVerification(t *testing.T) {
|
||||||
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
res := probeHealth(srv.URL, "live", false, time.Second)
|
||||||
|
if !res.Healthy {
|
||||||
|
t.Fatalf("self-signed TLS probe must succeed, got %+v", res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeHealthUnreachableAndTimeout(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
deadURL := srv.URL
|
||||||
|
srv.Close()
|
||||||
|
|
||||||
|
res := probeHealth(deadURL, "live", false, time.Second)
|
||||||
|
if res.Healthy || res.Err == "" {
|
||||||
|
t.Fatalf("probe of a closed server must report unreachable, got %+v", res)
|
||||||
|
}
|
||||||
|
|
||||||
|
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer slow.Close()
|
||||||
|
|
||||||
|
res = probeHealth(slow.URL, "live", false, 50*time.Millisecond)
|
||||||
|
if res.Healthy || res.Err == "" {
|
||||||
|
t.Fatalf("probe past its deadline must fail, got %+v", res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthcheckResultLine(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
res healthcheckResult
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{healthcheckResult{Check: "live", Healthy: true, StatusCode: 200, DurationMS: 2}, "live: ok (200, 2ms)"},
|
||||||
|
{healthcheckResult{Check: "cluster", StatusCode: 503, ServerStatus: "iam-offline", WriteQuorum: "3", HealingDrives: "2"}, "cluster: unhealthy (503) server-status=iam-offline write-quorum=3 healing-drives=2"},
|
||||||
|
{healthcheckResult{Check: "cluster", StatusCode: 412, WriteQuorum: "3"}, "cluster: not safe for maintenance (412) write-quorum=3"},
|
||||||
|
{healthcheckResult{Check: "ready", Err: "connection refused"}, "ready: unreachable (connection refused)"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
if got := test.res.line(); got != test.want {
|
||||||
|
t.Fatalf("line() = %q, want %q", got, test.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -138,6 +138,7 @@ func newApp(name string) *cli.App {
|
|||||||
// Register all commands.
|
// Register all commands.
|
||||||
registerCommand(serverCmd)
|
registerCommand(serverCmd)
|
||||||
registerCommand(fmtGenCmd)
|
registerCommand(fmtGenCmd)
|
||||||
|
registerCommand(healthcheckCmd)
|
||||||
|
|
||||||
// Set up app.
|
// Set up app.
|
||||||
cli.HelpFlag = cli.BoolFlag{
|
cli.HelpFlag = cli.BoolFlag{
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ case "${1:-}" in
|
|||||||
;;
|
;;
|
||||||
silo)
|
silo)
|
||||||
;;
|
;;
|
||||||
-*|server|fmt-gen)
|
-*|server|fmt-gen|healthcheck)
|
||||||
set -- silo "$@"
|
set -- silo "$@"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ run_case() {
|
|||||||
run_case default $'silo\n'
|
run_case default $'silo\n'
|
||||||
run_case server $'silo\nserver\n/data\n' server /data
|
run_case server $'silo\nserver\n/data\n' server /data
|
||||||
run_case option $'silo\n--version\n' --version
|
run_case option $'silo\n--version\n' --version
|
||||||
|
run_case healthcheck $'silo\nhealthcheck\nready\n' healthcheck ready
|
||||||
run_case explicit-silo $'silo\nserver\n/data\n' silo server /data
|
run_case explicit-silo $'silo\nserver\n/data\n' silo server /data
|
||||||
run_case legacy-minio $'silo\nserver\n/data\n' minio server /data
|
run_case legacy-minio $'silo\nserver\n/data\n' minio server /data
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,35 @@
|
|||||||
|
|
||||||
Silo server exposes three un-authenticated, healthcheck endpoints liveness probe and a cluster probe at `/minio/health/live` and `/minio/health/cluster` respectively.
|
Silo server exposes three un-authenticated, healthcheck endpoints liveness probe and a cluster probe at `/minio/health/live` and `/minio/health/cluster` respectively.
|
||||||
|
|
||||||
|
## Native CLI probe
|
||||||
|
|
||||||
|
The `silo` binary can probe those endpoints itself, which makes health checking possible in containers that ship no shell, `curl`, or `mc`:
|
||||||
|
|
||||||
|
```
|
||||||
|
silo healthcheck [FLAGS] [live|ready|cluster|cluster-read]
|
||||||
|
```
|
||||||
|
|
||||||
|
The check name maps 1:1 onto `/minio/health/<path>`; `live` is the default. The exit code is `0` when healthy and `1` otherwise, and one diagnostic line (including the `x-minio-server-status` and quorum headers on failure) is printed for `docker inspect` to capture. The probe target is derived the same way the server derives its own listen address — `--address` / `MINIO_ADDRESS`, with HTTPS auto-detected from `public.crt` and `private.key` in `--certs-dir` — or overridden wholesale with `--url`. Certificate verification is skipped, matching the kubelet's behavior for HTTPS probes.
|
||||||
|
|
||||||
|
Use it as an image `HEALTHCHECK` (exec form, since there may be no shell):
|
||||||
|
|
||||||
|
```
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=2m --start-interval=2s --retries=3 \
|
||||||
|
CMD ["/usr/bin/silo", "healthcheck", "ready"]
|
||||||
|
```
|
||||||
|
|
||||||
|
or as a Docker Compose healthcheck:
|
||||||
|
|
||||||
|
```
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "/usr/bin/silo", "healthcheck", "ready"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
```
|
||||||
|
|
||||||
|
`silo healthcheck --maintenance cluster` answers the pre-drain question documented below: exit `0` when the node can be taken down safely, exit `1` (HTTP 412) when doing so would lose HA. Keep the `cluster` checks out of per-container liveness probes — they reflect cluster-wide quorum, not this process.
|
||||||
|
|
||||||
## Liveness probe
|
## Liveness probe
|
||||||
|
|
||||||
This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When liveness probe fails, Kubernetes like platforms restart the container.
|
This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When liveness probe fails, Kubernetes like platforms restart the container.
|
||||||
|
|||||||
Reference in New Issue
Block a user