diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 14c5e2ee4..f258bb39a 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -220,6 +220,21 @@ jobs: org.opencontainers.image.created=${{ env.PUBLISHED_AT }} org.opencontainers.image.revision=${{ env.RELEASE_REVISION }} + - name: Verify HEALTHCHECK survived the distroless push + run: | + set -euo pipefail + # HEALTHCHECK is a Docker extension absent from the OCI image + # spec, and a publish path can drop it silently. Check the pushed + # architecture image now, before the versioned and rolling + # multi-arch manifests are created, so a broken health config + # stops their promotion. (The architecture-suffixed tags above + # are already public by this point - full staging-then-promote + # would be a workflow-wide redesign shared with the classic + # image lanes.) + docker pull "pgsty/silo:${RELEASE_TAG}-distroless-amd64" >/dev/null + test "$(docker inspect -f '{{json .Config.Healthcheck.Test}}' "pgsty/silo:${RELEASE_TAG}-distroless-amd64")" \ + = '["CMD","/usr/bin/silo","healthcheck","ready"]' + - name: Publish multi-architecture manifests run: | set -euo pipefail @@ -274,17 +289,6 @@ jobs: fi echo "SILO_DISTROLESS_DIGEST=${DISTROLESS_RELEASE_DIGEST}" >> "${GITHUB_ENV}" - - name: Verify HEALTHCHECK survived the distroless publish - run: | - set -euo pipefail - # HEALTHCHECK is a Docker extension absent from the OCI image - # spec, and OCI-media-type publishes can drop it silently. If the - # pushed image lost it, the pilot's zero-config compose health - # gating disappears - treat that as a release blocker. - docker pull "pgsty/silo:${RELEASE_TAG}-distroless" >/dev/null - docker inspect -f '{{json .Config.Healthcheck.Test}}' "pgsty/silo:${RELEASE_TAG}-distroless" \ - | grep -F '"healthcheck"' - - name: Generate architecture image SBOMs env: AMD64_DIGEST: ${{ steps.build-amd64.outputs.digest }} @@ -295,13 +299,21 @@ jobs: run: | set -euo pipefail mkdir -p docker-release/sbom + # Each per-architecture digest names an OCI index (image plus the + # provenance attestation buildx attaches), and Syft's platform + # default on an index follows the amd64 runner - an arm64-only + # index would fail outright. Select the platform explicitly. syft "registry:index.docker.io/pgsty/silo@${AMD64_DIGEST}" \ + --platform linux/amd64 \ --output "spdx-json=docker-release/sbom/linux-amd64.spdx.json" syft "registry:index.docker.io/pgsty/silo@${ARM64_DIGEST}" \ + --platform linux/arm64 \ --output "spdx-json=docker-release/sbom/linux-arm64.spdx.json" syft "registry:index.docker.io/pgsty/silo@${DISTROLESS_AMD64_DIGEST}" \ + --platform linux/amd64 \ --output "spdx-json=docker-release/sbom/linux-amd64-distroless.spdx.json" syft "registry:index.docker.io/pgsty/silo@${DISTROLESS_ARM64_DIGEST}" \ + --platform linux/arm64 \ --output "spdx-json=docker-release/sbom/linux-arm64-distroless.spdx.json" - name: Attest amd64 image SBOM diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index 9a9c23778..c3b397220 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -8,6 +8,8 @@ on: - ".github/nfpm.yml" - "Dockerfile.goreleaser" - "Dockerfile.distroless" + - "cmd/healthcheck-main.go" + - "cmd/main.go" - "dockerscripts/download-static-curl.sh" - "dockerscripts/docker-entrypoint.sh" - "dockerscripts/docker-entrypoint_test.sh" @@ -401,23 +403,33 @@ jobs: docker build -t silo-distroless-test:snapshot -f "${ctx}/Dockerfile.distroless" "${ctx}" # HEALTHCHECK is a Docker extension absent from the OCI image - # spec; assert it survived into the built image config. - docker inspect -f '{{json .Config.Healthcheck.Test}}' silo-distroless-test:snapshot \ - | grep -F '"healthcheck"' >/dev/null + # spec; assert the exact probe command survived into the image + # config, not merely a substring of it. + test "$(docker inspect -f '{{json .Config.Healthcheck.Test}}' silo-distroless-test:snapshot)" \ + = '["CMD","/usr/bin/silo","healthcheck","ready"]' # /data ships in the image layer world-writable (issue #55): - # there is no entrypoint left to repair ownership at runtime. The - # same export proves the image carries the binary and the license - # set, and neither a shell nor a legacy /usr/bin/minio. + # there is no entrypoint left to repair ownership at runtime. + # Export the rootfs once, then assert each required and each + # forbidden entry individually: tar's member-argument mode exits + # non-zero on any missing name, which under pipefail masks a + # found forbidden file, and -tv prints symlinks as 'name -> + # target' which defeats $-anchored greps. probe="$(docker create silo-distroless-test:snapshot server /data)" - docker export "${probe}" | tar -tvf - data usr/bin/silo licenses > "${ctx}/listing.txt" - grep -E '^drwxrwxrwx.* data/$' "${ctx}/listing.txt" >/dev/null - grep -E ' licenses/(LICENSE|NOTICE|CREDITS)$' "${ctx}/listing.txt" >/dev/null - if docker export "${probe}" | tar -tf - bin/sh usr/bin/minio 2>/dev/null | grep -q .; then - echo "distroless image unexpectedly contains a shell or /usr/bin/minio" - exit 1 - fi + docker export "${probe}" -o "${ctx}/rootfs.tar" docker rm "${probe}" >/dev/null + tar -tf "${ctx}/rootfs.tar" > "${ctx}/names.txt" + tar -tvf "${ctx}/rootfs.tar" > "${ctx}/verbose.txt" + grep -E '^drwxrwxrwx.* data/$' "${ctx}/verbose.txt" >/dev/null + for want in usr/bin/silo licenses/LICENSE licenses/NOTICE licenses/CREDITS; do + grep -Fxq "${want}" "${ctx}/names.txt" || { echo "missing ${want}"; exit 1; } + done + for forbid in bin/sh usr/bin/sh busybox/sh usr/bin/minio usr/bin/mc usr/bin/mcli; do + if grep -Fxq "${forbid}" "${ctx}/names.txt"; then + echo "distroless image unexpectedly contains ${forbid}" + exit 1 + fi + done # The baked-in healthcheck must drive Docker's health state on its # own, the probe binary must be directly exec-able without any diff --git a/Dockerfile.distroless b/Dockerfile.distroless index e540d43e0..03f4f7cc1 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -42,7 +42,9 @@ VOLUME ["/data"] # Exec form is mandatory: there is no /bin/sh in this image. `ready` # rather than `live` because Docker health feeds start-order gating # (readiness semantics); the two are identical unless KMS/etcd are used. -HEALTHCHECK --interval=30s --timeout=5s --start-period=2m --start-interval=2s --retries=3 \ +# The outer timeout stays above the probe's own 5s deadline so the +# probe can report its diagnostic line instead of being SIGKILLed. +HEALTHCHECK --interval=30s --timeout=10s --start-period=2m --start-interval=2s --retries=3 \ CMD ["/usr/bin/silo", "healthcheck", "ready"] ENTRYPOINT ["/usr/bin/silo"] diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index 70037e633..63859b1fc 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -252,6 +252,7 @@ "MINIO_FS_OSYNC", "MINIO_GID", "MINIO_GROUPNAME", + "MINIO_HEALTHCHECK_URL", "MINIO_HEAL_BITROTSCAN", "MINIO_HEAL_DRIVE_WORKERS", "MINIO_HEAL_MAX_IO", diff --git a/cmd/healthcheck-main.go b/cmd/healthcheck-main.go index 0137e819a..f58a74cff 100644 --- a/cmd/healthcheck-main.go +++ b/cmd/healthcheck-main.go @@ -61,8 +61,9 @@ var healthcheckFlags = []cli.Flag{ EnvVar: "MINIO_ADDRESS", }, cli.StringFlag{ - Name: "url", - Usage: "probe this base URL (http[s]://HOST:PORT) instead of deriving one from --address and the certs directory", + Name: "url", + Usage: "probe this base URL (http[s]://HOST:PORT) instead of deriving one from --address and the certs directory", + EnvVar: "MINIO_HEALTHCHECK_URL", }, cli.BoolFlag{ Name: "maintenance", @@ -152,7 +153,8 @@ func (r healthcheckResult) line() string { // healthcheckTarget derives the base URL to probe. An explicit rawURL wins; // otherwise the address' host:port is used, with the scheme decided by the -// same certificate presence check the server performs at startup. +// same certificate presence check the server performs at startup. URLs are +// serialized via url.URL so IPv6 zone identifiers survive as %25-escapes. func healthcheckTarget(rawURL, address, certsDir string) (string, error) { if rawURL != "" { u, err := url.Parse(rawURL) @@ -162,7 +164,7 @@ func healthcheckTarget(rawURL, address, certsDir string) (string, error) { if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return "", fmt.Errorf("invalid --url %q: expected http[s]://HOST:PORT", rawURL) } - return u.Scheme + "://" + u.Host, nil + return (&url.URL{Scheme: u.Scheme, Host: u.Host}).String(), nil } host, port, err := net.SplitHostPort(address) @@ -176,7 +178,7 @@ func healthcheckTarget(rawURL, address, certsDir string) (string, error) { if isFile(filepath.Join(certsDir, publicCertFile)) && isFile(filepath.Join(certsDir, privateKeyFile)) { scheme = "https" } - return scheme + "://" + net.JoinHostPort(host, port), nil + return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port)}).String(), nil } // probeHealth performs one bounded, strictly anonymous GET against the @@ -282,8 +284,8 @@ func healthcheckMain(ctx *cli.Context) { res := probeHealth(baseURL, check, ctx.Bool("maintenance"), timeout) - quiet := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet") - if ctx.IsSet("json") || ctx.GlobalIsSet("json") { + quiet := ctx.Bool("quiet") || ctx.GlobalBool("quiet") + if ctx.Bool("json") || ctx.GlobalBool("json") { buf, jerr := json.Marshal(res) if jerr != nil { fail("%v", jerr) diff --git a/cmd/healthcheck-main_test.go b/cmd/healthcheck-main_test.go index 50f61acfc..2c41bdcf1 100644 --- a/cmd/healthcheck-main_test.go +++ b/cmd/healthcheck-main_test.go @@ -54,6 +54,9 @@ func TestHealthcheckTarget(t *testing.T) { }{ {name: "default address", address: ":9000", certsDir: plainDir, want: "http://127.0.0.1:9000"}, {name: "explicit host", address: "10.0.0.7:9010", certsDir: plainDir, want: "http://10.0.0.7:9010"}, + {name: "ipv6 address", address: "[::1]:9000", certsDir: plainDir, want: "http://[::1]:9000"}, + {name: "ipv6 zone is escaped", address: "[fe80::1%eth0]:9000", certsDir: plainDir, want: "http://[fe80::1%25eth0]:9000"}, + {name: "ipv6 zone in url", rawURL: "http://[fe80::1%25eth0]:9000", certsDir: plainDir, want: "http://[fe80::1%25eth0]:9000"}, {name: "tls certs present", address: ":9000", certsDir: tlsDir, want: "https://127.0.0.1:9000"}, {name: "cert without key stays http", address: ":9000", certsDir: halfDir, want: "http://127.0.0.1:9000"}, {name: "url override wins", rawURL: "https://silo.internal:9000", address: ":9000", certsDir: plainDir, want: "https://silo.internal:9000"}, diff --git a/docs/metrics/healthcheck/README.md b/docs/metrics/healthcheck/README.md index 8343ac155..d2f5d185f 100644 --- a/docs/metrics/healthcheck/README.md +++ b/docs/metrics/healthcheck/README.md @@ -10,12 +10,12 @@ The `silo` binary can probe those endpoints itself, which makes health checking silo healthcheck [FLAGS] [live|ready|cluster|cluster-read] ``` -The check name maps 1:1 onto `/minio/health/`; `live` is the default. The exit code is `0` when healthy and `1` otherwise, and one diagnostic line (including the `x-minio-server-status` and quorum headers on failure) is printed for `docker inspect` to capture. The probe target is derived the same way the server derives its own listen address — `--address` / `MINIO_ADDRESS`, with HTTPS auto-detected from `public.crt` and `private.key` in `--certs-dir` — or overridden wholesale with `--url`. Certificate verification is skipped, matching the kubelet's behavior for HTTPS probes. +The check name maps 1:1 onto `/minio/health/`; `live` is the default. The exit code is `0` when healthy and `1` otherwise, and one diagnostic line (including the `x-minio-server-status` and quorum headers on failure) is printed for `docker inspect` to capture. The probe target is derived the same way the server derives its own listen address — `--address` / `MINIO_ADDRESS`, with HTTPS auto-detected from `public.crt` and `private.key` in `--certs-dir` — or overridden wholesale with `--url` / `MINIO_HEALTHCHECK_URL`. The environment form exists for containers with a baked-in `HEALTHCHECK`: a probe process cannot see the server's command line, so when the server's address or TLS setup comes from CLI arguments rather than the environment, set `MINIO_HEALTHCHECK_URL` (e.g. `https://127.0.0.1:9010`) to point the built-in probe at it. Certificate verification is skipped, matching the kubelet's behavior for HTTPS probes. -Use it as an image `HEALTHCHECK` (exec form, since there may be no shell): +Use it as an image `HEALTHCHECK` (exec form, since there may be no shell; keep the outer timeout above the probe's own 5s deadline so its diagnostic line survives): ``` -HEALTHCHECK --interval=30s --timeout=5s --start-period=2m --start-interval=2s --retries=3 \ +HEALTHCHECK --interval=30s --timeout=10s --start-period=2m --start-interval=2s --retries=3 \ CMD ["/usr/bin/silo", "healthcheck", "ready"] ``` @@ -25,7 +25,7 @@ or as a Docker Compose healthcheck: healthcheck: test: ["CMD", "/usr/bin/silo", "healthcheck", "ready"] interval: 5s - timeout: 5s + timeout: 10s retries: 5 ```