Prepare SILO 20261112 release candidate

Harden IAM/bucket/import/replication policy validation, enforce tag and version authorization across read/copy/list/archive endpoints, bind SigV4 credential-scope dates and streaming trailer limits, validate HTTP SSE-KMS contexts as strict UTF-8 JSON, add release-commit and coordinated dependency gates, and pin silo-pkg v3.16.1, Console v2.7.1, mcli 20261112, Go 1.27.4, and Helm chart 7.0.5. Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-11-12 02:25:34 +03:00 · 2026-11-12 03:06:15 +08:00
parent bae0ba0b75
commit 34ba7cc1ff
58 changed files with 3536 additions and 418 deletions
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+# Fail closed before a release job builds a commit that main never accepted or
+# that CI never validated.
+#
+# check-release-state.sh answers "is this tag safe to write to". This answers
+# the other half: "is this commit safe to ship". The release workflow already
+# proves the tag resolves to the checkout, but a tag can be pushed to any
+# commit at any time, including one that never landed on main and one whose
+# tests have not finished.
+
+set -euo pipefail
+
+release_commit="${1:-}"
+fixture="${2:-}"
+repository="${GITHUB_REPOSITORY:-pgsty/silo}"
+release_branch="${RELEASE_BRANCH:-origin/main}"
+
+# Workflows that must have a successful run for this exact commit, keyed by the
+# workflow file path rather than the display name: a name is free text, so a
+# second workflow could otherwise be added that calls itself "Go CI", succeeds
+# trivially, and satisfies this check while the real one fails.
+required_workflow_paths=(
+	".github/workflows/go.yml"
+	".github/workflows/vulncheck.yml"
+)
+
+if ! command -v jq >/dev/null 2>&1; then
+	echo "jq is required to inspect workflow runs" >&2
+	exit 1
+fi
+
+if [[ ! "${release_commit}" =~ ^[0-9a-f]{40}$ ]]; then
+	echo "Invalid release commit: ${release_commit:-<empty>}" >&2
+	exit 1
+fi
+
+if [ -z "${fixture}" ]; then
+	if ! git rev-parse --verify --quiet "${release_branch}" >/dev/null; then
+		echo "Cannot resolve ${release_branch}; fetch it before releasing" >&2
+		exit 1
+	fi
+	# The exact tip, not merely an ancestor: an older commit on main has no
+	# push run of its own when it arrived as part of a multi-commit push, and a
+	# release must not trail behind what main already contains.
+	tip="$(git rev-parse "${release_branch}^{commit}")"
+	if [ "${tip}" != "${release_commit}" ]; then
+		echo "Refusing to release ${release_commit}: ${release_branch} is at ${tip}" >&2
+		exit 1
+	fi
+	echo "Release commit ${release_commit} is the tip of ${release_branch}."
+
+	error_file="$(mktemp)"
+	trap 'rm -f "${error_file}"' EXIT
+	# A pull_request run checks out GitHub's synthetic merge ref, so a green
+	# run proves the merge result was good, not that this commit was ever built
+	# on its own. exclude_pull_requests only trims the pull_requests field of
+	# each run; the event/head_branch filter below is what drops those runs.
+	if ! runs_json="$(
+		gh api --paginate \
+			"repos/${repository}/actions/runs?head_sha=${release_commit}&exclude_pull_requests=true&per_page=100" \
+			--jq '.workflow_runs[] | {path, event, head_branch, head_sha, status, conclusion, run_number, run_attempt}' \
+			2>"${error_file}" | jq -s '.'
+	)"; then
+		cat "${error_file}" >&2
+		exit 1
+	fi
+else
+	runs_json="$(<"${fixture}")"
+fi
+
+if ! jq -e 'type == "array" and all(.[]; type == "object" and (.path | type == "string"))' \
+	<<<"${runs_json}" >/dev/null 2>&1; then
+	echo "Invalid workflow run response for ${release_commit}" >&2
+	exit 1
+fi
+
+fail=0
+for workflow in "${required_workflow_paths[@]}"; do
+	# Only runs that actually built this commit count: the right workflow file,
+	# triggered by the push that put this SHA on main. A manual dispatch is not
+	# substitute evidence - it can be run from any ref. Take the newest such
+	# run; an older success must not paper over a later failure.
+	latest="$(jq -c --arg path "${workflow}" --arg sha "${release_commit}" '
+		[ .[]
+		  | select(.path == $path)
+		  | select(.head_sha == $sha)
+		  | select(.event == "push" and .head_branch == "main")
+		]
+		| sort_by([.run_number // 0, .run_attempt // 0])
+		| last // null' <<<"${runs_json}")"
+
+	if [ "${latest}" = "null" ]; then
+		echo "Required workflow ${workflow} has no qualifying run for ${release_commit} (no run)" >&2
+		fail=1
+		continue
+	fi
+
+	status="$(jq -r '.status // "unknown"' <<<"${latest}")"
+	conclusion="$(jq -r '.conclusion // "none"' <<<"${latest}")"
+	if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then
+		observed="${conclusion}"
+		if [ "${status}" != "completed" ]; then
+			observed="${status}"
+		fi
+		echo "Required workflow ${workflow} did not succeed for ${release_commit} (${observed})" >&2
+		fail=1
+	fi
+done
+
+if [ "${fail}" -ne 0 ]; then
+	echo "Refusing to release ${release_commit}: required checks are not green" >&2
+	exit 1
+fi
+
+echo "Required workflows are green for ${release_commit}."
@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+checker="${script_dir}/check-release-commit.sh"
+commit="0123456789abcdef0123456789abcdef01234567"
+other_commit="89abcdef0123456789abcdef0123456789abcdef"
+fixture="$(mktemp)"
+stdout_file="$(mktemp)"
+stderr_file="$(mktemp)"
+trap 'rm -f "${fixture}" "${stdout_file}" "${stderr_file}"' EXIT
+
+expect_success() {
+	if ! "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
+		cat "${stderr_file}" >&2
+		return 1
+	fi
+}
+
+expect_failure() {
+	if "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
+		echo "Expected release-commit check to fail: $*" >&2
+		cat "${stdout_file}" >&2
+		return 1
+	fi
+}
+
+# run <path> <conclusion> [event] [branch] [sha] [run_number] [status]
+run() {
+	local path="$1" conclusion="$2"
+	local event="${3:-push}" branch="${4:-main}" sha="${5:-${commit}}"
+	local number="${6:-1}" status="${7:-completed}"
+	local conclusion_json="\"${conclusion}\""
+	[ "${conclusion}" = "null" ] && conclusion_json=null
+	printf '{"path":"%s","event":"%s","head_branch":"%s","head_sha":"%s","status":"%s","conclusion":%s,"run_number":%s,"run_attempt":1}' \
+		"${path}" "${event}" "${branch}" "${sha}" "${status}" "${conclusion_json}" "${number}"
+}
+
+go_ok="$(run .github/workflows/go.yml success)"
+vuln_ok="$(run .github/workflows/vulncheck.yml success)"
+
+all_green() {
+	printf '[%s,%s]\n' "${go_ok}" "${vuln_ok}" >"${fixture}"
+}
+
+# Baseline: both required workflows green on a push to main.
+all_green
+expect_success "${commit}" "${fixture}"
+grep -qF "Required workflows are green for ${commit}." "${stdout_file}"
+
+# A manual dispatch is not evidence: it can be run from any ref.
+printf '[%s,%s]\n' \
+	"$(run .github/workflows/go.yml success workflow_dispatch main)" \
+	"${vuln_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "no qualifying run" "${stderr_file}"
+
+# A re-run that finally succeeded is enough; the earlier failure is history.
+printf '[%s,%s,%s]\n' \
+	"$(run .github/workflows/go.yml failure push main "${commit}" 1)" \
+	"$(run .github/workflows/go.yml success push main "${commit}" 2)" \
+	"${vuln_ok}" >"${fixture}"
+expect_success "${commit}" "${fixture}"
+
+# ...but a LATER failure must not be papered over by an earlier success.
+printf '[%s,%s,%s]\n' \
+	"$(run .github/workflows/go.yml success push main "${commit}" 1)" \
+	"$(run .github/workflows/go.yml failure push main "${commit}" 2)" \
+	"${vuln_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "Required workflow .github/workflows/go.yml did not succeed" "${stderr_file}"
+grep -qF "(failure)" "${stderr_file}"
+
+# A decoy workflow that merely calls itself "Go CI" must not satisfy the gate.
+printf '[%s,%s]\n' \
+	"$(run .github/workflows/decoy.yml success)" "${vuln_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "Required workflow .github/workflows/go.yml has no qualifying run" "${stderr_file}"
+
+# A pull_request run tests GitHub's merge ref, not this commit on its own.
+printf '[%s,%s]\n' \
+	"$(run .github/workflows/go.yml success pull_request feature)" "${vuln_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "no qualifying run" "${stderr_file}"
+
+# A push run on some other branch is not evidence that main accepted it.
+printf '[%s,%s]\n' \
+	"$(run .github/workflows/go.yml success push feature)" "${vuln_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "no qualifying run" "${stderr_file}"
+
+# A run reporting a different head_sha must be ignored.
+printf '[%s,%s]\n' \
+	"$(run .github/workflows/go.yml success push main "${other_commit}")" "${vuln_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "no qualifying run" "${stderr_file}"
+
+# Missing workflow entirely.
+printf '[%s]\n' "${go_ok}" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "Required workflow .github/workflows/vulncheck.yml has no qualifying run" "${stderr_file}"
+
+# Still running.
+printf '[%s,%s]\n' "${go_ok}" \
+	"$(run .github/workflows/vulncheck.yml null push main "${commit}" 1 in_progress)" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "(in_progress)" "${stderr_file}"
+
+# Cancelled counts as not green.
+printf '[%s,%s]\n' "${go_ok}" \
+	"$(run .github/workflows/vulncheck.yml cancelled)" >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "(cancelled)" "${stderr_file}"
+
+# Malformed API response.
+printf '{not-json}\n' >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "Invalid workflow run response for ${commit}" "${stderr_file}"
+
+# An empty array is a valid response that satisfies nothing.
+printf '[]\n' >"${fixture}"
+expect_failure "${commit}" "${fixture}"
+grep -qF "Required workflow .github/workflows/go.yml has no qualifying run" "${stderr_file}"
+
+expect_failure "not-a-commit" "${fixture}"
+grep -qF "Invalid release commit" "${stderr_file}"
+
+expect_failure "" "${fixture}"
+grep -qF "Invalid release commit: <empty>" "${stderr_file}"
+
+echo "release-commit decision tests passed"
@@ -7,7 +7,7 @@ _init() {

 	## Minimum required versions for build dependencies
 	GIT_VERSION="1.0"
-	GO_VERSION="1.27.3"
+	GO_VERSION="1.27.4"
 	OSX_VERSION="10.8"
 	KNAME=$(uname -s)
 	ARCH=$(uname -m)
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
 	exit 0
 fi

-release=${MCLI_RELEASE:-RELEASE.2026-11-09T00-00-00Z}
+release=${MCLI_RELEASE:-RELEASE.2026-11-12T00-00-00Z}
 version_hyphen=${release#RELEASE.}
 package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
 if [ "${package_version}" = "${version_hyphen}" ]; then
@@ -0,0 +1,156 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+// release-dependency-guard checks the modules and bundled client actually used
+// by a release. A local workspace must not substitute unreleased source trees.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"golang.org/x/mod/semver"
+)
+
+type module struct {
+	Path, Version, Dir string
+	Main               bool
+	Replace            *module
+}
+
+func selected(m module, path string) (module, error) {
+	if m.Replace != nil {
+		m = *m.Replace
+	}
+	if m.Main || m.Path != path || m.Version == "" {
+		return module{}, fmt.Errorf("%s must select an immutable maintained module, got %s@%s", path, m.Path, m.Version)
+	}
+	return m, nil
+}
+
+func atLeast(version, minimum string) bool {
+	return semver.IsValid(version) && semver.Prerelease(version) == "" && semver.Compare(version, minimum) >= 0
+}
+
+func validate(modules map[string]module, consoleVersion, mcTag, mcCommit string) error {
+	pkg, err := selected(modules["github.com/pgsty/silo-pkg/v3"], "github.com/pgsty/silo-pkg/v3")
+	if err != nil {
+		return err
+	}
+	if !atLeast(pkg.Version, "v3.16.1") {
+		return fmt.Errorf("silo-pkg %s lacks the strict policy release; need v3.16.1 or newer", pkg.Version)
+	}
+	if _, err := selected(modules["github.com/minio/console"], "github.com/pgsty/silo-console"); err != nil {
+		return err
+	}
+	if !atLeast("v"+consoleVersion, "v2.7.1") {
+		return fmt.Errorf("embedded Console %s lacks the policy fixes; need 2.7.1 or newer", consoleVersion)
+	}
+	mc, err := selected(modules["github.com/minio/mc"], "github.com/pgsty/mc")
+	if err != nil {
+		return err
+	}
+	if !regexp.MustCompile(`^RELEASE\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z$`).MatchString(mcTag) || mcTag <= "RELEASE.2026-11-09T00-00-00Z" {
+		return fmt.Errorf("bundled mcli %s must be a release after 20261109", mcTag)
+	}
+	if len(mcCommit) != 40 || !strings.HasSuffix(mc.Version, "-"+mcCommit[:12]) {
+		return fmt.Errorf("mc module %s does not match Dockerfile.goreleaser tag %s (%s)", mc.Version, mcTag, mcCommit)
+	}
+	return nil
+}
+
+func command(name string, args ...string) ([]byte, error) {
+	cmd := exec.Command(name, args...)
+	cmd.Stderr = os.Stderr
+	return cmd.Output()
+}
+
+func check() error {
+	if err := os.Setenv("GOWORK", "off"); err != nil {
+		return err
+	}
+	data, err := command("go", "list", "-mod=readonly", "-m", "-json", "github.com/pgsty/silo-pkg/v3", "github.com/minio/console", "github.com/minio/mc")
+	if err != nil {
+		return err
+	}
+	modules := make(map[string]module)
+	dec := json.NewDecoder(bytes.NewReader(data))
+	for {
+		var m module
+		if err := dec.Decode(&m); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		modules[m.Path] = m
+	}
+	pkg, err := selected(modules["github.com/pgsty/silo-pkg/v3"], "github.com/pgsty/silo-pkg/v3")
+	if err != nil {
+		return err
+	}
+	if !atLeast(pkg.Version, "v3.16.1") {
+		return fmt.Errorf("release blocked: select silo-pkg v3.16.1 or newer (currently %s)", pkg.Version)
+	}
+	console, err := selected(modules["github.com/minio/console"], "github.com/pgsty/silo-console")
+	if err != nil {
+		return err
+	}
+	if console.Dir == "" {
+		data, err = command("go", "mod", "download", "-json", console.Path+"@"+console.Version)
+		if err != nil {
+			return err
+		}
+		if err := json.Unmarshal(data, &console); err != nil {
+			return err
+		}
+	}
+	data, err = os.ReadFile(filepath.Join(console.Dir, "web-app", "package.json"))
+	if err != nil {
+		return err
+	}
+	var metadata struct{ Version string }
+	if err := json.Unmarshal(data, &metadata); err != nil {
+		return err
+	}
+	data, err = os.ReadFile("Dockerfile.goreleaser")
+	if err != nil {
+		return err
+	}
+	match := regexp.MustCompile(`(?m)^ARG MC_VERSION=(\S+)\s*$`).FindSubmatch(data)
+	if len(match) != 2 {
+		return fmt.Errorf("Dockerfile.goreleaser must pin MC_VERSION")
+	}
+	tag := string(match[1])
+	data, err = command("gh", "api", "repos/pgsty/mc/commits/"+tag, "--jq", ".sha")
+	if err != nil {
+		return err
+	}
+	commit := strings.TrimSpace(string(data))
+	if err := validate(modules, metadata.Version, tag, commit); err != nil {
+		return err
+	}
+	data, err = command("gh", "api", "repos/pgsty/mc/compare/d7262c2a5fbdf2b9503cc0283f1d37ae8da41baf..."+commit, "--jq", ".status")
+	if err != nil {
+		return err
+	}
+	if status := strings.TrimSpace(string(data)); status != "ahead" && status != "identical" {
+		return fmt.Errorf("mcli %s does not include the access-key policy repair", tag)
+	}
+	fmt.Printf("Release dependencies verified: silo-pkg %s, Console %s, mcli %s (%s)\n", pkg.Version, metadata.Version, tag, commit)
+	return nil
+}
+
+func main() {
+	if err := check(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
@@ -0,0 +1,44 @@
+// Copyright (c) 2026 PGSTY
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package main
+
+import "testing"
+
+func TestReleaseDependencySelection(t *testing.T) {
+	const commit = "0123456789abcdef0123456789abcdef01234567"
+	for _, tc := range []struct {
+		name, pkg, console, tag, mcSHA string
+		local, upstream, wantError     bool
+	}{
+		{name: "coordinated", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit},
+		{name: "old-package", pkg: "v3.16.0", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true},
+		{name: "prerelease-package", pkg: "v3.16.1-rc.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true},
+		{name: "old-embedded-console", pkg: "v3.16.1", console: "2.7.0", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, wantError: true},
+		{name: "old-container-client", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-09T00-00-00Z", mcSHA: commit, wantError: true},
+		{name: "source-binary-mismatch", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: "ffffffffffffffffffffffffffffffffffffffff", wantError: true},
+		{name: "workspace-console", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, local: true, wantError: true},
+		{name: "upstream-console", pkg: "v3.16.1", console: "2.7.1", tag: "RELEASE.2026-11-12T00-00-00Z", mcSHA: commit, upstream: true, wantError: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			modules := map[string]module{
+				"github.com/pgsty/silo-pkg/v3": {Path: "github.com/pgsty/silo-pkg/v3", Version: tc.pkg},
+				"github.com/minio/console": {Path: "github.com/minio/console", Replace: &module{
+					Path: "github.com/pgsty/silo-console", Version: "v0.0.0-20261112000000-0123456789ab",
+				}},
+				"github.com/minio/mc": {Path: "github.com/minio/mc", Replace: &module{
+					Path: "github.com/pgsty/mc", Version: "v0.0.0-20261112000000-0123456789ab",
+				}},
+			}
+			if tc.local {
+				modules["github.com/minio/console"].Replace.Version = ""
+			}
+			if tc.upstream {
+				modules["github.com/minio/console"].Replace.Path = "github.com/minio/console"
+			}
+			if err := validate(modules, tc.console, tc.tag, tc.mcSHA); (err != nil) != tc.wantError {
+				t.Fatalf("got %v, wantError=%t", err, tc.wantError)
+			}
+		})
+	}
+}