astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-08-09 15:53:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

test: pin the external fixtures and run the suites against the silo binary

Browse Source 
The test and verification scripts invoked ./minio and pulled their tooling from
upstream infrastructure with no integrity check. Every `curl | tar` of a client
or an old server binary was an unverified execution path in a script that
regularly runs as a privileged user, and several fetched a floating "latest".

Two installers replace all of it:

- install-mcli.sh resolves a pinned pgsty/mc release, downloads the archive and
  its checksum manifest, requires exactly one valid manifest entry for the
  asset, verifies it, and installs. MCLI_BIN with a mandatory MCLI_SHA256 lets
  an offline or air-gapped run supply its own binary, still checksum-checked.
- install-verified-fixture.sh takes source, expected SHA-256 and target, and
  refuses anything that does not match. Sources may be a URL or a local file.

Every script that previously downloaded mc now calls install-mcli.sh. The three
places that genuinely need an upstream artifact - the old MinIO server binary
for the LDAP IAM upgrade-import test, the 2021 mc for the three-site
replication test, and the functional-tests.sh fixture - go through
install-verified-fixture.sh with the digest recorded inline. Those dl.min.io
URLs remain on purpose: they are historical upstream artifacts needed to prove
upgrade compatibility, and they are now pinned and verified rather than
trusted.

The scripts otherwise switch to ./silo, silo.service, the silo container and
compose service names, and SILO_CONFIG_DIR. run-multi-site-minio-idp.sh is
renamed to run-multi-site-silo-idp.sh with the Makefile target following.
buildscripts/minio-upgrade.sh keeps its name and its `minio server` argv - it
exists to test the MinIO-to-Silo upgrade, so the old side must stay old - but
it is now pinned to an image digest rather than a tag, and its `docker system
prune` and `docker volume prune` calls are removed. Those ran unfiltered
against the developer's whole Docker installation; the resiliency tests had the
same problem and lose their prune and `docker ps -q` sweeps too.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Feng Ruohang
2026-08-06 08:48:58 +08:00
parent bd8df51665
commit 6613c2a3cb
52 changed files with 2046 additions and 1989 deletions
Download Patch File Download Diff File Expand all files Collapse all files
buildscripts/verify-healing.sh
+30 -31
View File
@@ -4,23 +4,23 @@
set -E
set -o pipefail
if [ ! -x "$PWD/minio" ]; then
echo "minio executable binary not found in current directory"
if [ ! -x "$PWD/silo" ]; then
echo "Silo executable binary not found in current directory"
exit 1
fi
WORK_DIR="$PWD/.verify-$RANDOM"
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
SILO_CONFIG_DIR="$WORK_DIR/.silo"
SILO=("$PWD/silo" --config-dir "$SILO_CONFIG_DIR" server)
GOPATH=/tmp/gopath
function start_minio_3_node() {
function start_silo_3_node() {
for i in $(seq 1 3); do
rm "${WORK_DIR}/dist-minio-server$i.log"
rm "${WORK_DIR}/dist-silo-server$i.log"
done
export MINIO_ROOT_USER=minio
export MINIO_ROOT_PASSWORD=minio123
export MINIO_ROOT_USER=silo
export MINIO_ROOT_PASSWORD=silo123
export MINIO_ERASURE_SET_DRIVE_COUNT=6
export MINIO_CI_CD=1
@@ -34,51 +34,51 @@ function start_minio_3_node() {
args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
done
"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
"${SILO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-silo-server1.log" 2>&1 &
pid1=$!
disown ${pid1}
"${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 &
"${SILO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-silo-server2.log" 2>&1 &
pid2=$!
disown $pid2
"${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 &
"${SILO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-silo-server3.log" 2>&1 &
pid3=$!
disown $pid3
export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
timeout 15m /tmp/mc ready myminio || fail
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
timeout 15m /tmp/mc ready mysilo || fail
[ ${first_time} -eq 0 ] && upload_objects
[ ${first_time} -ne 0 ] && sleep 120
if ! ps -p $pid1 1>&2 >/dev/null; then
echo "minio server 1 is not running" && fail
echo "silo server 1 is not running" && fail
fi
if ! ps -p $pid2 1>&2 >/dev/null; then
echo "minio server 2 is not running" && fail
echo "silo server 2 is not running" && fail
fi
if ! ps -p $pid3 1>&2 >/dev/null; then
echo "minio server 3 is not running" && fail
echo "silo server 3 is not running" && fail
fi
if ! pkill minio; then
if ! pkill silo; then
fail
fi
sleep 1
if pgrep minio; then
if pgrep silo; then
# forcibly killing, to proceed further properly.
if ! pkill -9 minio; then
echo "no minio process running anymore, proceed."
if ! pkill -9 silo; then
echo "no Silo process running anymore, proceed."
fi
fi
}
function check_heal() {
if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
if ! grep -q 'API:' ${WORK_DIR}/dist-silo-*.log; then
return 1
fi
@@ -103,9 +103,9 @@ function purge() {
function fail() {
for i in $(seq 1 3); do
echo "server$i log:"
cat "${WORK_DIR}/dist-minio-server$i.log"
cat "${WORK_DIR}/dist-silo-server$i.log"
done
pkill -9 minio
pkill -9 silo
echo "FAILED"
purge "$WORK_DIR"
exit 1
@@ -114,28 +114,27 @@ function fail() {
function __init__() {
echo "Initializing environment"
mkdir -p "$WORK_DIR"
mkdir -p "$MINIO_CONFIG_DIR"
mkdir -p "$SILO_CONFIG_DIR"
## version is purposefully set to '3' for minio to migrate configuration file
echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
if [ ! -f /tmp/mc ]; then
wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
chmod +x /tmp/mc
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
fi
}
function upload_objects() {
/tmp/mc mb myminio/testbucket/
/tmp/mc mb mysilo/testbucket/
for ((i = 0; i < 20; i++)); do
echo "my content" | /tmp/mc pipe myminio/testbucket/file-$i
echo "my content" | /tmp/mc pipe mysilo/testbucket/file-$i
done
}
function perform_test() {
start_port=$2
start_minio_3_node $start_port
start_silo_3_node $start_port
echo "Testing Distributed Erasure setup healing of drives"
echo "Remove the contents of the disks belonging to '${1}' node"
@@ -143,7 +142,7 @@ function perform_test() {
rm -rf ${WORK_DIR}/${1}/*/
set -x
start_minio_3_node $start_port
start_silo_3_node $start_port
check_heal ${1}
rv=$?