mirror of
https://github.com/pgsty/minio.git
synced 2026-08-09 15:53:28 +03:00
test: pin the external fixtures and run the suites against the silo binary
The test and verification scripts invoked ./minio and pulled their tooling from upstream infrastructure with no integrity check. Every `curl | tar` of a client or an old server binary was an unverified execution path in a script that regularly runs as a privileged user, and several fetched a floating "latest". Two installers replace all of it: - install-mcli.sh resolves a pinned pgsty/mc release, downloads the archive and its checksum manifest, requires exactly one valid manifest entry for the asset, verifies it, and installs. MCLI_BIN with a mandatory MCLI_SHA256 lets an offline or air-gapped run supply its own binary, still checksum-checked. - install-verified-fixture.sh takes source, expected SHA-256 and target, and refuses anything that does not match. Sources may be a URL or a local file. Every script that previously downloaded mc now calls install-mcli.sh. The three places that genuinely need an upstream artifact - the old MinIO server binary for the LDAP IAM upgrade-import test, the 2021 mc for the three-site replication test, and the functional-tests.sh fixture - go through install-verified-fixture.sh with the digest recorded inline. Those dl.min.io URLs remain on purpose: they are historical upstream artifacts needed to prove upgrade compatibility, and they are now pinned and verified rather than trusted. The scripts otherwise switch to ./silo, silo.service, the silo container and compose service names, and SILO_CONFIG_DIR. run-multi-site-minio-idp.sh is renamed to run-multi-site-silo-idp.sh with the Makefile target following. buildscripts/minio-upgrade.sh keeps its name and its `minio server` argv - it exists to test the MinIO-to-Silo upgrade, so the old side must stay old - but it is now pinned to an image digest rather than a tag, and its `docker system prune` and `docker volume prune` calls are removed. Those ran unfiltered against the developer's whole Docker installation; the resiliency tests had the same problem and lose their prune and `docker ps -q` sweeps too. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Feng Ruohang
@@ -3,19 +3,8 @@
|
||||
TESTS_RUN_STATUS=1
|
||||
|
||||
function cleanup() {
|
||||
echo "Cleaning up MinIO deployment"
|
||||
docker compose -f "${DOCKER_COMPOSE_FILE}" down --volumes
|
||||
for container in $(docker ps -q); do
|
||||
echo Removing docker $container
|
||||
docker rm -f $container >/dev/null 2>&1
|
||||
docker wait $container
|
||||
done
|
||||
}
|
||||
|
||||
function cleanup_and_prune() {
|
||||
cleanup
|
||||
docker system prune --volumes --force
|
||||
docker image prune --all --force
|
||||
echo "Cleaning up Silo deployment"
|
||||
docker compose -f "${DOCKER_COMPOSE_FILE}" down --volumes --remove-orphans
|
||||
}
|
||||
|
||||
function verify_resiliency() {
|
||||
@@ -56,53 +45,53 @@ function test_resiliency_success_with_server_down() {
|
||||
echo
|
||||
echo -e "${GREEN}Running test_resiliency_success_with_server_down ...${NC}"
|
||||
# Stop one node
|
||||
docker stop resiliency-minio1-1
|
||||
docker stop resiliency-silo1-1
|
||||
sleep 10
|
||||
|
||||
verify_resiliency "${FUNCNAME[0]}"
|
||||
|
||||
# Finally restart the node
|
||||
docker start resiliency-minio1-1
|
||||
docker start resiliency-silo1-1
|
||||
|
||||
./mc ready myminio
|
||||
./mc ready mysilo
|
||||
}
|
||||
|
||||
function test_resiliency_failure_with_server_down_and_single_disk_offline() {
|
||||
echo
|
||||
echo -e "${GREEN}Running test_resiliency_failure_with_server_down_and_single_disk_offline ...${NC}"
|
||||
# Stop one node
|
||||
docker stop resiliency-minio1-1
|
||||
docker stop resiliency-silo1-1
|
||||
# In additional, suspend one more disk per set in order to induce a failure
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "mv /data2/.minio.sys /data2/.minio.bkp && touch /data2/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "mv /data6/.minio.sys /data6/.minio.bkp && touch /data6/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "mv /data2/.minio.sys /data2/.minio.bkp && touch /data2/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "mv /data6/.minio.sys /data6/.minio.bkp && touch /data6/.minio.sys"
|
||||
sleep 10
|
||||
|
||||
verify_resiliency_failure "${FUNCNAME[0]}"
|
||||
|
||||
# Enable the disks back on nodes
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "rm -rf /data2/.minio.sys && mv /data2/.minio.bkp /data2/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "rm -rf /data6/.minio.sys && mv /data6/.minio.bkp /data6/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data2/.minio.sys && mv /data2/.minio.bkp /data2/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data6/.minio.sys && mv /data6/.minio.bkp /data6/.minio.sys"
|
||||
|
||||
# Finally restart the node
|
||||
docker start resiliency-minio1-1
|
||||
./mc ready myminio
|
||||
docker start resiliency-silo1-1
|
||||
./mc ready mysilo
|
||||
}
|
||||
|
||||
function test_resiliency_failure_with_servers_down() {
|
||||
echo
|
||||
echo -e "${GREEN}Running test_resiliency_failure_with_servers_down ...${NC}"
|
||||
# Stop two nodes
|
||||
docker stop resiliency-minio1-1
|
||||
docker stop resiliency-minio2-1
|
||||
docker stop resiliency-silo1-1
|
||||
docker stop resiliency-silo2-1
|
||||
sleep 10
|
||||
|
||||
verify_resiliency_failure "${FUNCNAME[0]}"
|
||||
|
||||
# Restart the nodes
|
||||
docker start resiliency-minio1-1
|
||||
docker start resiliency-minio2-1
|
||||
docker start resiliency-silo1-1
|
||||
docker start resiliency-silo2-1
|
||||
|
||||
./mc ready myminio
|
||||
./mc ready mysilo
|
||||
}
|
||||
|
||||
function test_resiliency_success_with_disks_offline() {
|
||||
@@ -110,29 +99,29 @@ function test_resiliency_success_with_disks_offline() {
|
||||
echo -e "${GREEN}Running test_resiliency_success_with_disks_offline ...${NC}"
|
||||
# There are 8 disks on each node with EC:4 and two erasure sets.
|
||||
# We should be able to safely suspend one disk per set from each server.
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
sleep 10
|
||||
|
||||
verify_resiliency "${FUNCNAME[0]}"
|
||||
|
||||
# Finally enable the disks back on nodes
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
|
||||
./mc ready myminio
|
||||
./mc ready mysilo
|
||||
}
|
||||
|
||||
function test_resiliency_failure_with_too_many_disks_offline() {
|
||||
@@ -141,33 +130,33 @@ function test_resiliency_failure_with_too_many_disks_offline() {
|
||||
# There are 8 disks on each node with EC:4 and two erasure sets.
|
||||
# We should be able to safely suspend one disk per set from each server.
|
||||
# suspending one additional disk from each set should cause failures
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "mv /data2/.minio.sys /data2/.minio.bkp && touch /data2/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "mv /data6/.minio.sys /data6/.minio.bkp && touch /data6/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "mv /data2/.minio.sys /data2/.minio.bkp && touch /data2/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "mv /data6/.minio.sys /data6/.minio.bkp && touch /data6/.minio.sys"
|
||||
sleep 10
|
||||
|
||||
verify_resiliency_failure "${FUNCNAME[0]}"
|
||||
|
||||
# Finally enable the disks back on nodes
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio4-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-minio2-1 /bin/sh -c "rm -rf /data2/.minio.sys && mv /data2/.minio.bkp /data2/.minio.sys"
|
||||
docker exec resiliency-minio3-1 /bin/sh -c "rm -rf /data6/.minio.sys && mv /data6/.minio.bkp /data6/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
||||
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data2/.minio.sys && mv /data2/.minio.bkp /data2/.minio.sys"
|
||||
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data6/.minio.sys && mv /data6/.minio.bkp /data6/.minio.sys"
|
||||
|
||||
./mc ready myminio
|
||||
./mc ready mysilo
|
||||
}
|
||||
|
||||
function find_erasure_set_for_file() {
|
||||
@@ -175,7 +164,7 @@ function find_erasure_set_for_file() {
|
||||
local FILE=$1
|
||||
local DIR=$2
|
||||
# Check for existence of file in erasure set 1
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "stat /data1/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "stat /data1/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1
|
||||
STATUS=$?
|
||||
if [ $STATUS -eq 0 ]; then
|
||||
DATA_DRIVE=1
|
||||
@@ -183,7 +172,7 @@ function find_erasure_set_for_file() {
|
||||
|
||||
if [ $DATA_DRIVE -eq -1 ]; then
|
||||
# Check for existence of file in erasure set 2
|
||||
docker exec resiliency-minio1-1 /bin/sh -c "stat /data5/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1
|
||||
docker exec resiliency-silo1-1 /bin/sh -c "stat /data5/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1
|
||||
STATUS=$?
|
||||
if [ $STATUS -eq 0 ]; then
|
||||
DATA_DRIVE=5
|
||||
@@ -208,28 +197,28 @@ function test_resiliency_healing_missing_xl_metas() {
|
||||
fi
|
||||
|
||||
# Remove single xl.meta -- status still green
|
||||
OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
WANT='{ "before": { "color": "green", "missing": 1, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
|
||||
# Remove two xl.meta's -- status becomes yellow
|
||||
OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
WANT='{ "before": { "color": "yellow", "missing": 2, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
|
||||
# Remove three xl.meta's -- status becomes red (3 missing)
|
||||
OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-minio3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
WANT='{ "before": { "color": "red", "missing": 3, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
|
||||
# Remove four xl.meta's -- status becomes red (4 missing)
|
||||
OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-minio3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-minio4-1 /bin/sh -c "rm /data$((DATA_DRIVE + 3))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
OUTPUT=$(docker exec resiliency-silo4-1 /bin/sh -c "rm /data$((DATA_DRIVE + 3))/test-bucket/initial-data/$FILE/xl.meta")
|
||||
WANT='{ "before": { "color": "red", "missing": 4, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
}
|
||||
@@ -250,22 +239,22 @@ function test_resiliency_healing_truncated_parts() {
|
||||
fi
|
||||
|
||||
# Truncate single part -- status still green
|
||||
OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "truncate --size=10K /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/*/part.1")
|
||||
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "truncate --size=10K /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/*/part.1")
|
||||
WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
|
||||
# Truncate two parts -- status becomes yellow (2 missing)
|
||||
OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 1))}/test-bucket/initial-data/$FILE/*/part.1")
|
||||
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 1))}/test-bucket/initial-data/$FILE/*/part.1")
|
||||
WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
|
||||
# Truncate three parts -- status becomes red (3 missing)
|
||||
OUTPUT=$(docker exec resiliency-minio3-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 2))}/test-bucket/initial-data/$FILE/*/part.1")
|
||||
OUTPUT=$(docker exec resiliency-silo3-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 2))}/test-bucket/initial-data/$FILE/*/part.1")
|
||||
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
|
||||
# Truncate four parts -- status becomes red (4 missing)
|
||||
OUTPUT=$(docker exec resiliency-minio4-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 3))}/test-bucket/initial-data/$FILE/*/part.1")
|
||||
OUTPUT=$(docker exec resiliency-silo4-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 3))}/test-bucket/initial-data/$FILE/*/part.1")
|
||||
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
||||
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
||||
}
|
||||
@@ -275,19 +264,19 @@ function induce_bitrot() {
|
||||
local DIR=$2
|
||||
local FILE=$3
|
||||
# Figure out the UUID of the directory where the `part.*` files are stored
|
||||
UUID=$(docker exec resiliency-minio$NODE-1 /bin/sh -c "ls -l $DIR/test-bucket/initial-data/$FILE/*/part.1")
|
||||
UUID=$(docker exec resiliency-silo$NODE-1 /bin/sh -c "ls -l $DIR/test-bucket/initial-data/$FILE/*/part.1")
|
||||
UUID=$(echo $UUID | cut -d " " -f 9 | cut -d "/" -f 6)
|
||||
|
||||
# Determine head and tail size of file where we will introduce bitrot
|
||||
FILE_SIZE=$(docker exec resiliency-minio$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/initial-data/$FILE/$UUID/part.1")
|
||||
FILE_SIZE=$(docker exec resiliency-silo$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/initial-data/$FILE/$UUID/part.1")
|
||||
TAIL_SIZE=$((FILE_SIZE - 32 * 2))
|
||||
|
||||
# Extract head and tail of file
|
||||
$(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | head --bytes 32 > /tmp/head")
|
||||
$(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | tail --bytes $TAIL_SIZE > /tmp/tail")
|
||||
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | head --bytes 32 > /tmp/head")
|
||||
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | tail --bytes $TAIL_SIZE > /tmp/tail")
|
||||
|
||||
# Corrupt the part by writing head twice followed by tail
|
||||
$(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat /tmp/head /tmp/head /tmp/tail > $DIR/test-bucket/initial-data/$FILE/$UUID/part.1")
|
||||
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat /tmp/head /tmp/head /tmp/tail > $DIR/test-bucket/initial-data/$FILE/$UUID/part.1")
|
||||
}
|
||||
|
||||
function test_resiliency_healing_induced_bitrot() {
|
||||
@@ -338,15 +327,15 @@ function induce_bitrot_for_xlmeta() {
|
||||
local FILE=$3
|
||||
|
||||
# Determine head and tail size of file where we will introduce bitrot
|
||||
FILE_SIZE=$(docker exec resiliency-minio$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/inlined-data/$FILE/xl.meta")
|
||||
FILE_SIZE=$(docker exec resiliency-silo$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/inlined-data/$FILE/xl.meta")
|
||||
HEAD_SIZE=$((FILE_SIZE - 32 * 2))
|
||||
|
||||
# Extract head and tail of file
|
||||
$(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | head --bytes $HEAD_SIZE > /head")
|
||||
$(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | tail --bytes 32 > /tail")
|
||||
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | head --bytes $HEAD_SIZE > /head")
|
||||
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | tail --bytes 32 > /tail")
|
||||
|
||||
# Corrupt xl.meta by writing head followed by tail twice
|
||||
$(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat /head /tail tmp/tail > $DIR/test-bucket/inlined-data/$FILE/xl.meta")
|
||||
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat /head /tail tmp/tail > $DIR/test-bucket/inlined-data/$FILE/xl.meta")
|
||||
}
|
||||
|
||||
function test_resiliency_healing_inlined_metadata() {
|
||||
@@ -393,21 +382,21 @@ function test_resiliency_healing_inlined_metadata() {
|
||||
|
||||
function main() {
|
||||
if [ ! -f ./mc ]; then
|
||||
wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x ./mc
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" ./mc
|
||||
fi
|
||||
|
||||
export MC_HOST_myminio=http://minioadmin:minioadmin@localhost:9000
|
||||
export MC_HOST_mysilo=http://minioadmin:minioadmin@localhost:9000
|
||||
|
||||
cleanup_and_prune
|
||||
cleanup
|
||||
|
||||
# Run resiliency tests against MinIO
|
||||
# Run resiliency tests against Silo
|
||||
docker compose -f "${DOCKER_COMPOSE_FILE}" up -d
|
||||
|
||||
# Initial setup
|
||||
docs/resiliency/resiliency-initial-script.sh
|
||||
RESULT=$(grep "script passed" <resiliency-initial.log)
|
||||
if [ "$RESULT" != "script passed" ]; then
|
||||
cleanup_and_prune
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -426,7 +415,7 @@ function main() {
|
||||
rv=1
|
||||
fi
|
||||
|
||||
cleanup_and_prune
|
||||
cleanup
|
||||
exit $rv
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user