mirror of
https://github.com/pgsty/minio.git
synced 2026-08-09 15:53:28 +03:00
Feng Ruohang
6613c2a3cb
The test and verification scripts invoked ./minio and pulled their tooling from upstream infrastructure with no integrity check. Every `curl | tar` of a client or an old server binary was an unverified execution path in a script that regularly runs as a privileged user, and several fetched a floating "latest". Two installers replace all of it: - install-mcli.sh resolves a pinned pgsty/mc release, downloads the archive and its checksum manifest, requires exactly one valid manifest entry for the asset, verifies it, and installs. MCLI_BIN with a mandatory MCLI_SHA256 lets an offline or air-gapped run supply its own binary, still checksum-checked. - install-verified-fixture.sh takes source, expected SHA-256 and target, and refuses anything that does not match. Sources may be a URL or a local file. Every script that previously downloaded mc now calls install-mcli.sh. The three places that genuinely need an upstream artifact - the old MinIO server binary for the LDAP IAM upgrade-import test, the 2021 mc for the three-site replication test, and the functional-tests.sh fixture - go through install-verified-fixture.sh with the digest recorded inline. Those dl.min.io URLs remain on purpose: they are historical upstream artifacts needed to prove upgrade compatibility, and they are now pinned and verified rather than trusted. The scripts otherwise switch to ./silo, silo.service, the silo container and compose service names, and SILO_CONFIG_DIR. run-multi-site-minio-idp.sh is renamed to run-multi-site-silo-idp.sh with the Makefile target following. buildscripts/minio-upgrade.sh keeps its name and its `minio server` argv - it exists to test the MinIO-to-Silo upgrade, so the old side must stay old - but it is now pinned to an image digest rather than a tag, and its `docker system prune` and `docker volume prune` calls are removed. Those ran unfiltered against the developer's whole Docker installation; the resiliency tests had the same problem and lose their prune and `docker ps -q` sweeps too. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
423 lines
20 KiB
Bash
Executable File
423 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
TESTS_RUN_STATUS=1
|
|
|
|
function cleanup() {
|
|
echo "Cleaning up Silo deployment"
|
|
docker compose -f "${DOCKER_COMPOSE_FILE}" down --volumes --remove-orphans
|
|
}
|
|
|
|
function verify_resiliency() {
|
|
docs/resiliency/resiliency-verify-script.sh
|
|
RESULT=$(grep "script passed" <resiliency-verify.log)
|
|
if [ "$RESULT" != "script passed" ]; then
|
|
echo -e "${RED}${1} Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
else
|
|
echo -e "${GREEN}${1} Passed${NC}"
|
|
fi
|
|
}
|
|
|
|
function verify_resiliency_failure() {
|
|
docs/resiliency/resiliency-verify-failure-script.sh
|
|
RESULT=$(grep "script passed" <resiliency-verify-failure.log)
|
|
if [ "$RESULT" != "script passed" ]; then
|
|
echo -e "${RED}${1} Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
else
|
|
echo -e "${GREEN}${1} Passed${NC}"
|
|
fi
|
|
}
|
|
|
|
function verify_resiliency_healing() {
|
|
local WANT=$2
|
|
docs/resiliency/resiliency-verify-healing-script.sh "$WANT"
|
|
RESULT=$(grep "script passed" <resiliency-verify-healing.log)
|
|
if [ "$RESULT" != "script passed" ]; then
|
|
echo -e "${RED}${1} Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
else
|
|
echo -e "${GREEN}${1} Passed${NC}"
|
|
fi
|
|
}
|
|
|
|
function test_resiliency_success_with_server_down() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_success_with_server_down ...${NC}"
|
|
# Stop one node
|
|
docker stop resiliency-silo1-1
|
|
sleep 10
|
|
|
|
verify_resiliency "${FUNCNAME[0]}"
|
|
|
|
# Finally restart the node
|
|
docker start resiliency-silo1-1
|
|
|
|
./mc ready mysilo
|
|
}
|
|
|
|
function test_resiliency_failure_with_server_down_and_single_disk_offline() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_failure_with_server_down_and_single_disk_offline ...${NC}"
|
|
# Stop one node
|
|
docker stop resiliency-silo1-1
|
|
# In additional, suspend one more disk per set in order to induce a failure
|
|
docker exec resiliency-silo2-1 /bin/sh -c "mv /data2/.minio.sys /data2/.minio.bkp && touch /data2/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "mv /data6/.minio.sys /data6/.minio.bkp && touch /data6/.minio.sys"
|
|
sleep 10
|
|
|
|
verify_resiliency_failure "${FUNCNAME[0]}"
|
|
|
|
# Enable the disks back on nodes
|
|
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data2/.minio.sys && mv /data2/.minio.bkp /data2/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data6/.minio.sys && mv /data6/.minio.bkp /data6/.minio.sys"
|
|
|
|
# Finally restart the node
|
|
docker start resiliency-silo1-1
|
|
./mc ready mysilo
|
|
}
|
|
|
|
function test_resiliency_failure_with_servers_down() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_failure_with_servers_down ...${NC}"
|
|
# Stop two nodes
|
|
docker stop resiliency-silo1-1
|
|
docker stop resiliency-silo2-1
|
|
sleep 10
|
|
|
|
verify_resiliency_failure "${FUNCNAME[0]}"
|
|
|
|
# Restart the nodes
|
|
docker start resiliency-silo1-1
|
|
docker start resiliency-silo2-1
|
|
|
|
./mc ready mysilo
|
|
}
|
|
|
|
function test_resiliency_success_with_disks_offline() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_success_with_disks_offline ...${NC}"
|
|
# There are 8 disks on each node with EC:4 and two erasure sets.
|
|
# We should be able to safely suspend one disk per set from each server.
|
|
docker exec resiliency-silo1-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo1-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
sleep 10
|
|
|
|
verify_resiliency "${FUNCNAME[0]}"
|
|
|
|
# Finally enable the disks back on nodes
|
|
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
|
|
./mc ready mysilo
|
|
}
|
|
|
|
function test_resiliency_failure_with_too_many_disks_offline() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_failure_with_too_many_disks_offline ...${NC}"
|
|
# There are 8 disks on each node with EC:4 and two erasure sets.
|
|
# We should be able to safely suspend one disk per set from each server.
|
|
# suspending one additional disk from each set should cause failures
|
|
docker exec resiliency-silo1-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "mv /data1/.minio.sys /data1/.minio.bkp && touch /data1/.minio.sys"
|
|
docker exec resiliency-silo1-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "mv /data5/.minio.sys /data5/.minio.bkp && touch /data5/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "mv /data2/.minio.sys /data2/.minio.bkp && touch /data2/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "mv /data6/.minio.sys /data6/.minio.bkp && touch /data6/.minio.sys"
|
|
sleep 10
|
|
|
|
verify_resiliency_failure "${FUNCNAME[0]}"
|
|
|
|
# Finally enable the disks back on nodes
|
|
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data1/.minio.sys && mv /data1/.minio.bkp /data1/.minio.sys"
|
|
docker exec resiliency-silo1-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo4-1 /bin/sh -c "rm -rf /data5/.minio.sys && mv /data5/.minio.bkp /data5/.minio.sys"
|
|
docker exec resiliency-silo2-1 /bin/sh -c "rm -rf /data2/.minio.sys && mv /data2/.minio.bkp /data2/.minio.sys"
|
|
docker exec resiliency-silo3-1 /bin/sh -c "rm -rf /data6/.minio.sys && mv /data6/.minio.bkp /data6/.minio.sys"
|
|
|
|
./mc ready mysilo
|
|
}
|
|
|
|
function find_erasure_set_for_file() {
|
|
local DATA_DRIVE=-1
|
|
local FILE=$1
|
|
local DIR=$2
|
|
# Check for existence of file in erasure set 1
|
|
docker exec resiliency-silo1-1 /bin/sh -c "stat /data1/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1
|
|
STATUS=$?
|
|
if [ $STATUS -eq 0 ]; then
|
|
DATA_DRIVE=1
|
|
fi
|
|
|
|
if [ $DATA_DRIVE -eq -1 ]; then
|
|
# Check for existence of file in erasure set 2
|
|
docker exec resiliency-silo1-1 /bin/sh -c "stat /data5/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1
|
|
STATUS=$?
|
|
if [ $STATUS -eq 0 ]; then
|
|
DATA_DRIVE=5
|
|
fi
|
|
fi
|
|
echo $DATA_DRIVE
|
|
}
|
|
|
|
function test_resiliency_healing_missing_xl_metas() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_healing_missing_xl_metas ...${NC}"
|
|
|
|
DIR="initial-data"
|
|
FILE="file1"
|
|
DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR)
|
|
STATUS=$?
|
|
if [ $STATUS -ne 0 ]; then
|
|
echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}"
|
|
echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
return 1
|
|
fi
|
|
|
|
# Remove single xl.meta -- status still green
|
|
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
|
WANT='{ "before": { "color": "green", "missing": 1, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Remove two xl.meta's -- status becomes yellow
|
|
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
|
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
|
WANT='{ "before": { "color": "yellow", "missing": 2, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Remove three xl.meta's -- status becomes red (3 missing)
|
|
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
|
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
|
OUTPUT=$(docker exec resiliency-silo3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta")
|
|
WANT='{ "before": { "color": "red", "missing": 3, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Remove four xl.meta's -- status becomes red (4 missing)
|
|
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta")
|
|
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta")
|
|
OUTPUT=$(docker exec resiliency-silo3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta")
|
|
OUTPUT=$(docker exec resiliency-silo4-1 /bin/sh -c "rm /data$((DATA_DRIVE + 3))/test-bucket/initial-data/$FILE/xl.meta")
|
|
WANT='{ "before": { "color": "red", "missing": 4, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
}
|
|
|
|
function test_resiliency_healing_truncated_parts() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_healing_truncated_parts ...${NC}"
|
|
|
|
DIR="initial-data"
|
|
FILE="file2"
|
|
DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR)
|
|
STATUS=$?
|
|
if [ $STATUS -ne 0 ]; then
|
|
echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}"
|
|
echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
return 1
|
|
fi
|
|
|
|
# Truncate single part -- status still green
|
|
OUTPUT=$(docker exec resiliency-silo1-1 /bin/sh -c "truncate --size=10K /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/*/part.1")
|
|
WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Truncate two parts -- status becomes yellow (2 missing)
|
|
OUTPUT=$(docker exec resiliency-silo2-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 1))}/test-bucket/initial-data/$FILE/*/part.1")
|
|
WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Truncate three parts -- status becomes red (3 missing)
|
|
OUTPUT=$(docker exec resiliency-silo3-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 2))}/test-bucket/initial-data/$FILE/*/part.1")
|
|
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Truncate four parts -- status becomes red (4 missing)
|
|
OUTPUT=$(docker exec resiliency-silo4-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 3))}/test-bucket/initial-data/$FILE/*/part.1")
|
|
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
}
|
|
|
|
function induce_bitrot() {
|
|
local NODE=$1
|
|
local DIR=$2
|
|
local FILE=$3
|
|
# Figure out the UUID of the directory where the `part.*` files are stored
|
|
UUID=$(docker exec resiliency-silo$NODE-1 /bin/sh -c "ls -l $DIR/test-bucket/initial-data/$FILE/*/part.1")
|
|
UUID=$(echo $UUID | cut -d " " -f 9 | cut -d "/" -f 6)
|
|
|
|
# Determine head and tail size of file where we will introduce bitrot
|
|
FILE_SIZE=$(docker exec resiliency-silo$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/initial-data/$FILE/$UUID/part.1")
|
|
TAIL_SIZE=$((FILE_SIZE - 32 * 2))
|
|
|
|
# Extract head and tail of file
|
|
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | head --bytes 32 > /tmp/head")
|
|
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | tail --bytes $TAIL_SIZE > /tmp/tail")
|
|
|
|
# Corrupt the part by writing head twice followed by tail
|
|
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat /tmp/head /tmp/head /tmp/tail > $DIR/test-bucket/initial-data/$FILE/$UUID/part.1")
|
|
}
|
|
|
|
function test_resiliency_healing_induced_bitrot() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_healing_induced_bitrot ...${NC}"
|
|
|
|
DIR="initial-data"
|
|
FILE="file3"
|
|
DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR)
|
|
STATUS=$?
|
|
if [ $STATUS -ne 0 ]; then
|
|
echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}"
|
|
echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
return 1
|
|
fi
|
|
|
|
# Induce bitrot in single part -- status still green
|
|
induce_bitrot "2" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Induce bitrot in two parts -- status becomes yellow (2 corrupted)
|
|
induce_bitrot "2" "/data"$((DATA_DRIVE)) $FILE
|
|
induce_bitrot "1" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Induce bitrot in three parts -- status becomes red (3 corrupted)
|
|
induce_bitrot "3" "/data"$((DATA_DRIVE)) $FILE
|
|
induce_bitrot "2" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
induce_bitrot "1" "/data"$((DATA_DRIVE + 2)) $FILE
|
|
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Induce bitrot in four parts -- status becomes red (4 corrupted)
|
|
induce_bitrot "4" "/data"$((DATA_DRIVE)) $FILE
|
|
induce_bitrot "3" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
induce_bitrot "2" "/data"$((DATA_DRIVE + 2)) $FILE
|
|
induce_bitrot "1" "/data"$((DATA_DRIVE + 3)) $FILE
|
|
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
}
|
|
|
|
function induce_bitrot_for_xlmeta() {
|
|
local NODE=$1
|
|
local DIR=$2
|
|
local FILE=$3
|
|
|
|
# Determine head and tail size of file where we will introduce bitrot
|
|
FILE_SIZE=$(docker exec resiliency-silo$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/inlined-data/$FILE/xl.meta")
|
|
HEAD_SIZE=$((FILE_SIZE - 32 * 2))
|
|
|
|
# Extract head and tail of file
|
|
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | head --bytes $HEAD_SIZE > /head")
|
|
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | tail --bytes 32 > /tail")
|
|
|
|
# Corrupt xl.meta by writing head followed by tail twice
|
|
$(docker exec resiliency-silo$NODE-1 /bin/sh -c "cat /head /tail tmp/tail > $DIR/test-bucket/inlined-data/$FILE/xl.meta")
|
|
}
|
|
|
|
function test_resiliency_healing_inlined_metadata() {
|
|
echo
|
|
echo -e "${GREEN}Running test_resiliency_healing_inlined_metadata ...${NC}"
|
|
|
|
DIR="inlined-data"
|
|
FILE="inlined"
|
|
DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR)
|
|
STATUS=$?
|
|
if [ $STATUS -ne 0 ]; then
|
|
echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}"
|
|
echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}"
|
|
TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0))
|
|
return 1
|
|
fi
|
|
|
|
# Induce bitrot in single inlined xl.meta -- status still green
|
|
induce_bitrot_for_xlmeta "2" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Induce bitrot in two inlined xl.meta's -- status becomes yellow (2 corrupted)
|
|
induce_bitrot_for_xlmeta "3" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
induce_bitrot_for_xlmeta "3" "/data"$((DATA_DRIVE + 2)) $FILE
|
|
WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Induce bitrot in three inlined xl.meta's -- status becomes red (3 corrupted)
|
|
induce_bitrot_for_xlmeta "4" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
induce_bitrot_for_xlmeta "4" "/data"$((DATA_DRIVE + 2)) $FILE
|
|
induce_bitrot_for_xlmeta "4" "/data"$((DATA_DRIVE + 3)) $FILE
|
|
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
|
|
# Induce bitrot in four inlined xl.meta's -- status becomes red (4 corrupted)
|
|
induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE)) $FILE
|
|
induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE + 1)) $FILE
|
|
induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE + 2)) $FILE
|
|
induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE + 3)) $FILE
|
|
WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }'
|
|
verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}"
|
|
}
|
|
|
|
function main() {
|
|
if [ ! -f ./mc ]; then
|
|
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" ./mc
|
|
fi
|
|
|
|
export MC_HOST_mysilo=http://minioadmin:minioadmin@localhost:9000
|
|
|
|
cleanup
|
|
|
|
# Run resiliency tests against Silo
|
|
docker compose -f "${DOCKER_COMPOSE_FILE}" up -d
|
|
|
|
# Initial setup
|
|
docs/resiliency/resiliency-initial-script.sh
|
|
RESULT=$(grep "script passed" <resiliency-initial.log)
|
|
if [ "$RESULT" != "script passed" ]; then
|
|
cleanup
|
|
exit 1
|
|
fi
|
|
|
|
test_resiliency_healing_missing_xl_metas
|
|
test_resiliency_healing_truncated_parts
|
|
test_resiliency_healing_induced_bitrot
|
|
test_resiliency_healing_inlined_metadata
|
|
test_resiliency_success_with_disks_offline
|
|
test_resiliency_failure_with_too_many_disks_offline
|
|
test_resiliency_success_with_server_down
|
|
test_resiliency_failure_with_server_down_and_single_disk_offline
|
|
test_resiliency_failure_with_servers_down
|
|
|
|
local rv=0
|
|
if [ ${TESTS_RUN_STATUS} -ne 1 ]; then
|
|
rv=1
|
|
fi
|
|
|
|
cleanup
|
|
exit $rv
|
|
}
|
|
|
|
main "$@"
|