minio/buildscripts/verify-healing.sh

#!/bin/bash -e
#

set -E
set -o pipefail

if [ ! -x "$PWD/silo" ]; then
	echo "Silo executable binary not found in current directory"
	exit 1
fi

WORK_DIR="$PWD/.verify-$RANDOM"
SILO_CONFIG_DIR="$WORK_DIR/.silo"
SILO=("$PWD/silo" --config-dir "$SILO_CONFIG_DIR" server)
GOPATH=/tmp/gopath

function start_silo_3_node() {
	for i in $(seq 1 3); do
		rm "${WORK_DIR}/dist-silo-server$i.log"
	done

	export MINIO_ROOT_USER=silo
	export MINIO_ROOT_PASSWORD=silo123
	export MINIO_ERASURE_SET_DRIVE_COUNT=6
	export MINIO_CI_CD=1

	first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l)

	start_port=$1
	args=""
	for d in $(seq 1 3 5); do
		args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
		d=$((d + 1))
		args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
	done

	"${SILO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-silo-server1.log" 2>&1 &
	pid1=$!
	disown ${pid1}

	"${SILO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-silo-server2.log" 2>&1 &
	pid2=$!
	disown $pid2

	"${SILO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-silo-server3.log" 2>&1 &
	pid3=$!
	disown $pid3

	export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
	timeout 15m /tmp/mc ready mysilo || fail

	[ ${first_time} -eq 0 ] && upload_objects
	[ ${first_time} -ne 0 ] && sleep 120

	if ! ps -p $pid1 1>&2 >/dev/null; then
		echo "silo server 1 is not running" && fail
	fi

	if ! ps -p $pid2 1>&2 >/dev/null; then
		echo "silo server 2 is not running" && fail
	fi

	if ! ps -p $pid3 1>&2 >/dev/null; then
		echo "silo server 3 is not running" && fail
	fi

	if ! pkill silo; then
		fail
	fi

	sleep 1
	if pgrep silo; then
		# forcibly killing, to proceed further properly.
		if ! pkill -9 silo; then
			echo "no Silo process running anymore, proceed."
		fi
	fi
}

function check_heal() {
	if ! grep -q 'API:' ${WORK_DIR}/dist-silo-*.log; then
		return 1
	fi

	for ((i = 0; i < 20; i++)); do
		test -f ${WORK_DIR}/$1/1/.minio.sys/format.json
		v1=$?
		nextInES=$(($1 + 1)) && [ $nextInES -gt 3 ] && nextInES=1
		foundFiles1=$(find ${WORK_DIR}/$1/1/ | grep -v .minio.sys | grep xl.meta | wc -l)
		foundFiles2=$(find ${WORK_DIR}/$nextInES/1/ | grep -v .minio.sys | grep xl.meta | wc -l)
		test $foundFiles1 -eq $foundFiles2
		v2=$?
		[ $v1 == 0 -a $v2 == 0 ] && return 0
		sleep 10
	done
	return 1
}

function purge() {
	rm -rf "$1"
}

function fail() {
	for i in $(seq 1 3); do
		echo "server$i log:"
		cat "${WORK_DIR}/dist-silo-server$i.log"
	done
	pkill -9 silo
	echo "FAILED"
	purge "$WORK_DIR"
	exit 1
}

function __init__() {
	echo "Initializing environment"
	mkdir -p "$WORK_DIR"
	mkdir -p "$SILO_CONFIG_DIR"

	## version is purposefully set to '3' for minio to migrate configuration file
	echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"

	if [ ! -f /tmp/mc ]; then
		"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
	fi
}

function upload_objects() {
	/tmp/mc mb mysilo/testbucket/
	for ((i = 0; i < 20; i++)); do
		echo "my content" | /tmp/mc pipe mysilo/testbucket/file-$i
	done
}

function perform_test() {
	start_port=$2

	start_silo_3_node $start_port

	echo "Testing Distributed Erasure setup healing of drives"
	echo "Remove the contents of the disks belonging to '${1}' node"

	rm -rf ${WORK_DIR}/${1}/*/

	set -x
	start_silo_3_node $start_port

	check_heal ${1}
	rv=$?
	if [ "$rv" == "1" ]; then
		fail
	fi
}

function main() {
	# use same ports for all tests
	start_port=$(shuf -i 10000-65000 -n 1)

	perform_test "2" ${start_port}
	perform_test "1" ${start_port}
	perform_test "3" ${start_port}
}

(__init__ "$@" && main "$@")
rv=$?
purge "$WORK_DIR"
exit "$rv"