mirror of https://github.com/telemt/telemt.git
Compare commits
No commits in common. "67dc1e8d187ba223de5d30c34b477b596dfeb06f" and "049db1196f4a5b4dad5a19561a662cfe11bc61c0" have entirely different histories.
67dc1e8d18
...
049db1196f
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.25"
|
version = "3.3.23"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,6 @@ USER telemt
|
||||||
|
|
||||||
EXPOSE 443
|
EXPOSE 443
|
||||||
EXPOSE 9090
|
EXPOSE 9090
|
||||||
EXPOSE 9091
|
|
||||||
|
|
||||||
ENTRYPOINT ["/app/telemt"]
|
ENTRYPOINT ["/app/telemt"]
|
||||||
CMD ["config.toml"]
|
CMD ["config.toml"]
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ services:
|
||||||
ports:
|
ports:
|
||||||
- "443:443"
|
- "443:443"
|
||||||
- "127.0.0.1:9090:9090"
|
- "127.0.0.1:9090:9090"
|
||||||
- "127.0.0.1:9091:9091"
|
|
||||||
# Allow caching 'proxy-secret' in read-only container
|
# Allow caching 'proxy-secret' in read-only container
|
||||||
working_dir: /run/telemt
|
working_dir: /run/telemt
|
||||||
volumes:
|
volumes:
|
||||||
|
|
|
||||||
|
|
@ -181,8 +181,6 @@ docker compose down
|
||||||
docker build -t telemt:local .
|
docker build -t telemt:local .
|
||||||
docker run --name telemt --restart unless-stopped \
|
docker run --name telemt --restart unless-stopped \
|
||||||
-p 443:443 \
|
-p 443:443 \
|
||||||
-p 9090:9090 \
|
|
||||||
-p 9091:9091 \
|
|
||||||
-e RUST_LOG=info \
|
-e RUST_LOG=info \
|
||||||
-v "$PWD/config.toml:/app/config.toml:ro" \
|
-v "$PWD/config.toml:/app/config.toml:ro" \
|
||||||
--read-only \
|
--read-only \
|
||||||
|
|
|
||||||
|
|
@ -183,8 +183,6 @@ docker compose down
|
||||||
docker build -t telemt:local .
|
docker build -t telemt:local .
|
||||||
docker run --name telemt --restart unless-stopped \
|
docker run --name telemt --restart unless-stopped \
|
||||||
-p 443:443 \
|
-p 443:443 \
|
||||||
-p 9090:9090 \
|
|
||||||
-p 9091:9091 \
|
|
||||||
-e RUST_LOG=info \
|
-e RUST_LOG=info \
|
||||||
-v "$PWD/config.toml:/app/config.toml:ro" \
|
-v "$PWD/config.toml:/app/config.toml:ro" \
|
||||||
--read-only \
|
--read-only \
|
||||||
|
|
|
||||||
561
install.sh
561
install.sh
|
|
@ -1,190 +1,154 @@
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
# --- Global Configurations ---
|
||||||
REPO="${REPO:-telemt/telemt}"
|
REPO="${REPO:-telemt/telemt}"
|
||||||
BIN_NAME="${BIN_NAME:-telemt}"
|
BIN_NAME="${BIN_NAME:-telemt}"
|
||||||
INSTALL_DIR="${INSTALL_DIR:-/bin}"
|
INSTALL_DIR="${INSTALL_DIR:-/bin}"
|
||||||
CONFIG_DIR="${CONFIG_DIR:-/etc/telemt}"
|
CONFIG_DIR="${CONFIG_DIR:-/etc/telemt}"
|
||||||
CONFIG_FILE="${CONFIG_FILE:-${CONFIG_DIR}/telemt.toml}"
|
CONFIG_FILE="${CONFIG_FILE:-${CONFIG_DIR}/telemt.toml}"
|
||||||
WORK_DIR="${WORK_DIR:-/opt/telemt}"
|
WORK_DIR="${WORK_DIR:-/opt/telemt}"
|
||||||
TLS_DOMAIN="${TLS_DOMAIN:-petrovich.ru}"
|
|
||||||
SERVICE_NAME="telemt"
|
SERVICE_NAME="telemt"
|
||||||
TEMP_DIR=""
|
TEMP_DIR=""
|
||||||
SUDO=""
|
SUDO=""
|
||||||
CONFIG_PARENT_DIR=""
|
|
||||||
SERVICE_START_FAILED=0
|
|
||||||
|
|
||||||
|
# --- Argument Parsing ---
|
||||||
ACTION="install"
|
ACTION="install"
|
||||||
TARGET_VERSION="${VERSION:-latest}"
|
TARGET_VERSION="${VERSION:-latest}"
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
-h|--help) ACTION="help"; shift ;;
|
-h|--help)
|
||||||
|
ACTION="help"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
uninstall|--uninstall)
|
uninstall|--uninstall)
|
||||||
if [ "$ACTION" != "purge" ]; then ACTION="uninstall"; fi
|
[ "$ACTION" != "purge" ] && ACTION="uninstall"
|
||||||
shift ;;
|
shift
|
||||||
purge|--purge) ACTION="purge"; shift ;;
|
;;
|
||||||
install|--install) ACTION="install"; shift ;;
|
--purge)
|
||||||
-*) printf '[ERROR] Unknown option: %s\n' "$1" >&2; exit 1 ;;
|
ACTION="purge"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
install|--install)
|
||||||
|
ACTION="install"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-*)
|
||||||
|
printf '[ERROR] Unknown option: %s\n' "$1" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
if [ "$ACTION" = "install" ]; then TARGET_VERSION="$1"
|
if [ "$ACTION" = "install" ]; then
|
||||||
else printf '[WARNING] Ignoring extra argument: %s\n' "$1" >&2; fi
|
TARGET_VERSION="$1"
|
||||||
shift ;;
|
fi
|
||||||
|
shift
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
say() {
|
# --- Core Functions ---
|
||||||
if [ "$#" -eq 0 ] || [ -z "${1:-}" ]; then
|
say() { printf '[INFO] %s\n' "$*"; }
|
||||||
printf '\n'
|
|
||||||
else
|
|
||||||
printf '[INFO] %s\n' "$*"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; }
|
die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
write_root() { $SUDO sh -c 'cat > "$1"' _ "$1"; }
|
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
if [ -n "${TEMP_DIR:-}" ] && [ -d "$TEMP_DIR" ]; then
|
if [ -n "${TEMP_DIR:-}" ] && [ -d "$TEMP_DIR" ]; then
|
||||||
rm -rf -- "$TEMP_DIR"
|
rm -rf -- "$TEMP_DIR"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
trap cleanup EXIT INT TERM
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
show_help() {
|
show_help() {
|
||||||
say "Usage: $0 [ <version> | install | uninstall | purge | --help ]"
|
say "Usage: $0 [version | install | uninstall | --purge | --help]"
|
||||||
say " <version> Install specific version (e.g. 3.3.15, default: latest)"
|
say " version Install specific version (e.g. 1.0.0, default: latest)"
|
||||||
say " install Install the latest version"
|
say " uninstall Remove the binary and service (keeps config)"
|
||||||
say " uninstall Remove the binary and service (keeps config and user)"
|
say " --purge Remove everything including configuration"
|
||||||
say " purge Remove everything including configuration, data, and user"
|
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
||||||
check_os_entity() {
|
user_exists() {
|
||||||
if command -v getent >/dev/null 2>&1; then getent "$1" "$2" >/dev/null 2>&1
|
if command -v getent >/dev/null 2>&1; then
|
||||||
else grep -q "^${2}:" "/etc/$1" 2>/dev/null; fi
|
getent passwd "$1" >/dev/null 2>&1
|
||||||
}
|
|
||||||
|
|
||||||
normalize_path() {
|
|
||||||
printf '%s\n' "$1" | tr -s '/' | sed 's|/$||; s|^$|/|'
|
|
||||||
}
|
|
||||||
|
|
||||||
get_realpath() {
|
|
||||||
path_in="$1"
|
|
||||||
case "$path_in" in /*) ;; *) path_in="$(pwd)/$path_in" ;; esac
|
|
||||||
|
|
||||||
if command -v realpath >/dev/null 2>&1; then
|
|
||||||
if realpath_out="$(realpath -m "$path_in" 2>/dev/null)"; then
|
|
||||||
printf '%s\n' "$realpath_out"
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if command -v readlink >/dev/null 2>&1; then
|
|
||||||
resolved_path="$(readlink -f "$path_in" 2>/dev/null || true)"
|
|
||||||
if [ -n "$resolved_path" ]; then
|
|
||||||
printf '%s\n' "$resolved_path"
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
d="${path_in%/*}"; b="${path_in##*/}"
|
|
||||||
if [ -z "$d" ]; then d="/"; fi
|
|
||||||
if [ "$d" = "$path_in" ]; then d="/"; b="$path_in"; fi
|
|
||||||
|
|
||||||
if [ -d "$d" ]; then
|
|
||||||
abs_d="$(cd "$d" >/dev/null 2>&1 && pwd || true)"
|
|
||||||
if [ -n "$abs_d" ]; then
|
|
||||||
if [ "$b" = "." ] || [ -z "$b" ]; then printf '%s\n' "$abs_d"
|
|
||||||
elif [ "$abs_d" = "/" ]; then printf '/%s\n' "$b"
|
|
||||||
else printf '%s/%s\n' "$abs_d" "$b"; fi
|
|
||||||
else
|
|
||||||
normalize_path "$path_in"
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
normalize_path "$path_in"
|
grep -q "^${1}:" /etc/passwd 2>/dev/null
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
get_svc_mgr() {
|
group_exists() {
|
||||||
if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then echo "systemd"
|
if command -v getent >/dev/null 2>&1; then
|
||||||
elif command -v rc-service >/dev/null 2>&1; then echo "openrc"
|
getent group "$1" >/dev/null 2>&1
|
||||||
else echo "none"; fi
|
else
|
||||||
|
grep -q "^${1}:" /etc/group 2>/dev/null
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
verify_common() {
|
verify_common() {
|
||||||
[ -n "$BIN_NAME" ] || die "BIN_NAME cannot be empty."
|
[ -z "$BIN_NAME" ] && die "BIN_NAME cannot be empty."
|
||||||
[ -n "$INSTALL_DIR" ] || die "INSTALL_DIR cannot be empty."
|
[ -z "$INSTALL_DIR" ] && die "INSTALL_DIR cannot be empty."
|
||||||
[ -n "$CONFIG_DIR" ] || die "CONFIG_DIR cannot be empty."
|
[ -z "$CONFIG_DIR" ] && die "CONFIG_DIR cannot be empty."
|
||||||
[ -n "$CONFIG_FILE" ] || die "CONFIG_FILE cannot be empty."
|
|
||||||
|
|
||||||
case "${INSTALL_DIR}${CONFIG_DIR}${WORK_DIR}${CONFIG_FILE}" in
|
|
||||||
*[!a-zA-Z0-9_./-]*) die "Invalid characters in paths. Only alphanumeric, _, ., -, and / allowed." ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
case "$TARGET_VERSION" in *[!a-zA-Z0-9_.-]*) die "Invalid characters in version." ;; esac
|
|
||||||
case "$BIN_NAME" in *[!a-zA-Z0-9_-]*) die "Invalid characters in BIN_NAME." ;; esac
|
|
||||||
|
|
||||||
INSTALL_DIR="$(get_realpath "$INSTALL_DIR")"
|
|
||||||
CONFIG_DIR="$(get_realpath "$CONFIG_DIR")"
|
|
||||||
WORK_DIR="$(get_realpath "$WORK_DIR")"
|
|
||||||
CONFIG_FILE="$(get_realpath "$CONFIG_FILE")"
|
|
||||||
|
|
||||||
CONFIG_PARENT_DIR="${CONFIG_FILE%/*}"
|
|
||||||
if [ -z "$CONFIG_PARENT_DIR" ]; then CONFIG_PARENT_DIR="/"; fi
|
|
||||||
if [ "$CONFIG_PARENT_DIR" = "$CONFIG_FILE" ]; then CONFIG_PARENT_DIR="."; fi
|
|
||||||
|
|
||||||
if [ "$(id -u)" -eq 0 ]; then
|
if [ "$(id -u)" -eq 0 ]; then
|
||||||
SUDO=""
|
SUDO=""
|
||||||
else
|
else
|
||||||
command -v sudo >/dev/null 2>&1 || die "This script requires root or sudo. Neither found."
|
if ! command -v sudo >/dev/null 2>&1; then
|
||||||
|
die "This script requires root or sudo. Neither found."
|
||||||
|
fi
|
||||||
SUDO="sudo"
|
SUDO="sudo"
|
||||||
if ! sudo -n true 2>/dev/null; then
|
say "sudo is available. Caching credentials..."
|
||||||
if ! [ -t 0 ]; then
|
if ! sudo -v; then
|
||||||
die "sudo requires a password, but no TTY detected. Aborting to prevent hang."
|
die "Failed to cache sudo credentials"
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -n "$SUDO" ]; then
|
case "${INSTALL_DIR}${CONFIG_DIR}${WORK_DIR}" in
|
||||||
if $SUDO sh -c '[ -d "$1" ]' _ "$CONFIG_FILE"; then
|
*[!a-zA-Z0-9_./-]*)
|
||||||
die "Safety check failed: CONFIG_FILE '$CONFIG_FILE' is a directory."
|
die "Invalid characters in path variables. Only alphanumeric, _, ., -, and / are allowed."
|
||||||
fi
|
;;
|
||||||
elif [ -d "$CONFIG_FILE" ]; then
|
esac
|
||||||
die "Safety check failed: CONFIG_FILE '$CONFIG_FILE' is a directory."
|
|
||||||
fi
|
case "$BIN_NAME" in
|
||||||
|
*[!a-zA-Z0-9_-]*) die "Invalid characters in BIN_NAME: $BIN_NAME" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
for path in "$CONFIG_DIR" "$WORK_DIR"; do
|
||||||
|
check_path="$path"
|
||||||
|
|
||||||
|
while [ "$check_path" != "/" ] && [ "${check_path%"/"}" != "$check_path" ]; do
|
||||||
|
check_path="${check_path%"/"}"
|
||||||
|
done
|
||||||
|
[ -z "$check_path" ] && check_path="/"
|
||||||
|
|
||||||
for path in "$CONFIG_DIR" "$CONFIG_PARENT_DIR" "$WORK_DIR"; do
|
|
||||||
check_path="$(get_realpath "$path")"
|
|
||||||
case "$check_path" in
|
case "$check_path" in
|
||||||
/|/bin|/sbin|/usr|/usr/bin|/usr/sbin|/usr/local|/usr/local/bin|/usr/local/sbin|/usr/local/etc|/usr/local/share|/etc|/var|/var/lib|/var/log|/var/run|/home|/root|/tmp|/lib|/lib64|/opt|/run|/boot|/dev|/sys|/proc)
|
/|/bin|/sbin|/usr|/usr/bin|/usr/local|/etc|/opt|/var|/home|/root|/tmp)
|
||||||
die "Safety check failed: '$path' (resolved to '$check_path') is a critical system directory." ;;
|
die "Safety check failed: '$path' is a critical system directory."
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
check_install_dir="$(get_realpath "$INSTALL_DIR")"
|
for cmd in uname grep find rm chown chmod mv head mktemp; do
|
||||||
case "$check_install_dir" in
|
|
||||||
/|/etc|/var|/home|/root|/tmp|/usr|/usr/local|/opt|/boot|/dev|/sys|/proc|/run)
|
|
||||||
die "Safety check failed: INSTALL_DIR '$INSTALL_DIR' is a critical system directory." ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
for cmd in id uname grep find rm chown chmod mv mktemp mkdir tr dd sed ps head sleep cat tar gzip rmdir; do
|
|
||||||
command -v "$cmd" >/dev/null 2>&1 || die "Required command not found: $cmd"
|
command -v "$cmd" >/dev/null 2>&1 || die "Required command not found: $cmd"
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
verify_install_deps() {
|
verify_install_deps() {
|
||||||
command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1 || die "Neither curl nor wget is installed."
|
if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then
|
||||||
|
die "Neither curl nor wget is installed."
|
||||||
|
fi
|
||||||
|
command -v tar >/dev/null 2>&1 || die "Required command not found: tar"
|
||||||
|
command -v gzip >/dev/null 2>&1 || die "Required command not found: gzip"
|
||||||
command -v cp >/dev/null 2>&1 || command -v install >/dev/null 2>&1 || die "Need cp or install"
|
command -v cp >/dev/null 2>&1 || command -v install >/dev/null 2>&1 || die "Need cp or install"
|
||||||
|
|
||||||
if ! command -v setcap >/dev/null 2>&1; then
|
if ! command -v setcap >/dev/null 2>&1; then
|
||||||
|
say "setcap is missing. Installing required capability tools..."
|
||||||
if command -v apk >/dev/null 2>&1; then
|
if command -v apk >/dev/null 2>&1; then
|
||||||
$SUDO apk add --no-cache libcap-utils >/dev/null 2>&1 || $SUDO apk add --no-cache libcap >/dev/null 2>&1 || true
|
$SUDO apk add --no-cache libcap || die "Failed to install libcap"
|
||||||
elif command -v apt-get >/dev/null 2>&1; then
|
elif command -v apt-get >/dev/null 2>&1; then
|
||||||
$SUDO apt-get update -q >/dev/null 2>&1 || true
|
$SUDO apt-get update -qq && $SUDO apt-get install -y -qq libcap2-bin || die "Failed to install libcap2-bin"
|
||||||
$SUDO apt-get install -y -q libcap2-bin >/dev/null 2>&1 || true
|
elif command -v dnf >/dev/null 2>&1 || command -v yum >/dev/null 2>&1; then
|
||||||
elif command -v dnf >/dev/null 2>&1; then $SUDO dnf install -y -q libcap >/dev/null 2>&1 || true
|
$SUDO ${YUM_CMD:-yum} install -y -q libcap || die "Failed to install libcap"
|
||||||
elif command -v yum >/dev/null 2>&1; then $SUDO yum install -y -q libcap >/dev/null 2>&1 || true
|
else
|
||||||
|
die "Cannot install 'setcap'. Package manager not found. Please install libcap manually."
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
@ -199,96 +163,122 @@ detect_arch() {
|
||||||
}
|
}
|
||||||
|
|
||||||
detect_libc() {
|
detect_libc() {
|
||||||
|
if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -qi musl; then
|
||||||
|
echo "musl"; return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if grep -q '^ID=alpine' /etc/os-release 2>/dev/null || grep -q '^ID="alpine"' /etc/os-release 2>/dev/null; then
|
||||||
|
echo "musl"; return 0
|
||||||
|
fi
|
||||||
for f in /lib/ld-musl-*.so.* /lib64/ld-musl-*.so.*; do
|
for f in /lib/ld-musl-*.so.* /lib64/ld-musl-*.so.*; do
|
||||||
if [ -e "$f" ]; then echo "musl"; return 0; fi
|
if [ -e "$f" ]; then
|
||||||
|
echo "musl"; return 0
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
if grep -qE '^ID="?alpine"?' /etc/os-release 2>/dev/null; then echo "musl"; return 0; fi
|
|
||||||
if command -v ldd >/dev/null 2>&1 && (ldd --version 2>&1 || true) | grep -qi musl; then echo "musl"; return 0; fi
|
|
||||||
echo "gnu"
|
echo "gnu"
|
||||||
}
|
}
|
||||||
|
|
||||||
fetch_file() {
|
fetch_file() {
|
||||||
if command -v curl >/dev/null 2>&1; then curl -fsSL "$1" -o "$2"
|
fetch_url="$1"
|
||||||
else wget -q -O "$2" "$1"; fi
|
fetch_out="$2"
|
||||||
|
|
||||||
|
if command -v curl >/dev/null 2>&1; then
|
||||||
|
curl -fsSL "$fetch_url" -o "$fetch_out" || return 1
|
||||||
|
elif command -v wget >/dev/null 2>&1; then
|
||||||
|
wget -qO "$fetch_out" "$fetch_url" || return 1
|
||||||
|
else
|
||||||
|
die "curl or wget required"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
ensure_user_group() {
|
ensure_user_group() {
|
||||||
nologin_bin="$(command -v nologin 2>/dev/null || command -v false 2>/dev/null || echo /bin/false)"
|
nologin_bin="/bin/false"
|
||||||
|
|
||||||
if ! check_os_entity group telemt; then
|
cmd_nologin="$(command -v nologin 2>/dev/null || true)"
|
||||||
if command -v groupadd >/dev/null 2>&1; then $SUDO groupadd -r telemt
|
if [ -n "$cmd_nologin" ] && [ -x "$cmd_nologin" ]; then
|
||||||
elif command -v addgroup >/dev/null 2>&1; then $SUDO addgroup -S telemt
|
nologin_bin="$cmd_nologin"
|
||||||
else die "Cannot create group"; fi
|
else
|
||||||
|
for bin in /sbin/nologin /usr/sbin/nologin; do
|
||||||
|
if [ -x "$bin" ]; then
|
||||||
|
nologin_bin="$bin"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! check_os_entity passwd telemt; then
|
if ! group_exists telemt; then
|
||||||
|
if command -v groupadd >/dev/null 2>&1; then
|
||||||
|
$SUDO groupadd -r telemt || die "Failed to create group via groupadd"
|
||||||
|
elif command -v addgroup >/dev/null 2>&1; then
|
||||||
|
$SUDO addgroup -S telemt || die "Failed to create group via addgroup"
|
||||||
|
else
|
||||||
|
die "Cannot create group: neither groupadd nor addgroup found"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! user_exists telemt; then
|
||||||
if command -v useradd >/dev/null 2>&1; then
|
if command -v useradd >/dev/null 2>&1; then
|
||||||
$SUDO useradd -r -g telemt -d "$WORK_DIR" -s "$nologin_bin" -c "Telemt Proxy" telemt
|
$SUDO useradd -r -g telemt -d "$WORK_DIR" -s "$nologin_bin" -c "Telemt Proxy" telemt || die "Failed to create user via useradd"
|
||||||
elif command -v adduser >/dev/null 2>&1; then
|
elif command -v adduser >/dev/null 2>&1; then
|
||||||
if adduser --help 2>&1 | grep -q -- '-S'; then
|
$SUDO adduser -S -D -H -h "$WORK_DIR" -s "$nologin_bin" -G telemt telemt || die "Failed to create user via adduser"
|
||||||
$SUDO adduser -S -D -H -h "$WORK_DIR" -s "$nologin_bin" -G telemt telemt
|
else
|
||||||
else
|
die "Cannot create user: neither useradd nor adduser found"
|
||||||
$SUDO adduser --system --home "$WORK_DIR" --shell "$nologin_bin" --no-create-home --ingroup telemt --disabled-password telemt
|
fi
|
||||||
fi
|
|
||||||
else die "Cannot create user"; fi
|
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
setup_dirs() {
|
setup_dirs() {
|
||||||
$SUDO mkdir -p "$WORK_DIR" "$CONFIG_DIR" "$CONFIG_PARENT_DIR" || die "Failed to create directories"
|
say "Setting up directories..."
|
||||||
|
$SUDO mkdir -p "$WORK_DIR" "$CONFIG_DIR" || die "Failed to create directories"
|
||||||
$SUDO chown telemt:telemt "$WORK_DIR" && $SUDO chmod 750 "$WORK_DIR"
|
$SUDO chown telemt:telemt "$WORK_DIR" || die "Failed to set owner on WORK_DIR"
|
||||||
$SUDO chown root:telemt "$CONFIG_DIR" && $SUDO chmod 750 "$CONFIG_DIR"
|
$SUDO chmod 750 "$WORK_DIR" || die "Failed to set permissions on WORK_DIR"
|
||||||
|
|
||||||
if [ "$CONFIG_PARENT_DIR" != "$CONFIG_DIR" ] && [ "$CONFIG_PARENT_DIR" != "." ] && [ "$CONFIG_PARENT_DIR" != "/" ]; then
|
|
||||||
$SUDO chown root:telemt "$CONFIG_PARENT_DIR" && $SUDO chmod 750 "$CONFIG_PARENT_DIR"
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
stop_service() {
|
stop_service() {
|
||||||
svc="$(get_svc_mgr)"
|
say "Stopping service if running..."
|
||||||
if [ "$svc" = "systemd" ] && systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
|
if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
|
||||||
$SUDO systemctl stop "$SERVICE_NAME" 2>/dev/null || true
|
$SUDO systemctl stop "$SERVICE_NAME" 2>/dev/null || true
|
||||||
elif [ "$svc" = "openrc" ] && rc-service "$SERVICE_NAME" status >/dev/null 2>&1; then
|
elif command -v rc-service >/dev/null 2>&1; then
|
||||||
$SUDO rc-service "$SERVICE_NAME" stop 2>/dev/null || true
|
$SUDO rc-service "$SERVICE_NAME" stop 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
install_binary() {
|
install_binary() {
|
||||||
bin_src="$1"; bin_dst="$2"
|
bin_src="$1"
|
||||||
if [ -e "$INSTALL_DIR" ] && [ ! -d "$INSTALL_DIR" ]; then
|
bin_dst="$2"
|
||||||
die "'$INSTALL_DIR' is not a directory."
|
|
||||||
fi
|
|
||||||
|
|
||||||
$SUDO mkdir -p "$INSTALL_DIR" || die "Failed to create install directory"
|
$SUDO mkdir -p "$INSTALL_DIR" || die "Failed to create install directory"
|
||||||
if command -v install >/dev/null 2>&1; then
|
if command -v install >/dev/null 2>&1; then
|
||||||
$SUDO install -m 0755 "$bin_src" "$bin_dst" || die "Failed to install binary"
|
$SUDO install -m 0755 "$bin_src" "$bin_dst" || die "Failed to install binary"
|
||||||
else
|
else
|
||||||
$SUDO rm -f "$bin_dst" 2>/dev/null || true
|
$SUDO rm -f "$bin_dst"
|
||||||
$SUDO cp "$bin_src" "$bin_dst" && $SUDO chmod 0755 "$bin_dst" || die "Failed to copy binary"
|
$SUDO cp "$bin_src" "$bin_dst" || die "Failed to copy binary"
|
||||||
|
$SUDO chmod 0755 "$bin_dst" || die "Failed to set permissions"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
$SUDO sh -c '[ -x "$1" ]' _ "$bin_dst" || die "Binary not executable: $bin_dst"
|
if [ ! -x "$bin_dst" ]; then
|
||||||
|
die "Failed to install binary or it is not executable: $bin_dst"
|
||||||
|
fi
|
||||||
|
|
||||||
if command -v setcap >/dev/null 2>&1; then
|
say "Granting network bind capabilities to bind port 443..."
|
||||||
$SUDO setcap cap_net_bind_service=+ep "$bin_dst" 2>/dev/null || true
|
if ! $SUDO setcap cap_net_bind_service=+ep "$bin_dst" 2>/dev/null; then
|
||||||
|
say "[WARNING] Failed to apply setcap. The service will NOT be able to open port 443!"
|
||||||
|
say "[WARNING] This usually happens inside unprivileged Docker/LXC containers."
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
generate_secret() {
|
generate_secret() {
|
||||||
secret="$(command -v openssl >/dev/null 2>&1 && openssl rand -hex 16 2>/dev/null || true)"
|
if command -v openssl >/dev/null 2>&1; then
|
||||||
if [ -z "$secret" ] || [ "${#secret}" -ne 32 ]; then
|
secret="$(openssl rand -hex 16 2>/dev/null)" && [ -n "$secret" ] && { echo "$secret"; return 0; }
|
||||||
if command -v od >/dev/null 2>&1; then secret="$(dd if=/dev/urandom bs=16 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
|
|
||||||
elif command -v hexdump >/dev/null 2>&1; then secret="$(dd if=/dev/urandom bs=16 count=1 2>/dev/null | hexdump -e '1/1 "%02x"')"
|
|
||||||
elif command -v xxd >/dev/null 2>&1; then secret="$(dd if=/dev/urandom bs=16 count=1 2>/dev/null | xxd -p | tr -d '\n')"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
if [ "${#secret}" -eq 32 ]; then echo "$secret"; else return 1; fi
|
if command -v xxd >/dev/null 2>&1; then
|
||||||
|
secret="$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | xxd -p | tr -d '\n')" && [ -n "$secret" ] && { echo "$secret"; return 0; }
|
||||||
|
fi
|
||||||
|
secret="$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | od -An -tx1 | tr -d ' \n')" && [ -n "$secret" ] && { echo "$secret"; return 0; }
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
generate_config_content() {
|
generate_config_content() {
|
||||||
escaped_tls_domain="$(printf '%s\n' "$TLS_DOMAIN" | tr -d '[:cntrl:]' | sed 's/\\/\\\\/g; s/"/\\"/g')"
|
|
||||||
|
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
[general]
|
[general]
|
||||||
use_middle_proxy = false
|
use_middle_proxy = false
|
||||||
|
|
@ -307,7 +297,7 @@ listen = "127.0.0.1:9091"
|
||||||
whitelist = ["127.0.0.1/32"]
|
whitelist = ["127.0.0.1/32"]
|
||||||
|
|
||||||
[censorship]
|
[censorship]
|
||||||
tls_domain = "${escaped_tls_domain}"
|
tls_domain = "petrovich.ru"
|
||||||
|
|
||||||
[access.users]
|
[access.users]
|
||||||
hello = "$1"
|
hello = "$1"
|
||||||
|
|
@ -315,38 +305,44 @@ EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
install_config() {
|
install_config() {
|
||||||
|
config_exists=0
|
||||||
|
|
||||||
if [ -n "$SUDO" ]; then
|
if [ -n "$SUDO" ]; then
|
||||||
if $SUDO sh -c '[ -f "$1" ]' _ "$CONFIG_FILE"; then
|
$SUDO sh -c "[ -f '$CONFIG_FILE' ]" 2>/dev/null && config_exists=1 || true
|
||||||
say " -> Config already exists at $CONFIG_FILE. Skipping creation."
|
else
|
||||||
return 0
|
[ -f "$CONFIG_FILE" ] && config_exists=1 || true
|
||||||
fi
|
fi
|
||||||
elif [ -f "$CONFIG_FILE" ]; then
|
|
||||||
say " -> Config already exists at $CONFIG_FILE. Skipping creation."
|
if [ "$config_exists" -eq 1 ]; then
|
||||||
|
say "Config already exists, skipping generation."
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
toml_secret="$(generate_secret)" || die "Failed to generate secret."
|
toml_secret="$(generate_secret)" || die "Failed to generate secret"
|
||||||
|
say "Creating config at $CONFIG_FILE..."
|
||||||
|
|
||||||
generate_config_content "$toml_secret" | write_root "$CONFIG_FILE" || die "Failed to install config"
|
tmp_conf="$(mktemp "${TEMP_DIR:-/tmp}/telemt_conf.XXXXXX")" || die "Failed to create temp config"
|
||||||
$SUDO chown root:telemt "$CONFIG_FILE" && $SUDO chmod 640 "$CONFIG_FILE"
|
generate_config_content "$toml_secret" > "$tmp_conf" || die "Failed to write temp config"
|
||||||
|
|
||||||
say " -> Config created successfully."
|
$SUDO mv "$tmp_conf" "$CONFIG_FILE" || die "Failed to install config file"
|
||||||
say " -> Generated secret for default user 'hello': $toml_secret"
|
$SUDO chown root:telemt "$CONFIG_FILE" || die "Failed to set owner"
|
||||||
|
$SUDO chmod 640 "$CONFIG_FILE" || die "Failed to set config permissions"
|
||||||
|
|
||||||
|
say "Secret for user 'hello': $toml_secret"
|
||||||
}
|
}
|
||||||
|
|
||||||
generate_systemd_content() {
|
generate_systemd_content() {
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=Telemt
|
Description=Telemt Proxy Service
|
||||||
After=network-online.target
|
After=network-online.target
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
User=telemt
|
User=telemt
|
||||||
Group=telemt
|
Group=telemt
|
||||||
WorkingDirectory=$WORK_DIR
|
WorkingDirectory=$WORK_DIR
|
||||||
ExecStart="${INSTALL_DIR}/${BIN_NAME}" "${CONFIG_FILE}"
|
ExecStart=${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
LimitNOFILE=65536
|
LimitNOFILE=65536
|
||||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
||||||
|
|
@ -374,119 +370,111 @@ EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
install_service() {
|
install_service() {
|
||||||
svc="$(get_svc_mgr)"
|
if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
|
||||||
if [ "$svc" = "systemd" ]; then
|
say "Installing systemd service..."
|
||||||
generate_systemd_content | write_root "/etc/systemd/system/${SERVICE_NAME}.service"
|
tmp_svc="$(mktemp "${TEMP_DIR:-/tmp}/${SERVICE_NAME}.service.XXXXXX")" || die "Failed to create temp service"
|
||||||
$SUDO chown root:root "/etc/systemd/system/${SERVICE_NAME}.service" && $SUDO chmod 644 "/etc/systemd/system/${SERVICE_NAME}.service"
|
generate_systemd_content > "$tmp_svc" || die "Failed to generate service content"
|
||||||
|
|
||||||
$SUDO systemctl daemon-reload || true
|
$SUDO mv "$tmp_svc" "/etc/systemd/system/${SERVICE_NAME}.service" || die "Failed to move service file"
|
||||||
$SUDO systemctl enable "$SERVICE_NAME" || true
|
$SUDO chown root:root "/etc/systemd/system/${SERVICE_NAME}.service"
|
||||||
|
$SUDO chmod 644 "/etc/systemd/system/${SERVICE_NAME}.service"
|
||||||
|
|
||||||
if ! $SUDO systemctl start "$SERVICE_NAME"; then
|
$SUDO systemctl daemon-reload || die "Failed to reload systemd"
|
||||||
say "[WARNING] Failed to start service"
|
$SUDO systemctl enable "$SERVICE_NAME" || die "Failed to enable service"
|
||||||
SERVICE_START_FAILED=1
|
$SUDO systemctl start "$SERVICE_NAME" || die "Failed to start service"
|
||||||
fi
|
|
||||||
elif [ "$svc" = "openrc" ]; then
|
|
||||||
generate_openrc_content | write_root "/etc/init.d/${SERVICE_NAME}"
|
|
||||||
$SUDO chown root:root "/etc/init.d/${SERVICE_NAME}" && $SUDO chmod 0755 "/etc/init.d/${SERVICE_NAME}"
|
|
||||||
|
|
||||||
$SUDO rc-update add "$SERVICE_NAME" default 2>/dev/null || true
|
elif command -v rc-update >/dev/null 2>&1; then
|
||||||
|
say "Installing OpenRC service..."
|
||||||
|
tmp_svc="$(mktemp "${TEMP_DIR:-/tmp}/${SERVICE_NAME}.init.XXXXXX")" || die "Failed to create temp file"
|
||||||
|
generate_openrc_content > "$tmp_svc" || die "Failed to generate init content"
|
||||||
|
|
||||||
if ! $SUDO rc-service "$SERVICE_NAME" start 2>/dev/null; then
|
$SUDO mv "$tmp_svc" "/etc/init.d/${SERVICE_NAME}" || die "Failed to move service file"
|
||||||
say "[WARNING] Failed to start service"
|
$SUDO chown root:root "/etc/init.d/${SERVICE_NAME}"
|
||||||
SERVICE_START_FAILED=1
|
$SUDO chmod 0755 "/etc/init.d/${SERVICE_NAME}"
|
||||||
fi
|
|
||||||
|
$SUDO rc-update add "$SERVICE_NAME" default 2>/dev/null || die "Failed to register service"
|
||||||
|
$SUDO rc-service "$SERVICE_NAME" start 2>/dev/null || die "Failed to start OpenRC service"
|
||||||
else
|
else
|
||||||
cmd="\"${INSTALL_DIR}/${BIN_NAME}\" \"${CONFIG_FILE}\""
|
say "No service manager found. You can start it manually with:"
|
||||||
if [ -n "$SUDO" ]; then
|
if [ -n "$SUDO" ]; then
|
||||||
say " -> Service manager not found. Start manually: sudo -u telemt $cmd"
|
say " sudo -u telemt ${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}"
|
||||||
else
|
else
|
||||||
say " -> Service manager not found. Start manually: su -s /bin/sh telemt -c '$cmd'"
|
say " su -s /bin/sh telemt -c '${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}'"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
kill_user_procs() {
|
kill_user_procs() {
|
||||||
if command -v pkill >/dev/null 2>&1; then
|
say "Ensuring $BIN_NAME processes are killed..."
|
||||||
$SUDO pkill -u telemt "$BIN_NAME" 2>/dev/null || true
|
|
||||||
sleep 1
|
|
||||||
$SUDO pkill -9 -u telemt "$BIN_NAME" 2>/dev/null || true
|
|
||||||
else
|
|
||||||
if command -v pgrep >/dev/null 2>&1; then
|
|
||||||
pids="$(pgrep -u telemt 2>/dev/null || true)"
|
|
||||||
else
|
|
||||||
pids="$(ps -u telemt -o pid= 2>/dev/null || true)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$pids" ]; then
|
if pkill_cmd="$(command -v pkill 2>/dev/null)"; then
|
||||||
for pid in $pids; do
|
$SUDO "$pkill_cmd" -u telemt "$BIN_NAME" 2>/dev/null || true
|
||||||
case "$pid" in ''|*[!0-9]*) continue ;; *) $SUDO kill "$pid" 2>/dev/null || true ;; esac
|
sleep 1
|
||||||
done
|
$SUDO "$pkill_cmd" -9 -u telemt "$BIN_NAME" 2>/dev/null || true
|
||||||
sleep 1
|
elif killall_cmd="$(command -v killall 2>/dev/null)"; then
|
||||||
for pid in $pids; do
|
$SUDO "$killall_cmd" "$BIN_NAME" 2>/dev/null || true
|
||||||
case "$pid" in ''|*[!0-9]*) continue ;; *) $SUDO kill -9 "$pid" 2>/dev/null || true ;; esac
|
sleep 1
|
||||||
done
|
$SUDO "$killall_cmd" -9 "$BIN_NAME" 2>/dev/null || true
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
uninstall() {
|
uninstall() {
|
||||||
say "Starting uninstallation of $BIN_NAME..."
|
purge_data=0
|
||||||
|
[ "$ACTION" = "purge" ] && purge_data=1
|
||||||
|
|
||||||
say ">>> Stage 1: Stopping services"
|
say "Uninstalling $BIN_NAME..."
|
||||||
stop_service
|
stop_service
|
||||||
|
|
||||||
say ">>> Stage 2: Removing service configuration"
|
if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
|
||||||
svc="$(get_svc_mgr)"
|
|
||||||
if [ "$svc" = "systemd" ]; then
|
|
||||||
$SUDO systemctl disable "$SERVICE_NAME" 2>/dev/null || true
|
$SUDO systemctl disable "$SERVICE_NAME" 2>/dev/null || true
|
||||||
$SUDO rm -f "/etc/systemd/system/${SERVICE_NAME}.service"
|
$SUDO rm -f "/etc/systemd/system/${SERVICE_NAME}.service"
|
||||||
$SUDO systemctl daemon-reload 2>/dev/null || true
|
$SUDO systemctl daemon-reload || true
|
||||||
elif [ "$svc" = "openrc" ]; then
|
elif command -v rc-update >/dev/null 2>&1; then
|
||||||
$SUDO rc-update del "$SERVICE_NAME" 2>/dev/null || true
|
$SUDO rc-update del "$SERVICE_NAME" 2>/dev/null || true
|
||||||
$SUDO rm -f "/etc/init.d/${SERVICE_NAME}"
|
$SUDO rm -f "/etc/init.d/${SERVICE_NAME}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
say ">>> Stage 3: Terminating user processes"
|
|
||||||
kill_user_procs
|
kill_user_procs
|
||||||
|
|
||||||
say ">>> Stage 4: Removing binary"
|
|
||||||
$SUDO rm -f "${INSTALL_DIR}/${BIN_NAME}"
|
$SUDO rm -f "${INSTALL_DIR}/${BIN_NAME}"
|
||||||
|
|
||||||
if [ "$ACTION" = "purge" ]; then
|
$SUDO userdel telemt 2>/dev/null || $SUDO deluser telemt 2>/dev/null || true
|
||||||
say ">>> Stage 5: Purging configuration, data, and user"
|
$SUDO groupdel telemt 2>/dev/null || $SUDO delgroup telemt 2>/dev/null || true
|
||||||
|
|
||||||
|
if [ "$purge_data" -eq 1 ]; then
|
||||||
|
say "Purging configuration and data..."
|
||||||
$SUDO rm -rf "$CONFIG_DIR" "$WORK_DIR"
|
$SUDO rm -rf "$CONFIG_DIR" "$WORK_DIR"
|
||||||
$SUDO rm -f "$CONFIG_FILE"
|
|
||||||
if [ "$CONFIG_PARENT_DIR" != "$CONFIG_DIR" ] && [ "$CONFIG_PARENT_DIR" != "." ] && [ "$CONFIG_PARENT_DIR" != "/" ]; then
|
|
||||||
$SUDO rmdir "$CONFIG_PARENT_DIR" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
$SUDO userdel telemt 2>/dev/null || $SUDO deluser telemt 2>/dev/null || true
|
|
||||||
$SUDO groupdel telemt 2>/dev/null || $SUDO delgroup telemt 2>/dev/null || true
|
|
||||||
else
|
else
|
||||||
say "Note: Configuration and user kept. Run with 'purge' to remove completely."
|
say "Note: Configuration in $CONFIG_DIR was kept. Run with '--purge' to remove it."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
printf '\n====================================================================\n'
|
say "Uninstallation complete."
|
||||||
printf ' UNINSTALLATION COMPLETE\n'
|
|
||||||
printf '====================================================================\n\n'
|
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Main Entry Point
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
case "$ACTION" in
|
case "$ACTION" in
|
||||||
help) show_help ;;
|
help)
|
||||||
uninstall|purge) verify_common; uninstall ;;
|
show_help
|
||||||
|
;;
|
||||||
|
uninstall|purge)
|
||||||
|
verify_common
|
||||||
|
uninstall
|
||||||
|
;;
|
||||||
install)
|
install)
|
||||||
say "Starting installation of $BIN_NAME (Version: $TARGET_VERSION)"
|
say "Starting installation..."
|
||||||
|
verify_common
|
||||||
|
verify_install_deps
|
||||||
|
|
||||||
say ">>> Stage 1: Verifying environment and dependencies"
|
ARCH="$(detect_arch)"
|
||||||
verify_common; verify_install_deps
|
LIBC="$(detect_libc)"
|
||||||
|
say "Detected system: $ARCH-linux-$LIBC"
|
||||||
|
|
||||||
if [ "$TARGET_VERSION" != "latest" ]; then
|
|
||||||
TARGET_VERSION="${TARGET_VERSION#v}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
ARCH="$(detect_arch)"; LIBC="$(detect_libc)"
|
|
||||||
FILE_NAME="${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz"
|
FILE_NAME="${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz"
|
||||||
|
FILE_NAME="$(printf '%s' "$FILE_NAME" | tr -d ' \t\n\r')"
|
||||||
|
|
||||||
if [ "$TARGET_VERSION" = "latest" ]; then
|
if [ "$TARGET_VERSION" = "latest" ]; then
|
||||||
DL_URL="https://github.com/${REPO}/releases/latest/download/${FILE_NAME}"
|
DL_URL="https://github.com/${REPO}/releases/latest/download/${FILE_NAME}"
|
||||||
|
|
@ -494,63 +482,44 @@ case "$ACTION" in
|
||||||
DL_URL="https://github.com/${REPO}/releases/download/${TARGET_VERSION}/${FILE_NAME}"
|
DL_URL="https://github.com/${REPO}/releases/download/${TARGET_VERSION}/${FILE_NAME}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
say ">>> Stage 2: Downloading archive"
|
TEMP_DIR="$(mktemp -d)" || die "Failed to create temp directory"
|
||||||
TEMP_DIR="$(mktemp -d)" || die "Temp directory creation failed"
|
|
||||||
if [ -z "$TEMP_DIR" ] || [ ! -d "$TEMP_DIR" ]; then
|
if [ -z "$TEMP_DIR" ] || [ ! -d "$TEMP_DIR" ]; then
|
||||||
die "Temp directory is invalid or was not created"
|
die "Temp directory creation failed"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
fetch_file "$DL_URL" "${TEMP_DIR}/${FILE_NAME}" || die "Download failed"
|
say "Downloading from $DL_URL..."
|
||||||
|
fetch_file "$DL_URL" "${TEMP_DIR}/archive.tar.gz" || die "Download failed (check version or network)"
|
||||||
|
|
||||||
say ">>> Stage 3: Extracting archive"
|
gzip -dc "${TEMP_DIR}/archive.tar.gz" | tar -xf - -C "$TEMP_DIR" || die "Extraction failed"
|
||||||
if ! gzip -dc "${TEMP_DIR}/${FILE_NAME}" | tar -xf - -C "$TEMP_DIR" 2>/dev/null; then
|
|
||||||
die "Extraction failed (downloaded archive might be invalid or 404)."
|
|
||||||
fi
|
|
||||||
|
|
||||||
EXTRACTED_BIN="$(find "$TEMP_DIR" -type f -name "$BIN_NAME" -print 2>/dev/null | head -n 1 || true)"
|
EXTRACTED_BIN="$(find "$TEMP_DIR" -type f -name "$BIN_NAME" -print 2>/dev/null | head -n 1)"
|
||||||
[ -n "$EXTRACTED_BIN" ] || die "Binary '$BIN_NAME' not found in archive"
|
[ -z "$EXTRACTED_BIN" ] && die "Binary '$BIN_NAME' not found in archive"
|
||||||
|
|
||||||
say ">>> Stage 4: Setting up environment (User, Group, Directories)"
|
ensure_user_group
|
||||||
ensure_user_group; setup_dirs; stop_service
|
setup_dirs
|
||||||
|
stop_service
|
||||||
|
|
||||||
say ">>> Stage 5: Installing binary"
|
say "Installing binary..."
|
||||||
install_binary "$EXTRACTED_BIN" "${INSTALL_DIR}/${BIN_NAME}"
|
install_binary "$EXTRACTED_BIN" "${INSTALL_DIR}/${BIN_NAME}"
|
||||||
|
|
||||||
say ">>> Stage 6: Generating configuration"
|
|
||||||
install_config
|
install_config
|
||||||
|
|
||||||
say ">>> Stage 7: Installing and starting service"
|
|
||||||
install_service
|
install_service
|
||||||
|
|
||||||
if [ "${SERVICE_START_FAILED:-0}" -eq 1 ]; then
|
say ""
|
||||||
printf '\n====================================================================\n'
|
say "============================================="
|
||||||
printf ' INSTALLATION COMPLETED WITH WARNINGS\n'
|
say "Installation complete!"
|
||||||
printf '====================================================================\n\n'
|
say "============================================="
|
||||||
printf 'The service was installed but failed to start automatically.\n'
|
if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
|
||||||
printf 'Please check the logs to determine the issue.\n\n'
|
say "To check the logs, run:"
|
||||||
else
|
say " journalctl -u $SERVICE_NAME -f"
|
||||||
printf '\n====================================================================\n'
|
say ""
|
||||||
printf ' INSTALLATION SUCCESS\n'
|
|
||||||
printf '====================================================================\n\n'
|
|
||||||
fi
|
fi
|
||||||
|
say "To get user connection links, run:"
|
||||||
svc="$(get_svc_mgr)"
|
|
||||||
if [ "$svc" = "systemd" ]; then
|
|
||||||
printf 'To check the status of your proxy service, run:\n'
|
|
||||||
printf ' systemctl status %s\n\n' "$SERVICE_NAME"
|
|
||||||
elif [ "$svc" = "openrc" ]; then
|
|
||||||
printf 'To check the status of your proxy service, run:\n'
|
|
||||||
printf ' rc-service %s status\n\n' "$SERVICE_NAME"
|
|
||||||
fi
|
|
||||||
|
|
||||||
printf 'To get your user connection links (for Telegram), run:\n'
|
|
||||||
if command -v jq >/dev/null 2>&1; then
|
if command -v jq >/dev/null 2>&1; then
|
||||||
printf ' curl -s http://127.0.0.1:9091/v1/users | jq -r '\''.data[] | "User: \\(.username)\\n\\(.links.tls[0] // empty)\\n"'\''\n'
|
say " curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | \"User: \\(.username)\\n\\(.links.tls[0] // empty)\"'"
|
||||||
else
|
else
|
||||||
printf ' curl -s http://127.0.0.1:9091/v1/users\n'
|
say " curl -s http://127.0.0.1:9091/v1/users"
|
||||||
printf ' (Tip: Install '\''jq'\'' for a much cleaner output)\n'
|
say " (Note: Install 'jq' package to see the links nicely formatted)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
printf '\n====================================================================\n'
|
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
|
|
@ -364,7 +364,6 @@ pub(super) struct MinimalMeRuntimeData {
|
||||||
pub(super) me_reconnect_backoff_cap_ms: u64,
|
pub(super) me_reconnect_backoff_cap_ms: u64,
|
||||||
pub(super) me_reconnect_fast_retry_count: u32,
|
pub(super) me_reconnect_fast_retry_count: u32,
|
||||||
pub(super) me_pool_drain_ttl_secs: u64,
|
pub(super) me_pool_drain_ttl_secs: u64,
|
||||||
pub(super) me_instadrain: bool,
|
|
||||||
pub(super) me_pool_drain_soft_evict_enabled: bool,
|
pub(super) me_pool_drain_soft_evict_enabled: bool,
|
||||||
pub(super) me_pool_drain_soft_evict_grace_secs: u64,
|
pub(super) me_pool_drain_soft_evict_grace_secs: u64,
|
||||||
pub(super) me_pool_drain_soft_evict_per_writer: u8,
|
pub(super) me_pool_drain_soft_evict_per_writer: u8,
|
||||||
|
|
|
||||||
|
|
@ -431,7 +431,6 @@ async fn get_minimal_payload_cached(
|
||||||
me_reconnect_backoff_cap_ms: runtime.me_reconnect_backoff_cap_ms,
|
me_reconnect_backoff_cap_ms: runtime.me_reconnect_backoff_cap_ms,
|
||||||
me_reconnect_fast_retry_count: runtime.me_reconnect_fast_retry_count,
|
me_reconnect_fast_retry_count: runtime.me_reconnect_fast_retry_count,
|
||||||
me_pool_drain_ttl_secs: runtime.me_pool_drain_ttl_secs,
|
me_pool_drain_ttl_secs: runtime.me_pool_drain_ttl_secs,
|
||||||
me_instadrain: runtime.me_instadrain,
|
|
||||||
me_pool_drain_soft_evict_enabled: runtime.me_pool_drain_soft_evict_enabled,
|
me_pool_drain_soft_evict_enabled: runtime.me_pool_drain_soft_evict_enabled,
|
||||||
me_pool_drain_soft_evict_grace_secs: runtime.me_pool_drain_soft_evict_grace_secs,
|
me_pool_drain_soft_evict_grace_secs: runtime.me_pool_drain_soft_evict_grace_secs,
|
||||||
me_pool_drain_soft_evict_per_writer: runtime.me_pool_drain_soft_evict_per_writer,
|
me_pool_drain_soft_evict_per_writer: runtime.me_pool_drain_soft_evict_per_writer,
|
||||||
|
|
|
||||||
|
|
@ -198,7 +198,6 @@ desync_all_full = false
|
||||||
update_every = 43200
|
update_every = 43200
|
||||||
hardswap = false
|
hardswap = false
|
||||||
me_pool_drain_ttl_secs = 90
|
me_pool_drain_ttl_secs = 90
|
||||||
me_instadrain = false
|
|
||||||
me_pool_min_fresh_ratio = 0.8
|
me_pool_min_fresh_ratio = 0.8
|
||||||
me_reinit_drain_timeout_secs = 120
|
me_reinit_drain_timeout_secs = 120
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -613,10 +613,6 @@ pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||||
90
|
90
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_me_instadrain() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
||||||
128
|
128
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,6 @@ pub struct HotFields {
|
||||||
pub me_reinit_coalesce_window_ms: u64,
|
pub me_reinit_coalesce_window_ms: u64,
|
||||||
pub hardswap: bool,
|
pub hardswap: bool,
|
||||||
pub me_pool_drain_ttl_secs: u64,
|
pub me_pool_drain_ttl_secs: u64,
|
||||||
pub me_instadrain: bool,
|
|
||||||
pub me_pool_drain_threshold: u64,
|
pub me_pool_drain_threshold: u64,
|
||||||
pub me_pool_drain_soft_evict_enabled: bool,
|
pub me_pool_drain_soft_evict_enabled: bool,
|
||||||
pub me_pool_drain_soft_evict_grace_secs: u64,
|
pub me_pool_drain_soft_evict_grace_secs: u64,
|
||||||
|
|
@ -144,7 +143,6 @@ impl HotFields {
|
||||||
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
|
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
|
||||||
hardswap: cfg.general.hardswap,
|
hardswap: cfg.general.hardswap,
|
||||||
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
|
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
|
||||||
me_instadrain: cfg.general.me_instadrain,
|
|
||||||
me_pool_drain_threshold: cfg.general.me_pool_drain_threshold,
|
me_pool_drain_threshold: cfg.general.me_pool_drain_threshold,
|
||||||
me_pool_drain_soft_evict_enabled: cfg.general.me_pool_drain_soft_evict_enabled,
|
me_pool_drain_soft_evict_enabled: cfg.general.me_pool_drain_soft_evict_enabled,
|
||||||
me_pool_drain_soft_evict_grace_secs: cfg.general.me_pool_drain_soft_evict_grace_secs,
|
me_pool_drain_soft_evict_grace_secs: cfg.general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
@ -479,7 +477,6 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
||||||
cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms;
|
cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms;
|
||||||
cfg.general.hardswap = new.general.hardswap;
|
cfg.general.hardswap = new.general.hardswap;
|
||||||
cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs;
|
cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs;
|
||||||
cfg.general.me_instadrain = new.general.me_instadrain;
|
|
||||||
cfg.general.me_pool_drain_threshold = new.general.me_pool_drain_threshold;
|
cfg.general.me_pool_drain_threshold = new.general.me_pool_drain_threshold;
|
||||||
cfg.general.me_pool_drain_soft_evict_enabled = new.general.me_pool_drain_soft_evict_enabled;
|
cfg.general.me_pool_drain_soft_evict_enabled = new.general.me_pool_drain_soft_evict_enabled;
|
||||||
cfg.general.me_pool_drain_soft_evict_grace_secs =
|
cfg.general.me_pool_drain_soft_evict_grace_secs =
|
||||||
|
|
@ -872,12 +869,6 @@ fn log_changes(
|
||||||
old_hot.me_pool_drain_ttl_secs, new_hot.me_pool_drain_ttl_secs,
|
old_hot.me_pool_drain_ttl_secs, new_hot.me_pool_drain_ttl_secs,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if old_hot.me_instadrain != new_hot.me_instadrain {
|
|
||||||
info!(
|
|
||||||
"config reload: me_instadrain: {} → {}",
|
|
||||||
old_hot.me_instadrain, new_hot.me_instadrain,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if old_hot.me_pool_drain_threshold != new_hot.me_pool_drain_threshold {
|
if old_hot.me_pool_drain_threshold != new_hot.me_pool_drain_threshold {
|
||||||
info!(
|
info!(
|
||||||
|
|
|
||||||
|
|
@ -612,11 +612,6 @@ impl ProxyConfig {
|
||||||
"general.me_route_backpressure_base_timeout_ms must be > 0".to_string(),
|
"general.me_route_backpressure_base_timeout_ms must be > 0".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if config.general.me_route_backpressure_base_timeout_ms > 5000 {
|
|
||||||
return Err(ProxyError::Config(
|
|
||||||
"general.me_route_backpressure_base_timeout_ms must be within [1, 5000]".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
if config.general.me_route_backpressure_high_timeout_ms
|
if config.general.me_route_backpressure_high_timeout_ms
|
||||||
< config.general.me_route_backpressure_base_timeout_ms
|
< config.general.me_route_backpressure_base_timeout_ms
|
||||||
|
|
@ -625,11 +620,6 @@ impl ProxyConfig {
|
||||||
"general.me_route_backpressure_high_timeout_ms must be >= general.me_route_backpressure_base_timeout_ms".to_string(),
|
"general.me_route_backpressure_high_timeout_ms must be >= general.me_route_backpressure_base_timeout_ms".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if config.general.me_route_backpressure_high_timeout_ms > 5000 {
|
|
||||||
return Err(ProxyError::Config(
|
|
||||||
"general.me_route_backpressure_high_timeout_ms must be within [1, 5000]".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
if !(1..=100).contains(&config.general.me_route_backpressure_high_watermark_pct) {
|
if !(1..=100).contains(&config.general.me_route_backpressure_high_watermark_pct) {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
|
|
@ -1634,47 +1624,6 @@ mod tests {
|
||||||
let _ = std::fs::remove_file(path_valid);
|
let _ = std::fs::remove_file(path_valid);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn me_route_backpressure_base_timeout_ms_out_of_range_is_rejected() {
|
|
||||||
let toml = r#"
|
|
||||||
[general]
|
|
||||||
me_route_backpressure_base_timeout_ms = 5001
|
|
||||||
|
|
||||||
[censorship]
|
|
||||||
tls_domain = "example.com"
|
|
||||||
|
|
||||||
[access.users]
|
|
||||||
user = "00000000000000000000000000000000"
|
|
||||||
"#;
|
|
||||||
let dir = std::env::temp_dir();
|
|
||||||
let path = dir.join("telemt_me_route_backpressure_base_timeout_ms_out_of_range_test.toml");
|
|
||||||
std::fs::write(&path, toml).unwrap();
|
|
||||||
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
|
||||||
assert!(err.contains("general.me_route_backpressure_base_timeout_ms must be within [1, 5000]"));
|
|
||||||
let _ = std::fs::remove_file(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn me_route_backpressure_high_timeout_ms_out_of_range_is_rejected() {
|
|
||||||
let toml = r#"
|
|
||||||
[general]
|
|
||||||
me_route_backpressure_base_timeout_ms = 100
|
|
||||||
me_route_backpressure_high_timeout_ms = 5001
|
|
||||||
|
|
||||||
[censorship]
|
|
||||||
tls_domain = "example.com"
|
|
||||||
|
|
||||||
[access.users]
|
|
||||||
user = "00000000000000000000000000000000"
|
|
||||||
"#;
|
|
||||||
let dir = std::env::temp_dir();
|
|
||||||
let path = dir.join("telemt_me_route_backpressure_high_timeout_ms_out_of_range_test.toml");
|
|
||||||
std::fs::write(&path, toml).unwrap();
|
|
||||||
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
|
||||||
assert!(err.contains("general.me_route_backpressure_high_timeout_ms must be within [1, 5000]"));
|
|
||||||
let _ = std::fs::remove_file(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn me_route_no_writer_wait_ms_out_of_range_is_rejected() {
|
fn me_route_no_writer_wait_ms_out_of_range_is_rejected() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
|
|
|
||||||
|
|
@ -812,10 +812,6 @@ pub struct GeneralConfig {
|
||||||
#[serde(default = "default_me_pool_drain_ttl_secs")]
|
#[serde(default = "default_me_pool_drain_ttl_secs")]
|
||||||
pub me_pool_drain_ttl_secs: u64,
|
pub me_pool_drain_ttl_secs: u64,
|
||||||
|
|
||||||
/// Force-remove any draining writer on the next cleanup tick, regardless of age/deadline.
|
|
||||||
#[serde(default = "default_me_instadrain")]
|
|
||||||
pub me_instadrain: bool,
|
|
||||||
|
|
||||||
/// Maximum allowed number of draining ME writers before oldest ones are force-closed in batches.
|
/// Maximum allowed number of draining ME writers before oldest ones are force-closed in batches.
|
||||||
/// Set to 0 to disable threshold-based draining cleanup and keep timeout-only behavior.
|
/// Set to 0 to disable threshold-based draining cleanup and keep timeout-only behavior.
|
||||||
#[serde(default = "default_me_pool_drain_threshold")]
|
#[serde(default = "default_me_pool_drain_threshold")]
|
||||||
|
|
@ -1024,7 +1020,6 @@ impl Default for GeneralConfig {
|
||||||
me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(),
|
me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(),
|
||||||
proxy_secret_len_max: default_proxy_secret_len_max(),
|
proxy_secret_len_max: default_proxy_secret_len_max(),
|
||||||
me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(),
|
me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(),
|
||||||
me_instadrain: default_me_instadrain(),
|
|
||||||
me_pool_drain_threshold: default_me_pool_drain_threshold(),
|
me_pool_drain_threshold: default_me_pool_drain_threshold(),
|
||||||
me_pool_drain_soft_evict_enabled: default_me_pool_drain_soft_evict_enabled(),
|
me_pool_drain_soft_evict_enabled: default_me_pool_drain_soft_evict_enabled(),
|
||||||
me_pool_drain_soft_evict_grace_secs: default_me_pool_drain_soft_evict_grace_secs(),
|
me_pool_drain_soft_evict_grace_secs: default_me_pool_drain_soft_evict_grace_secs(),
|
||||||
|
|
|
||||||
|
|
@ -237,7 +237,6 @@ pub(crate) async fn initialize_me_pool(
|
||||||
config.general.me_adaptive_floor_max_warm_writers_global,
|
config.general.me_adaptive_floor_max_warm_writers_global,
|
||||||
config.general.hardswap,
|
config.general.hardswap,
|
||||||
config.general.me_pool_drain_ttl_secs,
|
config.general.me_pool_drain_ttl_secs,
|
||||||
config.general.me_instadrain,
|
|
||||||
config.general.me_pool_drain_threshold,
|
config.general.me_pool_drain_threshold,
|
||||||
config.general.me_pool_drain_soft_evict_enabled,
|
config.general.me_pool_drain_soft_evict_enabled,
|
||||||
config.general.me_pool_drain_soft_evict_grace_secs,
|
config.general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
@ -343,13 +342,6 @@ pub(crate) async fn initialize_me_pool(
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
let pool_drain_enforcer = pool_bg.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(
|
|
||||||
pool_drain_enforcer,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -417,13 +409,6 @@ pub(crate) async fn initialize_me_pool(
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
let pool_drain_enforcer = pool.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(
|
|
||||||
pool_drain_enforcer,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
|
|
||||||
break Some(pool);
|
break Some(pool);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1692,57 +1692,6 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# HELP telemt_me_writer_close_signal_drop_total Close-signal drops for already-removed ME writers"
|
|
||||||
);
|
|
||||||
let _ = writeln!(out, "# TYPE telemt_me_writer_close_signal_drop_total counter");
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"telemt_me_writer_close_signal_drop_total {}",
|
|
||||||
if me_allows_normal {
|
|
||||||
stats.get_me_writer_close_signal_drop_total()
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# HELP telemt_me_writer_close_signal_channel_full_total Close-signal drops caused by full writer command channels"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# TYPE telemt_me_writer_close_signal_channel_full_total counter"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"telemt_me_writer_close_signal_channel_full_total {}",
|
|
||||||
if me_allows_normal {
|
|
||||||
stats.get_me_writer_close_signal_channel_full_total()
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# HELP telemt_me_draining_writers_reap_progress_total Draining-writer removals processed by reap cleanup"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# TYPE telemt_me_draining_writers_reap_progress_total counter"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"telemt_me_draining_writers_reap_progress_total {}",
|
|
||||||
if me_allows_normal {
|
|
||||||
stats.get_me_draining_writers_reap_progress_total()
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let _ = writeln!(out, "# HELP telemt_me_writer_removed_total Total ME writer removals");
|
let _ = writeln!(out, "# HELP telemt_me_writer_removed_total Total ME writer removals");
|
||||||
let _ = writeln!(out, "# TYPE telemt_me_writer_removed_total counter");
|
let _ = writeln!(out, "# TYPE telemt_me_writer_removed_total counter");
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
|
|
@ -2175,13 +2124,6 @@ mod tests {
|
||||||
assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"));
|
assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter"));
|
assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_me_writer_removed_total counter"));
|
assert!(output.contains("# TYPE telemt_me_writer_removed_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_me_writer_close_signal_drop_total counter"));
|
|
||||||
assert!(output.contains(
|
|
||||||
"# TYPE telemt_me_writer_close_signal_channel_full_total counter"
|
|
||||||
));
|
|
||||||
assert!(output.contains(
|
|
||||||
"# TYPE telemt_me_draining_writers_reap_progress_total counter"
|
|
||||||
));
|
|
||||||
assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_total counter"));
|
assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_writer_total counter"));
|
assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_writer_total counter"));
|
||||||
assert!(output.contains(
|
assert!(output.contains(
|
||||||
|
|
|
||||||
|
|
@ -123,9 +123,6 @@ pub struct Stats {
|
||||||
pool_drain_soft_evict_total: AtomicU64,
|
pool_drain_soft_evict_total: AtomicU64,
|
||||||
pool_drain_soft_evict_writer_total: AtomicU64,
|
pool_drain_soft_evict_writer_total: AtomicU64,
|
||||||
pool_stale_pick_total: AtomicU64,
|
pool_stale_pick_total: AtomicU64,
|
||||||
me_writer_close_signal_drop_total: AtomicU64,
|
|
||||||
me_writer_close_signal_channel_full_total: AtomicU64,
|
|
||||||
me_draining_writers_reap_progress_total: AtomicU64,
|
|
||||||
me_writer_removed_total: AtomicU64,
|
me_writer_removed_total: AtomicU64,
|
||||||
me_writer_removed_unexpected_total: AtomicU64,
|
me_writer_removed_unexpected_total: AtomicU64,
|
||||||
me_refill_triggered_total: AtomicU64,
|
me_refill_triggered_total: AtomicU64,
|
||||||
|
|
@ -737,24 +734,6 @@ impl Stats {
|
||||||
self.pool_stale_pick_total.fetch_add(1, Ordering::Relaxed);
|
self.pool_stale_pick_total.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn increment_me_writer_close_signal_drop_total(&self) {
|
|
||||||
if self.telemetry_me_allows_normal() {
|
|
||||||
self.me_writer_close_signal_drop_total
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn increment_me_writer_close_signal_channel_full_total(&self) {
|
|
||||||
if self.telemetry_me_allows_normal() {
|
|
||||||
self.me_writer_close_signal_channel_full_total
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn increment_me_draining_writers_reap_progress_total(&self) {
|
|
||||||
if self.telemetry_me_allows_normal() {
|
|
||||||
self.me_draining_writers_reap_progress_total
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn increment_me_writer_removed_total(&self) {
|
pub fn increment_me_writer_removed_total(&self) {
|
||||||
if self.telemetry_me_allows_debug() {
|
if self.telemetry_me_allows_debug() {
|
||||||
self.me_writer_removed_total.fetch_add(1, Ordering::Relaxed);
|
self.me_writer_removed_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
@ -1280,17 +1259,6 @@ impl Stats {
|
||||||
pub fn get_pool_stale_pick_total(&self) -> u64 {
|
pub fn get_pool_stale_pick_total(&self) -> u64 {
|
||||||
self.pool_stale_pick_total.load(Ordering::Relaxed)
|
self.pool_stale_pick_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
pub fn get_me_writer_close_signal_drop_total(&self) -> u64 {
|
|
||||||
self.me_writer_close_signal_drop_total.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
pub fn get_me_writer_close_signal_channel_full_total(&self) -> u64 {
|
|
||||||
self.me_writer_close_signal_channel_full_total
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
pub fn get_me_draining_writers_reap_progress_total(&self) -> u64 {
|
|
||||||
self.me_draining_writers_reap_progress_total
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
pub fn get_me_writer_removed_total(&self) -> u64 {
|
pub fn get_me_writer_removed_total(&self) -> u64 {
|
||||||
self.me_writer_removed_total.load(Ordering::Relaxed)
|
self.me_writer_removed_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -298,7 +298,6 @@ async fn run_update_cycle(
|
||||||
pool.update_runtime_reinit_policy(
|
pool.update_runtime_reinit_policy(
|
||||||
cfg.general.hardswap,
|
cfg.general.hardswap,
|
||||||
cfg.general.me_pool_drain_ttl_secs,
|
cfg.general.me_pool_drain_ttl_secs,
|
||||||
cfg.general.me_instadrain,
|
|
||||||
cfg.general.me_pool_drain_threshold,
|
cfg.general.me_pool_drain_threshold,
|
||||||
cfg.general.me_pool_drain_soft_evict_enabled,
|
cfg.general.me_pool_drain_soft_evict_enabled,
|
||||||
cfg.general.me_pool_drain_soft_evict_grace_secs,
|
cfg.general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
@ -531,7 +530,6 @@ pub async fn me_config_updater(
|
||||||
pool.update_runtime_reinit_policy(
|
pool.update_runtime_reinit_policy(
|
||||||
cfg.general.hardswap,
|
cfg.general.hardswap,
|
||||||
cfg.general.me_pool_drain_ttl_secs,
|
cfg.general.me_pool_drain_ttl_secs,
|
||||||
cfg.general.me_instadrain,
|
|
||||||
cfg.general.me_pool_drain_threshold,
|
cfg.general.me_pool_drain_threshold,
|
||||||
cfg.general.me_pool_drain_soft_evict_enabled,
|
cfg.general.me_pool_drain_soft_evict_enabled,
|
||||||
cfg.general.me_pool_drain_soft_evict_grace_secs,
|
cfg.general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ use crate::crypto::SecureRandom;
|
||||||
use crate::network::IpFamily;
|
use crate::network::IpFamily;
|
||||||
|
|
||||||
use super::MePool;
|
use super::MePool;
|
||||||
use super::pool::MeWriter;
|
|
||||||
|
|
||||||
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
|
|
@ -31,8 +30,6 @@ const HEALTH_DRAIN_CLOSE_BUDGET_MIN: usize = 16;
|
||||||
const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256;
|
const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256;
|
||||||
const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN: usize = 8;
|
const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN: usize = 8;
|
||||||
const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX: usize = 256;
|
const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX: usize = 256;
|
||||||
const HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS: u64 = 1;
|
|
||||||
const HEALTH_DRAIN_TIMEOUT_ENFORCER_INTERVAL_SECS: u64 = 1;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct DcFloorPlanEntry {
|
struct DcFloorPlanEntry {
|
||||||
|
|
@ -102,8 +99,6 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
||||||
&mut adaptive_idle_since,
|
&mut adaptive_idle_since,
|
||||||
&mut adaptive_recover_until,
|
&mut adaptive_recover_until,
|
||||||
&mut floor_warn_next_allowed,
|
&mut floor_warn_next_allowed,
|
||||||
&mut drain_warn_next_allowed,
|
|
||||||
&mut drain_soft_evict_next_allowed,
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let v6_degraded = check_family(
|
let v6_degraded = check_family(
|
||||||
|
|
@ -121,63 +116,12 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
||||||
&mut adaptive_idle_since,
|
&mut adaptive_idle_since,
|
||||||
&mut adaptive_recover_until,
|
&mut adaptive_recover_until,
|
||||||
&mut floor_warn_next_allowed,
|
&mut floor_warn_next_allowed,
|
||||||
&mut drain_warn_next_allowed,
|
|
||||||
&mut drain_soft_evict_next_allowed,
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
degraded_interval = v4_degraded || v6_degraded;
|
degraded_interval = v4_degraded || v6_degraded;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn me_drain_timeout_enforcer(pool: Arc<MePool>) {
|
|
||||||
let mut drain_warn_next_allowed: HashMap<u64, Instant> = HashMap::new();
|
|
||||||
let mut drain_soft_evict_next_allowed: HashMap<u64, Instant> = HashMap::new();
|
|
||||||
loop {
|
|
||||||
tokio::time::sleep(Duration::from_secs(
|
|
||||||
HEALTH_DRAIN_TIMEOUT_ENFORCER_INTERVAL_SECS,
|
|
||||||
))
|
|
||||||
.await;
|
|
||||||
reap_draining_writers(
|
|
||||||
&pool,
|
|
||||||
&mut drain_warn_next_allowed,
|
|
||||||
&mut drain_soft_evict_next_allowed,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn draining_writer_timeout_expired(
|
|
||||||
pool: &MePool,
|
|
||||||
writer: &MeWriter,
|
|
||||||
now_epoch_secs: u64,
|
|
||||||
drain_ttl_secs: u64,
|
|
||||||
) -> bool {
|
|
||||||
if pool
|
|
||||||
.me_instadrain
|
|
||||||
.load(std::sync::atomic::Ordering::Relaxed)
|
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
let deadline_epoch_secs = writer
|
|
||||||
.drain_deadline_epoch_secs
|
|
||||||
.load(std::sync::atomic::Ordering::Relaxed);
|
|
||||||
if deadline_epoch_secs != 0 {
|
|
||||||
return now_epoch_secs >= deadline_epoch_secs;
|
|
||||||
}
|
|
||||||
|
|
||||||
if drain_ttl_secs == 0 {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let drain_started_at_epoch_secs = writer
|
|
||||||
.draining_started_at_epoch_secs
|
|
||||||
.load(std::sync::atomic::Ordering::Relaxed);
|
|
||||||
if drain_started_at_epoch_secs == 0 {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
now_epoch_secs.saturating_sub(drain_started_at_epoch_secs) > drain_ttl_secs
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn reap_draining_writers(
|
pub(super) async fn reap_draining_writers(
|
||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
warn_next_allowed: &mut HashMap<u64, Instant>,
|
warn_next_allowed: &mut HashMap<u64, Instant>,
|
||||||
|
|
@ -193,16 +137,11 @@ pub(super) async fn reap_draining_writers(
|
||||||
let activity = pool.registry.writer_activity_snapshot().await;
|
let activity = pool.registry.writer_activity_snapshot().await;
|
||||||
let mut draining_writers = Vec::new();
|
let mut draining_writers = Vec::new();
|
||||||
let mut empty_writer_ids = Vec::<u64>::new();
|
let mut empty_writer_ids = Vec::<u64>::new();
|
||||||
let mut timeout_expired_writer_ids = Vec::<u64>::new();
|
|
||||||
let mut force_close_writer_ids = Vec::<u64>::new();
|
let mut force_close_writer_ids = Vec::<u64>::new();
|
||||||
for writer in writers {
|
for writer in writers {
|
||||||
if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) {
|
if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if draining_writer_timeout_expired(pool, &writer, now_epoch_secs, drain_ttl_secs) {
|
|
||||||
timeout_expired_writer_ids.push(writer.id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if activity
|
if activity
|
||||||
.bound_clients_by_writer
|
.bound_clients_by_writer
|
||||||
.get(&writer.id)
|
.get(&writer.id)
|
||||||
|
|
@ -268,6 +207,14 @@ pub(super) async fn reap_draining_writers(
|
||||||
"ME draining writer remains non-empty past drain TTL"
|
"ME draining writer remains non-empty past drain TTL"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
let deadline_epoch_secs = writer
|
||||||
|
.drain_deadline_epoch_secs
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
|
if deadline_epoch_secs != 0 && now_epoch_secs >= deadline_epoch_secs {
|
||||||
|
warn!(writer_id = writer.id, "Drain timeout, force-closing");
|
||||||
|
force_close_writer_ids.push(writer.id);
|
||||||
|
active_draining_writer_ids.remove(&writer.id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
warn_next_allowed.retain(|writer_id, _| active_draining_writer_ids.contains(writer_id));
|
warn_next_allowed.retain(|writer_id, _| active_draining_writer_ids.contains(writer_id));
|
||||||
|
|
@ -352,21 +299,11 @@ pub(super) async fn reap_draining_writers(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut closed_writer_ids = HashSet::<u64>::new();
|
let close_budget = health_drain_close_budget();
|
||||||
for writer_id in timeout_expired_writer_ids {
|
|
||||||
if !closed_writer_ids.insert(writer_id) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
pool.stats.increment_pool_force_close_total();
|
|
||||||
pool.remove_writer_and_close_clients(writer_id).await;
|
|
||||||
pool.stats
|
|
||||||
.increment_me_draining_writers_reap_progress_total();
|
|
||||||
}
|
|
||||||
|
|
||||||
let requested_force_close = force_close_writer_ids.len();
|
let requested_force_close = force_close_writer_ids.len();
|
||||||
let requested_empty_close = empty_writer_ids.len();
|
let requested_empty_close = empty_writer_ids.len();
|
||||||
let requested_close_total = requested_force_close.saturating_add(requested_empty_close);
|
let requested_close_total = requested_force_close.saturating_add(requested_empty_close);
|
||||||
let close_budget = health_drain_close_budget();
|
let mut closed_writer_ids = HashSet::<u64>::new();
|
||||||
let mut closed_total = 0usize;
|
let mut closed_total = 0usize;
|
||||||
for writer_id in force_close_writer_ids {
|
for writer_id in force_close_writer_ids {
|
||||||
if closed_total >= close_budget {
|
if closed_total >= close_budget {
|
||||||
|
|
@ -377,8 +314,6 @@ pub(super) async fn reap_draining_writers(
|
||||||
}
|
}
|
||||||
pool.stats.increment_pool_force_close_total();
|
pool.stats.increment_pool_force_close_total();
|
||||||
pool.remove_writer_and_close_clients(writer_id).await;
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
pool.stats
|
|
||||||
.increment_me_draining_writers_reap_progress_total();
|
|
||||||
closed_total = closed_total.saturating_add(1);
|
closed_total = closed_total.saturating_add(1);
|
||||||
}
|
}
|
||||||
for writer_id in empty_writer_ids {
|
for writer_id in empty_writer_ids {
|
||||||
|
|
@ -389,8 +324,6 @@ pub(super) async fn reap_draining_writers(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
pool.remove_writer_and_close_clients(writer_id).await;
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
pool.stats
|
|
||||||
.increment_me_draining_writers_reap_progress_total();
|
|
||||||
closed_total = closed_total.saturating_add(1);
|
closed_total = closed_total.saturating_add(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -459,8 +392,6 @@ async fn check_family(
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
drain_warn_next_allowed: &mut HashMap<u64, Instant>,
|
|
||||||
drain_soft_evict_next_allowed: &mut HashMap<u64, Instant>,
|
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let enabled = match family {
|
let enabled = match family {
|
||||||
IpFamily::V4 => pool.decision.ipv4_me,
|
IpFamily::V4 => pool.decision.ipv4_me,
|
||||||
|
|
@ -541,15 +472,8 @@ async fn check_family(
|
||||||
floor_plan.active_writers_current,
|
floor_plan.active_writers_current,
|
||||||
floor_plan.warm_writers_current,
|
floor_plan.warm_writers_current,
|
||||||
);
|
);
|
||||||
let mut next_drain_reap_at = Instant::now();
|
|
||||||
|
|
||||||
for (dc, endpoints) in dc_endpoints {
|
for (dc, endpoints) in dc_endpoints {
|
||||||
if Instant::now() >= next_drain_reap_at {
|
|
||||||
reap_draining_writers(pool, drain_warn_next_allowed, drain_soft_evict_next_allowed)
|
|
||||||
.await;
|
|
||||||
next_drain_reap_at = Instant::now()
|
|
||||||
+ Duration::from_secs(HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS);
|
|
||||||
}
|
|
||||||
if endpoints.is_empty() {
|
if endpoints.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -693,12 +617,6 @@ async fn check_family(
|
||||||
|
|
||||||
let mut restored = 0usize;
|
let mut restored = 0usize;
|
||||||
for _ in 0..missing {
|
for _ in 0..missing {
|
||||||
if Instant::now() >= next_drain_reap_at {
|
|
||||||
reap_draining_writers(pool, drain_warn_next_allowed, drain_soft_evict_next_allowed)
|
|
||||||
.await;
|
|
||||||
next_drain_reap_at = Instant::now()
|
|
||||||
+ Duration::from_secs(HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS);
|
|
||||||
}
|
|
||||||
if reconnect_budget == 0 {
|
if reconnect_budget == 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
@ -1626,7 +1544,6 @@ mod tests {
|
||||||
general.me_adaptive_floor_max_warm_writers_global,
|
general.me_adaptive_floor_max_warm_writers_global,
|
||||||
general.hardswap,
|
general.hardswap,
|
||||||
general.me_pool_drain_ttl_secs,
|
general.me_pool_drain_ttl_secs,
|
||||||
general.me_instadrain,
|
|
||||||
general.me_pool_drain_threshold,
|
general.me_pool_drain_threshold,
|
||||||
general.me_pool_drain_soft_evict_enabled,
|
general.me_pool_drain_soft_evict_enabled,
|
||||||
general.me_pool_drain_soft_evict_grace_secs,
|
general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,6 @@ async fn make_pool(
|
||||||
general.me_adaptive_floor_max_warm_writers_global,
|
general.me_adaptive_floor_max_warm_writers_global,
|
||||||
general.hardswap,
|
general.hardswap,
|
||||||
general.me_pool_drain_ttl_secs,
|
general.me_pool_drain_ttl_secs,
|
||||||
general.me_instadrain,
|
|
||||||
general.me_pool_drain_threshold,
|
general.me_pool_drain_threshold,
|
||||||
general.me_pool_drain_soft_evict_enabled,
|
general.me_pool_drain_soft_evict_enabled,
|
||||||
general.me_pool_drain_soft_evict_grace_secs,
|
general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
@ -214,7 +213,7 @@ async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycle
|
||||||
insert_draining_writer(
|
insert_draining_writer(
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
now_epoch_secs.saturating_sub(20),
|
now_epoch_secs.saturating_sub(600).saturating_add(writer_id),
|
||||||
1,
|
1,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
|
|
@ -231,7 +230,7 @@ async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycle
|
||||||
}
|
}
|
||||||
|
|
||||||
assert_eq!(writer_count(&pool).await, threshold as usize);
|
assert_eq!(writer_count(&pool).await, threshold as usize);
|
||||||
assert_eq!(sorted_writer_ids(&pool).await, vec![1, 2, 3]);
|
assert_eq!(sorted_writer_ids(&pool).await, vec![58, 59, 60]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
|
||||||
|
|
@ -80,7 +80,6 @@ async fn make_pool(
|
||||||
general.me_adaptive_floor_max_warm_writers_global,
|
general.me_adaptive_floor_max_warm_writers_global,
|
||||||
general.hardswap,
|
general.hardswap,
|
||||||
general.me_pool_drain_ttl_secs,
|
general.me_pool_drain_ttl_secs,
|
||||||
general.me_instadrain,
|
|
||||||
general.me_pool_drain_threshold,
|
general.me_pool_drain_threshold,
|
||||||
general.me_pool_drain_soft_evict_enabled,
|
general.me_pool_drain_soft_evict_enabled,
|
||||||
general.me_pool_drain_soft_evict_grace_secs,
|
general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@ use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use bytes::Bytes;
|
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
|
@ -74,7 +73,6 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc<MePool> {
|
||||||
general.me_adaptive_floor_max_warm_writers_global,
|
general.me_adaptive_floor_max_warm_writers_global,
|
||||||
general.hardswap,
|
general.hardswap,
|
||||||
general.me_pool_drain_ttl_secs,
|
general.me_pool_drain_ttl_secs,
|
||||||
general.me_instadrain,
|
|
||||||
general.me_pool_drain_threshold,
|
general.me_pool_drain_threshold,
|
||||||
general.me_pool_drain_soft_evict_enabled,
|
general.me_pool_drain_soft_evict_enabled,
|
||||||
general.me_pool_drain_soft_evict_grace_secs,
|
general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
|
@ -181,14 +179,8 @@ async fn current_writer_ids(pool: &Arc<MePool>) -> Vec<u64> {
|
||||||
async fn reap_draining_writers_drops_warn_state_for_removed_writer() {
|
async fn reap_draining_writers_drops_warn_state_for_removed_writer() {
|
||||||
let pool = make_pool(128).await;
|
let pool = make_pool(128).await;
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
let conn_ids = insert_draining_writer(
|
let conn_ids =
|
||||||
&pool,
|
insert_draining_writer(&pool, 7, now_epoch_secs.saturating_sub(180), 1, 0).await;
|
||||||
7,
|
|
||||||
now_epoch_secs.saturating_sub(180),
|
|
||||||
1,
|
|
||||||
now_epoch_secs.saturating_add(3_600),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let mut warn_next_allowed = HashMap::new();
|
let mut warn_next_allowed = HashMap::new();
|
||||||
let mut soft_evict_next_allowed = HashMap::new();
|
let mut soft_evict_next_allowed = HashMap::new();
|
||||||
|
|
||||||
|
|
@ -217,89 +209,6 @@ async fn reap_draining_writers_removes_empty_draining_writers() {
|
||||||
assert_eq!(current_writer_ids(&pool).await, vec![3]);
|
assert_eq!(current_writer_ids(&pool).await, vec![3]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn reap_draining_writers_does_not_block_on_stuck_writer_close_signal() {
|
|
||||||
let pool = make_pool(128).await;
|
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
|
||||||
|
|
||||||
let (blocked_tx, blocked_rx) = mpsc::channel::<WriterCommand>(1);
|
|
||||||
assert!(
|
|
||||||
blocked_tx
|
|
||||||
.try_send(WriterCommand::Data(Bytes::from_static(b"stuck")))
|
|
||||||
.is_ok()
|
|
||||||
);
|
|
||||||
let blocked_rx_guard = tokio::spawn(async move {
|
|
||||||
let _hold_rx = blocked_rx;
|
|
||||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
|
||||||
});
|
|
||||||
|
|
||||||
let blocked_writer_id = 90u64;
|
|
||||||
let blocked_writer = MeWriter {
|
|
||||||
id: blocked_writer_id,
|
|
||||||
addr: SocketAddr::new(
|
|
||||||
IpAddr::V4(Ipv4Addr::LOCALHOST),
|
|
||||||
4500 + blocked_writer_id as u16,
|
|
||||||
),
|
|
||||||
source_ip: IpAddr::V4(Ipv4Addr::LOCALHOST),
|
|
||||||
writer_dc: 2,
|
|
||||||
generation: 1,
|
|
||||||
contour: Arc::new(AtomicU8::new(WriterContour::Draining.as_u8())),
|
|
||||||
created_at: Instant::now() - Duration::from_secs(blocked_writer_id),
|
|
||||||
tx: blocked_tx.clone(),
|
|
||||||
cancel: CancellationToken::new(),
|
|
||||||
degraded: Arc::new(AtomicBool::new(false)),
|
|
||||||
rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)),
|
|
||||||
draining: Arc::new(AtomicBool::new(true)),
|
|
||||||
draining_started_at_epoch_secs: Arc::new(AtomicU64::new(
|
|
||||||
now_epoch_secs.saturating_sub(120),
|
|
||||||
)),
|
|
||||||
drain_deadline_epoch_secs: Arc::new(AtomicU64::new(0)),
|
|
||||||
allow_drain_fallback: Arc::new(AtomicBool::new(false)),
|
|
||||||
};
|
|
||||||
pool.writers.write().await.push(blocked_writer);
|
|
||||||
pool.registry
|
|
||||||
.register_writer(blocked_writer_id, blocked_tx)
|
|
||||||
.await;
|
|
||||||
pool.conn_count.fetch_add(1, Ordering::Relaxed);
|
|
||||||
|
|
||||||
insert_draining_writer(&pool, 91, now_epoch_secs.saturating_sub(110), 0, 0).await;
|
|
||||||
|
|
||||||
let mut warn_next_allowed = HashMap::new();
|
|
||||||
let mut soft_evict_next_allowed = HashMap::new();
|
|
||||||
|
|
||||||
let reap_res = tokio::time::timeout(
|
|
||||||
Duration::from_millis(500),
|
|
||||||
reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
blocked_rx_guard.abort();
|
|
||||||
|
|
||||||
assert!(reap_res.is_ok(), "reap should not block on close signal");
|
|
||||||
assert!(current_writer_ids(&pool).await.is_empty());
|
|
||||||
assert_eq!(pool.stats.get_me_writer_close_signal_drop_total(), 2);
|
|
||||||
assert_eq!(pool.stats.get_me_writer_close_signal_channel_full_total(), 1);
|
|
||||||
assert_eq!(pool.stats.get_me_draining_writers_reap_progress_total(), 2);
|
|
||||||
let activity = pool.registry.writer_activity_snapshot().await;
|
|
||||||
assert!(!activity.bound_clients_by_writer.contains_key(&blocked_writer_id));
|
|
||||||
assert!(!activity.bound_clients_by_writer.contains_key(&91));
|
|
||||||
let (probe_conn_id, _rx) = pool.registry.register().await;
|
|
||||||
assert!(
|
|
||||||
!pool.registry
|
|
||||||
.bind_writer(
|
|
||||||
probe_conn_id,
|
|
||||||
blocked_writer_id,
|
|
||||||
ConnMeta {
|
|
||||||
target_dc: 2,
|
|
||||||
client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6400),
|
|
||||||
our_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443),
|
|
||||||
proto_flags: 0,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
);
|
|
||||||
let _ = pool.registry.unregister(probe_conn_id).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn reap_draining_writers_overflow_closes_oldest_non_empty_writers() {
|
async fn reap_draining_writers_overflow_closes_oldest_non_empty_writers() {
|
||||||
let pool = make_pool(2).await;
|
let pool = make_pool(2).await;
|
||||||
|
|
@ -338,17 +247,17 @@ async fn reap_draining_writers_deadline_force_close_applies_under_threshold() {
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn reap_draining_writers_limits_closes_per_health_tick() {
|
async fn reap_draining_writers_limits_closes_per_health_tick() {
|
||||||
let pool = make_pool(1).await;
|
let pool = make_pool(128).await;
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
let close_budget = health_drain_close_budget();
|
let close_budget = health_drain_close_budget();
|
||||||
let writer_total = close_budget.saturating_add(20);
|
let writer_total = close_budget.saturating_add(19);
|
||||||
for writer_id in 1..=writer_total as u64 {
|
for writer_id in 1..=writer_total as u64 {
|
||||||
insert_draining_writer(
|
insert_draining_writer(
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
now_epoch_secs.saturating_sub(20),
|
now_epoch_secs.saturating_sub(20),
|
||||||
1,
|
1,
|
||||||
0,
|
now_epoch_secs.saturating_sub(1),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
@ -371,8 +280,8 @@ async fn reap_draining_writers_backlog_drains_across_ticks() {
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
now_epoch_secs.saturating_sub(20),
|
now_epoch_secs.saturating_sub(20),
|
||||||
0,
|
1,
|
||||||
0,
|
now_epoch_secs.saturating_sub(1),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
@ -400,7 +309,7 @@ async fn reap_draining_writers_threshold_backlog_converges_to_threshold() {
|
||||||
insert_draining_writer(
|
insert_draining_writer(
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
now_epoch_secs.saturating_sub(20),
|
now_epoch_secs.saturating_sub(200).saturating_add(writer_id),
|
||||||
1,
|
1,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
|
|
@ -436,27 +345,27 @@ async fn reap_draining_writers_threshold_zero_preserves_non_expired_non_empty_wr
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn reap_draining_writers_prioritizes_force_close_before_empty_cleanup() {
|
async fn reap_draining_writers_prioritizes_force_close_before_empty_cleanup() {
|
||||||
let pool = make_pool(1).await;
|
let pool = make_pool(128).await;
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
let close_budget = health_drain_close_budget();
|
let close_budget = health_drain_close_budget();
|
||||||
for writer_id in 1..=close_budget.saturating_add(1) as u64 {
|
for writer_id in 1..=close_budget as u64 {
|
||||||
insert_draining_writer(
|
insert_draining_writer(
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
now_epoch_secs.saturating_sub(20),
|
now_epoch_secs.saturating_sub(20),
|
||||||
1,
|
1,
|
||||||
0,
|
now_epoch_secs.saturating_sub(1),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
let empty_writer_id = close_budget.saturating_add(2) as u64;
|
let empty_writer_id = close_budget as u64 + 1;
|
||||||
insert_draining_writer(&pool, empty_writer_id, now_epoch_secs.saturating_sub(20), 0, 0).await;
|
insert_draining_writer(&pool, empty_writer_id, now_epoch_secs.saturating_sub(20), 0, 0).await;
|
||||||
let mut warn_next_allowed = HashMap::new();
|
let mut warn_next_allowed = HashMap::new();
|
||||||
let mut soft_evict_next_allowed = HashMap::new();
|
let mut soft_evict_next_allowed = HashMap::new();
|
||||||
|
|
||||||
reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await;
|
reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await;
|
||||||
|
|
||||||
assert_eq!(current_writer_ids(&pool).await, vec![1, empty_writer_id]);
|
assert_eq!(current_writer_ids(&pool).await, vec![empty_writer_id]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
@ -578,14 +487,7 @@ async fn reap_draining_writers_soft_evicts_stuck_writer_with_per_writer_cap() {
|
||||||
.store(1, Ordering::Relaxed);
|
.store(1, Ordering::Relaxed);
|
||||||
|
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
insert_draining_writer(
|
insert_draining_writer(&pool, 77, now_epoch_secs.saturating_sub(240), 3, 0).await;
|
||||||
&pool,
|
|
||||||
77,
|
|
||||||
now_epoch_secs.saturating_sub(240),
|
|
||||||
3,
|
|
||||||
now_epoch_secs.saturating_add(3_600),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let mut warn_next_allowed = HashMap::new();
|
let mut warn_next_allowed = HashMap::new();
|
||||||
let mut soft_evict_next_allowed = HashMap::new();
|
let mut soft_evict_next_allowed = HashMap::new();
|
||||||
|
|
||||||
|
|
@ -609,14 +511,7 @@ async fn reap_draining_writers_soft_evict_respects_cooldown_per_writer() {
|
||||||
.store(60_000, Ordering::Relaxed);
|
.store(60_000, Ordering::Relaxed);
|
||||||
|
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
insert_draining_writer(
|
insert_draining_writer(&pool, 88, now_epoch_secs.saturating_sub(240), 3, 0).await;
|
||||||
&pool,
|
|
||||||
88,
|
|
||||||
now_epoch_secs.saturating_sub(240),
|
|
||||||
3,
|
|
||||||
now_epoch_secs.saturating_add(3_600),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let mut warn_next_allowed = HashMap::new();
|
let mut warn_next_allowed = HashMap::new();
|
||||||
let mut soft_evict_next_allowed = HashMap::new();
|
let mut soft_evict_next_allowed = HashMap::new();
|
||||||
|
|
||||||
|
|
@ -629,21 +524,6 @@ async fn reap_draining_writers_soft_evict_respects_cooldown_per_writer() {
|
||||||
assert_eq!(pool.stats.get_pool_drain_soft_evict_writer_total(), 1);
|
assert_eq!(pool.stats.get_pool_drain_soft_evict_writer_total(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn reap_draining_writers_instadrain_removes_non_expired_writers_immediately() {
|
|
||||||
let pool = make_pool(0).await;
|
|
||||||
pool.me_instadrain.store(true, Ordering::Relaxed);
|
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
|
||||||
insert_draining_writer(&pool, 101, now_epoch_secs.saturating_sub(5), 1, 0).await;
|
|
||||||
insert_draining_writer(&pool, 102, now_epoch_secs.saturating_sub(4), 1, 0).await;
|
|
||||||
let mut warn_next_allowed = HashMap::new();
|
|
||||||
let mut soft_evict_next_allowed = HashMap::new();
|
|
||||||
|
|
||||||
reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await;
|
|
||||||
|
|
||||||
assert!(current_writer_ids(&pool).await.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn general_config_default_drain_threshold_remains_enabled() {
|
fn general_config_default_drain_threshold_remains_enabled() {
|
||||||
assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 128);
|
assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 128);
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ mod health_adversarial_tests;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
|
||||||
pub use health::{me_drain_timeout_enforcer, me_health_monitor};
|
pub use health::me_health_monitor;
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub use ping::{run_me_ping, format_sample_line, format_me_route, MePingReport, MePingSample, MePingFamily};
|
pub use ping::{run_me_ping, format_sample_line, format_me_route, MePingReport, MePingSample, MePingFamily};
|
||||||
pub use pool::MePool;
|
pub use pool::MePool;
|
||||||
|
|
|
||||||
|
|
@ -171,7 +171,6 @@ pub struct MePool {
|
||||||
pub(super) endpoint_quarantine: Arc<Mutex<HashMap<SocketAddr, Instant>>>,
|
pub(super) endpoint_quarantine: Arc<Mutex<HashMap<SocketAddr, Instant>>>,
|
||||||
pub(super) kdf_material_fingerprint: Arc<RwLock<HashMap<SocketAddr, (u64, u16)>>>,
|
pub(super) kdf_material_fingerprint: Arc<RwLock<HashMap<SocketAddr, (u64, u16)>>>,
|
||||||
pub(super) me_pool_drain_ttl_secs: AtomicU64,
|
pub(super) me_pool_drain_ttl_secs: AtomicU64,
|
||||||
pub(super) me_instadrain: AtomicBool,
|
|
||||||
pub(super) me_pool_drain_threshold: AtomicU64,
|
pub(super) me_pool_drain_threshold: AtomicU64,
|
||||||
pub(super) me_pool_drain_soft_evict_enabled: AtomicBool,
|
pub(super) me_pool_drain_soft_evict_enabled: AtomicBool,
|
||||||
pub(super) me_pool_drain_soft_evict_grace_secs: AtomicU64,
|
pub(super) me_pool_drain_soft_evict_grace_secs: AtomicU64,
|
||||||
|
|
@ -280,7 +279,6 @@ impl MePool {
|
||||||
me_adaptive_floor_max_warm_writers_global: u32,
|
me_adaptive_floor_max_warm_writers_global: u32,
|
||||||
hardswap: bool,
|
hardswap: bool,
|
||||||
me_pool_drain_ttl_secs: u64,
|
me_pool_drain_ttl_secs: u64,
|
||||||
me_instadrain: bool,
|
|
||||||
me_pool_drain_threshold: u64,
|
me_pool_drain_threshold: u64,
|
||||||
me_pool_drain_soft_evict_enabled: bool,
|
me_pool_drain_soft_evict_enabled: bool,
|
||||||
me_pool_drain_soft_evict_grace_secs: u64,
|
me_pool_drain_soft_evict_grace_secs: u64,
|
||||||
|
|
@ -464,7 +462,6 @@ impl MePool {
|
||||||
endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())),
|
endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())),
|
||||||
kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())),
|
kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())),
|
||||||
me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs),
|
me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs),
|
||||||
me_instadrain: AtomicBool::new(me_instadrain),
|
|
||||||
me_pool_drain_threshold: AtomicU64::new(me_pool_drain_threshold),
|
me_pool_drain_threshold: AtomicU64::new(me_pool_drain_threshold),
|
||||||
me_pool_drain_soft_evict_enabled: AtomicBool::new(me_pool_drain_soft_evict_enabled),
|
me_pool_drain_soft_evict_enabled: AtomicBool::new(me_pool_drain_soft_evict_enabled),
|
||||||
me_pool_drain_soft_evict_grace_secs: AtomicU64::new(me_pool_drain_soft_evict_grace_secs),
|
me_pool_drain_soft_evict_grace_secs: AtomicU64::new(me_pool_drain_soft_evict_grace_secs),
|
||||||
|
|
@ -527,7 +524,6 @@ impl MePool {
|
||||||
&self,
|
&self,
|
||||||
hardswap: bool,
|
hardswap: bool,
|
||||||
drain_ttl_secs: u64,
|
drain_ttl_secs: u64,
|
||||||
instadrain: bool,
|
|
||||||
pool_drain_threshold: u64,
|
pool_drain_threshold: u64,
|
||||||
pool_drain_soft_evict_enabled: bool,
|
pool_drain_soft_evict_enabled: bool,
|
||||||
pool_drain_soft_evict_grace_secs: u64,
|
pool_drain_soft_evict_grace_secs: u64,
|
||||||
|
|
@ -572,7 +568,6 @@ impl MePool {
|
||||||
self.hardswap.store(hardswap, Ordering::Relaxed);
|
self.hardswap.store(hardswap, Ordering::Relaxed);
|
||||||
self.me_pool_drain_ttl_secs
|
self.me_pool_drain_ttl_secs
|
||||||
.store(drain_ttl_secs, Ordering::Relaxed);
|
.store(drain_ttl_secs, Ordering::Relaxed);
|
||||||
self.me_instadrain.store(instadrain, Ordering::Relaxed);
|
|
||||||
self.me_pool_drain_threshold
|
self.me_pool_drain_threshold
|
||||||
.store(pool_drain_threshold, Ordering::Relaxed);
|
.store(pool_drain_threshold, Ordering::Relaxed);
|
||||||
self.me_pool_drain_soft_evict_enabled
|
self.me_pool_drain_soft_evict_enabled
|
||||||
|
|
|
||||||
|
|
@ -126,7 +126,6 @@ pub(crate) struct MeApiRuntimeSnapshot {
|
||||||
pub me_reconnect_backoff_cap_ms: u64,
|
pub me_reconnect_backoff_cap_ms: u64,
|
||||||
pub me_reconnect_fast_retry_count: u32,
|
pub me_reconnect_fast_retry_count: u32,
|
||||||
pub me_pool_drain_ttl_secs: u64,
|
pub me_pool_drain_ttl_secs: u64,
|
||||||
pub me_instadrain: bool,
|
|
||||||
pub me_pool_drain_soft_evict_enabled: bool,
|
pub me_pool_drain_soft_evict_enabled: bool,
|
||||||
pub me_pool_drain_soft_evict_grace_secs: u64,
|
pub me_pool_drain_soft_evict_grace_secs: u64,
|
||||||
pub me_pool_drain_soft_evict_per_writer: u8,
|
pub me_pool_drain_soft_evict_per_writer: u8,
|
||||||
|
|
@ -584,7 +583,6 @@ impl MePool {
|
||||||
me_reconnect_backoff_cap_ms: self.me_reconnect_backoff_cap.as_millis() as u64,
|
me_reconnect_backoff_cap_ms: self.me_reconnect_backoff_cap.as_millis() as u64,
|
||||||
me_reconnect_fast_retry_count: self.me_reconnect_fast_retry_count,
|
me_reconnect_fast_retry_count: self.me_reconnect_fast_retry_count,
|
||||||
me_pool_drain_ttl_secs: self.me_pool_drain_ttl_secs.load(Ordering::Relaxed),
|
me_pool_drain_ttl_secs: self.me_pool_drain_ttl_secs.load(Ordering::Relaxed),
|
||||||
me_instadrain: self.me_instadrain.load(Ordering::Relaxed),
|
|
||||||
me_pool_drain_soft_evict_enabled: self
|
me_pool_drain_soft_evict_enabled: self
|
||||||
.me_pool_drain_soft_evict_enabled
|
.me_pool_drain_soft_evict_enabled
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@ use bytes::Bytes;
|
||||||
use bytes::BytesMut;
|
use bytes::BytesMut;
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio::sync::mpsc::error::TrySendError;
|
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
|
|
@ -492,9 +491,11 @@ impl MePool {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn remove_writer_and_close_clients(self: &Arc<Self>, writer_id: u64) {
|
pub(crate) async fn remove_writer_and_close_clients(self: &Arc<Self>, writer_id: u64) {
|
||||||
// Full client cleanup now happens inside `registry.writer_lost` to keep
|
let conns = self.remove_writer_only(writer_id).await;
|
||||||
// writer reap/remove paths strictly non-blocking per connection.
|
for bound in conns {
|
||||||
let _ = self.remove_writer_only(writer_id).await;
|
let _ = self.registry.route(bound.conn_id, super::MeResponse::Close).await;
|
||||||
|
let _ = self.registry.unregister(bound.conn_id).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn remove_writer_only(self: &Arc<Self>, writer_id: u64) -> Vec<BoundConn> {
|
async fn remove_writer_only(self: &Arc<Self>, writer_id: u64) -> Vec<BoundConn> {
|
||||||
|
|
@ -524,11 +525,6 @@ impl MePool {
|
||||||
self.conn_count.fetch_sub(1, Ordering::Relaxed);
|
self.conn_count.fetch_sub(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// State invariant:
|
|
||||||
// - writer is removed from `self.writers` (pool visibility),
|
|
||||||
// - writer is removed from registry routing/binding maps via `writer_lost`.
|
|
||||||
// The close command below is only a best-effort accelerator for task shutdown.
|
|
||||||
// Cleanup progress must never depend on command-channel availability.
|
|
||||||
let conns = self.registry.writer_lost(writer_id).await;
|
let conns = self.registry.writer_lost(writer_id).await;
|
||||||
{
|
{
|
||||||
let mut tracker = self.ping_tracker.lock().await;
|
let mut tracker = self.ping_tracker.lock().await;
|
||||||
|
|
@ -536,25 +532,7 @@ impl MePool {
|
||||||
}
|
}
|
||||||
self.rtt_stats.lock().await.remove(&writer_id);
|
self.rtt_stats.lock().await.remove(&writer_id);
|
||||||
if let Some(tx) = close_tx {
|
if let Some(tx) = close_tx {
|
||||||
match tx.try_send(WriterCommand::Close) {
|
let _ = tx.send(WriterCommand::Close).await;
|
||||||
Ok(()) => {}
|
|
||||||
Err(TrySendError::Full(_)) => {
|
|
||||||
self.stats.increment_me_writer_close_signal_drop_total();
|
|
||||||
self.stats
|
|
||||||
.increment_me_writer_close_signal_channel_full_total();
|
|
||||||
debug!(
|
|
||||||
writer_id,
|
|
||||||
"Skipping close signal for removed writer: command channel is full"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(TrySendError::Closed(_)) => {
|
|
||||||
self.stats.increment_me_writer_close_signal_drop_total();
|
|
||||||
debug!(
|
|
||||||
writer_id,
|
|
||||||
"Skipping close signal for removed writer: command channel is closed"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if trigger_refill
|
if trigger_refill
|
||||||
&& let Some(addr) = removed_addr
|
&& let Some(addr) = removed_addr
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@ use bytes::{Bytes, BytesMut};
|
||||||
use tokio::io::AsyncReadExt;
|
use tokio::io::AsyncReadExt;
|
||||||
use tokio::net::TcpStream;
|
use tokio::net::TcpStream;
|
||||||
use tokio::sync::{Mutex, mpsc};
|
use tokio::sync::{Mutex, mpsc};
|
||||||
use tokio::sync::mpsc::error::TrySendError;
|
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tracing::{debug, trace, warn};
|
use tracing::{debug, trace, warn};
|
||||||
|
|
||||||
|
|
@ -174,12 +173,12 @@ pub(crate) async fn reader_loop(
|
||||||
} else if pt == RPC_CLOSE_EXT_U32 && body.len() >= 8 {
|
} else if pt == RPC_CLOSE_EXT_U32 && body.len() >= 8 {
|
||||||
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
|
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
debug!(cid, "RPC_CLOSE_EXT from ME");
|
debug!(cid, "RPC_CLOSE_EXT from ME");
|
||||||
let _ = reg.route_nowait(cid, MeResponse::Close).await;
|
reg.route(cid, MeResponse::Close).await;
|
||||||
reg.unregister(cid).await;
|
reg.unregister(cid).await;
|
||||||
} else if pt == RPC_CLOSE_CONN_U32 && body.len() >= 8 {
|
} else if pt == RPC_CLOSE_CONN_U32 && body.len() >= 8 {
|
||||||
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
|
let cid = u64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
debug!(cid, "RPC_CLOSE_CONN from ME");
|
debug!(cid, "RPC_CLOSE_CONN from ME");
|
||||||
let _ = reg.route_nowait(cid, MeResponse::Close).await;
|
reg.route(cid, MeResponse::Close).await;
|
||||||
reg.unregister(cid).await;
|
reg.unregister(cid).await;
|
||||||
} else if pt == RPC_PING_U32 && body.len() >= 8 {
|
} else if pt == RPC_PING_U32 && body.len() >= 8 {
|
||||||
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
|
|
@ -187,15 +186,13 @@ pub(crate) async fn reader_loop(
|
||||||
let mut pong = Vec::with_capacity(12);
|
let mut pong = Vec::with_capacity(12);
|
||||||
pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes());
|
pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes());
|
||||||
pong.extend_from_slice(&ping_id.to_le_bytes());
|
pong.extend_from_slice(&ping_id.to_le_bytes());
|
||||||
match tx.try_send(WriterCommand::DataAndFlush(Bytes::from(pong))) {
|
if tx
|
||||||
Ok(()) => {}
|
.send(WriterCommand::DataAndFlush(Bytes::from(pong)))
|
||||||
Err(TrySendError::Full(_)) => {
|
.await
|
||||||
debug!(ping_id, "PONG dropped: writer command channel is full");
|
.is_err()
|
||||||
}
|
{
|
||||||
Err(TrySendError::Closed(_)) => {
|
warn!("PONG send failed");
|
||||||
warn!("PONG send failed: writer channel closed");
|
break;
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else if pt == RPC_PONG_U32 && body.len() >= 8 {
|
} else if pt == RPC_PONG_U32 && body.len() >= 8 {
|
||||||
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
|
|
@ -235,13 +232,6 @@ async fn send_close_conn(tx: &mpsc::Sender<WriterCommand>, conn_id: u64) {
|
||||||
let mut p = Vec::with_capacity(12);
|
let mut p = Vec::with_capacity(12);
|
||||||
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
p.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
match tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
|
||||||
Ok(()) => {}
|
let _ = tx.send(WriterCommand::DataAndFlush(Bytes::from(p))).await;
|
||||||
Err(TrySendError::Full(_)) => {
|
|
||||||
debug!(conn_id, "ME close_conn signal skipped: writer command channel is full");
|
|
||||||
}
|
|
||||||
Err(TrySendError::Closed(_)) => {
|
|
||||||
debug!(conn_id, "ME close_conn signal skipped: writer command channel is closed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -169,7 +169,6 @@ impl ConnRegistry {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult {
|
pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult {
|
||||||
let tx = {
|
let tx = {
|
||||||
let inner = self.inner.read().await;
|
let inner = self.inner.read().await;
|
||||||
|
|
@ -446,38 +445,30 @@ impl ConnRegistry {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn writer_lost(&self, writer_id: u64) -> Vec<BoundConn> {
|
pub async fn writer_lost(&self, writer_id: u64) -> Vec<BoundConn> {
|
||||||
let mut close_txs = Vec::<mpsc::Sender<MeResponse>>::new();
|
let mut inner = self.inner.write().await;
|
||||||
let mut out = Vec::new();
|
inner.writers.remove(&writer_id);
|
||||||
{
|
inner.last_meta_for_writer.remove(&writer_id);
|
||||||
let mut inner = self.inner.write().await;
|
inner.writer_idle_since_epoch_secs.remove(&writer_id);
|
||||||
inner.writers.remove(&writer_id);
|
let conns = inner
|
||||||
inner.last_meta_for_writer.remove(&writer_id);
|
.conns_for_writer
|
||||||
inner.writer_idle_since_epoch_secs.remove(&writer_id);
|
.remove(&writer_id)
|
||||||
let conns = inner
|
.unwrap_or_default()
|
||||||
.conns_for_writer
|
.into_iter()
|
||||||
.remove(&writer_id)
|
.collect::<Vec<_>>();
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
for conn_id in conns {
|
let mut out = Vec::new();
|
||||||
if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) {
|
for conn_id in conns {
|
||||||
continue;
|
if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) {
|
||||||
}
|
continue;
|
||||||
inner.writer_for_conn.remove(&conn_id);
|
}
|
||||||
if let Some(client_tx) = inner.map.remove(&conn_id) {
|
inner.writer_for_conn.remove(&conn_id);
|
||||||
close_txs.push(client_tx);
|
if let Some(m) = inner.meta.get(&conn_id) {
|
||||||
}
|
out.push(BoundConn {
|
||||||
if let Some(meta) = inner.meta.remove(&conn_id) {
|
conn_id,
|
||||||
out.push(BoundConn { conn_id, meta });
|
meta: m.clone(),
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for client_tx in close_txs {
|
|
||||||
let _ = client_tx.try_send(MeResponse::Close);
|
|
||||||
}
|
|
||||||
|
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -500,7 +491,6 @@ impl ConnRegistry {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use super::ConnMeta;
|
use super::ConnMeta;
|
||||||
use super::ConnRegistry;
|
use super::ConnRegistry;
|
||||||
|
|
@ -673,39 +663,6 @@ mod tests {
|
||||||
assert!(registry.is_writer_empty(20).await);
|
assert!(registry.is_writer_empty(20).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn writer_lost_removes_bound_conn_from_registry_and_signals_close() {
|
|
||||||
let registry = ConnRegistry::new();
|
|
||||||
let (conn_id, mut rx) = registry.register().await;
|
|
||||||
let (writer_tx, _writer_rx) = tokio::sync::mpsc::channel(8);
|
|
||||||
registry.register_writer(10, writer_tx).await;
|
|
||||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
registry
|
|
||||||
.bind_writer(
|
|
||||||
conn_id,
|
|
||||||
10,
|
|
||||||
ConnMeta {
|
|
||||||
target_dc: 2,
|
|
||||||
client_addr: addr,
|
|
||||||
our_addr: addr,
|
|
||||||
proto_flags: 0,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
);
|
|
||||||
|
|
||||||
let lost = registry.writer_lost(10).await;
|
|
||||||
assert_eq!(lost.len(), 1);
|
|
||||||
assert_eq!(lost[0].conn_id, conn_id);
|
|
||||||
assert!(registry.get_writer(conn_id).await.is_none());
|
|
||||||
assert!(registry.get_meta(conn_id).await.is_none());
|
|
||||||
assert_eq!(registry.unregister(conn_id).await, None);
|
|
||||||
let close = tokio::time::timeout(Duration::from_millis(50), rx.recv()).await;
|
|
||||||
assert!(matches!(close, Ok(Some(MeResponse::Close))));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bind_writer_rejects_unregistered_writer() {
|
async fn bind_writer_rejects_unregistered_writer() {
|
||||||
let registry = ConnRegistry::new();
|
let registry = ConnRegistry::new();
|
||||||
|
|
|
||||||
|
|
@ -643,19 +643,13 @@ impl MePool {
|
||||||
let mut p = Vec::with_capacity(12);
|
let mut p = Vec::with_capacity(12);
|
||||||
p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
p.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
if w.tx
|
||||||
Ok(()) => {}
|
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
||||||
Err(TrySendError::Full(_)) => {
|
.await
|
||||||
debug!(
|
.is_err()
|
||||||
conn_id,
|
{
|
||||||
writer_id = w.writer_id,
|
debug!("ME close write failed");
|
||||||
"ME close skipped: writer command channel is full"
|
self.remove_writer_and_close_clients(w.writer_id).await;
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(TrySendError::Closed(_)) => {
|
|
||||||
debug!("ME close write failed");
|
|
||||||
self.remove_writer_and_close_clients(w.writer_id).await;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
debug!(conn_id, "ME close skipped (writer missing)");
|
debug!(conn_id, "ME close skipped (writer missing)");
|
||||||
|
|
@ -672,12 +666,8 @@ impl MePool {
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
p.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(TrySendError::Full(_)) => {
|
Err(TrySendError::Full(cmd)) => {
|
||||||
debug!(
|
let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await;
|
||||||
conn_id,
|
|
||||||
writer_id = w.writer_id,
|
|
||||||
"ME close_conn skipped: writer command channel is full"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
Err(TrySendError::Closed(_)) => {
|
Err(TrySendError::Closed(_)) => {
|
||||||
debug!(conn_id, "ME close_conn skipped: writer channel closed");
|
debug!(conn_id, "ME close_conn skipped: writer channel closed");
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue