mirror of
https://github.com/telemt/telemt.git
synced 2026-04-15 01:24:09 +03:00
Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7bdc80956 | ||
|
|
d641137537 | ||
|
|
4fd22b3219 | ||
|
|
fca0e3f619 | ||
|
|
9401c46727 | ||
|
|
6b3697ee87 | ||
|
|
c08160600e | ||
|
|
cd5c60ce1e | ||
|
|
ae1c97e27a | ||
|
|
cfee7de66b | ||
|
|
c942c492ad | ||
|
|
0e4be43b2b | ||
|
|
7eb2b60855 | ||
|
|
373ae3281e | ||
|
|
178630e3bf | ||
|
|
67f307cd43 | ||
|
|
ca2eaa9ead | ||
|
|
3c78daea0c | ||
|
|
d2baa8e721 | ||
|
|
a0cf4b4713 | ||
|
|
1bd249b0a9 | ||
|
|
2f47ec5797 | ||
|
|
80f3661b8e | ||
|
|
32eeb4a98c | ||
|
|
a74cc14ed9 | ||
|
|
5f77f83b48 | ||
|
|
d543dbca92 | ||
|
|
02f9d59f5a | ||
|
|
7b745bc7bc | ||
|
|
5ac0ef1ffd | ||
|
|
e1f3efb619 | ||
|
|
508eea0131 | ||
|
|
9e7f80b9b3 | ||
|
|
ee2def2e62 | ||
|
|
258191ab87 | ||
|
|
27e6dec018 |
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.7"
|
version = "3.3.11"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
# === General Settings ===
|
# === General Settings ===
|
||||||
[general]
|
[general]
|
||||||
use_middle_proxy = false
|
use_middle_proxy = true
|
||||||
# Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags]
|
# Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags]
|
||||||
# ad_tag = "00000000000000000000000000000000"
|
# ad_tag = "00000000000000000000000000000000"
|
||||||
# Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot)
|
# Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot)
|
||||||
|
|||||||
@@ -126,7 +126,7 @@ WantedBy=multi-user.target
|
|||||||
```bash
|
```bash
|
||||||
curl -s http://127.0.0.1:9091/v1/users | jq
|
curl -s http://127.0.0.1:9091/v1/users | jq
|
||||||
```
|
```
|
||||||
> Одной ссылкой модет пользоваться сколько угодно человек.
|
> Одной ссылкой может пользоваться сколько угодно человек.
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете!
|
> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете!
|
||||||
|
|||||||
138
install.sh
138
install.sh
@@ -1,73 +1,93 @@
|
|||||||
sudo bash -c '
|
#!/bin/sh
|
||||||
set -e
|
set -eu
|
||||||
|
|
||||||
# --- Проверка на существующую установку ---
|
REPO="${REPO:-telemt/telemt}"
|
||||||
if systemctl list-unit-files | grep -q telemt.service; then
|
BIN_NAME="${BIN_NAME:-telemt}"
|
||||||
# --- РЕЖИМ ОБНОВЛЕНИЯ ---
|
VERSION="${1:-${VERSION:-latest}}"
|
||||||
echo "--- Обнаружена существующая установка Telemt. Запускаю обновление... ---"
|
INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
|
||||||
|
|
||||||
echo "[*] Остановка службы telemt..."
|
say() {
|
||||||
systemctl stop telemt || true # Игнорируем ошибку, если служба уже остановлена
|
printf '%s\n' "$*"
|
||||||
|
}
|
||||||
|
|
||||||
echo "[1/2] Скачивание последней версии Telemt..."
|
die() {
|
||||||
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
|
printf 'Error: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
echo "[1/2] Замена исполняемого файла в /usr/local/bin..."
|
need_cmd() {
|
||||||
mv telemt /usr/local/bin/telemt
|
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
|
||||||
chmod +x /usr/local/bin/telemt
|
}
|
||||||
|
|
||||||
echo "[2/2] Запуск службы..."
|
detect_arch() {
|
||||||
systemctl start telemt
|
arch="$(uname -m)"
|
||||||
|
case "$arch" in
|
||||||
|
x86_64|amd64) printf 'x86_64\n' ;;
|
||||||
|
aarch64|arm64) printf 'aarch64\n' ;;
|
||||||
|
*) die "unsupported architecture: $arch" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
echo "--- Обновление Telemt успешно завершено! ---"
|
detect_libc() {
|
||||||
echo
|
case "$(ldd --version 2>&1 || true)" in
|
||||||
echo "Для проверки статуса службы выполните:"
|
*musl*) printf 'musl\n' ;;
|
||||||
echo " systemctl status telemt"
|
*) printf 'gnu\n' ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
else
|
fetch_to_stdout() {
|
||||||
# --- РЕЖИМ НОВОЙ УСТАНОВКИ ---
|
url="$1"
|
||||||
echo "--- Начало автоматической установки Telemt ---"
|
if command -v curl >/dev/null 2>&1; then
|
||||||
|
curl -fsSL "$url"
|
||||||
|
elif command -v wget >/dev/null 2>&1; then
|
||||||
|
wget -qO- "$url"
|
||||||
|
else
|
||||||
|
die "neither curl nor wget is installed"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# Шаг 1: Скачивание и установка бинарного файла
|
install_binary() {
|
||||||
echo "[1/5] Скачивание последней версии Telemt..."
|
src="$1"
|
||||||
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz
|
dst="$2"
|
||||||
|
|
||||||
echo "[1/5] Перемещение исполняемого файла в /usr/local/bin и установка прав..."
|
if [ -w "$INSTALL_DIR" ] || { [ ! -e "$INSTALL_DIR" ] && [ -w "$(dirname "$INSTALL_DIR")" ]; }; then
|
||||||
mv telemt /usr/local/bin/telemt
|
mkdir -p "$INSTALL_DIR"
|
||||||
chmod +x /usr/local/bin/telemt
|
install -m 0755 "$src" "$dst"
|
||||||
|
elif command -v sudo >/dev/null 2>&1; then
|
||||||
|
sudo mkdir -p "$INSTALL_DIR"
|
||||||
|
sudo install -m 0755 "$src" "$dst"
|
||||||
|
else
|
||||||
|
die "cannot write to $INSTALL_DIR and sudo is not available"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# Шаг 2: Генерация секрета
|
need_cmd uname
|
||||||
echo "[2/5] Генерация секретного ключа..."
|
need_cmd tar
|
||||||
SECRET=$(openssl rand -hex 16)
|
need_cmd mktemp
|
||||||
|
need_cmd grep
|
||||||
|
need_cmd install
|
||||||
|
|
||||||
# Шаг 3: Создание файла конфигурации
|
ARCH="$(detect_arch)"
|
||||||
echo "[3/5] Создание файла конфигурации /etc/telemt.toml..."
|
LIBC="$(detect_libc)"
|
||||||
printf "# === General Settings ===\n[general]\n[general.modes]\nclassic = false\nsecure = false\ntls = true\n\n# === Anti-Censorship & Masking ===\n[censorship]\n# !!! ВАЖНО: Замените на ваш домен или домен, который вы хотите использовать для маскировки !!!\ntls_domain = \"petrovich.ru\"\n\n[access.users]\nhello = \"%s\"\n" "$SECRET" > /etc/telemt.toml
|
|
||||||
|
|
||||||
# Шаг 4: Создание службы Systemd
|
case "$VERSION" in
|
||||||
echo "[4/5] Создание службы systemd..."
|
latest)
|
||||||
printf "[Unit]\nDescription=Telemt Proxy\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/telemt /etc/telemt.toml\nRestart=on-failure\nRestartSec=5\nLimitNOFILE=65536\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/telemt.service
|
URL="https://github.com/$REPO/releases/latest/download/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
URL="https://github.com/$REPO/releases/download/${VERSION}/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
# Шаг 5: Запуск службы
|
TMPDIR="$(mktemp -d)"
|
||||||
echo "[5/5] Перезагрузка systemd, запуск и включение службы telemt..."
|
trap 'rm -rf "$TMPDIR"' EXIT INT TERM
|
||||||
systemctl daemon-reload
|
|
||||||
systemctl start telemt
|
|
||||||
systemctl enable telemt
|
|
||||||
|
|
||||||
echo "--- Установка и запуск Telemt успешно завершены! ---"
|
say "Installing $BIN_NAME ($VERSION) for $ARCH-linux-$LIBC..."
|
||||||
echo
|
fetch_to_stdout "$URL" | tar -xzf - -C "$TMPDIR"
|
||||||
echo "ВАЖНАЯ ИНФОРМАЦИЯ:"
|
|
||||||
echo "==================="
|
[ -f "$TMPDIR/$BIN_NAME" ] || die "archive did not contain $BIN_NAME"
|
||||||
echo "1. Вам НЕОБХОДИМО отредактировать файл /etc/telemt.toml и заменить '\''petrovich.ru'\'' на другой домен"
|
|
||||||
echo " с помощью команды:"
|
install_binary "$TMPDIR/$BIN_NAME" "$INSTALL_DIR/$BIN_NAME"
|
||||||
echo " nano /etc/telemt.toml"
|
|
||||||
echo " После редактирования файла перезапустите службу командой:"
|
say "Installed: $INSTALL_DIR/$BIN_NAME"
|
||||||
echo " sudo systemctl restart telemt"
|
"$INSTALL_DIR/$BIN_NAME" --version 2>/dev/null || true
|
||||||
echo
|
|
||||||
echo "2. Для проверки статуса службы выполните команду:"
|
|
||||||
echo " systemctl status telemt"
|
|
||||||
echo
|
|
||||||
echo "3. Для получения ссылок на подключение выполните команду:"
|
|
||||||
echo " journalctl -u telemt -n -g '\''links'\'' --no-pager -o cat | tac"
|
|
||||||
fi
|
|
||||||
'
|
|
||||||
|
|||||||
@@ -11,11 +11,12 @@ use hyper::server::conn::http1;
|
|||||||
use hyper::service::service_fn;
|
use hyper::service::service_fn;
|
||||||
use hyper::{Method, Request, Response, StatusCode};
|
use hyper::{Method, Request, Response, StatusCode};
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::sync::{Mutex, watch};
|
use tokio::sync::{Mutex, RwLock, watch};
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
|
use crate::startup::StartupTracker;
|
||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
@@ -25,6 +26,7 @@ mod events;
|
|||||||
mod http_utils;
|
mod http_utils;
|
||||||
mod model;
|
mod model;
|
||||||
mod runtime_edge;
|
mod runtime_edge;
|
||||||
|
mod runtime_init;
|
||||||
mod runtime_min;
|
mod runtime_min;
|
||||||
mod runtime_stats;
|
mod runtime_stats;
|
||||||
mod runtime_watch;
|
mod runtime_watch;
|
||||||
@@ -41,6 +43,7 @@ use runtime_edge::{
|
|||||||
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
||||||
build_runtime_events_recent_data,
|
build_runtime_events_recent_data,
|
||||||
};
|
};
|
||||||
|
use runtime_init::build_runtime_initialization_data;
|
||||||
use runtime_min::{
|
use runtime_min::{
|
||||||
build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data,
|
build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data,
|
||||||
build_runtime_upstream_quality_data, build_security_whitelist_data,
|
build_runtime_upstream_quality_data, build_security_whitelist_data,
|
||||||
@@ -67,7 +70,7 @@ pub(super) struct ApiRuntimeState {
|
|||||||
pub(super) struct ApiShared {
|
pub(super) struct ApiShared {
|
||||||
pub(super) stats: Arc<Stats>,
|
pub(super) stats: Arc<Stats>,
|
||||||
pub(super) ip_tracker: Arc<UserIpTracker>,
|
pub(super) ip_tracker: Arc<UserIpTracker>,
|
||||||
pub(super) me_pool: Option<Arc<MePool>>,
|
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
pub(super) upstream_manager: Arc<UpstreamManager>,
|
pub(super) upstream_manager: Arc<UpstreamManager>,
|
||||||
pub(super) config_path: PathBuf,
|
pub(super) config_path: PathBuf,
|
||||||
pub(super) startup_detected_ip_v4: Option<IpAddr>,
|
pub(super) startup_detected_ip_v4: Option<IpAddr>,
|
||||||
@@ -79,6 +82,7 @@ pub(super) struct ApiShared {
|
|||||||
pub(super) runtime_events: Arc<ApiEventStore>,
|
pub(super) runtime_events: Arc<ApiEventStore>,
|
||||||
pub(super) request_id: Arc<AtomicU64>,
|
pub(super) request_id: Arc<AtomicU64>,
|
||||||
pub(super) runtime_state: Arc<ApiRuntimeState>,
|
pub(super) runtime_state: Arc<ApiRuntimeState>,
|
||||||
|
pub(super) startup_tracker: Arc<StartupTracker>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiShared {
|
impl ApiShared {
|
||||||
@@ -91,7 +95,7 @@ pub async fn serve(
|
|||||||
listen: SocketAddr,
|
listen: SocketAddr,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
upstream_manager: Arc<UpstreamManager>,
|
upstream_manager: Arc<UpstreamManager>,
|
||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
admission_rx: watch::Receiver<bool>,
|
admission_rx: watch::Receiver<bool>,
|
||||||
@@ -99,6 +103,7 @@ pub async fn serve(
|
|||||||
startup_detected_ip_v4: Option<IpAddr>,
|
startup_detected_ip_v4: Option<IpAddr>,
|
||||||
startup_detected_ip_v6: Option<IpAddr>,
|
startup_detected_ip_v6: Option<IpAddr>,
|
||||||
process_started_at_epoch_secs: u64,
|
process_started_at_epoch_secs: u64,
|
||||||
|
startup_tracker: Arc<StartupTracker>,
|
||||||
) {
|
) {
|
||||||
let listener = match TcpListener::bind(listen).await {
|
let listener = match TcpListener::bind(listen).await {
|
||||||
Ok(listener) => listener,
|
Ok(listener) => listener,
|
||||||
@@ -138,6 +143,7 @@ pub async fn serve(
|
|||||||
)),
|
)),
|
||||||
request_id: Arc::new(AtomicU64::new(1)),
|
request_id: Arc::new(AtomicU64::new(1)),
|
||||||
runtime_state: runtime_state.clone(),
|
runtime_state: runtime_state.clone(),
|
||||||
|
startup_tracker,
|
||||||
});
|
});
|
||||||
|
|
||||||
spawn_runtime_watchers(
|
spawn_runtime_watchers(
|
||||||
@@ -248,7 +254,12 @@ async fn handle(
|
|||||||
}
|
}
|
||||||
("GET", "/v1/runtime/gates") => {
|
("GET", "/v1/runtime/gates") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref());
|
let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/initialization") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_initialization_data(shared.as_ref()).await;
|
||||||
Ok(success_response(StatusCode::OK, data, revision))
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
}
|
}
|
||||||
("GET", "/v1/limits/effective") => {
|
("GET", "/v1/limits/effective") => {
|
||||||
|
|||||||
@@ -266,6 +266,7 @@ pub(super) struct MeWritersData {
|
|||||||
pub(super) struct DcStatus {
|
pub(super) struct DcStatus {
|
||||||
pub(super) dc: i16,
|
pub(super) dc: i16,
|
||||||
pub(super) endpoints: Vec<String>,
|
pub(super) endpoints: Vec<String>,
|
||||||
|
pub(super) endpoint_writers: Vec<DcEndpointWriters>,
|
||||||
pub(super) available_endpoints: usize,
|
pub(super) available_endpoints: usize,
|
||||||
pub(super) available_pct: f64,
|
pub(super) available_pct: f64,
|
||||||
pub(super) required_writers: usize,
|
pub(super) required_writers: usize,
|
||||||
@@ -279,6 +280,12 @@ pub(super) struct DcStatus {
|
|||||||
pub(super) load: usize,
|
pub(super) load: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Clone)]
|
||||||
|
pub(super) struct DcEndpointWriters {
|
||||||
|
pub(super) endpoint: String,
|
||||||
|
pub(super) active_writers: usize,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Clone)]
|
#[derive(Serialize, Clone)]
|
||||||
pub(super) struct DcStatusData {
|
pub(super) struct DcStatusData {
|
||||||
pub(super) middle_proxy_enabled: bool,
|
pub(super) middle_proxy_enabled: bool,
|
||||||
@@ -318,11 +325,21 @@ pub(super) struct MinimalMeRuntimeData {
|
|||||||
pub(super) adaptive_floor_cpu_cores_override: u16,
|
pub(super) adaptive_floor_cpu_cores_override: u16,
|
||||||
pub(super) adaptive_floor_max_extra_writers_single_per_core: u16,
|
pub(super) adaptive_floor_max_extra_writers_single_per_core: u16,
|
||||||
pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16,
|
pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
pub(super) adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
pub(super) adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
pub(super) adaptive_floor_max_active_writers_global: u32,
|
||||||
|
pub(super) adaptive_floor_max_warm_writers_global: u32,
|
||||||
pub(super) adaptive_floor_cpu_cores_detected: u32,
|
pub(super) adaptive_floor_cpu_cores_detected: u32,
|
||||||
pub(super) adaptive_floor_cpu_cores_effective: u32,
|
pub(super) adaptive_floor_cpu_cores_effective: u32,
|
||||||
pub(super) adaptive_floor_global_cap_raw: u64,
|
pub(super) adaptive_floor_global_cap_raw: u64,
|
||||||
pub(super) adaptive_floor_global_cap_effective: u64,
|
pub(super) adaptive_floor_global_cap_effective: u64,
|
||||||
pub(super) adaptive_floor_target_writers_total: u64,
|
pub(super) adaptive_floor_target_writers_total: u64,
|
||||||
|
pub(super) adaptive_floor_active_cap_configured: u64,
|
||||||
|
pub(super) adaptive_floor_active_cap_effective: u64,
|
||||||
|
pub(super) adaptive_floor_warm_cap_configured: u64,
|
||||||
|
pub(super) adaptive_floor_warm_cap_effective: u64,
|
||||||
|
pub(super) adaptive_floor_active_writers_current: u64,
|
||||||
|
pub(super) adaptive_floor_warm_writers_current: u64,
|
||||||
pub(super) me_keepalive_enabled: bool,
|
pub(super) me_keepalive_enabled: bool,
|
||||||
pub(super) me_keepalive_interval_secs: u64,
|
pub(super) me_keepalive_interval_secs: u64,
|
||||||
pub(super) me_keepalive_jitter_secs: u64,
|
pub(super) me_keepalive_jitter_secs: u64,
|
||||||
@@ -344,6 +361,8 @@ pub(super) struct MinimalMeRuntimeData {
|
|||||||
pub(super) me_single_endpoint_outage_backoff_max_ms: u64,
|
pub(super) me_single_endpoint_outage_backoff_max_ms: u64,
|
||||||
pub(super) me_single_endpoint_shadow_rotate_every_secs: u64,
|
pub(super) me_single_endpoint_shadow_rotate_every_secs: u64,
|
||||||
pub(super) me_deterministic_writer_sort: bool,
|
pub(super) me_deterministic_writer_sort: bool,
|
||||||
|
pub(super) me_writer_pick_mode: &'static str,
|
||||||
|
pub(super) me_writer_pick_sample_size: u8,
|
||||||
pub(super) me_socks_kdf_policy: &'static str,
|
pub(super) me_socks_kdf_policy: &'static str,
|
||||||
pub(super) quarantined_endpoints_total: usize,
|
pub(super) quarantined_endpoints_total: usize,
|
||||||
pub(super) quarantined_endpoints: Vec<MinimalQuarantineData>,
|
pub(super) quarantined_endpoints: Vec<MinimalQuarantineData>,
|
||||||
|
|||||||
186
src/api/runtime_init.rs
Normal file
186
src/api/runtime_init.rs
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::startup::{
|
||||||
|
COMPONENT_ME_CONNECTIVITY_PING, COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V4, COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH,
|
||||||
|
StartupComponentStatus, StartupMeStatus, compute_progress_pct,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::ApiShared;
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeInitializationComponentData {
|
||||||
|
pub(super) id: &'static str,
|
||||||
|
pub(super) title: &'static str,
|
||||||
|
pub(super) status: &'static str,
|
||||||
|
pub(super) started_at_epoch_ms: Option<u64>,
|
||||||
|
pub(super) finished_at_epoch_ms: Option<u64>,
|
||||||
|
pub(super) duration_ms: Option<u64>,
|
||||||
|
pub(super) attempts: u32,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) details: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeInitializationMeData {
|
||||||
|
pub(super) status: &'static str,
|
||||||
|
pub(super) current_stage: String,
|
||||||
|
pub(super) progress_pct: f64,
|
||||||
|
pub(super) init_attempt: u32,
|
||||||
|
pub(super) retry_limit: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) last_error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct RuntimeInitializationData {
|
||||||
|
pub(super) status: &'static str,
|
||||||
|
pub(super) degraded: bool,
|
||||||
|
pub(super) current_stage: String,
|
||||||
|
pub(super) progress_pct: f64,
|
||||||
|
pub(super) started_at_epoch_secs: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub(super) ready_at_epoch_secs: Option<u64>,
|
||||||
|
pub(super) total_elapsed_ms: u64,
|
||||||
|
pub(super) transport_mode: String,
|
||||||
|
pub(super) me: RuntimeInitializationMeData,
|
||||||
|
pub(super) components: Vec<RuntimeInitializationComponentData>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(super) struct RuntimeStartupSummaryData {
|
||||||
|
pub(super) status: &'static str,
|
||||||
|
pub(super) stage: String,
|
||||||
|
pub(super) progress_pct: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_startup_summary(shared: &ApiShared) -> RuntimeStartupSummaryData {
|
||||||
|
let snapshot = shared.startup_tracker.snapshot().await;
|
||||||
|
let me_pool_progress = current_me_pool_stage_progress(shared).await;
|
||||||
|
let progress_pct = compute_progress_pct(&snapshot, me_pool_progress);
|
||||||
|
RuntimeStartupSummaryData {
|
||||||
|
status: snapshot.status.as_str(),
|
||||||
|
stage: snapshot.current_stage,
|
||||||
|
progress_pct,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn build_runtime_initialization_data(
|
||||||
|
shared: &ApiShared,
|
||||||
|
) -> RuntimeInitializationData {
|
||||||
|
let snapshot = shared.startup_tracker.snapshot().await;
|
||||||
|
let me_pool_progress = current_me_pool_stage_progress(shared).await;
|
||||||
|
let progress_pct = compute_progress_pct(&snapshot, me_pool_progress);
|
||||||
|
let me_progress_pct = compute_me_progress_pct(&snapshot, me_pool_progress);
|
||||||
|
|
||||||
|
RuntimeInitializationData {
|
||||||
|
status: snapshot.status.as_str(),
|
||||||
|
degraded: snapshot.degraded,
|
||||||
|
current_stage: snapshot.current_stage,
|
||||||
|
progress_pct,
|
||||||
|
started_at_epoch_secs: snapshot.started_at_epoch_secs,
|
||||||
|
ready_at_epoch_secs: snapshot.ready_at_epoch_secs,
|
||||||
|
total_elapsed_ms: snapshot.total_elapsed_ms,
|
||||||
|
transport_mode: snapshot.transport_mode,
|
||||||
|
me: RuntimeInitializationMeData {
|
||||||
|
status: snapshot.me.status.as_str(),
|
||||||
|
current_stage: snapshot.me.current_stage,
|
||||||
|
progress_pct: me_progress_pct,
|
||||||
|
init_attempt: snapshot.me.init_attempt,
|
||||||
|
retry_limit: snapshot.me.retry_limit,
|
||||||
|
last_error: snapshot.me.last_error,
|
||||||
|
},
|
||||||
|
components: snapshot
|
||||||
|
.components
|
||||||
|
.into_iter()
|
||||||
|
.map(|component| RuntimeInitializationComponentData {
|
||||||
|
id: component.id,
|
||||||
|
title: component.title,
|
||||||
|
status: component.status.as_str(),
|
||||||
|
started_at_epoch_ms: component.started_at_epoch_ms,
|
||||||
|
finished_at_epoch_ms: component.finished_at_epoch_ms,
|
||||||
|
duration_ms: component.duration_ms,
|
||||||
|
attempts: component.attempts,
|
||||||
|
details: component.details,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compute_me_progress_pct(
|
||||||
|
snapshot: &crate::startup::StartupSnapshot,
|
||||||
|
me_pool_progress: Option<f64>,
|
||||||
|
) -> f64 {
|
||||||
|
match snapshot.me.status {
|
||||||
|
StartupMeStatus::Pending => 0.0,
|
||||||
|
StartupMeStatus::Ready | StartupMeStatus::Failed | StartupMeStatus::Skipped => 100.0,
|
||||||
|
StartupMeStatus::Initializing => {
|
||||||
|
let mut total_weight = 0.0f64;
|
||||||
|
let mut completed_weight = 0.0f64;
|
||||||
|
for component in &snapshot.components {
|
||||||
|
if !is_me_component(component.id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
total_weight += component.weight;
|
||||||
|
let unit_progress = match component.status {
|
||||||
|
StartupComponentStatus::Pending => 0.0,
|
||||||
|
StartupComponentStatus::Running => {
|
||||||
|
if component.id == COMPONENT_ME_POOL_INIT_STAGE1 {
|
||||||
|
me_pool_progress.unwrap_or(0.0).clamp(0.0, 1.0)
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
StartupComponentStatus::Ready
|
||||||
|
| StartupComponentStatus::Failed
|
||||||
|
| StartupComponentStatus::Skipped => 1.0,
|
||||||
|
};
|
||||||
|
completed_weight += component.weight * unit_progress;
|
||||||
|
}
|
||||||
|
if total_weight <= f64::EPSILON {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
((completed_weight / total_weight) * 100.0).clamp(0.0, 100.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_me_component(component_id: &str) -> bool {
|
||||||
|
matches!(
|
||||||
|
component_id,
|
||||||
|
COMPONENT_ME_SECRET_FETCH
|
||||||
|
| COMPONENT_ME_PROXY_CONFIG_V4
|
||||||
|
| COMPONENT_ME_PROXY_CONFIG_V6
|
||||||
|
| COMPONENT_ME_POOL_CONSTRUCT
|
||||||
|
| COMPONENT_ME_POOL_INIT_STAGE1
|
||||||
|
| COMPONENT_ME_CONNECTIVITY_PING
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn current_me_pool_stage_progress(shared: &ApiShared) -> Option<f64> {
|
||||||
|
let snapshot = shared.startup_tracker.snapshot().await;
|
||||||
|
if snapshot.me.status != StartupMeStatus::Initializing {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool = shared.me_pool.read().await.clone()?;
|
||||||
|
let status = pool.api_status_snapshot().await;
|
||||||
|
let configured_dc_groups = status.configured_dc_groups;
|
||||||
|
let covered_dc_groups = status
|
||||||
|
.dcs
|
||||||
|
.iter()
|
||||||
|
.filter(|dc| dc.alive_writers > 0)
|
||||||
|
.count();
|
||||||
|
|
||||||
|
let dc_coverage = ratio_01(covered_dc_groups, configured_dc_groups);
|
||||||
|
let writer_coverage = ratio_01(status.alive_writers, status.required_writers);
|
||||||
|
Some((0.7 * dc_coverage + 0.3 * writer_coverage).clamp(0.0, 1.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ratio_01(part: usize, total: usize) -> f64 {
|
||||||
|
if total == 0 {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
((part as f64) / (total as f64)).clamp(0.0, 1.0)
|
||||||
|
}
|
||||||
@@ -260,7 +260,7 @@ pub(super) fn build_security_whitelist_data(cfg: &ProxyConfig) -> SecurityWhitel
|
|||||||
|
|
||||||
pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData {
|
pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData {
|
||||||
let now_epoch_secs = now_epoch_secs();
|
let now_epoch_secs = now_epoch_secs();
|
||||||
let Some(pool) = &shared.me_pool else {
|
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||||
return RuntimeMePoolStateData {
|
return RuntimeMePoolStateData {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
@@ -350,7 +350,7 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
|||||||
|
|
||||||
pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData {
|
pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData {
|
||||||
let now_epoch_secs = now_epoch_secs();
|
let now_epoch_secs = now_epoch_secs();
|
||||||
let Some(pool) = &shared.me_pool else {
|
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||||
return RuntimeMeQualityData {
|
return RuntimeMeQualityData {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
@@ -486,7 +486,7 @@ pub(super) async fn build_runtime_upstream_quality_data(
|
|||||||
|
|
||||||
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||||
let now_epoch_secs = now_epoch_secs();
|
let now_epoch_secs = now_epoch_secs();
|
||||||
let Some(pool) = &shared.me_pool else {
|
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||||
return RuntimeNatStunData {
|
return RuntimeNatStunData {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ use crate::transport::UpstreamRouteKind;
|
|||||||
|
|
||||||
use super::ApiShared;
|
use super::ApiShared;
|
||||||
use super::model::{
|
use super::model::{
|
||||||
DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary, MinimalAllData,
|
DcEndpointWriters, DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary,
|
||||||
MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData, MinimalQuarantineData,
|
MinimalAllData, MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData,
|
||||||
UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData, ZeroAllData,
|
MinimalQuarantineData, UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData,
|
||||||
ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData,
|
ZeroAllData, ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData,
|
||||||
ZeroUpstreamData,
|
ZeroUpstreamData,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -297,7 +297,7 @@ async fn get_minimal_payload_cached(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let pool = shared.me_pool.as_ref()?;
|
let pool = shared.me_pool.read().await.clone()?;
|
||||||
let status = pool.api_status_snapshot().await;
|
let status = pool.api_status_snapshot().await;
|
||||||
let runtime = pool.api_runtime_snapshot().await;
|
let runtime = pool.api_runtime_snapshot().await;
|
||||||
let generated_at_epoch_secs = status.generated_at_epoch_secs;
|
let generated_at_epoch_secs = status.generated_at_epoch_secs;
|
||||||
@@ -346,6 +346,14 @@ async fn get_minimal_payload_cached(
|
|||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|value| value.to_string())
|
.map(|value| value.to_string())
|
||||||
.collect(),
|
.collect(),
|
||||||
|
endpoint_writers: entry
|
||||||
|
.endpoint_writers
|
||||||
|
.into_iter()
|
||||||
|
.map(|coverage| DcEndpointWriters {
|
||||||
|
endpoint: coverage.endpoint.to_string(),
|
||||||
|
active_writers: coverage.active_writers,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
available_endpoints: entry.available_endpoints,
|
available_endpoints: entry.available_endpoints,
|
||||||
available_pct: entry.available_pct,
|
available_pct: entry.available_pct,
|
||||||
required_writers: entry.required_writers,
|
required_writers: entry.required_writers,
|
||||||
@@ -380,11 +388,25 @@ async fn get_minimal_payload_cached(
|
|||||||
.adaptive_floor_max_extra_writers_single_per_core,
|
.adaptive_floor_max_extra_writers_single_per_core,
|
||||||
adaptive_floor_max_extra_writers_multi_per_core: runtime
|
adaptive_floor_max_extra_writers_multi_per_core: runtime
|
||||||
.adaptive_floor_max_extra_writers_multi_per_core,
|
.adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
adaptive_floor_max_active_writers_per_core: runtime
|
||||||
|
.adaptive_floor_max_active_writers_per_core,
|
||||||
|
adaptive_floor_max_warm_writers_per_core: runtime
|
||||||
|
.adaptive_floor_max_warm_writers_per_core,
|
||||||
|
adaptive_floor_max_active_writers_global: runtime
|
||||||
|
.adaptive_floor_max_active_writers_global,
|
||||||
|
adaptive_floor_max_warm_writers_global: runtime
|
||||||
|
.adaptive_floor_max_warm_writers_global,
|
||||||
adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected,
|
adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected,
|
||||||
adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective,
|
adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective,
|
||||||
adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw,
|
adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw,
|
||||||
adaptive_floor_global_cap_effective: runtime.adaptive_floor_global_cap_effective,
|
adaptive_floor_global_cap_effective: runtime.adaptive_floor_global_cap_effective,
|
||||||
adaptive_floor_target_writers_total: runtime.adaptive_floor_target_writers_total,
|
adaptive_floor_target_writers_total: runtime.adaptive_floor_target_writers_total,
|
||||||
|
adaptive_floor_active_cap_configured: runtime.adaptive_floor_active_cap_configured,
|
||||||
|
adaptive_floor_active_cap_effective: runtime.adaptive_floor_active_cap_effective,
|
||||||
|
adaptive_floor_warm_cap_configured: runtime.adaptive_floor_warm_cap_configured,
|
||||||
|
adaptive_floor_warm_cap_effective: runtime.adaptive_floor_warm_cap_effective,
|
||||||
|
adaptive_floor_active_writers_current: runtime.adaptive_floor_active_writers_current,
|
||||||
|
adaptive_floor_warm_writers_current: runtime.adaptive_floor_warm_writers_current,
|
||||||
me_keepalive_enabled: runtime.me_keepalive_enabled,
|
me_keepalive_enabled: runtime.me_keepalive_enabled,
|
||||||
me_keepalive_interval_secs: runtime.me_keepalive_interval_secs,
|
me_keepalive_interval_secs: runtime.me_keepalive_interval_secs,
|
||||||
me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs,
|
me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs,
|
||||||
@@ -408,6 +430,8 @@ async fn get_minimal_payload_cached(
|
|||||||
me_single_endpoint_shadow_rotate_every_secs: runtime
|
me_single_endpoint_shadow_rotate_every_secs: runtime
|
||||||
.me_single_endpoint_shadow_rotate_every_secs,
|
.me_single_endpoint_shadow_rotate_every_secs,
|
||||||
me_deterministic_writer_sort: runtime.me_deterministic_writer_sort,
|
me_deterministic_writer_sort: runtime.me_deterministic_writer_sort,
|
||||||
|
me_writer_pick_mode: runtime.me_writer_pick_mode,
|
||||||
|
me_writer_pick_sample_size: runtime.me_writer_pick_sample_size,
|
||||||
me_socks_kdf_policy: runtime.me_socks_kdf_policy,
|
me_socks_kdf_policy: runtime.me_socks_kdf_policy,
|
||||||
quarantined_endpoints_total: runtime.quarantined_endpoints.len(),
|
quarantined_endpoints_total: runtime.quarantined_endpoints.len(),
|
||||||
quarantined_endpoints: runtime
|
quarantined_endpoints: runtime
|
||||||
|
|||||||
@@ -2,9 +2,10 @@ use std::sync::atomic::Ordering;
|
|||||||
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::config::{MeFloorMode, ProxyConfig, UserMaxUniqueIpsMode};
|
use crate::config::{MeFloorMode, MeWriterPickMode, ProxyConfig, UserMaxUniqueIpsMode};
|
||||||
|
|
||||||
use super::ApiShared;
|
use super::ApiShared;
|
||||||
|
use super::runtime_init::build_runtime_startup_summary;
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub(super) struct SystemInfoData {
|
pub(super) struct SystemInfoData {
|
||||||
@@ -34,6 +35,9 @@ pub(super) struct RuntimeGatesData {
|
|||||||
pub(super) me_runtime_ready: bool,
|
pub(super) me_runtime_ready: bool,
|
||||||
pub(super) me2dc_fallback_enabled: bool,
|
pub(super) me2dc_fallback_enabled: bool,
|
||||||
pub(super) use_middle_proxy: bool,
|
pub(super) use_middle_proxy: bool,
|
||||||
|
pub(super) startup_status: &'static str,
|
||||||
|
pub(super) startup_stage: String,
|
||||||
|
pub(super) startup_progress_pct: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -66,10 +70,16 @@ pub(super) struct EffectiveMiddleProxyLimits {
|
|||||||
pub(super) adaptive_floor_cpu_cores_override: u16,
|
pub(super) adaptive_floor_cpu_cores_override: u16,
|
||||||
pub(super) adaptive_floor_max_extra_writers_single_per_core: u16,
|
pub(super) adaptive_floor_max_extra_writers_single_per_core: u16,
|
||||||
pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16,
|
pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
pub(super) adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
pub(super) adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
pub(super) adaptive_floor_max_active_writers_global: u32,
|
||||||
|
pub(super) adaptive_floor_max_warm_writers_global: u32,
|
||||||
pub(super) reconnect_max_concurrent_per_dc: u32,
|
pub(super) reconnect_max_concurrent_per_dc: u32,
|
||||||
pub(super) reconnect_backoff_base_ms: u64,
|
pub(super) reconnect_backoff_base_ms: u64,
|
||||||
pub(super) reconnect_backoff_cap_ms: u64,
|
pub(super) reconnect_backoff_cap_ms: u64,
|
||||||
pub(super) reconnect_fast_retry_count: u32,
|
pub(super) reconnect_fast_retry_count: u32,
|
||||||
|
pub(super) writer_pick_mode: &'static str,
|
||||||
|
pub(super) writer_pick_sample_size: u8,
|
||||||
pub(super) me2dc_fallback: bool,
|
pub(super) me2dc_fallback: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,12 +152,18 @@ pub(super) fn build_system_info_data(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) -> RuntimeGatesData {
|
pub(super) async fn build_runtime_gates_data(
|
||||||
|
shared: &ApiShared,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
) -> RuntimeGatesData {
|
||||||
|
let startup_summary = build_runtime_startup_summary(shared).await;
|
||||||
let me_runtime_ready = if !cfg.general.use_middle_proxy {
|
let me_runtime_ready = if !cfg.general.use_middle_proxy {
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
shared
|
shared
|
||||||
.me_pool
|
.me_pool
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|pool| pool.is_runtime_ready())
|
.map(|pool| pool.is_runtime_ready())
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
@@ -159,6 +175,9 @@ pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) ->
|
|||||||
me_runtime_ready,
|
me_runtime_ready,
|
||||||
me2dc_fallback_enabled: cfg.general.me2dc_fallback,
|
me2dc_fallback_enabled: cfg.general.me2dc_fallback,
|
||||||
use_middle_proxy: cfg.general.use_middle_proxy,
|
use_middle_proxy: cfg.general.use_middle_proxy,
|
||||||
|
startup_status: startup_summary.status,
|
||||||
|
startup_stage: startup_summary.stage,
|
||||||
|
startup_progress_pct: startup_summary.progress_pct,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -204,10 +223,24 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD
|
|||||||
adaptive_floor_max_extra_writers_multi_per_core: cfg
|
adaptive_floor_max_extra_writers_multi_per_core: cfg
|
||||||
.general
|
.general
|
||||||
.me_adaptive_floor_max_extra_writers_multi_per_core,
|
.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
adaptive_floor_max_active_writers_per_core: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
adaptive_floor_max_warm_writers_per_core: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
adaptive_floor_max_active_writers_global: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_active_writers_global,
|
||||||
|
adaptive_floor_max_warm_writers_global: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_warm_writers_global,
|
||||||
reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc,
|
reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc,
|
||||||
reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms,
|
reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms,
|
||||||
reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms,
|
reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms,
|
||||||
reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count,
|
reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count,
|
||||||
|
writer_pick_mode: me_writer_pick_mode_label(cfg.general.me_writer_pick_mode),
|
||||||
|
writer_pick_sample_size: cfg.general.me_writer_pick_sample_size,
|
||||||
me2dc_fallback: cfg.general.me2dc_fallback,
|
me2dc_fallback: cfg.general.me2dc_fallback,
|
||||||
},
|
},
|
||||||
user_ip_policy: EffectiveUserIpPolicyLimits {
|
user_ip_policy: EffectiveUserIpPolicyLimits {
|
||||||
@@ -245,3 +278,10 @@ fn me_floor_mode_label(mode: MeFloorMode) -> &'static str {
|
|||||||
MeFloorMode::Adaptive => "adaptive",
|
MeFloorMode::Adaptive => "adaptive",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn me_writer_pick_mode_label(mode: MeWriterPickMode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
MeWriterPickMode::SortedRr => "sorted_rr",
|
||||||
|
MeWriterPickMode::P2c => "p2c",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,6 +17,18 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_WRITERS_PER_CORE_TOTAL: u16 = 48;
|
|||||||
const DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE: u16 = 0;
|
const DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE: u16 = 0;
|
||||||
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE: u16 = 1;
|
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE: u16 = 1;
|
||||||
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE: u16 = 2;
|
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE: u16 = 2;
|
||||||
|
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64;
|
||||||
|
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64;
|
||||||
|
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256;
|
||||||
|
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256;
|
||||||
|
const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 1024;
|
||||||
|
const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 512;
|
||||||
|
const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 256;
|
||||||
|
const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3;
|
||||||
|
const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000;
|
||||||
|
const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000;
|
||||||
|
const DEFAULT_ME_ADMISSION_POLL_MS: u64 = 1000;
|
||||||
|
const DEFAULT_ME_WARN_RATE_LIMIT_MS: u64 = 5000;
|
||||||
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
||||||
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
|
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
|
||||||
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
|
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
|
||||||
@@ -276,6 +288,54 @@ pub(crate) fn default_me_adaptive_floor_max_extra_writers_multi_per_core() -> u1
|
|||||||
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE
|
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_adaptive_floor_max_active_writers_per_core() -> u16 {
|
||||||
|
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_adaptive_floor_max_warm_writers_per_core() -> u16 {
|
||||||
|
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_adaptive_floor_max_active_writers_global() -> u32 {
|
||||||
|
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_adaptive_floor_max_warm_writers_global() -> u32 {
|
||||||
|
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_writer_cmd_channel_capacity() -> usize {
|
||||||
|
DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_channel_capacity() -> usize {
|
||||||
|
DEFAULT_ME_ROUTE_CHANNEL_CAPACITY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_c2me_channel_capacity() -> usize {
|
||||||
|
DEFAULT_ME_C2ME_CHANNEL_CAPACITY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||||
|
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
||||||
|
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
||||||
|
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
||||||
|
DEFAULT_ME_ADMISSION_POLL_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
||||||
|
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,10 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher};
|
|||||||
use tokio::sync::{mpsc, watch};
|
use tokio::sync::{mpsc, watch};
|
||||||
use tracing::{error, info, warn};
|
use tracing::{error, info, warn};
|
||||||
|
|
||||||
use crate::config::{LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel};
|
use crate::config::{
|
||||||
|
LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
||||||
|
MeWriterPickMode,
|
||||||
|
};
|
||||||
use super::load::ProxyConfig;
|
use super::load::ProxyConfig;
|
||||||
|
|
||||||
// ── Hot fields ────────────────────────────────────────────────────────────────
|
// ── Hot fields ────────────────────────────────────────────────────────────────
|
||||||
@@ -57,6 +60,8 @@ pub struct HotFields {
|
|||||||
pub me_bind_stale_ttl_secs: u64,
|
pub me_bind_stale_ttl_secs: u64,
|
||||||
pub me_secret_atomic_snapshot: bool,
|
pub me_secret_atomic_snapshot: bool,
|
||||||
pub me_deterministic_writer_sort: bool,
|
pub me_deterministic_writer_sort: bool,
|
||||||
|
pub me_writer_pick_mode: MeWriterPickMode,
|
||||||
|
pub me_writer_pick_sample_size: u8,
|
||||||
pub me_single_endpoint_shadow_writers: u8,
|
pub me_single_endpoint_shadow_writers: u8,
|
||||||
pub me_single_endpoint_outage_mode_enabled: bool,
|
pub me_single_endpoint_outage_mode_enabled: bool,
|
||||||
pub me_single_endpoint_outage_disable_quarantine: bool,
|
pub me_single_endpoint_outage_disable_quarantine: bool,
|
||||||
@@ -84,9 +89,17 @@ pub struct HotFields {
|
|||||||
pub me_adaptive_floor_cpu_cores_override: u16,
|
pub me_adaptive_floor_cpu_cores_override: u16,
|
||||||
pub me_adaptive_floor_max_extra_writers_single_per_core: u16,
|
pub me_adaptive_floor_max_extra_writers_single_per_core: u16,
|
||||||
pub me_adaptive_floor_max_extra_writers_multi_per_core: u16,
|
pub me_adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
pub me_adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
pub me_adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
pub me_adaptive_floor_max_active_writers_global: u32,
|
||||||
|
pub me_adaptive_floor_max_warm_writers_global: u32,
|
||||||
pub me_route_backpressure_base_timeout_ms: u64,
|
pub me_route_backpressure_base_timeout_ms: u64,
|
||||||
pub me_route_backpressure_high_timeout_ms: u64,
|
pub me_route_backpressure_high_timeout_ms: u64,
|
||||||
pub me_route_backpressure_high_watermark_pct: u8,
|
pub me_route_backpressure_high_watermark_pct: u8,
|
||||||
|
pub me_health_interval_ms_unhealthy: u64,
|
||||||
|
pub me_health_interval_ms_healthy: u64,
|
||||||
|
pub me_admission_poll_ms: u64,
|
||||||
|
pub me_warn_rate_limit_ms: u64,
|
||||||
pub users: std::collections::HashMap<String, String>,
|
pub users: std::collections::HashMap<String, String>,
|
||||||
pub user_ad_tags: std::collections::HashMap<String, String>,
|
pub user_ad_tags: std::collections::HashMap<String, String>,
|
||||||
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
|
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
|
||||||
@@ -122,6 +135,8 @@ impl HotFields {
|
|||||||
me_bind_stale_ttl_secs: cfg.general.me_bind_stale_ttl_secs,
|
me_bind_stale_ttl_secs: cfg.general.me_bind_stale_ttl_secs,
|
||||||
me_secret_atomic_snapshot: cfg.general.me_secret_atomic_snapshot,
|
me_secret_atomic_snapshot: cfg.general.me_secret_atomic_snapshot,
|
||||||
me_deterministic_writer_sort: cfg.general.me_deterministic_writer_sort,
|
me_deterministic_writer_sort: cfg.general.me_deterministic_writer_sort,
|
||||||
|
me_writer_pick_mode: cfg.general.me_writer_pick_mode,
|
||||||
|
me_writer_pick_sample_size: cfg.general.me_writer_pick_sample_size,
|
||||||
me_single_endpoint_shadow_writers: cfg.general.me_single_endpoint_shadow_writers,
|
me_single_endpoint_shadow_writers: cfg.general.me_single_endpoint_shadow_writers,
|
||||||
me_single_endpoint_outage_mode_enabled: cfg
|
me_single_endpoint_outage_mode_enabled: cfg
|
||||||
.general
|
.general
|
||||||
@@ -173,9 +188,25 @@ impl HotFields {
|
|||||||
me_adaptive_floor_max_extra_writers_multi_per_core: cfg
|
me_adaptive_floor_max_extra_writers_multi_per_core: cfg
|
||||||
.general
|
.general
|
||||||
.me_adaptive_floor_max_extra_writers_multi_per_core,
|
.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
me_adaptive_floor_max_active_writers_per_core: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
me_adaptive_floor_max_warm_writers_per_core: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
me_adaptive_floor_max_active_writers_global: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_active_writers_global,
|
||||||
|
me_adaptive_floor_max_warm_writers_global: cfg
|
||||||
|
.general
|
||||||
|
.me_adaptive_floor_max_warm_writers_global,
|
||||||
me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms,
|
me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms,
|
||||||
me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms,
|
me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms,
|
||||||
me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct,
|
me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct,
|
||||||
|
me_health_interval_ms_unhealthy: cfg.general.me_health_interval_ms_unhealthy,
|
||||||
|
me_health_interval_ms_healthy: cfg.general.me_health_interval_ms_healthy,
|
||||||
|
me_admission_poll_ms: cfg.general.me_admission_poll_ms,
|
||||||
|
me_warn_rate_limit_ms: cfg.general.me_warn_rate_limit_ms,
|
||||||
users: cfg.access.users.clone(),
|
users: cfg.access.users.clone(),
|
||||||
user_ad_tags: cfg.access.user_ad_tags.clone(),
|
user_ad_tags: cfg.access.user_ad_tags.clone(),
|
||||||
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
|
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
|
||||||
@@ -268,6 +299,8 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
|||||||
cfg.general.me_bind_stale_ttl_secs = new.general.me_bind_stale_ttl_secs;
|
cfg.general.me_bind_stale_ttl_secs = new.general.me_bind_stale_ttl_secs;
|
||||||
cfg.general.me_secret_atomic_snapshot = new.general.me_secret_atomic_snapshot;
|
cfg.general.me_secret_atomic_snapshot = new.general.me_secret_atomic_snapshot;
|
||||||
cfg.general.me_deterministic_writer_sort = new.general.me_deterministic_writer_sort;
|
cfg.general.me_deterministic_writer_sort = new.general.me_deterministic_writer_sort;
|
||||||
|
cfg.general.me_writer_pick_mode = new.general.me_writer_pick_mode;
|
||||||
|
cfg.general.me_writer_pick_sample_size = new.general.me_writer_pick_sample_size;
|
||||||
cfg.general.me_single_endpoint_shadow_writers = new.general.me_single_endpoint_shadow_writers;
|
cfg.general.me_single_endpoint_shadow_writers = new.general.me_single_endpoint_shadow_writers;
|
||||||
cfg.general.me_single_endpoint_outage_mode_enabled =
|
cfg.general.me_single_endpoint_outage_mode_enabled =
|
||||||
new.general.me_single_endpoint_outage_mode_enabled;
|
new.general.me_single_endpoint_outage_mode_enabled;
|
||||||
@@ -305,12 +338,24 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
|||||||
new.general.me_adaptive_floor_max_extra_writers_single_per_core;
|
new.general.me_adaptive_floor_max_extra_writers_single_per_core;
|
||||||
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core =
|
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core =
|
||||||
new.general.me_adaptive_floor_max_extra_writers_multi_per_core;
|
new.general.me_adaptive_floor_max_extra_writers_multi_per_core;
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_per_core =
|
||||||
|
new.general.me_adaptive_floor_max_active_writers_per_core;
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_per_core =
|
||||||
|
new.general.me_adaptive_floor_max_warm_writers_per_core;
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_global =
|
||||||
|
new.general.me_adaptive_floor_max_active_writers_global;
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_global =
|
||||||
|
new.general.me_adaptive_floor_max_warm_writers_global;
|
||||||
cfg.general.me_route_backpressure_base_timeout_ms =
|
cfg.general.me_route_backpressure_base_timeout_ms =
|
||||||
new.general.me_route_backpressure_base_timeout_ms;
|
new.general.me_route_backpressure_base_timeout_ms;
|
||||||
cfg.general.me_route_backpressure_high_timeout_ms =
|
cfg.general.me_route_backpressure_high_timeout_ms =
|
||||||
new.general.me_route_backpressure_high_timeout_ms;
|
new.general.me_route_backpressure_high_timeout_ms;
|
||||||
cfg.general.me_route_backpressure_high_watermark_pct =
|
cfg.general.me_route_backpressure_high_watermark_pct =
|
||||||
new.general.me_route_backpressure_high_watermark_pct;
|
new.general.me_route_backpressure_high_watermark_pct;
|
||||||
|
cfg.general.me_health_interval_ms_unhealthy = new.general.me_health_interval_ms_unhealthy;
|
||||||
|
cfg.general.me_health_interval_ms_healthy = new.general.me_health_interval_ms_healthy;
|
||||||
|
cfg.general.me_admission_poll_ms = new.general.me_admission_poll_ms;
|
||||||
|
cfg.general.me_warn_rate_limit_ms = new.general.me_warn_rate_limit_ms;
|
||||||
|
|
||||||
cfg.access.users = new.access.users.clone();
|
cfg.access.users = new.access.users.clone();
|
||||||
cfg.access.user_ad_tags = new.access.user_ad_tags.clone();
|
cfg.access.user_ad_tags = new.access.user_ad_tags.clone();
|
||||||
@@ -647,11 +692,15 @@ fn log_changes(
|
|||||||
}
|
}
|
||||||
if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot
|
if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot
|
||||||
|| old_hot.me_deterministic_writer_sort != new_hot.me_deterministic_writer_sort
|
|| old_hot.me_deterministic_writer_sort != new_hot.me_deterministic_writer_sort
|
||||||
|
|| old_hot.me_writer_pick_mode != new_hot.me_writer_pick_mode
|
||||||
|
|| old_hot.me_writer_pick_sample_size != new_hot.me_writer_pick_sample_size
|
||||||
{
|
{
|
||||||
info!(
|
info!(
|
||||||
"config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={}",
|
"config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={} writer_pick_mode={:?} writer_pick_sample_size={}",
|
||||||
new_hot.me_secret_atomic_snapshot,
|
new_hot.me_secret_atomic_snapshot,
|
||||||
new_hot.me_deterministic_writer_sort
|
new_hot.me_deterministic_writer_sort,
|
||||||
|
new_hot.me_writer_pick_mode,
|
||||||
|
new_hot.me_writer_pick_sample_size,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if old_hot.me_single_endpoint_shadow_writers != new_hot.me_single_endpoint_shadow_writers
|
if old_hot.me_single_endpoint_shadow_writers != new_hot.me_single_endpoint_shadow_writers
|
||||||
@@ -739,9 +788,17 @@ fn log_changes(
|
|||||||
!= new_hot.me_adaptive_floor_max_extra_writers_single_per_core
|
!= new_hot.me_adaptive_floor_max_extra_writers_single_per_core
|
||||||
|| old_hot.me_adaptive_floor_max_extra_writers_multi_per_core
|
|| old_hot.me_adaptive_floor_max_extra_writers_multi_per_core
|
||||||
!= new_hot.me_adaptive_floor_max_extra_writers_multi_per_core
|
!= new_hot.me_adaptive_floor_max_extra_writers_multi_per_core
|
||||||
|
|| old_hot.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
!= new_hot.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
|| old_hot.me_adaptive_floor_max_warm_writers_per_core
|
||||||
|
!= new_hot.me_adaptive_floor_max_warm_writers_per_core
|
||||||
|
|| old_hot.me_adaptive_floor_max_active_writers_global
|
||||||
|
!= new_hot.me_adaptive_floor_max_active_writers_global
|
||||||
|
|| old_hot.me_adaptive_floor_max_warm_writers_global
|
||||||
|
!= new_hot.me_adaptive_floor_max_warm_writers_global
|
||||||
{
|
{
|
||||||
info!(
|
info!(
|
||||||
"config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={}",
|
"config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={} max_active_per_core={} max_warm_per_core={} max_active_global={} max_warm_global={}",
|
||||||
new_hot.me_floor_mode,
|
new_hot.me_floor_mode,
|
||||||
new_hot.me_adaptive_floor_idle_secs,
|
new_hot.me_adaptive_floor_idle_secs,
|
||||||
new_hot.me_adaptive_floor_min_writers_single_endpoint,
|
new_hot.me_adaptive_floor_min_writers_single_endpoint,
|
||||||
@@ -751,6 +808,10 @@ fn log_changes(
|
|||||||
new_hot.me_adaptive_floor_cpu_cores_override,
|
new_hot.me_adaptive_floor_cpu_cores_override,
|
||||||
new_hot.me_adaptive_floor_max_extra_writers_single_per_core,
|
new_hot.me_adaptive_floor_max_extra_writers_single_per_core,
|
||||||
new_hot.me_adaptive_floor_max_extra_writers_multi_per_core,
|
new_hot.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
new_hot.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
new_hot.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
new_hot.me_adaptive_floor_max_active_writers_global,
|
||||||
|
new_hot.me_adaptive_floor_max_warm_writers_global,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -760,12 +821,21 @@ fn log_changes(
|
|||||||
!= new_hot.me_route_backpressure_high_timeout_ms
|
!= new_hot.me_route_backpressure_high_timeout_ms
|
||||||
|| old_hot.me_route_backpressure_high_watermark_pct
|
|| old_hot.me_route_backpressure_high_watermark_pct
|
||||||
!= new_hot.me_route_backpressure_high_watermark_pct
|
!= new_hot.me_route_backpressure_high_watermark_pct
|
||||||
|
|| old_hot.me_health_interval_ms_unhealthy
|
||||||
|
!= new_hot.me_health_interval_ms_unhealthy
|
||||||
|
|| old_hot.me_health_interval_ms_healthy != new_hot.me_health_interval_ms_healthy
|
||||||
|
|| old_hot.me_admission_poll_ms != new_hot.me_admission_poll_ms
|
||||||
|
|| old_hot.me_warn_rate_limit_ms != new_hot.me_warn_rate_limit_ms
|
||||||
{
|
{
|
||||||
info!(
|
info!(
|
||||||
"config reload: me_route_backpressure: base={}ms high={}ms watermark={}%",
|
"config reload: me_route_backpressure: base={}ms high={}ms watermark={}%; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms",
|
||||||
new_hot.me_route_backpressure_base_timeout_ms,
|
new_hot.me_route_backpressure_base_timeout_ms,
|
||||||
new_hot.me_route_backpressure_high_timeout_ms,
|
new_hot.me_route_backpressure_high_timeout_ms,
|
||||||
new_hot.me_route_backpressure_high_watermark_pct,
|
new_hot.me_route_backpressure_high_watermark_pct,
|
||||||
|
new_hot.me_health_interval_ms_unhealthy,
|
||||||
|
new_hot.me_health_interval_ms_healthy,
|
||||||
|
new_hot.me_admission_poll_ms,
|
||||||
|
new_hot.me_warn_rate_limit_ms,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -285,6 +285,48 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.general.me_writer_cmd_channel_capacity == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_writer_cmd_channel_capacity must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_route_channel_capacity == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_route_channel_capacity must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_c2me_channel_capacity == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_c2me_channel_capacity must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_health_interval_ms_unhealthy == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_health_interval_ms_unhealthy must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_health_interval_ms_healthy == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_health_interval_ms_healthy must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_admission_poll_ms == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_admission_poll_ms must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_warn_rate_limit_ms == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_warn_rate_limit_ms must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if config.access.user_max_unique_ips_window_secs == 0 {
|
if config.access.user_max_unique_ips_window_secs == 0 {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"access.user_max_unique_ips_window_secs must be > 0".to_string(),
|
"access.user_max_unique_ips_window_secs must be > 0".to_string(),
|
||||||
@@ -327,6 +369,30 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.general.me_adaptive_floor_max_active_writers_per_core == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_adaptive_floor_max_active_writers_per_core must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_adaptive_floor_max_warm_writers_per_core == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_adaptive_floor_max_warm_writers_per_core must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_adaptive_floor_max_active_writers_global == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_adaptive_floor_max_active_writers_global must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.general.me_adaptive_floor_max_warm_writers_global == 0 {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_adaptive_floor_max_warm_writers_global must be > 0".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if config.general.me_single_endpoint_outage_backoff_min_ms == 0 {
|
if config.general.me_single_endpoint_outage_backoff_min_ms == 0 {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(),
|
"general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(),
|
||||||
@@ -453,6 +519,12 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !(2..=4).contains(&config.general.me_writer_pick_sample_size) {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_writer_pick_sample_size must be within [2, 4]".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if config.general.me_route_inline_recovery_attempts == 0 {
|
if config.general.me_route_inline_recovery_attempts == 0 {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"general.me_route_inline_recovery_attempts must be > 0".to_string(),
|
"general.me_route_inline_recovery_attempts must be > 0".to_string(),
|
||||||
@@ -1238,6 +1310,46 @@ mod tests {
|
|||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn me_adaptive_floor_max_active_writers_per_core_zero_is_rejected() {
|
||||||
|
let toml = r#"
|
||||||
|
[general]
|
||||||
|
me_adaptive_floor_max_active_writers_per_core = 0
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#;
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join("telemt_me_adaptive_floor_max_active_per_core_zero_test.toml");
|
||||||
|
std::fs::write(&path, toml).unwrap();
|
||||||
|
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||||
|
assert!(err.contains("general.me_adaptive_floor_max_active_writers_per_core must be > 0"));
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn me_adaptive_floor_max_warm_writers_global_zero_is_rejected() {
|
||||||
|
let toml = r#"
|
||||||
|
[general]
|
||||||
|
me_adaptive_floor_max_warm_writers_global = 0
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#;
|
||||||
|
let dir = std::env::temp_dir();
|
||||||
|
let path = dir.join("telemt_me_adaptive_floor_max_warm_global_zero_test.toml");
|
||||||
|
std::fs::write(&path, toml).unwrap();
|
||||||
|
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||||
|
assert!(err.contains("general.me_adaptive_floor_max_warm_writers_global must be > 0"));
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn upstream_connect_retry_attempts_zero_is_rejected() {
|
fn upstream_connect_retry_attempts_zero_is_rejected() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
|
|||||||
@@ -212,6 +212,32 @@ impl MeRouteNoWriterMode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Middle-End writer selection mode for new client bindings.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum MeWriterPickMode {
|
||||||
|
SortedRr,
|
||||||
|
#[default]
|
||||||
|
P2c,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MeWriterPickMode {
|
||||||
|
pub fn as_u8(self) -> u8 {
|
||||||
|
match self {
|
||||||
|
MeWriterPickMode::SortedRr => 0,
|
||||||
|
MeWriterPickMode::P2c => 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_u8(raw: u8) -> Self {
|
||||||
|
match raw {
|
||||||
|
0 => MeWriterPickMode::SortedRr,
|
||||||
|
1 => MeWriterPickMode::P2c,
|
||||||
|
_ => MeWriterPickMode::P2c,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-user unique source IP limit mode.
|
/// Per-user unique source IP limit mode.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -420,6 +446,18 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_rpc_proxy_req_every")]
|
#[serde(default = "default_rpc_proxy_req_every")]
|
||||||
pub rpc_proxy_req_every: u64,
|
pub rpc_proxy_req_every: u64,
|
||||||
|
|
||||||
|
/// Capacity of per-ME writer command channel.
|
||||||
|
#[serde(default = "default_me_writer_cmd_channel_capacity")]
|
||||||
|
pub me_writer_cmd_channel_capacity: usize,
|
||||||
|
|
||||||
|
/// Capacity of per-connection ME response route channel.
|
||||||
|
#[serde(default = "default_me_route_channel_capacity")]
|
||||||
|
pub me_route_channel_capacity: usize,
|
||||||
|
|
||||||
|
/// Capacity of per-client command queue from client reader to ME sender task.
|
||||||
|
#[serde(default = "default_me_c2me_channel_capacity")]
|
||||||
|
pub me_c2me_channel_capacity: usize,
|
||||||
|
|
||||||
/// Max pending ciphertext buffer per client writer (bytes).
|
/// Max pending ciphertext buffer per client writer (bytes).
|
||||||
/// Controls FakeTLS backpressure vs throughput.
|
/// Controls FakeTLS backpressure vs throughput.
|
||||||
#[serde(default = "default_crypto_pending_buffer")]
|
#[serde(default = "default_crypto_pending_buffer")]
|
||||||
@@ -545,6 +583,22 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_me_adaptive_floor_max_extra_writers_multi_per_core")]
|
#[serde(default = "default_me_adaptive_floor_max_extra_writers_multi_per_core")]
|
||||||
pub me_adaptive_floor_max_extra_writers_multi_per_core: u16,
|
pub me_adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
|
||||||
|
/// Hard cap for active ME writers per logical CPU core.
|
||||||
|
#[serde(default = "default_me_adaptive_floor_max_active_writers_per_core")]
|
||||||
|
pub me_adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
|
||||||
|
/// Hard cap for warm ME writers per logical CPU core.
|
||||||
|
#[serde(default = "default_me_adaptive_floor_max_warm_writers_per_core")]
|
||||||
|
pub me_adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
|
||||||
|
/// Hard global cap for active ME writers.
|
||||||
|
#[serde(default = "default_me_adaptive_floor_max_active_writers_global")]
|
||||||
|
pub me_adaptive_floor_max_active_writers_global: u32,
|
||||||
|
|
||||||
|
/// Hard global cap for warm ME writers.
|
||||||
|
#[serde(default = "default_me_adaptive_floor_max_warm_writers_global")]
|
||||||
|
pub me_adaptive_floor_max_warm_writers_global: u32,
|
||||||
|
|
||||||
/// Connect attempts for the selected upstream before returning error/fallback.
|
/// Connect attempts for the selected upstream before returning error/fallback.
|
||||||
#[serde(default = "default_upstream_connect_retry_attempts")]
|
#[serde(default = "default_upstream_connect_retry_attempts")]
|
||||||
pub upstream_connect_retry_attempts: u32,
|
pub upstream_connect_retry_attempts: u32,
|
||||||
@@ -604,6 +658,22 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_me_route_backpressure_high_watermark_pct")]
|
#[serde(default = "default_me_route_backpressure_high_watermark_pct")]
|
||||||
pub me_route_backpressure_high_watermark_pct: u8,
|
pub me_route_backpressure_high_watermark_pct: u8,
|
||||||
|
|
||||||
|
/// Health monitor interval in milliseconds while writer coverage is degraded.
|
||||||
|
#[serde(default = "default_me_health_interval_ms_unhealthy")]
|
||||||
|
pub me_health_interval_ms_unhealthy: u64,
|
||||||
|
|
||||||
|
/// Health monitor interval in milliseconds while writer coverage is stable.
|
||||||
|
#[serde(default = "default_me_health_interval_ms_healthy")]
|
||||||
|
pub me_health_interval_ms_healthy: u64,
|
||||||
|
|
||||||
|
/// Poll interval in milliseconds for conditional-admission state checks.
|
||||||
|
#[serde(default = "default_me_admission_poll_ms")]
|
||||||
|
pub me_admission_poll_ms: u64,
|
||||||
|
|
||||||
|
/// Cooldown for repetitive ME warning logs in milliseconds.
|
||||||
|
#[serde(default = "default_me_warn_rate_limit_ms")]
|
||||||
|
pub me_warn_rate_limit_ms: u64,
|
||||||
|
|
||||||
/// ME route behavior when no writer is immediately available.
|
/// ME route behavior when no writer is immediately available.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub me_route_no_writer_mode: MeRouteNoWriterMode,
|
pub me_route_no_writer_mode: MeRouteNoWriterMode,
|
||||||
@@ -738,6 +808,14 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_me_deterministic_writer_sort")]
|
#[serde(default = "default_me_deterministic_writer_sort")]
|
||||||
pub me_deterministic_writer_sort: bool,
|
pub me_deterministic_writer_sort: bool,
|
||||||
|
|
||||||
|
/// Writer selection mode for ME route bind path.
|
||||||
|
#[serde(default)]
|
||||||
|
pub me_writer_pick_mode: MeWriterPickMode,
|
||||||
|
|
||||||
|
/// Number of candidates sampled by writer picker in `p2c` mode.
|
||||||
|
#[serde(default = "default_me_writer_pick_sample_size")]
|
||||||
|
pub me_writer_pick_sample_size: u8,
|
||||||
|
|
||||||
/// Enable NTP drift check at startup.
|
/// Enable NTP drift check at startup.
|
||||||
#[serde(default = "default_ntp_check")]
|
#[serde(default = "default_ntp_check")]
|
||||||
pub ntp_check: bool,
|
pub ntp_check: bool,
|
||||||
@@ -780,6 +858,9 @@ impl Default for GeneralConfig {
|
|||||||
me_keepalive_jitter_secs: default_keepalive_jitter(),
|
me_keepalive_jitter_secs: default_keepalive_jitter(),
|
||||||
me_keepalive_payload_random: default_true(),
|
me_keepalive_payload_random: default_true(),
|
||||||
rpc_proxy_req_every: default_rpc_proxy_req_every(),
|
rpc_proxy_req_every: default_rpc_proxy_req_every(),
|
||||||
|
me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(),
|
||||||
|
me_route_channel_capacity: default_me_route_channel_capacity(),
|
||||||
|
me_c2me_channel_capacity: default_me_c2me_channel_capacity(),
|
||||||
me_warmup_stagger_enabled: default_true(),
|
me_warmup_stagger_enabled: default_true(),
|
||||||
me_warmup_step_delay_ms: default_warmup_step_delay_ms(),
|
me_warmup_step_delay_ms: default_warmup_step_delay_ms(),
|
||||||
me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(),
|
me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(),
|
||||||
@@ -802,6 +883,10 @@ impl Default for GeneralConfig {
|
|||||||
me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(),
|
me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(),
|
||||||
me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(),
|
me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(),
|
||||||
me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(),
|
me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(),
|
||||||
|
me_adaptive_floor_max_active_writers_per_core: default_me_adaptive_floor_max_active_writers_per_core(),
|
||||||
|
me_adaptive_floor_max_warm_writers_per_core: default_me_adaptive_floor_max_warm_writers_per_core(),
|
||||||
|
me_adaptive_floor_max_active_writers_global: default_me_adaptive_floor_max_active_writers_global(),
|
||||||
|
me_adaptive_floor_max_warm_writers_global: default_me_adaptive_floor_max_warm_writers_global(),
|
||||||
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
||||||
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
||||||
upstream_connect_budget_ms: default_upstream_connect_budget_ms(),
|
upstream_connect_budget_ms: default_upstream_connect_budget_ms(),
|
||||||
@@ -817,6 +902,10 @@ impl Default for GeneralConfig {
|
|||||||
me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(),
|
me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(),
|
||||||
me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(),
|
me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(),
|
||||||
me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(),
|
me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(),
|
||||||
|
me_health_interval_ms_unhealthy: default_me_health_interval_ms_unhealthy(),
|
||||||
|
me_health_interval_ms_healthy: default_me_health_interval_ms_healthy(),
|
||||||
|
me_admission_poll_ms: default_me_admission_poll_ms(),
|
||||||
|
me_warn_rate_limit_ms: default_me_warn_rate_limit_ms(),
|
||||||
me_route_no_writer_mode: MeRouteNoWriterMode::default(),
|
me_route_no_writer_mode: MeRouteNoWriterMode::default(),
|
||||||
me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(),
|
me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(),
|
||||||
me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(),
|
me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(),
|
||||||
@@ -857,6 +946,8 @@ impl Default for GeneralConfig {
|
|||||||
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
|
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
|
||||||
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
|
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
|
||||||
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
|
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
|
||||||
|
me_writer_pick_mode: MeWriterPickMode::default(),
|
||||||
|
me_writer_pick_sample_size: default_me_writer_pick_sample_size(),
|
||||||
ntp_check: default_ntp_check(),
|
ntp_check: default_ntp_check(),
|
||||||
ntp_servers: default_ntp_servers(),
|
ntp_servers: default_ntp_servers(),
|
||||||
auto_degradation_enabled: default_true(),
|
auto_degradation_enabled: default_true(),
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
@@ -18,6 +19,7 @@ pub struct UserIpTracker {
|
|||||||
max_ips: Arc<RwLock<HashMap<String, usize>>>,
|
max_ips: Arc<RwLock<HashMap<String, usize>>>,
|
||||||
limit_mode: Arc<RwLock<UserMaxUniqueIpsMode>>,
|
limit_mode: Arc<RwLock<UserMaxUniqueIpsMode>>,
|
||||||
limit_window: Arc<RwLock<Duration>>,
|
limit_window: Arc<RwLock<Duration>>,
|
||||||
|
last_compact_epoch_secs: Arc<AtomicU64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UserIpTracker {
|
impl UserIpTracker {
|
||||||
@@ -28,6 +30,54 @@ impl UserIpTracker {
|
|||||||
max_ips: Arc::new(RwLock::new(HashMap::new())),
|
max_ips: Arc::new(RwLock::new(HashMap::new())),
|
||||||
limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)),
|
limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)),
|
||||||
limit_window: Arc::new(RwLock::new(Duration::from_secs(30))),
|
limit_window: Arc::new(RwLock::new(Duration::from_secs(30))),
|
||||||
|
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn maybe_compact_empty_users(&self) {
|
||||||
|
const COMPACT_INTERVAL_SECS: u64 = 60;
|
||||||
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
|
let last_compact_epoch_secs = self.last_compact_epoch_secs.load(Ordering::Relaxed);
|
||||||
|
if now_epoch_secs.saturating_sub(last_compact_epoch_secs) < COMPACT_INTERVAL_SECS {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.last_compact_epoch_secs
|
||||||
|
.compare_exchange(
|
||||||
|
last_compact_epoch_secs,
|
||||||
|
now_epoch_secs,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut active_ips = self.active_ips.write().await;
|
||||||
|
let mut recent_ips = self.recent_ips.write().await;
|
||||||
|
let mut users = Vec::<String>::with_capacity(active_ips.len().saturating_add(recent_ips.len()));
|
||||||
|
users.extend(active_ips.keys().cloned());
|
||||||
|
for user in recent_ips.keys() {
|
||||||
|
if !active_ips.contains_key(user) {
|
||||||
|
users.push(user.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for user in users {
|
||||||
|
let active_empty = active_ips.get(&user).map(|ips| ips.is_empty()).unwrap_or(true);
|
||||||
|
let recent_empty = recent_ips.get(&user).map(|ips| ips.is_empty()).unwrap_or(true);
|
||||||
|
if active_empty && recent_empty {
|
||||||
|
active_ips.remove(&user);
|
||||||
|
recent_ips.remove(&user);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,6 +113,7 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
|
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
|
||||||
|
self.maybe_compact_empty_users().await;
|
||||||
let limit = {
|
let limit = {
|
||||||
let max_ips = self.max_ips.read().await;
|
let max_ips = self.max_ips.read().await;
|
||||||
max_ips.get(username).copied()
|
max_ips.get(username).copied()
|
||||||
@@ -116,6 +167,7 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn remove_ip(&self, username: &str, ip: IpAddr) {
|
pub async fn remove_ip(&self, username: &str, ip: IpAddr) {
|
||||||
|
self.maybe_compact_empty_users().await;
|
||||||
let mut active_ips = self.active_ips.write().await;
|
let mut active_ips = self.active_ips.write().await;
|
||||||
if let Some(user_ips) = active_ips.get_mut(username) {
|
if let Some(user_ips) = active_ips.get_mut(username) {
|
||||||
if let Some(count) = user_ips.get_mut(&ip) {
|
if let Some(count) = user_ips.get_mut(&ip) {
|
||||||
|
|||||||
569
src/main.rs
569
src/main.rs
@@ -8,7 +8,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::signal;
|
use tokio::signal;
|
||||||
use tokio::sync::{Semaphore, mpsc, watch};
|
use tokio::sync::{RwLock, Semaphore, mpsc, watch};
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
|
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -26,6 +26,7 @@ mod protocol;
|
|||||||
mod proxy;
|
mod proxy;
|
||||||
mod stats;
|
mod stats;
|
||||||
mod stream;
|
mod stream;
|
||||||
|
mod startup;
|
||||||
mod transport;
|
mod transport;
|
||||||
mod tls_front;
|
mod tls_front;
|
||||||
mod util;
|
mod util;
|
||||||
@@ -39,6 +40,14 @@ use crate::proxy::ClientHandler;
|
|||||||
use crate::stats::beobachten::BeobachtenStore;
|
use crate::stats::beobachten::BeobachtenStore;
|
||||||
use crate::stats::telemetry::TelemetryPolicy;
|
use crate::stats::telemetry::TelemetryPolicy;
|
||||||
use crate::stats::{ReplayChecker, Stats};
|
use crate::stats::{ReplayChecker, Stats};
|
||||||
|
use crate::startup::{
|
||||||
|
COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, COMPONENT_CONFIG_WATCHER_START,
|
||||||
|
COMPONENT_DC_CONNECTIVITY_PING, COMPONENT_LISTENERS_BIND, COMPONENT_ME_CONNECTIVITY_PING,
|
||||||
|
COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, COMPONENT_ME_PROXY_CONFIG_V4,
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, COMPONENT_METRICS_START,
|
||||||
|
COMPONENT_NETWORK_PROBE, COMPONENT_RUNTIME_READY, COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||||
|
COMPONENT_TRACING_INIT, StartupMeStatus, StartupTracker,
|
||||||
|
};
|
||||||
use crate::stream::BufferPool;
|
use crate::stream::BufferPool;
|
||||||
use crate::transport::middle_proxy::{
|
use crate::transport::middle_proxy::{
|
||||||
MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line,
|
MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line,
|
||||||
@@ -373,6 +382,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.as_secs();
|
.as_secs();
|
||||||
|
let startup_tracker = Arc::new(StartupTracker::new(process_started_at_epoch_secs));
|
||||||
|
startup_tracker
|
||||||
|
.start_component(COMPONENT_CONFIG_LOAD, Some("load and validate config".to_string()))
|
||||||
|
.await;
|
||||||
let (config_path, cli_silent, cli_log_level) = parse_cli();
|
let (config_path, cli_silent, cli_log_level) = parse_cli();
|
||||||
|
|
||||||
let mut config = match ProxyConfig::load(&config_path) {
|
let mut config = match ProxyConfig::load(&config_path) {
|
||||||
@@ -399,6 +412,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
|
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
|
||||||
|
.await;
|
||||||
|
|
||||||
let has_rust_log = std::env::var("RUST_LOG").is_ok();
|
let has_rust_log = std::env::var("RUST_LOG").is_ok();
|
||||||
let effective_log_level = if cli_silent {
|
let effective_log_level = if cli_silent {
|
||||||
@@ -410,6 +426,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info"));
|
let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info"));
|
||||||
|
startup_tracker
|
||||||
|
.start_component(COMPONENT_TRACING_INIT, Some("initialize tracing subscriber".to_string()))
|
||||||
|
.await;
|
||||||
|
|
||||||
// Configure color output based on config
|
// Configure color output based on config
|
||||||
let fmt_layer = if config.general.disable_colors {
|
let fmt_layer = if config.general.disable_colors {
|
||||||
@@ -422,6 +441,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
.with(filter_layer)
|
.with(filter_layer)
|
||||||
.with(fmt_layer)
|
.with(fmt_layer)
|
||||||
.init();
|
.init();
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(COMPONENT_TRACING_INIT, Some("tracing initialized".to_string()))
|
||||||
|
.await;
|
||||||
|
|
||||||
info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION"));
|
info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION"));
|
||||||
info!("Log level: {}", effective_log_level);
|
info!("Log level: {}", effective_log_level);
|
||||||
@@ -473,6 +495,95 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
config.general.upstream_connect_failfast_hard_errors,
|
config.general.upstream_connect_failfast_hard_errors,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
));
|
));
|
||||||
|
let ip_tracker = Arc::new(UserIpTracker::new());
|
||||||
|
ip_tracker.load_limits(&config.access.user_max_unique_ips).await;
|
||||||
|
ip_tracker
|
||||||
|
.set_limit_policy(
|
||||||
|
config.access.user_max_unique_ips_mode,
|
||||||
|
config.access.user_max_unique_ips_window_secs,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if !config.access.user_max_unique_ips.is_empty() {
|
||||||
|
info!(
|
||||||
|
"IP limits configured for {} users",
|
||||||
|
config.access.user_max_unique_ips.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !config.network.dns_overrides.is_empty() {
|
||||||
|
info!(
|
||||||
|
"Runtime DNS overrides configured: {} entries",
|
||||||
|
config.network.dns_overrides.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone()));
|
||||||
|
let initial_admission_open = !config.general.use_middle_proxy;
|
||||||
|
let (admission_tx, admission_rx) = watch::channel(initial_admission_open);
|
||||||
|
let api_me_pool = Arc::new(RwLock::new(None::<Arc<MePool>>));
|
||||||
|
startup_tracker
|
||||||
|
.start_component(COMPONENT_API_BOOTSTRAP, Some("spawn API listener task".to_string()))
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if config.server.api.enabled {
|
||||||
|
let listen = match config.server.api.listen.parse::<SocketAddr>() {
|
||||||
|
Ok(listen) => listen,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error,
|
||||||
|
listen = %config.server.api.listen,
|
||||||
|
"Invalid server.api.listen; API is disabled"
|
||||||
|
);
|
||||||
|
SocketAddr::from(([127, 0, 0, 1], 0))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if listen.port() != 0 {
|
||||||
|
let stats_api = stats.clone();
|
||||||
|
let ip_tracker_api = ip_tracker.clone();
|
||||||
|
let me_pool_api = api_me_pool.clone();
|
||||||
|
let upstream_manager_api = upstream_manager.clone();
|
||||||
|
let config_rx_api = api_config_rx.clone();
|
||||||
|
let admission_rx_api = admission_rx.clone();
|
||||||
|
let config_path_api = std::path::PathBuf::from(&config_path);
|
||||||
|
let startup_tracker_api = startup_tracker.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
api::serve(
|
||||||
|
listen,
|
||||||
|
stats_api,
|
||||||
|
ip_tracker_api,
|
||||||
|
me_pool_api,
|
||||||
|
upstream_manager_api,
|
||||||
|
config_rx_api,
|
||||||
|
admission_rx_api,
|
||||||
|
config_path_api,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
process_started_at_epoch_secs,
|
||||||
|
startup_tracker_api,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_API_BOOTSTRAP,
|
||||||
|
Some(format!("api task spawned on {}", listen)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_API_BOOTSTRAP,
|
||||||
|
Some("server.api.listen has zero port".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_API_BOOTSTRAP,
|
||||||
|
Some("server.api.enabled is false".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
||||||
tls_domains.push(config.censorship.tls_domain.clone());
|
tls_domains.push(config.censorship.tls_domain.clone());
|
||||||
@@ -483,6 +594,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Start TLS front fetching in background immediately, in parallel with STUN probing.
|
// Start TLS front fetching in background immediately, in parallel with STUN probing.
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||||
|
Some("initialize TLS front cache/bootstrap tasks".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let tls_cache: Option<Arc<TlsFrontCache>> = if config.censorship.tls_emulation {
|
let tls_cache: Option<Arc<TlsFrontCache>> = if config.censorship.tls_emulation {
|
||||||
let cache = Arc::new(TlsFrontCache::new(
|
let cache = Arc::new(TlsFrontCache::new(
|
||||||
&tls_domains,
|
&tls_domains,
|
||||||
@@ -603,9 +720,26 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
Some(cache)
|
Some(cache)
|
||||||
} else {
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||||
|
Some("censorship.tls_emulation is false".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
if tls_cache.is_some() {
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||||
|
Some("tls front cache is initialized".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
startup_tracker
|
||||||
|
.start_component(COMPONENT_NETWORK_PROBE, Some("probe network capabilities".to_string()))
|
||||||
|
.await;
|
||||||
let probe = run_probe(
|
let probe = run_probe(
|
||||||
&config.network,
|
&config.network,
|
||||||
config.general.middle_proxy_nat_probe,
|
config.general.middle_proxy_nat_probe,
|
||||||
@@ -614,32 +748,18 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
.await?;
|
.await?;
|
||||||
let decision = decide_network_capabilities(&config.network, &probe);
|
let decision = decide_network_capabilities(&config.network, &probe);
|
||||||
log_probe_result(&probe, &decision);
|
log_probe_result(&probe, &decision);
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_NETWORK_PROBE,
|
||||||
|
Some("network capabilities determined".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let prefer_ipv6 = decision.prefer_ipv6();
|
let prefer_ipv6 = decision.prefer_ipv6();
|
||||||
let mut use_middle_proxy = config.general.use_middle_proxy;
|
let mut use_middle_proxy = config.general.use_middle_proxy;
|
||||||
let beobachten = Arc::new(BeobachtenStore::new());
|
let beobachten = Arc::new(BeobachtenStore::new());
|
||||||
let rng = Arc::new(SecureRandom::new());
|
let rng = Arc::new(SecureRandom::new());
|
||||||
|
|
||||||
// IP Tracker initialization
|
|
||||||
let ip_tracker = Arc::new(UserIpTracker::new());
|
|
||||||
ip_tracker.load_limits(&config.access.user_max_unique_ips).await;
|
|
||||||
ip_tracker
|
|
||||||
.set_limit_policy(
|
|
||||||
config.access.user_max_unique_ips_mode,
|
|
||||||
config.access.user_max_unique_ips_window_secs,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
if !config.access.user_max_unique_ips.is_empty() {
|
|
||||||
info!("IP limits configured for {} users", config.access.user_max_unique_ips.len());
|
|
||||||
}
|
|
||||||
if !config.network.dns_overrides.is_empty() {
|
|
||||||
info!(
|
|
||||||
"Runtime DNS overrides configured: {} entries",
|
|
||||||
config.network.dns_overrides.len()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connection concurrency limit
|
// Connection concurrency limit
|
||||||
let max_connections = Arc::new(Semaphore::new(10_000));
|
let max_connections = Arc::new(Semaphore::new(10_000));
|
||||||
|
|
||||||
@@ -657,6 +777,59 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if use_middle_proxy {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_SECRET_FETCH)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_ME_SECRET_FETCH,
|
||||||
|
Some("fetch proxy-secret from source/cache".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_retry_limit(if !me2dc_fallback || me_init_retry_attempts == 0 {
|
||||||
|
"unlimited".to_string()
|
||||||
|
} else {
|
||||||
|
me_init_retry_attempts.to_string()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Skipped, "skipped")
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_SECRET_FETCH,
|
||||||
|
Some("middle proxy mode disabled".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V4,
|
||||||
|
Some("middle proxy mode disabled".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V6,
|
||||||
|
Some("middle proxy mode disabled".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_POOL_CONSTRUCT,
|
||||||
|
Some("middle proxy mode disabled".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
Some("middle proxy mode disabled".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
// =====================================================================
|
// =====================================================================
|
||||||
// Middle Proxy initialization (if enabled)
|
// Middle Proxy initialization (if enabled)
|
||||||
// =====================================================================
|
// =====================================================================
|
||||||
@@ -694,6 +867,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
{
|
{
|
||||||
Ok(proxy_secret) => break Some(proxy_secret),
|
Ok(proxy_secret) => break Some(proxy_secret),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_last_error(Some(e.to_string()))
|
||||||
|
.await;
|
||||||
if me2dc_fallback {
|
if me2dc_fallback {
|
||||||
error!(
|
error!(
|
||||||
error = %e,
|
error = %e,
|
||||||
@@ -713,6 +889,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
};
|
};
|
||||||
match proxy_secret {
|
match proxy_secret {
|
||||||
Some(proxy_secret) => {
|
Some(proxy_secret) => {
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_ME_SECRET_FETCH,
|
||||||
|
Some("proxy-secret loaded".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
info!(
|
info!(
|
||||||
secret_len = proxy_secret.len(),
|
secret_len = proxy_secret.len(),
|
||||||
key_sig = format_args!(
|
key_sig = format_args!(
|
||||||
@@ -731,6 +913,15 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
"Proxy-secret loaded"
|
"Proxy-secret loaded"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V4,
|
||||||
|
Some("load startup proxy-config v4".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4)
|
||||||
|
.await;
|
||||||
let cfg_v4 = load_startup_proxy_config_snapshot(
|
let cfg_v4 = load_startup_proxy_config_snapshot(
|
||||||
"https://core.telegram.org/getProxyConfig",
|
"https://core.telegram.org/getProxyConfig",
|
||||||
config.general.proxy_config_v4_cache_path.as_deref(),
|
config.general.proxy_config_v4_cache_path.as_deref(),
|
||||||
@@ -738,6 +929,30 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
"getProxyConfig",
|
"getProxyConfig",
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
if cfg_v4.is_some() {
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V4,
|
||||||
|
Some("proxy-config v4 loaded".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V4,
|
||||||
|
Some("proxy-config v4 unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V6,
|
||||||
|
Some("load startup proxy-config v6".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6)
|
||||||
|
.await;
|
||||||
let cfg_v6 = load_startup_proxy_config_snapshot(
|
let cfg_v6 = load_startup_proxy_config_snapshot(
|
||||||
"https://core.telegram.org/getProxyConfigV6",
|
"https://core.telegram.org/getProxyConfigV6",
|
||||||
config.general.proxy_config_v6_cache_path.as_deref(),
|
config.general.proxy_config_v6_cache_path.as_deref(),
|
||||||
@@ -745,8 +960,32 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
"getProxyConfigV6",
|
"getProxyConfigV6",
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
if cfg_v6.is_some() {
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V6,
|
||||||
|
Some("proxy-config v6 loaded".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V6,
|
||||||
|
Some("proxy-config v6 unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) {
|
if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) {
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_ME_POOL_CONSTRUCT,
|
||||||
|
Some("construct ME pool".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_CONSTRUCT)
|
||||||
|
.await;
|
||||||
let pool = MePool::new(
|
let pool = MePool::new(
|
||||||
proxy_tag.clone(),
|
proxy_tag.clone(),
|
||||||
proxy_secret,
|
proxy_secret,
|
||||||
@@ -792,6 +1031,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
config.general.me_adaptive_floor_cpu_cores_override,
|
config.general.me_adaptive_floor_cpu_cores_override,
|
||||||
config.general.me_adaptive_floor_max_extra_writers_single_per_core,
|
config.general.me_adaptive_floor_max_extra_writers_single_per_core,
|
||||||
config.general.me_adaptive_floor_max_extra_writers_multi_per_core,
|
config.general.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
config.general.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
config.general.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
config.general.me_adaptive_floor_max_active_writers_global,
|
||||||
|
config.general.me_adaptive_floor_max_warm_writers_global,
|
||||||
config.general.hardswap,
|
config.general.hardswap,
|
||||||
config.general.me_pool_drain_ttl_secs,
|
config.general.me_pool_drain_ttl_secs,
|
||||||
config.general.effective_me_pool_force_close_secs(),
|
config.general.effective_me_pool_force_close_secs(),
|
||||||
@@ -804,21 +1047,60 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
config.general.me_bind_stale_ttl_secs,
|
config.general.me_bind_stale_ttl_secs,
|
||||||
config.general.me_secret_atomic_snapshot,
|
config.general.me_secret_atomic_snapshot,
|
||||||
config.general.me_deterministic_writer_sort,
|
config.general.me_deterministic_writer_sort,
|
||||||
|
config.general.me_writer_pick_mode,
|
||||||
|
config.general.me_writer_pick_sample_size,
|
||||||
config.general.me_socks_kdf_policy,
|
config.general.me_socks_kdf_policy,
|
||||||
|
config.general.me_writer_cmd_channel_capacity,
|
||||||
|
config.general.me_route_channel_capacity,
|
||||||
config.general.me_route_backpressure_base_timeout_ms,
|
config.general.me_route_backpressure_base_timeout_ms,
|
||||||
config.general.me_route_backpressure_high_timeout_ms,
|
config.general.me_route_backpressure_high_timeout_ms,
|
||||||
config.general.me_route_backpressure_high_watermark_pct,
|
config.general.me_route_backpressure_high_watermark_pct,
|
||||||
|
config.general.me_health_interval_ms_unhealthy,
|
||||||
|
config.general.me_health_interval_ms_healthy,
|
||||||
|
config.general.me_warn_rate_limit_ms,
|
||||||
config.general.me_route_no_writer_mode,
|
config.general.me_route_no_writer_mode,
|
||||||
config.general.me_route_no_writer_wait_ms,
|
config.general.me_route_no_writer_wait_ms,
|
||||||
config.general.me_route_inline_recovery_attempts,
|
config.general.me_route_inline_recovery_attempts,
|
||||||
config.general.me_route_inline_recovery_wait_ms,
|
config.general.me_route_inline_recovery_wait_ms,
|
||||||
);
|
);
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_ME_POOL_CONSTRUCT,
|
||||||
|
Some("ME pool object created".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
*api_me_pool.write().await = Some(pool.clone());
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
Some("initialize ME pool writers".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(
|
||||||
|
StartupMeStatus::Initializing,
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let mut init_attempt: u32 = 0;
|
let mut init_attempt: u32 = 0;
|
||||||
loop {
|
loop {
|
||||||
init_attempt = init_attempt.saturating_add(1);
|
init_attempt = init_attempt.saturating_add(1);
|
||||||
|
startup_tracker.set_me_init_attempt(init_attempt).await;
|
||||||
match pool.init(pool_size, &rng).await {
|
match pool.init(pool_size, &rng).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_last_error(None)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
Some("ME pool initialized".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Ready, "ready")
|
||||||
|
.await;
|
||||||
info!(
|
info!(
|
||||||
attempt = init_attempt,
|
attempt = init_attempt,
|
||||||
"Middle-End pool initialized successfully"
|
"Middle-End pool initialized successfully"
|
||||||
@@ -838,8 +1120,20 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
break Some(pool);
|
break Some(pool);
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_last_error(Some(e.to_string()))
|
||||||
|
.await;
|
||||||
let retries_limited = me2dc_fallback && me_init_retry_attempts > 0;
|
let retries_limited = me2dc_fallback && me_init_retry_attempts > 0;
|
||||||
if retries_limited && init_attempt >= me_init_retry_attempts {
|
if retries_limited && init_attempt >= me_init_retry_attempts {
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
Some("ME init retry budget exhausted".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Failed, "failed")
|
||||||
|
.await;
|
||||||
error!(
|
error!(
|
||||||
error = %e,
|
error = %e,
|
||||||
attempt = init_attempt,
|
attempt = init_attempt,
|
||||||
@@ -879,10 +1173,60 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_POOL_CONSTRUCT,
|
||||||
|
Some("ME configs are incomplete".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
Some("ME configs are incomplete".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Failed, "failed")
|
||||||
|
.await;
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => None,
|
None => {
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_ME_SECRET_FETCH,
|
||||||
|
Some("proxy-secret unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V4,
|
||||||
|
Some("proxy-secret unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_PROXY_CONFIG_V6,
|
||||||
|
Some("proxy-secret unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_POOL_CONSTRUCT,
|
||||||
|
Some("proxy-secret unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
|
Some("proxy-secret unavailable".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Failed, "failed")
|
||||||
|
.await;
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@@ -890,12 +1234,33 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
// If ME failed to initialize, force direct-only mode.
|
// If ME failed to initialize, force direct-only mode.
|
||||||
if me_pool.is_some() {
|
if me_pool.is_some() {
|
||||||
|
startup_tracker
|
||||||
|
.set_transport_mode("middle_proxy")
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_degraded(false)
|
||||||
|
.await;
|
||||||
info!("Transport: Middle-End Proxy - all DC-over-RPC");
|
info!("Transport: Middle-End Proxy - all DC-over-RPC");
|
||||||
} else {
|
} else {
|
||||||
let _ = use_middle_proxy;
|
let _ = use_middle_proxy;
|
||||||
use_middle_proxy = false;
|
use_middle_proxy = false;
|
||||||
// Make runtime config reflect direct-only mode for handlers.
|
// Make runtime config reflect direct-only mode for handlers.
|
||||||
config.general.use_middle_proxy = false;
|
config.general.use_middle_proxy = false;
|
||||||
|
startup_tracker
|
||||||
|
.set_transport_mode("direct")
|
||||||
|
.await;
|
||||||
|
startup_tracker
|
||||||
|
.set_degraded(true)
|
||||||
|
.await;
|
||||||
|
if me2dc_fallback {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Failed, "fallback_to_direct")
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.set_me_status(StartupMeStatus::Skipped, "skipped")
|
||||||
|
.await;
|
||||||
|
}
|
||||||
info!("Transport: Direct DC - TCP - standard DC-over-TCP");
|
info!("Transport: Direct DC - TCP - standard DC-over-TCP");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -910,6 +1275,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096));
|
let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096));
|
||||||
|
|
||||||
// Middle-End ping before DC connectivity
|
// Middle-End ping before DC connectivity
|
||||||
|
if me_pool.is_some() {
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_ME_CONNECTIVITY_PING,
|
||||||
|
Some("run startup ME connectivity check".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_ME_CONNECTIVITY_PING,
|
||||||
|
Some("ME pool is not available".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
if let Some(ref pool) = me_pool {
|
if let Some(ref pool) = me_pool {
|
||||||
let me_results = run_me_ping(pool, &rng).await;
|
let me_results = run_me_ping(pool, &rng).await;
|
||||||
|
|
||||||
@@ -979,9 +1359,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!("============================================================");
|
info!("============================================================");
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_ME_CONNECTIVITY_PING,
|
||||||
|
Some("startup ME connectivity check completed".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("================= Telegram DC Connectivity =================");
|
info!("================= Telegram DC Connectivity =================");
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_DC_CONNECTIVITY_PING,
|
||||||
|
Some("run startup DC connectivity check".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let ping_results = upstream_manager
|
let ping_results = upstream_manager
|
||||||
.ping_all_dcs(
|
.ping_all_dcs(
|
||||||
@@ -1061,9 +1453,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
info!("============================================================");
|
info!("============================================================");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_DC_CONNECTIVITY_PING,
|
||||||
|
Some("startup DC connectivity check completed".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
let initialized_secs = process_started_at.elapsed().as_secs();
|
let initialized_secs = process_started_at.elapsed().as_secs();
|
||||||
let second_suffix = if initialized_secs == 1 { "" } else { "s" };
|
let second_suffix = if initialized_secs == 1 { "" } else { "s" };
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_RUNTIME_READY,
|
||||||
|
Some("finalize startup runtime state".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
info!("===================== Telegram Startup =====================");
|
info!("===================== Telegram Startup =====================");
|
||||||
info!(
|
info!(
|
||||||
" DC/ME Initialized in {} second{}",
|
" DC/ME Initialized in {} second{}",
|
||||||
@@ -1074,6 +1478,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
if let Some(ref pool) = me_pool {
|
if let Some(ref pool) = me_pool {
|
||||||
pool.set_runtime_ready(true);
|
pool.set_runtime_ready(true);
|
||||||
}
|
}
|
||||||
|
*api_me_pool.write().await = me_pool.clone();
|
||||||
|
|
||||||
// Background tasks
|
// Background tasks
|
||||||
let um_clone = upstream_manager.clone();
|
let um_clone = upstream_manager.clone();
|
||||||
@@ -1105,6 +1510,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
// ── Hot-reload watcher ────────────────────────────────────────────────
|
// ── Hot-reload watcher ────────────────────────────────────────────────
|
||||||
// Uses inotify to detect file changes instantly (SIGHUP also works).
|
// Uses inotify to detect file changes instantly (SIGHUP also works).
|
||||||
// detected_ip_v4/v6 are passed so newly added users get correct TG links.
|
// detected_ip_v4/v6 are passed so newly added users get correct TG links.
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_CONFIG_WATCHER_START,
|
||||||
|
Some("spawn config hot-reload watcher".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let (config_rx, mut log_level_rx): (
|
let (config_rx, mut log_level_rx): (
|
||||||
tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
||||||
tokio::sync::watch::Receiver<LogLevel>,
|
tokio::sync::watch::Receiver<LogLevel>,
|
||||||
@@ -1114,6 +1525,23 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
detected_ip_v4,
|
detected_ip_v4,
|
||||||
detected_ip_v6,
|
detected_ip_v6,
|
||||||
);
|
);
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_CONFIG_WATCHER_START,
|
||||||
|
Some("config hot-reload watcher started".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut config_rx_api_bridge = config_rx.clone();
|
||||||
|
let api_config_tx_bridge = api_config_tx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
if config_rx_api_bridge.changed().await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let cfg = config_rx_api_bridge.borrow_and_update().clone();
|
||||||
|
api_config_tx_bridge.send_replace(cfg);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let stats_policy = stats.clone();
|
let stats_policy = stats.clone();
|
||||||
let mut config_rx_policy = config_rx.clone();
|
let mut config_rx_policy = config_rx.clone();
|
||||||
@@ -1244,6 +1672,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_LISTENERS_BIND,
|
||||||
|
Some("bind TCP/Unix listeners".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let mut listeners = Vec::new();
|
let mut listeners = Vec::new();
|
||||||
|
|
||||||
for listener_conf in &config.server.listeners {
|
for listener_conf in &config.server.listeners {
|
||||||
@@ -1345,7 +1779,6 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
print_proxy_links(&host, port, &config);
|
print_proxy_links(&host, port, &config);
|
||||||
}
|
}
|
||||||
|
|
||||||
let (admission_tx, admission_rx) = watch::channel(true);
|
|
||||||
if config.general.use_middle_proxy {
|
if config.general.use_middle_proxy {
|
||||||
if let Some(pool) = me_pool.as_ref() {
|
if let Some(pool) = me_pool.as_ref() {
|
||||||
let initial_open = pool.admission_ready_conditional_cast().await;
|
let initial_open = pool.admission_ready_conditional_cast().await;
|
||||||
@@ -1358,11 +1791,24 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
let pool_for_gate = pool.clone();
|
let pool_for_gate = pool.clone();
|
||||||
let admission_tx_gate = admission_tx.clone();
|
let admission_tx_gate = admission_tx.clone();
|
||||||
|
let mut config_rx_gate = config_rx.clone();
|
||||||
|
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut gate_open = initial_open;
|
let mut gate_open = initial_open;
|
||||||
let mut open_streak = if initial_open { 1u32 } else { 0u32 };
|
let mut open_streak = if initial_open { 1u32 } else { 0u32 };
|
||||||
let mut close_streak = if initial_open { 0u32 } else { 1u32 };
|
let mut close_streak = if initial_open { 0u32 } else { 1u32 };
|
||||||
loop {
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
changed = config_rx_gate.changed() => {
|
||||||
|
if changed.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let cfg = config_rx_gate.borrow_and_update().clone();
|
||||||
|
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
||||||
|
}
|
||||||
let ready = pool_for_gate.admission_ready_conditional_cast().await;
|
let ready = pool_for_gate.admission_ready_conditional_cast().await;
|
||||||
if ready {
|
if ready {
|
||||||
open_streak = open_streak.saturating_add(1);
|
open_streak = open_streak.saturating_add(1);
|
||||||
@@ -1387,7 +1833,6 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
@@ -1495,6 +1940,16 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_LISTENERS_BIND,
|
||||||
|
Some(format!(
|
||||||
|
"listeners configured tcp={} unix={}",
|
||||||
|
listeners.len(),
|
||||||
|
has_unix_listener
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
if listeners.is_empty() && !has_unix_listener {
|
if listeners.is_empty() && !has_unix_listener {
|
||||||
error!("No listeners. Exiting.");
|
error!("No listeners. Exiting.");
|
||||||
@@ -1528,6 +1983,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if let Some(port) = config.server.metrics_port {
|
if let Some(port) = config.server.metrics_port {
|
||||||
|
startup_tracker
|
||||||
|
.start_component(
|
||||||
|
COMPONENT_METRICS_START,
|
||||||
|
Some(format!("spawn metrics endpoint on {}", port)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let stats = stats.clone();
|
let stats = stats.clone();
|
||||||
let beobachten = beobachten.clone();
|
let beobachten = beobachten.clone();
|
||||||
let config_rx_metrics = config_rx.clone();
|
let config_rx_metrics = config_rx.clone();
|
||||||
@@ -1544,48 +2005,28 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
|
startup_tracker
|
||||||
|
.complete_component(
|
||||||
|
COMPONENT_METRICS_START,
|
||||||
|
Some("metrics task spawned".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
startup_tracker
|
||||||
|
.skip_component(
|
||||||
|
COMPONENT_METRICS_START,
|
||||||
|
Some("server.metrics_port is not configured".to_string()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.server.api.enabled {
|
startup_tracker
|
||||||
let listen = match config.server.api.listen.parse::<SocketAddr>() {
|
.complete_component(
|
||||||
Ok(listen) => listen,
|
COMPONENT_RUNTIME_READY,
|
||||||
Err(error) => {
|
Some("startup pipeline is fully initialized".to_string()),
|
||||||
warn!(
|
)
|
||||||
error = %error,
|
.await;
|
||||||
listen = %config.server.api.listen,
|
startup_tracker.mark_ready().await;
|
||||||
"Invalid server.api.listen; API is disabled"
|
|
||||||
);
|
|
||||||
SocketAddr::from(([127, 0, 0, 1], 0))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if listen.port() != 0 {
|
|
||||||
let stats = stats.clone();
|
|
||||||
let ip_tracker_api = ip_tracker.clone();
|
|
||||||
let me_pool_api = me_pool.clone();
|
|
||||||
let upstream_manager_api = upstream_manager.clone();
|
|
||||||
let config_rx_api = config_rx.clone();
|
|
||||||
let admission_rx_api = admission_rx.clone();
|
|
||||||
let config_path_api = std::path::PathBuf::from(&config_path);
|
|
||||||
let startup_detected_ip_v4 = detected_ip_v4;
|
|
||||||
let startup_detected_ip_v6 = detected_ip_v6;
|
|
||||||
tokio::spawn(async move {
|
|
||||||
api::serve(
|
|
||||||
listen,
|
|
||||||
stats,
|
|
||||||
ip_tracker_api,
|
|
||||||
me_pool_api,
|
|
||||||
upstream_manager_api,
|
|
||||||
config_rx_api,
|
|
||||||
admission_rx_api,
|
|
||||||
config_path_api,
|
|
||||||
startup_detected_ip_v4,
|
|
||||||
startup_detected_ip_v6,
|
|
||||||
process_started_at_epoch_secs,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (listener, listener_proxy_protocol) in listeners {
|
for (listener, listener_proxy_protocol) in listeners {
|
||||||
let mut config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
let mut config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
||||||
|
|||||||
225
src/metrics.rs
225
src/metrics.rs
@@ -689,6 +689,135 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_writer_pick_total ME writer-pick outcomes by mode and result"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_writer_pick_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_try\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_sorted_rr_success_try_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_fallback\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_sorted_rr_success_fallback_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"full\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_sorted_rr_full_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"closed\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_sorted_rr_closed_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"no_candidate\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_sorted_rr_no_candidate_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_try\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_p2c_success_try_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_fallback\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_p2c_success_fallback_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"full\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_p2c_full_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"closed\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_p2c_closed_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"no_candidate\"}} {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_p2c_no_candidate_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_writer_pick_blocking_fallback_total ME writer-pick blocking fallback attempts"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_me_writer_pick_blocking_fallback_total counter"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_blocking_fallback_total {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_blocking_fallback_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_writer_pick_mode_switch_total Writer-pick mode switches via runtime updates"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_writer_pick_mode_switch_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writer_pick_mode_switch_total {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writer_pick_mode_switch_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes"
|
"# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes"
|
||||||
@@ -1053,6 +1182,102 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_adaptive_floor_active_cap_configured Runtime configured active writer cap"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_me_adaptive_floor_active_cap_configured gauge"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_adaptive_floor_active_cap_configured {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_floor_active_cap_configured_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_adaptive_floor_active_cap_effective Runtime effective active writer cap"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_me_adaptive_floor_active_cap_effective gauge"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_adaptive_floor_active_cap_effective {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_floor_active_cap_effective_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_adaptive_floor_warm_cap_configured Runtime configured warm writer cap"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_me_adaptive_floor_warm_cap_configured gauge"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_adaptive_floor_warm_cap_configured {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_floor_warm_cap_configured_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_adaptive_floor_warm_cap_effective Runtime effective warm writer cap"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_me_adaptive_floor_warm_cap_effective gauge"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_adaptive_floor_warm_cap_effective {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_floor_warm_cap_effective_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_writers_active_current Current non-draining active ME writers"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_writers_active_current gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writers_active_current {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writers_active_current_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_writers_warm_current Current non-draining warm ME writers"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_writers_warm_current gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_writers_warm_current {}",
|
||||||
|
if me_allows_normal {
|
||||||
|
stats.get_me_writers_warm_current_gauge()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps"
|
"# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
|||||||
use std::sync::{Arc, Mutex, OnceLock};
|
use std::sync::{Arc, Mutex, OnceLock};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||||
use tokio::sync::{mpsc, oneshot};
|
use tokio::sync::{mpsc, oneshot};
|
||||||
use tracing::{debug, trace, warn};
|
use tracing::{debug, trace, warn};
|
||||||
@@ -20,13 +21,13 @@ use crate::stream::{BufferPool, CryptoReader, CryptoWriter};
|
|||||||
use crate::transport::middle_proxy::{MePool, MeResponse, proto_flags_for_tag};
|
use crate::transport::middle_proxy::{MePool, MeResponse, proto_flags_for_tag};
|
||||||
|
|
||||||
enum C2MeCommand {
|
enum C2MeCommand {
|
||||||
Data { payload: Vec<u8>, flags: u32 },
|
Data { payload: Bytes, flags: u32 },
|
||||||
Close,
|
Close,
|
||||||
}
|
}
|
||||||
|
|
||||||
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
|
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
|
||||||
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
|
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
|
||||||
const C2ME_CHANNEL_CAPACITY: usize = 1024;
|
const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128;
|
||||||
const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64;
|
const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64;
|
||||||
const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32;
|
const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32;
|
||||||
static DESYNC_DEDUP: OnceLock<Mutex<HashMap<u64, Instant>>> = OnceLock::new();
|
static DESYNC_DEDUP: OnceLock<Mutex<HashMap<u64, Instant>>> = OnceLock::new();
|
||||||
@@ -270,7 +271,11 @@ where
|
|||||||
|
|
||||||
let frame_limit = config.general.max_client_frame;
|
let frame_limit = config.general.max_client_frame;
|
||||||
|
|
||||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(C2ME_CHANNEL_CAPACITY);
|
let c2me_channel_capacity = config
|
||||||
|
.general
|
||||||
|
.me_c2me_channel_capacity
|
||||||
|
.max(C2ME_CHANNEL_CAPACITY_FALLBACK);
|
||||||
|
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||||
let me_pool_c2me = me_pool.clone();
|
let me_pool_c2me = me_pool.clone();
|
||||||
let effective_tag = effective_tag;
|
let effective_tag = effective_tag;
|
||||||
let c2me_sender = tokio::spawn(async move {
|
let c2me_sender = tokio::spawn(async move {
|
||||||
@@ -283,7 +288,7 @@ where
|
|||||||
success.dc_idx,
|
success.dc_idx,
|
||||||
peer,
|
peer,
|
||||||
translated_local_addr,
|
translated_local_addr,
|
||||||
&payload,
|
payload.as_ref(),
|
||||||
flags,
|
flags,
|
||||||
effective_tag.as_deref(),
|
effective_tag.as_deref(),
|
||||||
).await?;
|
).await?;
|
||||||
@@ -479,7 +484,7 @@ async fn read_client_payload<R>(
|
|||||||
forensics: &RelayForensicsState,
|
forensics: &RelayForensicsState,
|
||||||
frame_counter: &mut u64,
|
frame_counter: &mut u64,
|
||||||
stats: &Stats,
|
stats: &Stats,
|
||||||
) -> Result<Option<(Vec<u8>, bool)>>
|
) -> Result<Option<(Bytes, bool)>>
|
||||||
where
|
where
|
||||||
R: AsyncRead + Unpin + Send + 'static,
|
R: AsyncRead + Unpin + Send + 'static,
|
||||||
{
|
{
|
||||||
@@ -578,7 +583,7 @@ where
|
|||||||
payload.truncate(secure_payload_len);
|
payload.truncate(secure_payload_len);
|
||||||
}
|
}
|
||||||
*frame_counter += 1;
|
*frame_counter += 1;
|
||||||
return Ok(Some((payload, quickack)));
|
return Ok(Some((Bytes::from(payload), quickack)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -715,7 +720,7 @@ mod tests {
|
|||||||
enqueue_c2me_command(
|
enqueue_c2me_command(
|
||||||
&tx,
|
&tx,
|
||||||
C2MeCommand::Data {
|
C2MeCommand::Data {
|
||||||
payload: vec![1, 2, 3],
|
payload: Bytes::from_static(&[1, 2, 3]),
|
||||||
flags: 0,
|
flags: 0,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -728,7 +733,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
match recv {
|
match recv {
|
||||||
C2MeCommand::Data { payload, flags } => {
|
C2MeCommand::Data { payload, flags } => {
|
||||||
assert_eq!(payload, vec![1, 2, 3]);
|
assert_eq!(payload.as_ref(), &[1, 2, 3]);
|
||||||
assert_eq!(flags, 0);
|
assert_eq!(flags, 0);
|
||||||
}
|
}
|
||||||
C2MeCommand::Close => panic!("unexpected close command"),
|
C2MeCommand::Close => panic!("unexpected close command"),
|
||||||
@@ -739,7 +744,7 @@ mod tests {
|
|||||||
async fn enqueue_c2me_command_falls_back_to_send_when_queue_is_full() {
|
async fn enqueue_c2me_command_falls_back_to_send_when_queue_is_full() {
|
||||||
let (tx, mut rx) = mpsc::channel::<C2MeCommand>(1);
|
let (tx, mut rx) = mpsc::channel::<C2MeCommand>(1);
|
||||||
tx.send(C2MeCommand::Data {
|
tx.send(C2MeCommand::Data {
|
||||||
payload: vec![9],
|
payload: Bytes::from_static(&[9]),
|
||||||
flags: 9,
|
flags: 9,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
@@ -750,7 +755,7 @@ mod tests {
|
|||||||
enqueue_c2me_command(
|
enqueue_c2me_command(
|
||||||
&tx2,
|
&tx2,
|
||||||
C2MeCommand::Data {
|
C2MeCommand::Data {
|
||||||
payload: vec![7, 7],
|
payload: Bytes::from_static(&[7, 7]),
|
||||||
flags: 7,
|
flags: 7,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -769,7 +774,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
match recv {
|
match recv {
|
||||||
C2MeCommand::Data { payload, flags } => {
|
C2MeCommand::Data { payload, flags } => {
|
||||||
assert_eq!(payload, vec![7, 7]);
|
assert_eq!(payload.as_ref(), &[7, 7]);
|
||||||
assert_eq!(flags, 7);
|
assert_eq!(flags, 7);
|
||||||
}
|
}
|
||||||
C2MeCommand::Close => panic!("unexpected close command"),
|
C2MeCommand::Close => panic!("unexpected close command"),
|
||||||
|
|||||||
373
src/startup.rs
Normal file
373
src/startup.rs
Normal file
@@ -0,0 +1,373 @@
|
|||||||
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
pub const COMPONENT_CONFIG_LOAD: &str = "config_load";
|
||||||
|
pub const COMPONENT_TRACING_INIT: &str = "tracing_init";
|
||||||
|
pub const COMPONENT_API_BOOTSTRAP: &str = "api_bootstrap";
|
||||||
|
pub const COMPONENT_TLS_FRONT_BOOTSTRAP: &str = "tls_front_bootstrap";
|
||||||
|
pub const COMPONENT_NETWORK_PROBE: &str = "network_probe";
|
||||||
|
pub const COMPONENT_ME_SECRET_FETCH: &str = "me_secret_fetch";
|
||||||
|
pub const COMPONENT_ME_PROXY_CONFIG_V4: &str = "me_proxy_config_fetch_v4";
|
||||||
|
pub const COMPONENT_ME_PROXY_CONFIG_V6: &str = "me_proxy_config_fetch_v6";
|
||||||
|
pub const COMPONENT_ME_POOL_CONSTRUCT: &str = "me_pool_construct";
|
||||||
|
pub const COMPONENT_ME_POOL_INIT_STAGE1: &str = "me_pool_init_stage1";
|
||||||
|
pub const COMPONENT_ME_CONNECTIVITY_PING: &str = "me_connectivity_ping";
|
||||||
|
pub const COMPONENT_DC_CONNECTIVITY_PING: &str = "dc_connectivity_ping";
|
||||||
|
pub const COMPONENT_LISTENERS_BIND: &str = "listeners_bind";
|
||||||
|
pub const COMPONENT_CONFIG_WATCHER_START: &str = "config_watcher_start";
|
||||||
|
pub const COMPONENT_METRICS_START: &str = "metrics_start";
|
||||||
|
pub const COMPONENT_RUNTIME_READY: &str = "runtime_ready";
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum StartupStatus {
|
||||||
|
Initializing,
|
||||||
|
Ready,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StartupStatus {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Initializing => "initializing",
|
||||||
|
Self::Ready => "ready",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum StartupComponentStatus {
|
||||||
|
Pending,
|
||||||
|
Running,
|
||||||
|
Ready,
|
||||||
|
Failed,
|
||||||
|
Skipped,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StartupComponentStatus {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Pending => "pending",
|
||||||
|
Self::Running => "running",
|
||||||
|
Self::Ready => "ready",
|
||||||
|
Self::Failed => "failed",
|
||||||
|
Self::Skipped => "skipped",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum StartupMeStatus {
|
||||||
|
Pending,
|
||||||
|
Initializing,
|
||||||
|
Ready,
|
||||||
|
Failed,
|
||||||
|
Skipped,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StartupMeStatus {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Pending => "pending",
|
||||||
|
Self::Initializing => "initializing",
|
||||||
|
Self::Ready => "ready",
|
||||||
|
Self::Failed => "failed",
|
||||||
|
Self::Skipped => "skipped",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct StartupComponentSnapshot {
|
||||||
|
pub id: &'static str,
|
||||||
|
pub title: &'static str,
|
||||||
|
pub weight: f64,
|
||||||
|
pub status: StartupComponentStatus,
|
||||||
|
pub started_at_epoch_ms: Option<u64>,
|
||||||
|
pub finished_at_epoch_ms: Option<u64>,
|
||||||
|
pub duration_ms: Option<u64>,
|
||||||
|
pub attempts: u32,
|
||||||
|
pub details: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct StartupMeSnapshot {
|
||||||
|
pub status: StartupMeStatus,
|
||||||
|
pub current_stage: String,
|
||||||
|
pub init_attempt: u32,
|
||||||
|
pub retry_limit: String,
|
||||||
|
pub last_error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct StartupSnapshot {
|
||||||
|
pub status: StartupStatus,
|
||||||
|
pub degraded: bool,
|
||||||
|
pub current_stage: String,
|
||||||
|
pub started_at_epoch_secs: u64,
|
||||||
|
pub ready_at_epoch_secs: Option<u64>,
|
||||||
|
pub total_elapsed_ms: u64,
|
||||||
|
pub transport_mode: String,
|
||||||
|
pub me: StartupMeSnapshot,
|
||||||
|
pub components: Vec<StartupComponentSnapshot>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct StartupComponent {
|
||||||
|
id: &'static str,
|
||||||
|
title: &'static str,
|
||||||
|
weight: f64,
|
||||||
|
status: StartupComponentStatus,
|
||||||
|
started_at_epoch_ms: Option<u64>,
|
||||||
|
finished_at_epoch_ms: Option<u64>,
|
||||||
|
duration_ms: Option<u64>,
|
||||||
|
attempts: u32,
|
||||||
|
details: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct StartupState {
|
||||||
|
status: StartupStatus,
|
||||||
|
degraded: bool,
|
||||||
|
current_stage: String,
|
||||||
|
started_at_epoch_secs: u64,
|
||||||
|
ready_at_epoch_secs: Option<u64>,
|
||||||
|
transport_mode: String,
|
||||||
|
me: StartupMeSnapshot,
|
||||||
|
components: Vec<StartupComponent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct StartupTracker {
|
||||||
|
started_at_instant: Instant,
|
||||||
|
state: RwLock<StartupState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StartupTracker {
|
||||||
|
pub fn new(started_at_epoch_secs: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
started_at_instant: Instant::now(),
|
||||||
|
state: RwLock::new(StartupState {
|
||||||
|
status: StartupStatus::Initializing,
|
||||||
|
degraded: false,
|
||||||
|
current_stage: COMPONENT_CONFIG_LOAD.to_string(),
|
||||||
|
started_at_epoch_secs,
|
||||||
|
ready_at_epoch_secs: None,
|
||||||
|
transport_mode: "unknown".to_string(),
|
||||||
|
me: StartupMeSnapshot {
|
||||||
|
status: StartupMeStatus::Pending,
|
||||||
|
current_stage: "pending".to_string(),
|
||||||
|
init_attempt: 0,
|
||||||
|
retry_limit: "unlimited".to_string(),
|
||||||
|
last_error: None,
|
||||||
|
},
|
||||||
|
components: component_blueprint(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_transport_mode(&self, mode: &'static str) {
|
||||||
|
self.state.write().await.transport_mode = mode.to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_degraded(&self, degraded: bool) {
|
||||||
|
self.state.write().await.degraded = degraded;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn start_component(&self, id: &'static str, details: Option<String>) {
|
||||||
|
let mut guard = self.state.write().await;
|
||||||
|
guard.current_stage = id.to_string();
|
||||||
|
if let Some(component) = guard.components.iter_mut().find(|component| component.id == id) {
|
||||||
|
if component.started_at_epoch_ms.is_none() {
|
||||||
|
component.started_at_epoch_ms = Some(now_epoch_ms());
|
||||||
|
}
|
||||||
|
component.attempts = component.attempts.saturating_add(1);
|
||||||
|
component.status = StartupComponentStatus::Running;
|
||||||
|
component.details = normalize_details(details);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn complete_component(&self, id: &'static str, details: Option<String>) {
|
||||||
|
self.finish_component(id, StartupComponentStatus::Ready, details)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fail_component(&self, id: &'static str, details: Option<String>) {
|
||||||
|
self.finish_component(id, StartupComponentStatus::Failed, details)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn skip_component(&self, id: &'static str, details: Option<String>) {
|
||||||
|
self.finish_component(id, StartupComponentStatus::Skipped, details)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn finish_component(
|
||||||
|
&self,
|
||||||
|
id: &'static str,
|
||||||
|
status: StartupComponentStatus,
|
||||||
|
details: Option<String>,
|
||||||
|
) {
|
||||||
|
let mut guard = self.state.write().await;
|
||||||
|
let finished_at = now_epoch_ms();
|
||||||
|
if let Some(component) = guard.components.iter_mut().find(|component| component.id == id) {
|
||||||
|
if component.started_at_epoch_ms.is_none() {
|
||||||
|
component.started_at_epoch_ms = Some(finished_at);
|
||||||
|
component.attempts = component.attempts.saturating_add(1);
|
||||||
|
}
|
||||||
|
component.finished_at_epoch_ms = Some(finished_at);
|
||||||
|
component.duration_ms = component
|
||||||
|
.started_at_epoch_ms
|
||||||
|
.map(|started_at| finished_at.saturating_sub(started_at));
|
||||||
|
component.status = status;
|
||||||
|
component.details = normalize_details(details);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_me_status(&self, status: StartupMeStatus, stage: &'static str) {
|
||||||
|
let mut guard = self.state.write().await;
|
||||||
|
guard.me.status = status;
|
||||||
|
guard.me.current_stage = stage.to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_me_retry_limit(&self, retry_limit: String) {
|
||||||
|
self.state.write().await.me.retry_limit = retry_limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_me_init_attempt(&self, attempt: u32) {
|
||||||
|
self.state.write().await.me.init_attempt = attempt;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn set_me_last_error(&self, error: Option<String>) {
|
||||||
|
self.state.write().await.me.last_error = normalize_details(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn mark_ready(&self) {
|
||||||
|
let mut guard = self.state.write().await;
|
||||||
|
if guard.status == StartupStatus::Ready {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
guard.status = StartupStatus::Ready;
|
||||||
|
guard.current_stage = "ready".to_string();
|
||||||
|
guard.ready_at_epoch_secs = Some(now_epoch_secs());
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn snapshot(&self) -> StartupSnapshot {
|
||||||
|
let guard = self.state.read().await;
|
||||||
|
StartupSnapshot {
|
||||||
|
status: guard.status,
|
||||||
|
degraded: guard.degraded,
|
||||||
|
current_stage: guard.current_stage.clone(),
|
||||||
|
started_at_epoch_secs: guard.started_at_epoch_secs,
|
||||||
|
ready_at_epoch_secs: guard.ready_at_epoch_secs,
|
||||||
|
total_elapsed_ms: self.started_at_instant.elapsed().as_millis() as u64,
|
||||||
|
transport_mode: guard.transport_mode.clone(),
|
||||||
|
me: guard.me.clone(),
|
||||||
|
components: guard
|
||||||
|
.components
|
||||||
|
.iter()
|
||||||
|
.map(|component| StartupComponentSnapshot {
|
||||||
|
id: component.id,
|
||||||
|
title: component.title,
|
||||||
|
weight: component.weight,
|
||||||
|
status: component.status,
|
||||||
|
started_at_epoch_ms: component.started_at_epoch_ms,
|
||||||
|
finished_at_epoch_ms: component.finished_at_epoch_ms,
|
||||||
|
duration_ms: component.duration_ms,
|
||||||
|
attempts: component.attempts,
|
||||||
|
details: component.details.clone(),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compute_progress_pct(snapshot: &StartupSnapshot, me_stage_progress: Option<f64>) -> f64 {
|
||||||
|
if snapshot.status == StartupStatus::Ready {
|
||||||
|
return 100.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut total_weight = 0.0f64;
|
||||||
|
let mut completed_weight = 0.0f64;
|
||||||
|
|
||||||
|
for component in &snapshot.components {
|
||||||
|
total_weight += component.weight;
|
||||||
|
let unit_progress = match component.status {
|
||||||
|
StartupComponentStatus::Pending => 0.0,
|
||||||
|
StartupComponentStatus::Running => {
|
||||||
|
if component.id == COMPONENT_ME_POOL_INIT_STAGE1 {
|
||||||
|
me_stage_progress.unwrap_or(0.0).clamp(0.0, 1.0)
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
StartupComponentStatus::Ready
|
||||||
|
| StartupComponentStatus::Failed
|
||||||
|
| StartupComponentStatus::Skipped => 1.0,
|
||||||
|
};
|
||||||
|
completed_weight += component.weight * unit_progress;
|
||||||
|
}
|
||||||
|
|
||||||
|
if total_weight <= f64::EPSILON {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
((completed_weight / total_weight) * 100.0).clamp(0.0, 100.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn component_blueprint() -> Vec<StartupComponent> {
|
||||||
|
vec![
|
||||||
|
component(COMPONENT_CONFIG_LOAD, "Config load", 5.0),
|
||||||
|
component(COMPONENT_TRACING_INIT, "Tracing init", 3.0),
|
||||||
|
component(COMPONENT_API_BOOTSTRAP, "API bootstrap", 5.0),
|
||||||
|
component(COMPONENT_TLS_FRONT_BOOTSTRAP, "TLS front bootstrap", 5.0),
|
||||||
|
component(COMPONENT_NETWORK_PROBE, "Network probe", 10.0),
|
||||||
|
component(COMPONENT_ME_SECRET_FETCH, "ME secret fetch", 8.0),
|
||||||
|
component(COMPONENT_ME_PROXY_CONFIG_V4, "ME config v4 fetch", 4.0),
|
||||||
|
component(COMPONENT_ME_PROXY_CONFIG_V6, "ME config v6 fetch", 4.0),
|
||||||
|
component(COMPONENT_ME_POOL_CONSTRUCT, "ME pool construct", 6.0),
|
||||||
|
component(COMPONENT_ME_POOL_INIT_STAGE1, "ME pool init stage1", 24.0),
|
||||||
|
component(COMPONENT_ME_CONNECTIVITY_PING, "ME connectivity ping", 6.0),
|
||||||
|
component(COMPONENT_DC_CONNECTIVITY_PING, "DC connectivity ping", 8.0),
|
||||||
|
component(COMPONENT_LISTENERS_BIND, "Listener bind", 8.0),
|
||||||
|
component(COMPONENT_CONFIG_WATCHER_START, "Config watcher start", 2.0),
|
||||||
|
component(COMPONENT_METRICS_START, "Metrics start", 1.0),
|
||||||
|
component(COMPONENT_RUNTIME_READY, "Runtime ready", 1.0),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn component(id: &'static str, title: &'static str, weight: f64) -> StartupComponent {
|
||||||
|
StartupComponent {
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
weight,
|
||||||
|
status: StartupComponentStatus::Pending,
|
||||||
|
started_at_epoch_ms: None,
|
||||||
|
finished_at_epoch_ms: None,
|
||||||
|
duration_ms: None,
|
||||||
|
attempts: 0,
|
||||||
|
details: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_details(details: Option<String>) -> Option<String> {
|
||||||
|
details.map(|detail| {
|
||||||
|
if detail.len() <= 256 {
|
||||||
|
detail
|
||||||
|
} else {
|
||||||
|
detail[..256].to_string()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_ms() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis() as u64
|
||||||
|
}
|
||||||
327
src/stats/mod.rs
327
src/stats/mod.rs
@@ -6,7 +6,7 @@ pub mod beobachten;
|
|||||||
pub mod telemetry;
|
pub mod telemetry;
|
||||||
|
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
|
||||||
use std::time::{Instant, Duration};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use lru::LruCache;
|
use lru::LruCache;
|
||||||
@@ -16,7 +16,7 @@ use std::collections::hash_map::DefaultHasher;
|
|||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
use crate::config::MeTelemetryLevel;
|
use crate::config::{MeTelemetryLevel, MeWriterPickMode};
|
||||||
use self::telemetry::TelemetryPolicy;
|
use self::telemetry::TelemetryPolicy;
|
||||||
|
|
||||||
// ============= Stats =============
|
// ============= Stats =============
|
||||||
@@ -80,6 +80,12 @@ pub struct Stats {
|
|||||||
me_floor_global_cap_raw_gauge: AtomicU64,
|
me_floor_global_cap_raw_gauge: AtomicU64,
|
||||||
me_floor_global_cap_effective_gauge: AtomicU64,
|
me_floor_global_cap_effective_gauge: AtomicU64,
|
||||||
me_floor_target_writers_total_gauge: AtomicU64,
|
me_floor_target_writers_total_gauge: AtomicU64,
|
||||||
|
me_floor_active_cap_configured_gauge: AtomicU64,
|
||||||
|
me_floor_active_cap_effective_gauge: AtomicU64,
|
||||||
|
me_floor_warm_cap_configured_gauge: AtomicU64,
|
||||||
|
me_floor_warm_cap_effective_gauge: AtomicU64,
|
||||||
|
me_writers_active_current_gauge: AtomicU64,
|
||||||
|
me_writers_warm_current_gauge: AtomicU64,
|
||||||
me_floor_cap_block_total: AtomicU64,
|
me_floor_cap_block_total: AtomicU64,
|
||||||
me_floor_swap_idle_total: AtomicU64,
|
me_floor_swap_idle_total: AtomicU64,
|
||||||
me_floor_swap_idle_failed_total: AtomicU64,
|
me_floor_swap_idle_failed_total: AtomicU64,
|
||||||
@@ -89,6 +95,18 @@ pub struct Stats {
|
|||||||
me_route_drop_queue_full: AtomicU64,
|
me_route_drop_queue_full: AtomicU64,
|
||||||
me_route_drop_queue_full_base: AtomicU64,
|
me_route_drop_queue_full_base: AtomicU64,
|
||||||
me_route_drop_queue_full_high: AtomicU64,
|
me_route_drop_queue_full_high: AtomicU64,
|
||||||
|
me_writer_pick_sorted_rr_success_try_total: AtomicU64,
|
||||||
|
me_writer_pick_sorted_rr_success_fallback_total: AtomicU64,
|
||||||
|
me_writer_pick_sorted_rr_full_total: AtomicU64,
|
||||||
|
me_writer_pick_sorted_rr_closed_total: AtomicU64,
|
||||||
|
me_writer_pick_sorted_rr_no_candidate_total: AtomicU64,
|
||||||
|
me_writer_pick_p2c_success_try_total: AtomicU64,
|
||||||
|
me_writer_pick_p2c_success_fallback_total: AtomicU64,
|
||||||
|
me_writer_pick_p2c_full_total: AtomicU64,
|
||||||
|
me_writer_pick_p2c_closed_total: AtomicU64,
|
||||||
|
me_writer_pick_p2c_no_candidate_total: AtomicU64,
|
||||||
|
me_writer_pick_blocking_fallback_total: AtomicU64,
|
||||||
|
me_writer_pick_mode_switch_total: AtomicU64,
|
||||||
me_socks_kdf_strict_reject: AtomicU64,
|
me_socks_kdf_strict_reject: AtomicU64,
|
||||||
me_socks_kdf_compat_fallback: AtomicU64,
|
me_socks_kdf_compat_fallback: AtomicU64,
|
||||||
secure_padding_invalid: AtomicU64,
|
secure_padding_invalid: AtomicU64,
|
||||||
@@ -119,6 +137,7 @@ pub struct Stats {
|
|||||||
telemetry_user_enabled: AtomicBool,
|
telemetry_user_enabled: AtomicBool,
|
||||||
telemetry_me_level: AtomicU8,
|
telemetry_me_level: AtomicU8,
|
||||||
user_stats: DashMap<String, UserStats>,
|
user_stats: DashMap<String, UserStats>,
|
||||||
|
user_stats_last_cleanup_epoch_secs: AtomicU64,
|
||||||
start_time: parking_lot::RwLock<Option<Instant>>,
|
start_time: parking_lot::RwLock<Option<Instant>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,6 +149,7 @@ pub struct UserStats {
|
|||||||
pub octets_to_client: AtomicU64,
|
pub octets_to_client: AtomicU64,
|
||||||
pub msgs_from_client: AtomicU64,
|
pub msgs_from_client: AtomicU64,
|
||||||
pub msgs_to_client: AtomicU64,
|
pub msgs_to_client: AtomicU64,
|
||||||
|
pub last_seen_epoch_secs: AtomicU64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Stats {
|
impl Stats {
|
||||||
@@ -178,6 +198,54 @@ impl Stats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn touch_user_stats(stats: &UserStats) {
|
||||||
|
stats
|
||||||
|
.last_seen_epoch_secs
|
||||||
|
.store(Self::now_epoch_secs(), Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn maybe_cleanup_user_stats(&self) {
|
||||||
|
const USER_STATS_CLEANUP_INTERVAL_SECS: u64 = 60;
|
||||||
|
const USER_STATS_IDLE_TTL_SECS: u64 = 24 * 60 * 60;
|
||||||
|
|
||||||
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
|
let last_cleanup_epoch_secs = self
|
||||||
|
.user_stats_last_cleanup_epoch_secs
|
||||||
|
.load(Ordering::Relaxed);
|
||||||
|
if now_epoch_secs.saturating_sub(last_cleanup_epoch_secs)
|
||||||
|
< USER_STATS_CLEANUP_INTERVAL_SECS
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.user_stats_last_cleanup_epoch_secs
|
||||||
|
.compare_exchange(
|
||||||
|
last_cleanup_epoch_secs,
|
||||||
|
now_epoch_secs,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.user_stats.retain(|_, stats| {
|
||||||
|
if stats.curr_connects.load(Ordering::Relaxed) > 0 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let last_seen_epoch_secs = stats.last_seen_epoch_secs.load(Ordering::Relaxed);
|
||||||
|
now_epoch_secs.saturating_sub(last_seen_epoch_secs) <= USER_STATS_IDLE_TTL_SECS
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) {
|
pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) {
|
||||||
self.telemetry_core_enabled
|
self.telemetry_core_enabled
|
||||||
.store(policy.core_enabled, Ordering::Relaxed);
|
.store(policy.core_enabled, Ordering::Relaxed);
|
||||||
@@ -441,6 +509,93 @@ impl Stats {
|
|||||||
self.me_route_drop_queue_full_high.fetch_add(1, Ordering::Relaxed);
|
self.me_route_drop_queue_full_high.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pub fn increment_me_writer_pick_success_try_total(&self, mode: MeWriterPickMode) {
|
||||||
|
if !self.telemetry_me_allows_normal() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match mode {
|
||||||
|
MeWriterPickMode::SortedRr => {
|
||||||
|
self.me_writer_pick_sorted_rr_success_try_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
MeWriterPickMode::P2c => {
|
||||||
|
self.me_writer_pick_p2c_success_try_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_writer_pick_success_fallback_total(&self, mode: MeWriterPickMode) {
|
||||||
|
if !self.telemetry_me_allows_normal() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match mode {
|
||||||
|
MeWriterPickMode::SortedRr => {
|
||||||
|
self.me_writer_pick_sorted_rr_success_fallback_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
MeWriterPickMode::P2c => {
|
||||||
|
self.me_writer_pick_p2c_success_fallback_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_writer_pick_full_total(&self, mode: MeWriterPickMode) {
|
||||||
|
if !self.telemetry_me_allows_normal() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match mode {
|
||||||
|
MeWriterPickMode::SortedRr => {
|
||||||
|
self.me_writer_pick_sorted_rr_full_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
MeWriterPickMode::P2c => {
|
||||||
|
self.me_writer_pick_p2c_full_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_writer_pick_closed_total(&self, mode: MeWriterPickMode) {
|
||||||
|
if !self.telemetry_me_allows_normal() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match mode {
|
||||||
|
MeWriterPickMode::SortedRr => {
|
||||||
|
self.me_writer_pick_sorted_rr_closed_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
MeWriterPickMode::P2c => {
|
||||||
|
self.me_writer_pick_p2c_closed_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_writer_pick_no_candidate_total(&self, mode: MeWriterPickMode) {
|
||||||
|
if !self.telemetry_me_allows_normal() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match mode {
|
||||||
|
MeWriterPickMode::SortedRr => {
|
||||||
|
self.me_writer_pick_sorted_rr_no_candidate_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
MeWriterPickMode::P2c => {
|
||||||
|
self.me_writer_pick_p2c_no_candidate_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_writer_pick_blocking_fallback_total(&self) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_writer_pick_blocking_fallback_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_writer_pick_mode_switch_total(&self) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_writer_pick_mode_switch_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
pub fn increment_me_socks_kdf_strict_reject(&self) {
|
pub fn increment_me_socks_kdf_strict_reject(&self) {
|
||||||
if self.telemetry_me_allows_normal() {
|
if self.telemetry_me_allows_normal() {
|
||||||
self.me_socks_kdf_strict_reject.fetch_add(1, Ordering::Relaxed);
|
self.me_socks_kdf_strict_reject.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -714,6 +869,42 @@ impl Stats {
|
|||||||
.store(value, Ordering::Relaxed);
|
.store(value, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pub fn set_me_floor_active_cap_configured_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_floor_active_cap_configured_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_floor_active_cap_effective_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_floor_active_cap_effective_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_floor_warm_cap_configured_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_floor_warm_cap_configured_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_floor_warm_cap_effective_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_floor_warm_cap_effective_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_writers_active_current_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_writers_active_current_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn set_me_writers_warm_current_gauge(&self, value: u64) {
|
||||||
|
if self.telemetry_me_allows_normal() {
|
||||||
|
self.me_writers_warm_current_gauge
|
||||||
|
.store(value, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
pub fn increment_me_floor_cap_block_total(&self) {
|
pub fn increment_me_floor_cap_block_total(&self) {
|
||||||
if self.telemetry_me_allows_normal() {
|
if self.telemetry_me_allows_normal() {
|
||||||
self.me_floor_cap_block_total.fetch_add(1, Ordering::Relaxed);
|
self.me_floor_cap_block_total.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -854,6 +1045,30 @@ impl Stats {
|
|||||||
self.me_floor_target_writers_total_gauge
|
self.me_floor_target_writers_total_gauge
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
pub fn get_me_floor_active_cap_configured_gauge(&self) -> u64 {
|
||||||
|
self.me_floor_active_cap_configured_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_floor_active_cap_effective_gauge(&self) -> u64 {
|
||||||
|
self.me_floor_active_cap_effective_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_floor_warm_cap_configured_gauge(&self) -> u64 {
|
||||||
|
self.me_floor_warm_cap_configured_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_floor_warm_cap_effective_gauge(&self) -> u64 {
|
||||||
|
self.me_floor_warm_cap_effective_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writers_active_current_gauge(&self) -> u64 {
|
||||||
|
self.me_writers_active_current_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writers_warm_current_gauge(&self) -> u64 {
|
||||||
|
self.me_writers_warm_current_gauge
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
pub fn get_me_floor_cap_block_total(&self) -> u64 {
|
pub fn get_me_floor_cap_block_total(&self) -> u64 {
|
||||||
self.me_floor_cap_block_total.load(Ordering::Relaxed)
|
self.me_floor_cap_block_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
@@ -885,6 +1100,52 @@ impl Stats {
|
|||||||
pub fn get_me_route_drop_queue_full_high(&self) -> u64 {
|
pub fn get_me_route_drop_queue_full_high(&self) -> u64 {
|
||||||
self.me_route_drop_queue_full_high.load(Ordering::Relaxed)
|
self.me_route_drop_queue_full_high.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
pub fn get_me_writer_pick_sorted_rr_success_try_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_sorted_rr_success_try_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_sorted_rr_success_fallback_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_sorted_rr_success_fallback_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_sorted_rr_full_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_sorted_rr_full_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_sorted_rr_closed_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_sorted_rr_closed_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_sorted_rr_no_candidate_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_sorted_rr_no_candidate_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_p2c_success_try_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_p2c_success_try_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_p2c_success_fallback_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_p2c_success_fallback_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_p2c_full_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_p2c_full_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_p2c_closed_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_p2c_closed_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_p2c_no_candidate_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_p2c_no_candidate_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_blocking_fallback_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_blocking_fallback_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_writer_pick_mode_switch_total(&self) -> u64 {
|
||||||
|
self.me_writer_pick_mode_switch_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
pub fn get_me_socks_kdf_strict_reject(&self) -> u64 {
|
pub fn get_me_socks_kdf_strict_reject(&self) -> u64 {
|
||||||
self.me_socks_kdf_strict_reject.load(Ordering::Relaxed)
|
self.me_socks_kdf_strict_reject.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
@@ -970,34 +1231,36 @@ impl Stats {
|
|||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
stats.connects.fetch_add(1, Ordering::Relaxed);
|
stats.connects.fetch_add(1, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats
|
let stats = self.user_stats.entry(user.to_string()).or_default();
|
||||||
.entry(user.to_string())
|
Self::touch_user_stats(stats.value());
|
||||||
.or_default()
|
stats.connects.fetch_add(1, Ordering::Relaxed);
|
||||||
.connects
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn increment_user_curr_connects(&self, user: &str) {
|
pub fn increment_user_curr_connects(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
|
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats
|
let stats = self.user_stats.entry(user.to_string()).or_default();
|
||||||
.entry(user.to_string())
|
Self::touch_user_stats(stats.value());
|
||||||
.or_default()
|
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
|
||||||
.curr_connects
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn decrement_user_curr_connects(&self, user: &str) {
|
pub fn decrement_user_curr_connects(&self, user: &str) {
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
let counter = &stats.curr_connects;
|
let counter = &stats.curr_connects;
|
||||||
let mut current = counter.load(Ordering::Relaxed);
|
let mut current = counter.load(Ordering::Relaxed);
|
||||||
loop {
|
loop {
|
||||||
@@ -1027,60 +1290,60 @@ impl Stats {
|
|||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed);
|
stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats
|
let stats = self.user_stats.entry(user.to_string()).or_default();
|
||||||
.entry(user.to_string())
|
Self::touch_user_stats(stats.value());
|
||||||
.or_default()
|
stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed);
|
||||||
.octets_from_client
|
|
||||||
.fetch_add(bytes, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn add_user_octets_to(&self, user: &str, bytes: u64) {
|
pub fn add_user_octets_to(&self, user: &str, bytes: u64) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed);
|
stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats
|
let stats = self.user_stats.entry(user.to_string()).or_default();
|
||||||
.entry(user.to_string())
|
Self::touch_user_stats(stats.value());
|
||||||
.or_default()
|
stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed);
|
||||||
.octets_to_client
|
|
||||||
.fetch_add(bytes, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn increment_user_msgs_from(&self, user: &str) {
|
pub fn increment_user_msgs_from(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats
|
let stats = self.user_stats.entry(user.to_string()).or_default();
|
||||||
.entry(user.to_string())
|
Self::touch_user_stats(stats.value());
|
||||||
.or_default()
|
stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
||||||
.msgs_from_client
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn increment_user_msgs_to(&self, user: &str) {
|
pub fn increment_user_msgs_to(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
|
Self::touch_user_stats(stats.value());
|
||||||
stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.user_stats
|
let stats = self.user_stats.entry(user.to_string()).or_default();
|
||||||
.entry(user.to_string())
|
Self::touch_user_stats(stats.value());
|
||||||
.or_default()
|
stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
||||||
.msgs_to_client
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_user_total_octets(&self, user: &str) -> u64 {
|
pub fn get_user_total_octets(&self, user: &str) -> u64 {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
use crate::crypto::{AesCbc, crc32, crc32c};
|
use crate::crypto::{AesCbc, crc32, crc32c};
|
||||||
use crate::error::{ProxyError, Result};
|
use crate::error::{ProxyError, Result};
|
||||||
@@ -6,8 +7,8 @@ use crate::protocol::constants::*;
|
|||||||
|
|
||||||
/// Commands sent to dedicated writer tasks to avoid mutex contention on TCP writes.
|
/// Commands sent to dedicated writer tasks to avoid mutex contention on TCP writes.
|
||||||
pub(crate) enum WriterCommand {
|
pub(crate) enum WriterCommand {
|
||||||
Data(Vec<u8>),
|
Data(Bytes),
|
||||||
DataAndFlush(Vec<u8>),
|
DataAndFlush(Bytes),
|
||||||
Close,
|
Close,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -306,6 +306,8 @@ async fn run_update_cycle(
|
|||||||
cfg.general.me_bind_stale_ttl_secs,
|
cfg.general.me_bind_stale_ttl_secs,
|
||||||
cfg.general.me_secret_atomic_snapshot,
|
cfg.general.me_secret_atomic_snapshot,
|
||||||
cfg.general.me_deterministic_writer_sort,
|
cfg.general.me_deterministic_writer_sort,
|
||||||
|
cfg.general.me_writer_pick_mode,
|
||||||
|
cfg.general.me_writer_pick_sample_size,
|
||||||
cfg.general.me_single_endpoint_shadow_writers,
|
cfg.general.me_single_endpoint_shadow_writers,
|
||||||
cfg.general.me_single_endpoint_outage_mode_enabled,
|
cfg.general.me_single_endpoint_outage_mode_enabled,
|
||||||
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
||||||
@@ -321,6 +323,13 @@ async fn run_update_cycle(
|
|||||||
cfg.general.me_adaptive_floor_cpu_cores_override,
|
cfg.general.me_adaptive_floor_cpu_cores_override,
|
||||||
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core,
|
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core,
|
||||||
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core,
|
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_global,
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_global,
|
||||||
|
cfg.general.me_health_interval_ms_unhealthy,
|
||||||
|
cfg.general.me_health_interval_ms_healthy,
|
||||||
|
cfg.general.me_warn_rate_limit_ms,
|
||||||
);
|
);
|
||||||
|
|
||||||
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
|
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
|
||||||
@@ -523,6 +532,8 @@ pub async fn me_config_updater(
|
|||||||
cfg.general.me_bind_stale_ttl_secs,
|
cfg.general.me_bind_stale_ttl_secs,
|
||||||
cfg.general.me_secret_atomic_snapshot,
|
cfg.general.me_secret_atomic_snapshot,
|
||||||
cfg.general.me_deterministic_writer_sort,
|
cfg.general.me_deterministic_writer_sort,
|
||||||
|
cfg.general.me_writer_pick_mode,
|
||||||
|
cfg.general.me_writer_pick_sample_size,
|
||||||
cfg.general.me_single_endpoint_shadow_writers,
|
cfg.general.me_single_endpoint_shadow_writers,
|
||||||
cfg.general.me_single_endpoint_outage_mode_enabled,
|
cfg.general.me_single_endpoint_outage_mode_enabled,
|
||||||
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
||||||
@@ -538,6 +549,13 @@ pub async fn me_config_updater(
|
|||||||
cfg.general.me_adaptive_floor_cpu_cores_override,
|
cfg.general.me_adaptive_floor_cpu_cores_override,
|
||||||
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core,
|
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core,
|
||||||
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core,
|
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_global,
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_global,
|
||||||
|
cfg.general.me_health_interval_ms_unhealthy,
|
||||||
|
cfg.general.me_health_interval_ms_healthy,
|
||||||
|
cfg.general.me_warn_rate_limit_ms,
|
||||||
);
|
);
|
||||||
let new_secs = cfg.general.effective_update_every_secs().max(1);
|
let new_secs = cfg.general.effective_update_every_secs().max(1);
|
||||||
if new_secs == update_every_secs {
|
if new_secs == update_every_secs {
|
||||||
|
|||||||
@@ -135,10 +135,15 @@ impl MePool {
|
|||||||
pub(crate) async fn connect_tcp(
|
pub(crate) async fn connect_tcp(
|
||||||
&self,
|
&self,
|
||||||
addr: SocketAddr,
|
addr: SocketAddr,
|
||||||
|
dc_idx_override: Option<i16>,
|
||||||
) -> Result<(TcpStream, f64, Option<UpstreamEgressInfo>)> {
|
) -> Result<(TcpStream, f64, Option<UpstreamEgressInfo>)> {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let (stream, upstream_egress) = if let Some(upstream) = &self.upstream {
|
let (stream, upstream_egress) = if let Some(upstream) = &self.upstream {
|
||||||
let dc_idx = self.resolve_dc_idx_for_endpoint(addr).await;
|
let dc_idx = if let Some(dc_idx) = dc_idx_override {
|
||||||
|
Some(dc_idx)
|
||||||
|
} else {
|
||||||
|
self.resolve_dc_idx_for_endpoint(addr).await
|
||||||
|
};
|
||||||
let (stream, egress) = upstream.connect_with_details(addr, dc_idx, None).await?;
|
let (stream, egress) = upstream.connect_with_details(addr, dc_idx, None).await?;
|
||||||
(stream, Some(egress))
|
(stream, Some(egress))
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ use crate::network::IpFamily;
|
|||||||
|
|
||||||
use super::MePool;
|
use super::MePool;
|
||||||
|
|
||||||
const HEALTH_INTERVAL_SECS: u64 = 1;
|
|
||||||
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1;
|
const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1;
|
||||||
@@ -42,7 +41,12 @@ struct DcFloorPlanEntry {
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct FamilyFloorPlan {
|
struct FamilyFloorPlan {
|
||||||
by_dc: HashMap<i32, DcFloorPlanEntry>,
|
by_dc: HashMap<i32, DcFloorPlanEntry>,
|
||||||
global_cap_effective_total: usize,
|
active_cap_configured_total: usize,
|
||||||
|
active_cap_effective_total: usize,
|
||||||
|
warm_cap_configured_total: usize,
|
||||||
|
warm_cap_effective_total: usize,
|
||||||
|
active_writers_current: usize,
|
||||||
|
warm_writers_current: usize,
|
||||||
target_writers_total: usize,
|
target_writers_total: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,10 +61,18 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
|
let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
|
let mut degraded_interval = true;
|
||||||
loop {
|
loop {
|
||||||
tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await;
|
let interval = if degraded_interval {
|
||||||
|
pool.health_interval_unhealthy()
|
||||||
|
} else {
|
||||||
|
pool.health_interval_healthy()
|
||||||
|
};
|
||||||
|
tokio::time::sleep(interval).await;
|
||||||
pool.prune_closed_writers().await;
|
pool.prune_closed_writers().await;
|
||||||
check_family(
|
reap_draining_writers(&pool).await;
|
||||||
|
let v4_degraded = check_family(
|
||||||
IpFamily::V4,
|
IpFamily::V4,
|
||||||
&pool,
|
&pool,
|
||||||
&rng,
|
&rng,
|
||||||
@@ -74,9 +86,10 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
&mut idle_refresh_next_attempt,
|
&mut idle_refresh_next_attempt,
|
||||||
&mut adaptive_idle_since,
|
&mut adaptive_idle_since,
|
||||||
&mut adaptive_recover_until,
|
&mut adaptive_recover_until,
|
||||||
|
&mut floor_warn_next_allowed,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
check_family(
|
let v6_degraded = check_family(
|
||||||
IpFamily::V6,
|
IpFamily::V6,
|
||||||
&pool,
|
&pool,
|
||||||
&rng,
|
&rng,
|
||||||
@@ -90,8 +103,32 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
&mut idle_refresh_next_attempt,
|
&mut idle_refresh_next_attempt,
|
||||||
&mut adaptive_idle_since,
|
&mut adaptive_idle_since,
|
||||||
&mut adaptive_recover_until,
|
&mut adaptive_recover_until,
|
||||||
|
&mut floor_warn_next_allowed,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
degraded_interval = v4_degraded || v6_degraded;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reap_draining_writers(pool: &Arc<MePool>) {
|
||||||
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
|
let writers = pool.writers.read().await.clone();
|
||||||
|
for writer in writers {
|
||||||
|
if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if pool.registry.is_writer_empty(writer.id).await {
|
||||||
|
pool.remove_writer_and_close_clients(writer.id).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let deadline_epoch_secs = writer
|
||||||
|
.drain_deadline_epoch_secs
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
|
if deadline_epoch_secs != 0 && now_epoch_secs >= deadline_epoch_secs {
|
||||||
|
warn!(writer_id = writer.id, "Drain timeout, force-closing");
|
||||||
|
pool.stats.increment_pool_force_close_total();
|
||||||
|
pool.remove_writer_and_close_clients(writer.id).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,22 +146,25 @@ async fn check_family(
|
|||||||
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
) {
|
floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
|
) -> bool {
|
||||||
let enabled = match family {
|
let enabled = match family {
|
||||||
IpFamily::V4 => pool.decision.ipv4_me,
|
IpFamily::V4 => pool.decision.ipv4_me,
|
||||||
IpFamily::V6 => pool.decision.ipv6_me,
|
IpFamily::V6 => pool.decision.ipv6_me,
|
||||||
};
|
};
|
||||||
if !enabled {
|
if !enabled {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut family_degraded = false;
|
||||||
|
|
||||||
let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new();
|
let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new();
|
||||||
let map_guard = match family {
|
let map_guard = match family {
|
||||||
IpFamily::V4 => pool.proxy_map_v4.read().await,
|
IpFamily::V4 => pool.proxy_map_v4.read().await,
|
||||||
IpFamily::V6 => pool.proxy_map_v6.read().await,
|
IpFamily::V6 => pool.proxy_map_v6.read().await,
|
||||||
};
|
};
|
||||||
for (dc, addrs) in map_guard.iter() {
|
for (dc, addrs) in map_guard.iter() {
|
||||||
let entry = dc_endpoints.entry(dc.abs()).or_default();
|
let entry = dc_endpoints.entry(*dc).or_default();
|
||||||
for (ip, port) in addrs.iter().copied() {
|
for (ip, port) in addrs.iter().copied() {
|
||||||
entry.push(SocketAddr::new(ip, port));
|
entry.push(SocketAddr::new(ip, port));
|
||||||
}
|
}
|
||||||
@@ -141,31 +181,51 @@ async fn check_family(
|
|||||||
adaptive_recover_until.clear();
|
adaptive_recover_until.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut live_addr_counts = HashMap::<SocketAddr, usize>::new();
|
let mut live_addr_counts = HashMap::<(i32, SocketAddr), usize>::new();
|
||||||
let mut live_writer_ids_by_addr = HashMap::<SocketAddr, Vec<u64>>::new();
|
let mut live_writer_ids_by_addr = HashMap::<(i32, SocketAddr), Vec<u64>>::new();
|
||||||
for writer in pool.writers.read().await.iter().filter(|w| {
|
for writer in pool.writers.read().await.iter().filter(|w| {
|
||||||
!w.draining.load(std::sync::atomic::Ordering::Relaxed)
|
!w.draining.load(std::sync::atomic::Ordering::Relaxed)
|
||||||
}) {
|
}) {
|
||||||
*live_addr_counts.entry(writer.addr).or_insert(0) += 1;
|
if !matches!(
|
||||||
|
super::pool::WriterContour::from_u8(
|
||||||
|
writer.contour.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
|
),
|
||||||
|
super::pool::WriterContour::Active
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let key = (writer.writer_dc, writer.addr);
|
||||||
|
*live_addr_counts.entry(key).or_insert(0) += 1;
|
||||||
live_writer_ids_by_addr
|
live_writer_ids_by_addr
|
||||||
.entry(writer.addr)
|
.entry(key)
|
||||||
.or_default()
|
.or_default()
|
||||||
.push(writer.id);
|
.push(writer.id);
|
||||||
}
|
}
|
||||||
let writer_idle_since = pool.registry.writer_idle_since_snapshot().await;
|
let writer_idle_since = pool.registry.writer_idle_since_snapshot().await;
|
||||||
|
let bound_clients_by_writer = pool
|
||||||
|
.registry
|
||||||
|
.writer_activity_snapshot()
|
||||||
|
.await
|
||||||
|
.bound_clients_by_writer;
|
||||||
let floor_plan = build_family_floor_plan(
|
let floor_plan = build_family_floor_plan(
|
||||||
pool,
|
pool,
|
||||||
family,
|
family,
|
||||||
&dc_endpoints,
|
&dc_endpoints,
|
||||||
&live_addr_counts,
|
&live_addr_counts,
|
||||||
&live_writer_ids_by_addr,
|
&live_writer_ids_by_addr,
|
||||||
|
&bound_clients_by_writer,
|
||||||
adaptive_idle_since,
|
adaptive_idle_since,
|
||||||
adaptive_recover_until,
|
adaptive_recover_until,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
pool.set_adaptive_floor_runtime_caps(
|
pool.set_adaptive_floor_runtime_caps(
|
||||||
floor_plan.global_cap_effective_total,
|
floor_plan.active_cap_configured_total,
|
||||||
|
floor_plan.active_cap_effective_total,
|
||||||
|
floor_plan.warm_cap_configured_total,
|
||||||
|
floor_plan.warm_cap_effective_total,
|
||||||
floor_plan.target_writers_total,
|
floor_plan.target_writers_total,
|
||||||
|
floor_plan.active_writers_current,
|
||||||
|
floor_plan.warm_writers_current,
|
||||||
);
|
);
|
||||||
|
|
||||||
for (dc, endpoints) in dc_endpoints {
|
for (dc, endpoints) in dc_endpoints {
|
||||||
@@ -182,10 +242,11 @@ async fn check_family(
|
|||||||
});
|
});
|
||||||
let alive = endpoints
|
let alive = endpoints
|
||||||
.iter()
|
.iter()
|
||||||
.map(|addr| *live_addr_counts.get(addr).unwrap_or(&0))
|
.map(|addr| *live_addr_counts.get(&(dc, *addr)).unwrap_or(&0))
|
||||||
.sum::<usize>();
|
.sum::<usize>();
|
||||||
|
|
||||||
if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 {
|
if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 {
|
||||||
|
family_degraded = true;
|
||||||
if single_endpoint_outage.insert(key) {
|
if single_endpoint_outage.insert(key) {
|
||||||
pool.stats.increment_me_single_endpoint_outage_enter_total();
|
pool.stats.increment_me_single_endpoint_outage_enter_total();
|
||||||
warn!(
|
warn!(
|
||||||
@@ -241,6 +302,7 @@ async fn check_family(
|
|||||||
required,
|
required,
|
||||||
&live_writer_ids_by_addr,
|
&live_writer_ids_by_addr,
|
||||||
&writer_idle_since,
|
&writer_idle_since,
|
||||||
|
&bound_clients_by_writer,
|
||||||
idle_refresh_next_attempt,
|
idle_refresh_next_attempt,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -254,12 +316,14 @@ async fn check_family(
|
|||||||
alive,
|
alive,
|
||||||
required,
|
required,
|
||||||
&live_writer_ids_by_addr,
|
&live_writer_ids_by_addr,
|
||||||
|
&bound_clients_by_writer,
|
||||||
shadow_rotate_deadline,
|
shadow_rotate_deadline,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let missing = required - alive;
|
let missing = required - alive;
|
||||||
|
family_degraded = true;
|
||||||
|
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
if reconnect_budget == 0 {
|
if reconnect_budget == 0 {
|
||||||
@@ -290,7 +354,10 @@ async fn check_family(
|
|||||||
if *inflight.get(&key).unwrap_or(&0) >= max_concurrent {
|
if *inflight.get(&key).unwrap_or(&0) >= max_concurrent {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if pool.has_refill_inflight_for_endpoints(&endpoints).await {
|
if pool
|
||||||
|
.has_refill_inflight_for_dc_key(super::pool::RefillDcKey { dc, family })
|
||||||
|
.await
|
||||||
|
{
|
||||||
debug!(
|
debug!(
|
||||||
dc = %dc,
|
dc = %dc,
|
||||||
?family,
|
?family,
|
||||||
@@ -309,8 +376,8 @@ async fn check_family(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
reconnect_budget = reconnect_budget.saturating_sub(1);
|
reconnect_budget = reconnect_budget.saturating_sub(1);
|
||||||
if pool.floor_mode() == MeFloorMode::Adaptive
|
if pool.active_contour_writer_count_total().await
|
||||||
&& pool.active_writer_count_total().await >= floor_plan.global_cap_effective_total
|
>= floor_plan.active_cap_effective_total
|
||||||
{
|
{
|
||||||
let swapped = maybe_swap_idle_writer_for_cap(
|
let swapped = maybe_swap_idle_writer_for_cap(
|
||||||
pool,
|
pool,
|
||||||
@@ -320,6 +387,7 @@ async fn check_family(
|
|||||||
&endpoints,
|
&endpoints,
|
||||||
&live_writer_ids_by_addr,
|
&live_writer_ids_by_addr,
|
||||||
&writer_idle_since,
|
&writer_idle_since,
|
||||||
|
&bound_clients_by_writer,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
if swapped {
|
if swapped {
|
||||||
@@ -334,14 +402,14 @@ async fn check_family(
|
|||||||
?family,
|
?family,
|
||||||
alive,
|
alive,
|
||||||
required,
|
required,
|
||||||
global_cap_effective_total = floor_plan.global_cap_effective_total,
|
active_cap_effective_total = floor_plan.active_cap_effective_total,
|
||||||
"Adaptive floor cap reached, reconnect attempt blocked"
|
"Adaptive floor cap reached, reconnect attempt blocked"
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let res = tokio::time::timeout(
|
let res = tokio::time::timeout(
|
||||||
pool.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_endpoints_round_robin(&endpoints, rng.as_ref()),
|
pool.connect_endpoints_round_robin(dc, &endpoints, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
match res {
|
match res {
|
||||||
@@ -384,15 +452,23 @@ async fn check_family(
|
|||||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
||||||
next_attempt.insert(key, now + wait);
|
next_attempt.insert(key, now + wait);
|
||||||
if pool.is_runtime_ready() {
|
if pool.is_runtime_ready() {
|
||||||
warn!(
|
let warn_cooldown = pool.warn_rate_limit_duration();
|
||||||
dc = %dc,
|
if should_emit_rate_limited_warn(
|
||||||
?family,
|
floor_warn_next_allowed,
|
||||||
alive = now_alive,
|
key,
|
||||||
required,
|
now,
|
||||||
endpoint_count = endpoints.len(),
|
warn_cooldown,
|
||||||
backoff_ms = next_ms,
|
) {
|
||||||
"DC writer floor is below required level, scheduled reconnect"
|
warn!(
|
||||||
);
|
dc = %dc,
|
||||||
|
?family,
|
||||||
|
alive = now_alive,
|
||||||
|
required,
|
||||||
|
endpoint_count = endpoints.len(),
|
||||||
|
backoff_ms = next_ms,
|
||||||
|
"DC writer floor is below required level, scheduled reconnect"
|
||||||
|
);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
info!(
|
info!(
|
||||||
dc = %dc,
|
dc = %dc,
|
||||||
@@ -409,6 +485,8 @@ async fn check_family(
|
|||||||
*v = v.saturating_sub(1);
|
*v = v.saturating_sub(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
family_degraded
|
||||||
}
|
}
|
||||||
|
|
||||||
fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
|
fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
|
||||||
@@ -420,6 +498,23 @@ fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
|
|||||||
.clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX)
|
.clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn should_emit_rate_limited_warn(
|
||||||
|
next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
|
key: (i32, IpFamily),
|
||||||
|
now: Instant,
|
||||||
|
cooldown: Duration,
|
||||||
|
) -> bool {
|
||||||
|
let Some(ready_at) = next_allowed.get(&key).copied() else {
|
||||||
|
next_allowed.insert(key, now + cooldown);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
if now >= ready_at {
|
||||||
|
next_allowed.insert(key, now + cooldown);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
fn adaptive_floor_class_min(
|
fn adaptive_floor_class_min(
|
||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
endpoint_count: usize,
|
endpoint_count: usize,
|
||||||
@@ -452,12 +547,13 @@ fn adaptive_floor_class_max(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn list_writer_ids_for_endpoints(
|
fn list_writer_ids_for_endpoints(
|
||||||
|
dc: i32,
|
||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
) -> Vec<u64> {
|
) -> Vec<u64> {
|
||||||
let mut out = Vec::<u64>::new();
|
let mut out = Vec::<u64>::new();
|
||||||
for endpoint in endpoints {
|
for endpoint in endpoints {
|
||||||
if let Some(ids) = live_writer_ids_by_addr.get(endpoint) {
|
if let Some(ids) = live_writer_ids_by_addr.get(&(dc, *endpoint)) {
|
||||||
out.extend(ids.iter().copied());
|
out.extend(ids.iter().copied());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -468,8 +564,9 @@ async fn build_family_floor_plan(
|
|||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
family: IpFamily,
|
family: IpFamily,
|
||||||
dc_endpoints: &HashMap<i32, Vec<SocketAddr>>,
|
dc_endpoints: &HashMap<i32, Vec<SocketAddr>>,
|
||||||
live_addr_counts: &HashMap<SocketAddr, usize>,
|
live_addr_counts: &HashMap<(i32, SocketAddr), usize>,
|
||||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
) -> FamilyFloorPlan {
|
) -> FamilyFloorPlan {
|
||||||
@@ -480,6 +577,8 @@ async fn build_family_floor_plan(
|
|||||||
let floor_mode = pool.floor_mode();
|
let floor_mode = pool.floor_mode();
|
||||||
let is_adaptive = floor_mode == MeFloorMode::Adaptive;
|
let is_adaptive = floor_mode == MeFloorMode::Adaptive;
|
||||||
let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1);
|
let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1);
|
||||||
|
let (active_writers_current, warm_writers_current, _) =
|
||||||
|
pool.non_draining_writer_counts_by_contour().await;
|
||||||
|
|
||||||
for (dc, endpoints) in dc_endpoints {
|
for (dc, endpoints) in dc_endpoints {
|
||||||
if endpoints.is_empty() {
|
if endpoints.is_empty() {
|
||||||
@@ -489,8 +588,10 @@ async fn build_family_floor_plan(
|
|||||||
let reduce_for_idle = should_reduce_floor_for_idle(
|
let reduce_for_idle = should_reduce_floor_for_idle(
|
||||||
pool,
|
pool,
|
||||||
key,
|
key,
|
||||||
|
*dc,
|
||||||
endpoints,
|
endpoints,
|
||||||
live_writer_ids_by_addr,
|
live_writer_ids_by_addr,
|
||||||
|
bound_clients_by_writer,
|
||||||
adaptive_idle_since,
|
adaptive_idle_since,
|
||||||
adaptive_recover_until,
|
adaptive_recover_until,
|
||||||
)
|
)
|
||||||
@@ -517,11 +618,11 @@ async fn build_family_floor_plan(
|
|||||||
let target_required = desired_raw.clamp(min_required, max_required);
|
let target_required = desired_raw.clamp(min_required, max_required);
|
||||||
let alive = endpoints
|
let alive = endpoints
|
||||||
.iter()
|
.iter()
|
||||||
.map(|endpoint| live_addr_counts.get(endpoint).copied().unwrap_or(0))
|
.map(|endpoint| live_addr_counts.get(&(*dc, *endpoint)).copied().unwrap_or(0))
|
||||||
.sum::<usize>();
|
.sum::<usize>();
|
||||||
family_active_total = family_active_total.saturating_add(alive);
|
family_active_total = family_active_total.saturating_add(alive);
|
||||||
let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr);
|
let writer_ids = list_writer_ids_for_endpoints(*dc, endpoints, live_writer_ids_by_addr);
|
||||||
let has_bound_clients = has_bound_clients_on_endpoint(pool, &writer_ids).await;
|
let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer);
|
||||||
|
|
||||||
entries.push(DcFloorPlanEntry {
|
entries.push(DcFloorPlanEntry {
|
||||||
dc: *dc,
|
dc: *dc,
|
||||||
@@ -536,9 +637,16 @@ async fn build_family_floor_plan(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if entries.is_empty() {
|
if entries.is_empty() {
|
||||||
|
let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total();
|
||||||
|
let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total();
|
||||||
return FamilyFloorPlan {
|
return FamilyFloorPlan {
|
||||||
by_dc,
|
by_dc,
|
||||||
global_cap_effective_total: 0,
|
active_cap_configured_total,
|
||||||
|
active_cap_effective_total: active_cap_configured_total,
|
||||||
|
warm_cap_configured_total,
|
||||||
|
warm_cap_effective_total: warm_cap_configured_total,
|
||||||
|
active_writers_current,
|
||||||
|
warm_writers_current,
|
||||||
target_writers_total: 0,
|
target_writers_total: 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -548,20 +656,26 @@ async fn build_family_floor_plan(
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|entry| entry.target_required)
|
.map(|entry| entry.target_required)
|
||||||
.sum::<usize>();
|
.sum::<usize>();
|
||||||
let active_total = pool.active_writer_count_total().await;
|
let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total();
|
||||||
|
let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total();
|
||||||
for entry in entries {
|
for entry in entries {
|
||||||
by_dc.insert(entry.dc, entry);
|
by_dc.insert(entry.dc, entry);
|
||||||
}
|
}
|
||||||
return FamilyFloorPlan {
|
return FamilyFloorPlan {
|
||||||
by_dc,
|
by_dc,
|
||||||
global_cap_effective_total: active_total.max(target_total),
|
active_cap_configured_total,
|
||||||
|
active_cap_effective_total: active_cap_configured_total.max(target_total),
|
||||||
|
warm_cap_configured_total,
|
||||||
|
warm_cap_effective_total: warm_cap_configured_total,
|
||||||
|
active_writers_current,
|
||||||
|
warm_writers_current,
|
||||||
target_writers_total: target_total,
|
target_writers_total: target_total,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let global_cap_raw = pool.adaptive_floor_global_cap_raw();
|
let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total();
|
||||||
let total_active = pool.active_writer_count_total().await;
|
let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total();
|
||||||
let other_active = total_active.saturating_sub(family_active_total);
|
let other_active = active_writers_current.saturating_sub(family_active_total);
|
||||||
let min_sum = entries
|
let min_sum = entries
|
||||||
.iter()
|
.iter()
|
||||||
.map(|entry| entry.min_required)
|
.map(|entry| entry.min_required)
|
||||||
@@ -570,7 +684,7 @@ async fn build_family_floor_plan(
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|entry| entry.target_required)
|
.map(|entry| entry.target_required)
|
||||||
.sum::<usize>();
|
.sum::<usize>();
|
||||||
let family_cap = global_cap_raw
|
let family_cap = active_cap_configured_total
|
||||||
.saturating_sub(other_active)
|
.saturating_sub(other_active)
|
||||||
.max(min_sum);
|
.max(min_sum);
|
||||||
if target_sum > family_cap {
|
if target_sum > family_cap {
|
||||||
@@ -605,11 +719,17 @@ async fn build_family_floor_plan(
|
|||||||
for entry in entries {
|
for entry in entries {
|
||||||
by_dc.insert(entry.dc, entry);
|
by_dc.insert(entry.dc, entry);
|
||||||
}
|
}
|
||||||
let global_cap_effective_total = global_cap_raw.max(other_active.saturating_add(min_sum));
|
let active_cap_effective_total =
|
||||||
|
active_cap_configured_total.max(other_active.saturating_add(min_sum));
|
||||||
let target_writers_total = other_active.saturating_add(target_sum);
|
let target_writers_total = other_active.saturating_add(target_sum);
|
||||||
FamilyFloorPlan {
|
FamilyFloorPlan {
|
||||||
by_dc,
|
by_dc,
|
||||||
global_cap_effective_total,
|
active_cap_configured_total,
|
||||||
|
active_cap_effective_total,
|
||||||
|
warm_cap_configured_total,
|
||||||
|
warm_cap_effective_total: warm_cap_configured_total,
|
||||||
|
active_writers_current,
|
||||||
|
warm_writers_current,
|
||||||
target_writers_total,
|
target_writers_total,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -620,17 +740,18 @@ async fn maybe_swap_idle_writer_for_cap(
|
|||||||
dc: i32,
|
dc: i32,
|
||||||
family: IpFamily,
|
family: IpFamily,
|
||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
writer_idle_since: &HashMap<u64, u64>,
|
writer_idle_since: &HashMap<u64, u64>,
|
||||||
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
let mut candidate: Option<(u64, SocketAddr, u64)> = None;
|
let mut candidate: Option<(u64, SocketAddr, u64)> = None;
|
||||||
for endpoint in endpoints {
|
for endpoint in endpoints {
|
||||||
let Some(writer_ids) = live_writer_ids_by_addr.get(endpoint) else {
|
let Some(writer_ids) = live_writer_ids_by_addr.get(&(dc, *endpoint)) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
for writer_id in writer_ids {
|
for writer_id in writer_ids {
|
||||||
if !pool.registry.is_writer_empty(*writer_id).await {
|
if bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0 {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else {
|
let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else {
|
||||||
@@ -651,7 +772,12 @@ async fn maybe_swap_idle_writer_for_cap(
|
|||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
let connected = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
|
let connected = match tokio::time::timeout(
|
||||||
|
pool.me_one_timeout,
|
||||||
|
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(Ok(())) => true,
|
Ok(Ok(())) => true,
|
||||||
Ok(Err(error)) => {
|
Ok(Err(error)) => {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -703,8 +829,9 @@ async fn maybe_refresh_idle_writer_for_dc(
|
|||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
alive: usize,
|
alive: usize,
|
||||||
required: usize,
|
required: usize,
|
||||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
writer_idle_since: &HashMap<u64, u64>,
|
writer_idle_since: &HashMap<u64, u64>,
|
||||||
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
) {
|
) {
|
||||||
if alive < required {
|
if alive < required {
|
||||||
@@ -721,10 +848,13 @@ async fn maybe_refresh_idle_writer_for_dc(
|
|||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
let mut candidate: Option<(u64, SocketAddr, u64, u64)> = None;
|
let mut candidate: Option<(u64, SocketAddr, u64, u64)> = None;
|
||||||
for endpoint in endpoints {
|
for endpoint in endpoints {
|
||||||
let Some(writer_ids) = live_writer_ids_by_addr.get(endpoint) else {
|
let Some(writer_ids) = live_writer_ids_by_addr.get(&(dc, *endpoint)) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
for writer_id in writer_ids {
|
for writer_id in writer_ids {
|
||||||
|
if bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else {
|
let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
@@ -748,7 +878,12 @@ async fn maybe_refresh_idle_writer_for_dc(
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
|
let rotate_ok = match tokio::time::timeout(
|
||||||
|
pool.me_one_timeout,
|
||||||
|
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(Ok(())) => true,
|
Ok(Ok(())) => true,
|
||||||
Ok(Err(error)) => {
|
Ok(Err(error)) => {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -804,8 +939,10 @@ async fn maybe_refresh_idle_writer_for_dc(
|
|||||||
async fn should_reduce_floor_for_idle(
|
async fn should_reduce_floor_for_idle(
|
||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
key: (i32, IpFamily),
|
key: (i32, IpFamily),
|
||||||
|
dc: i32,
|
||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
@@ -816,8 +953,8 @@ async fn should_reduce_floor_for_idle(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr);
|
let writer_ids = list_writer_ids_for_endpoints(dc, endpoints, live_writer_ids_by_addr);
|
||||||
let has_bound_clients = has_bound_clients_on_endpoint(pool, &writer_ids).await;
|
let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer);
|
||||||
if has_bound_clients {
|
if has_bound_clients {
|
||||||
adaptive_idle_since.remove(&key);
|
adaptive_idle_since.remove(&key);
|
||||||
adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration());
|
adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration());
|
||||||
@@ -836,13 +973,13 @@ async fn should_reduce_floor_for_idle(
|
|||||||
now.saturating_duration_since(*idle_since) >= pool.adaptive_floor_idle_duration()
|
now.saturating_duration_since(*idle_since) >= pool.adaptive_floor_idle_duration()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn has_bound_clients_on_endpoint(pool: &Arc<MePool>, writer_ids: &[u64]) -> bool {
|
fn has_bound_clients_on_endpoint(
|
||||||
for writer_id in writer_ids {
|
writer_ids: &[u64],
|
||||||
if !pool.registry.is_writer_empty(*writer_id).await {
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
return true;
|
) -> bool {
|
||||||
}
|
writer_ids
|
||||||
}
|
.iter()
|
||||||
false
|
.any(|writer_id| bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn recover_single_endpoint_outage(
|
async fn recover_single_endpoint_outage(
|
||||||
@@ -882,7 +1019,12 @@ async fn recover_single_endpoint_outage(
|
|||||||
let attempt_ok = if bypass_quarantine {
|
let attempt_ok = if bypass_quarantine {
|
||||||
pool.stats
|
pool.stats
|
||||||
.increment_me_single_endpoint_quarantine_bypass_total();
|
.increment_me_single_endpoint_quarantine_bypass_total();
|
||||||
match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
|
match tokio::time::timeout(
|
||||||
|
pool.me_one_timeout,
|
||||||
|
pool.connect_one_for_dc(endpoint, key.0, rng.as_ref()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(Ok(())) => true,
|
Ok(Ok(())) => true,
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -908,7 +1050,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
let one_endpoint = [endpoint];
|
let one_endpoint = [endpoint];
|
||||||
match tokio::time::timeout(
|
match tokio::time::timeout(
|
||||||
pool.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_endpoints_round_robin(&one_endpoint, rng.as_ref()),
|
pool.connect_endpoints_round_robin(key.0, &one_endpoint, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -972,7 +1114,8 @@ async fn maybe_rotate_single_endpoint_shadow(
|
|||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
alive: usize,
|
alive: usize,
|
||||||
required: usize,
|
required: usize,
|
||||||
live_writer_ids_by_addr: &HashMap<SocketAddr, Vec<u64>>,
|
live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec<u64>>,
|
||||||
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
) {
|
) {
|
||||||
if endpoints.len() != 1 || alive < required {
|
if endpoints.len() != 1 || alive < required {
|
||||||
@@ -1004,14 +1147,14 @@ async fn maybe_rotate_single_endpoint_shadow(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(writer_ids) = live_writer_ids_by_addr.get(&endpoint) else {
|
let Some(writer_ids) = live_writer_ids_by_addr.get(&(dc, endpoint)) else {
|
||||||
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
|
shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS));
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut candidate_writer_id = None;
|
let mut candidate_writer_id = None;
|
||||||
for writer_id in writer_ids {
|
for writer_id in writer_ids {
|
||||||
if pool.registry.is_writer_empty(*writer_id).await {
|
if bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) == 0 {
|
||||||
candidate_writer_id = Some(*writer_id);
|
candidate_writer_id = Some(*writer_id);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1030,7 +1173,12 @@ async fn maybe_rotate_single_endpoint_shadow(
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await {
|
let rotate_ok = match tokio::time::timeout(
|
||||||
|
pool.me_one_timeout,
|
||||||
|
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(Ok(())) => true,
|
Ok(Ok(())) => true,
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
debug!(
|
debug!(
|
||||||
|
|||||||
@@ -331,7 +331,7 @@ pub async fn run_me_ping(pool: &Arc<MePool>, rng: &SecureRandom) -> Vec<MePingRe
|
|||||||
let mut error = None;
|
let mut error = None;
|
||||||
let mut route = None;
|
let mut route = None;
|
||||||
|
|
||||||
match pool.connect_tcp(addr).await {
|
match pool.connect_tcp(addr, None).await {
|
||||||
Ok((stream, conn_rtt, upstream_egress)) => {
|
Ok((stream, conn_rtt, upstream_egress)) => {
|
||||||
connect_ms = Some(conn_rtt);
|
connect_ms = Some(conn_rtt);
|
||||||
route = route_from_egress(upstream_egress);
|
route = route_from_egress(upstream_egress);
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|||||||
use tokio::sync::{Mutex, Notify, RwLock, mpsc};
|
use tokio::sync::{Mutex, Notify, RwLock, mpsc};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
use crate::config::{MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy};
|
use crate::config::{
|
||||||
|
MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode,
|
||||||
|
};
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
use crate::network::IpFamily;
|
use crate::network::IpFamily;
|
||||||
use crate::network::probe::NetworkDecision;
|
use crate::network::probe::NetworkDecision;
|
||||||
@@ -22,18 +24,27 @@ pub(super) struct RefillDcKey {
|
|||||||
pub family: IpFamily,
|
pub family: IpFamily,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
|
pub(super) struct RefillEndpointKey {
|
||||||
|
pub dc: i32,
|
||||||
|
pub addr: SocketAddr,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct MeWriter {
|
pub struct MeWriter {
|
||||||
pub id: u64,
|
pub id: u64,
|
||||||
pub addr: SocketAddr,
|
pub addr: SocketAddr,
|
||||||
|
pub writer_dc: i32,
|
||||||
pub generation: u64,
|
pub generation: u64,
|
||||||
pub contour: Arc<AtomicU8>,
|
pub contour: Arc<AtomicU8>,
|
||||||
pub created_at: Instant,
|
pub created_at: Instant,
|
||||||
pub tx: mpsc::Sender<WriterCommand>,
|
pub tx: mpsc::Sender<WriterCommand>,
|
||||||
pub cancel: CancellationToken,
|
pub cancel: CancellationToken,
|
||||||
pub degraded: Arc<AtomicBool>,
|
pub degraded: Arc<AtomicBool>,
|
||||||
|
pub rtt_ema_ms_x10: Arc<AtomicU32>,
|
||||||
pub draining: Arc<AtomicBool>,
|
pub draining: Arc<AtomicBool>,
|
||||||
pub draining_started_at_epoch_secs: Arc<AtomicU64>,
|
pub draining_started_at_epoch_secs: Arc<AtomicU64>,
|
||||||
|
pub drain_deadline_epoch_secs: Arc<AtomicU64>,
|
||||||
pub allow_drain_fallback: Arc<AtomicBool>,
|
pub allow_drain_fallback: Arc<AtomicBool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,6 +106,7 @@ pub struct MePool {
|
|||||||
pub(super) me_keepalive_jitter: Duration,
|
pub(super) me_keepalive_jitter: Duration,
|
||||||
pub(super) me_keepalive_payload_random: bool,
|
pub(super) me_keepalive_payload_random: bool,
|
||||||
pub(super) rpc_proxy_req_every_secs: AtomicU64,
|
pub(super) rpc_proxy_req_every_secs: AtomicU64,
|
||||||
|
pub(super) writer_cmd_channel_capacity: usize,
|
||||||
pub(super) me_warmup_stagger_enabled: bool,
|
pub(super) me_warmup_stagger_enabled: bool,
|
||||||
pub(super) me_warmup_step_delay: Duration,
|
pub(super) me_warmup_step_delay: Duration,
|
||||||
pub(super) me_warmup_step_jitter: Duration,
|
pub(super) me_warmup_step_jitter: Duration,
|
||||||
@@ -117,23 +129,34 @@ pub struct MePool {
|
|||||||
pub(super) me_adaptive_floor_cpu_cores_override: AtomicU32,
|
pub(super) me_adaptive_floor_cpu_cores_override: AtomicU32,
|
||||||
pub(super) me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32,
|
pub(super) me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32,
|
||||||
pub(super) me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32,
|
pub(super) me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32,
|
||||||
|
pub(super) me_adaptive_floor_max_active_writers_per_core: AtomicU32,
|
||||||
|
pub(super) me_adaptive_floor_max_warm_writers_per_core: AtomicU32,
|
||||||
|
pub(super) me_adaptive_floor_max_active_writers_global: AtomicU32,
|
||||||
|
pub(super) me_adaptive_floor_max_warm_writers_global: AtomicU32,
|
||||||
pub(super) me_adaptive_floor_cpu_cores_detected: AtomicU32,
|
pub(super) me_adaptive_floor_cpu_cores_detected: AtomicU32,
|
||||||
pub(super) me_adaptive_floor_cpu_cores_effective: AtomicU32,
|
pub(super) me_adaptive_floor_cpu_cores_effective: AtomicU32,
|
||||||
pub(super) me_adaptive_floor_global_cap_raw: AtomicU64,
|
pub(super) me_adaptive_floor_global_cap_raw: AtomicU64,
|
||||||
pub(super) me_adaptive_floor_global_cap_effective: AtomicU64,
|
pub(super) me_adaptive_floor_global_cap_effective: AtomicU64,
|
||||||
pub(super) me_adaptive_floor_target_writers_total: AtomicU64,
|
pub(super) me_adaptive_floor_target_writers_total: AtomicU64,
|
||||||
|
pub(super) me_adaptive_floor_active_cap_configured: AtomicU64,
|
||||||
|
pub(super) me_adaptive_floor_active_cap_effective: AtomicU64,
|
||||||
|
pub(super) me_adaptive_floor_warm_cap_configured: AtomicU64,
|
||||||
|
pub(super) me_adaptive_floor_warm_cap_effective: AtomicU64,
|
||||||
|
pub(super) me_adaptive_floor_active_writers_current: AtomicU64,
|
||||||
|
pub(super) me_adaptive_floor_warm_writers_current: AtomicU64,
|
||||||
pub(super) proxy_map_v4: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
|
pub(super) proxy_map_v4: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
|
||||||
pub(super) proxy_map_v6: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
|
pub(super) proxy_map_v6: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
|
||||||
pub(super) endpoint_dc_map: Arc<RwLock<HashMap<SocketAddr, Option<i32>>>>,
|
pub(super) endpoint_dc_map: Arc<RwLock<HashMap<SocketAddr, Option<i32>>>>,
|
||||||
pub(super) default_dc: AtomicI32,
|
pub(super) default_dc: AtomicI32,
|
||||||
pub(super) next_writer_id: AtomicU64,
|
pub(super) next_writer_id: AtomicU64,
|
||||||
pub(super) ping_tracker: Arc<Mutex<HashMap<i64, (std::time::Instant, u64)>>>,
|
pub(super) ping_tracker: Arc<Mutex<HashMap<i64, (std::time::Instant, u64)>>>,
|
||||||
|
pub(super) ping_tracker_last_cleanup_epoch_ms: AtomicU64,
|
||||||
pub(super) rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
pub(super) rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
||||||
pub(super) nat_reflection_cache: Arc<Mutex<NatReflectionCache>>,
|
pub(super) nat_reflection_cache: Arc<Mutex<NatReflectionCache>>,
|
||||||
pub(super) nat_reflection_singleflight_v4: Arc<Mutex<()>>,
|
pub(super) nat_reflection_singleflight_v4: Arc<Mutex<()>>,
|
||||||
pub(super) nat_reflection_singleflight_v6: Arc<Mutex<()>>,
|
pub(super) nat_reflection_singleflight_v6: Arc<Mutex<()>>,
|
||||||
pub(super) writer_available: Arc<Notify>,
|
pub(super) writer_available: Arc<Notify>,
|
||||||
pub(super) refill_inflight: Arc<Mutex<HashSet<SocketAddr>>>,
|
pub(super) refill_inflight: Arc<Mutex<HashSet<RefillEndpointKey>>>,
|
||||||
pub(super) refill_inflight_dc: Arc<Mutex<HashSet<RefillDcKey>>>,
|
pub(super) refill_inflight_dc: Arc<Mutex<HashSet<RefillDcKey>>>,
|
||||||
pub(super) conn_count: AtomicUsize,
|
pub(super) conn_count: AtomicUsize,
|
||||||
pub(super) stats: Arc<crate::stats::Stats>,
|
pub(super) stats: Arc<crate::stats::Stats>,
|
||||||
@@ -157,13 +180,19 @@ pub struct MePool {
|
|||||||
pub(super) me_bind_stale_ttl_secs: AtomicU64,
|
pub(super) me_bind_stale_ttl_secs: AtomicU64,
|
||||||
pub(super) secret_atomic_snapshot: AtomicBool,
|
pub(super) secret_atomic_snapshot: AtomicBool,
|
||||||
pub(super) me_deterministic_writer_sort: AtomicBool,
|
pub(super) me_deterministic_writer_sort: AtomicBool,
|
||||||
|
pub(super) me_writer_pick_mode: AtomicU8,
|
||||||
|
pub(super) me_writer_pick_sample_size: AtomicU8,
|
||||||
pub(super) me_socks_kdf_policy: AtomicU8,
|
pub(super) me_socks_kdf_policy: AtomicU8,
|
||||||
pub(super) me_route_no_writer_mode: AtomicU8,
|
pub(super) me_route_no_writer_mode: AtomicU8,
|
||||||
pub(super) me_route_no_writer_wait: Duration,
|
pub(super) me_route_no_writer_wait: Duration,
|
||||||
pub(super) me_route_inline_recovery_attempts: u32,
|
pub(super) me_route_inline_recovery_attempts: u32,
|
||||||
pub(super) me_route_inline_recovery_wait: Duration,
|
pub(super) me_route_inline_recovery_wait: Duration,
|
||||||
|
pub(super) me_health_interval_ms_unhealthy: AtomicU64,
|
||||||
|
pub(super) me_health_interval_ms_healthy: AtomicU64,
|
||||||
|
pub(super) me_warn_rate_limit_ms: AtomicU64,
|
||||||
pub(super) runtime_ready: AtomicBool,
|
pub(super) runtime_ready: AtomicBool,
|
||||||
pool_size: usize,
|
pool_size: usize,
|
||||||
|
pub(super) preferred_endpoints_by_dc: Arc<RwLock<HashMap<i32, Vec<SocketAddr>>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
@@ -234,6 +263,10 @@ impl MePool {
|
|||||||
me_adaptive_floor_cpu_cores_override: u16,
|
me_adaptive_floor_cpu_cores_override: u16,
|
||||||
me_adaptive_floor_max_extra_writers_single_per_core: u16,
|
me_adaptive_floor_max_extra_writers_single_per_core: u16,
|
||||||
me_adaptive_floor_max_extra_writers_multi_per_core: u16,
|
me_adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
me_adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
me_adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
me_adaptive_floor_max_active_writers_global: u32,
|
||||||
|
me_adaptive_floor_max_warm_writers_global: u32,
|
||||||
hardswap: bool,
|
hardswap: bool,
|
||||||
me_pool_drain_ttl_secs: u64,
|
me_pool_drain_ttl_secs: u64,
|
||||||
me_pool_force_close_secs: u64,
|
me_pool_force_close_secs: u64,
|
||||||
@@ -246,17 +279,28 @@ impl MePool {
|
|||||||
me_bind_stale_ttl_secs: u64,
|
me_bind_stale_ttl_secs: u64,
|
||||||
me_secret_atomic_snapshot: bool,
|
me_secret_atomic_snapshot: bool,
|
||||||
me_deterministic_writer_sort: bool,
|
me_deterministic_writer_sort: bool,
|
||||||
|
me_writer_pick_mode: MeWriterPickMode,
|
||||||
|
me_writer_pick_sample_size: u8,
|
||||||
me_socks_kdf_policy: MeSocksKdfPolicy,
|
me_socks_kdf_policy: MeSocksKdfPolicy,
|
||||||
|
me_writer_cmd_channel_capacity: usize,
|
||||||
|
me_route_channel_capacity: usize,
|
||||||
me_route_backpressure_base_timeout_ms: u64,
|
me_route_backpressure_base_timeout_ms: u64,
|
||||||
me_route_backpressure_high_timeout_ms: u64,
|
me_route_backpressure_high_timeout_ms: u64,
|
||||||
me_route_backpressure_high_watermark_pct: u8,
|
me_route_backpressure_high_watermark_pct: u8,
|
||||||
|
me_health_interval_ms_unhealthy: u64,
|
||||||
|
me_health_interval_ms_healthy: u64,
|
||||||
|
me_warn_rate_limit_ms: u64,
|
||||||
me_route_no_writer_mode: MeRouteNoWriterMode,
|
me_route_no_writer_mode: MeRouteNoWriterMode,
|
||||||
me_route_no_writer_wait_ms: u64,
|
me_route_no_writer_wait_ms: u64,
|
||||||
me_route_inline_recovery_attempts: u32,
|
me_route_inline_recovery_attempts: u32,
|
||||||
me_route_inline_recovery_wait_ms: u64,
|
me_route_inline_recovery_wait_ms: u64,
|
||||||
) -> Arc<Self> {
|
) -> Arc<Self> {
|
||||||
let endpoint_dc_map = Self::build_endpoint_dc_map_from_maps(&proxy_map_v4, &proxy_map_v6);
|
let endpoint_dc_map = Self::build_endpoint_dc_map_from_maps(&proxy_map_v4, &proxy_map_v6);
|
||||||
let registry = Arc::new(ConnRegistry::new());
|
let preferred_endpoints_by_dc =
|
||||||
|
Self::build_preferred_endpoints_by_dc(&decision, &proxy_map_v4, &proxy_map_v6);
|
||||||
|
let registry = Arc::new(ConnRegistry::with_route_channel_capacity(
|
||||||
|
me_route_channel_capacity,
|
||||||
|
));
|
||||||
registry.update_route_backpressure_policy(
|
registry.update_route_backpressure_policy(
|
||||||
me_route_backpressure_base_timeout_ms,
|
me_route_backpressure_base_timeout_ms,
|
||||||
me_route_backpressure_high_timeout_ms,
|
me_route_backpressure_high_timeout_ms,
|
||||||
@@ -303,6 +347,7 @@ impl MePool {
|
|||||||
me_keepalive_jitter: Duration::from_secs(me_keepalive_jitter_secs),
|
me_keepalive_jitter: Duration::from_secs(me_keepalive_jitter_secs),
|
||||||
me_keepalive_payload_random,
|
me_keepalive_payload_random,
|
||||||
rpc_proxy_req_every_secs: AtomicU64::new(rpc_proxy_req_every_secs),
|
rpc_proxy_req_every_secs: AtomicU64::new(rpc_proxy_req_every_secs),
|
||||||
|
writer_cmd_channel_capacity: me_writer_cmd_channel_capacity.max(1),
|
||||||
me_warmup_stagger_enabled,
|
me_warmup_stagger_enabled,
|
||||||
me_warmup_step_delay: Duration::from_millis(me_warmup_step_delay_ms),
|
me_warmup_step_delay: Duration::from_millis(me_warmup_step_delay_ms),
|
||||||
me_warmup_step_jitter: Duration::from_millis(me_warmup_step_jitter_ms),
|
me_warmup_step_jitter: Duration::from_millis(me_warmup_step_jitter_ms),
|
||||||
@@ -349,11 +394,29 @@ impl MePool {
|
|||||||
me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32::new(
|
me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32::new(
|
||||||
me_adaptive_floor_max_extra_writers_multi_per_core as u32,
|
me_adaptive_floor_max_extra_writers_multi_per_core as u32,
|
||||||
),
|
),
|
||||||
|
me_adaptive_floor_max_active_writers_per_core: AtomicU32::new(
|
||||||
|
me_adaptive_floor_max_active_writers_per_core as u32,
|
||||||
|
),
|
||||||
|
me_adaptive_floor_max_warm_writers_per_core: AtomicU32::new(
|
||||||
|
me_adaptive_floor_max_warm_writers_per_core as u32,
|
||||||
|
),
|
||||||
|
me_adaptive_floor_max_active_writers_global: AtomicU32::new(
|
||||||
|
me_adaptive_floor_max_active_writers_global,
|
||||||
|
),
|
||||||
|
me_adaptive_floor_max_warm_writers_global: AtomicU32::new(
|
||||||
|
me_adaptive_floor_max_warm_writers_global,
|
||||||
|
),
|
||||||
me_adaptive_floor_cpu_cores_detected: AtomicU32::new(1),
|
me_adaptive_floor_cpu_cores_detected: AtomicU32::new(1),
|
||||||
me_adaptive_floor_cpu_cores_effective: AtomicU32::new(1),
|
me_adaptive_floor_cpu_cores_effective: AtomicU32::new(1),
|
||||||
me_adaptive_floor_global_cap_raw: AtomicU64::new(0),
|
me_adaptive_floor_global_cap_raw: AtomicU64::new(0),
|
||||||
me_adaptive_floor_global_cap_effective: AtomicU64::new(0),
|
me_adaptive_floor_global_cap_effective: AtomicU64::new(0),
|
||||||
me_adaptive_floor_target_writers_total: AtomicU64::new(0),
|
me_adaptive_floor_target_writers_total: AtomicU64::new(0),
|
||||||
|
me_adaptive_floor_active_cap_configured: AtomicU64::new(0),
|
||||||
|
me_adaptive_floor_active_cap_effective: AtomicU64::new(0),
|
||||||
|
me_adaptive_floor_warm_cap_configured: AtomicU64::new(0),
|
||||||
|
me_adaptive_floor_warm_cap_effective: AtomicU64::new(0),
|
||||||
|
me_adaptive_floor_active_writers_current: AtomicU64::new(0),
|
||||||
|
me_adaptive_floor_warm_writers_current: AtomicU64::new(0),
|
||||||
pool_size: 2,
|
pool_size: 2,
|
||||||
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
|
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
|
||||||
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
|
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
|
||||||
@@ -361,6 +424,7 @@ impl MePool {
|
|||||||
default_dc: AtomicI32::new(default_dc.unwrap_or(2)),
|
default_dc: AtomicI32::new(default_dc.unwrap_or(2)),
|
||||||
next_writer_id: AtomicU64::new(1),
|
next_writer_id: AtomicU64::new(1),
|
||||||
ping_tracker: Arc::new(Mutex::new(HashMap::new())),
|
ping_tracker: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
ping_tracker_last_cleanup_epoch_ms: AtomicU64::new(0),
|
||||||
rtt_stats: Arc::new(Mutex::new(HashMap::new())),
|
rtt_stats: Arc::new(Mutex::new(HashMap::new())),
|
||||||
nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())),
|
nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())),
|
||||||
nat_reflection_singleflight_v4: Arc::new(Mutex::new(())),
|
nat_reflection_singleflight_v4: Arc::new(Mutex::new(())),
|
||||||
@@ -393,12 +457,18 @@ impl MePool {
|
|||||||
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
|
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
|
||||||
secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot),
|
secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot),
|
||||||
me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort),
|
me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort),
|
||||||
|
me_writer_pick_mode: AtomicU8::new(me_writer_pick_mode.as_u8()),
|
||||||
|
me_writer_pick_sample_size: AtomicU8::new(me_writer_pick_sample_size.clamp(2, 4)),
|
||||||
me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()),
|
me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()),
|
||||||
me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()),
|
me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()),
|
||||||
me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms),
|
me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms),
|
||||||
me_route_inline_recovery_attempts,
|
me_route_inline_recovery_attempts,
|
||||||
me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms),
|
me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms),
|
||||||
|
me_health_interval_ms_unhealthy: AtomicU64::new(me_health_interval_ms_unhealthy.max(1)),
|
||||||
|
me_health_interval_ms_healthy: AtomicU64::new(me_health_interval_ms_healthy.max(1)),
|
||||||
|
me_warn_rate_limit_ms: AtomicU64::new(me_warn_rate_limit_ms.max(1)),
|
||||||
runtime_ready: AtomicBool::new(false),
|
runtime_ready: AtomicBool::new(false),
|
||||||
|
preferred_endpoints_by_dc: Arc::new(RwLock::new(preferred_endpoints_by_dc)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -428,6 +498,8 @@ impl MePool {
|
|||||||
bind_stale_ttl_secs: u64,
|
bind_stale_ttl_secs: u64,
|
||||||
secret_atomic_snapshot: bool,
|
secret_atomic_snapshot: bool,
|
||||||
deterministic_writer_sort: bool,
|
deterministic_writer_sort: bool,
|
||||||
|
writer_pick_mode: MeWriterPickMode,
|
||||||
|
writer_pick_sample_size: u8,
|
||||||
single_endpoint_shadow_writers: u8,
|
single_endpoint_shadow_writers: u8,
|
||||||
single_endpoint_outage_mode_enabled: bool,
|
single_endpoint_outage_mode_enabled: bool,
|
||||||
single_endpoint_outage_disable_quarantine: bool,
|
single_endpoint_outage_disable_quarantine: bool,
|
||||||
@@ -443,6 +515,13 @@ impl MePool {
|
|||||||
adaptive_floor_cpu_cores_override: u16,
|
adaptive_floor_cpu_cores_override: u16,
|
||||||
adaptive_floor_max_extra_writers_single_per_core: u16,
|
adaptive_floor_max_extra_writers_single_per_core: u16,
|
||||||
adaptive_floor_max_extra_writers_multi_per_core: u16,
|
adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
adaptive_floor_max_active_writers_global: u32,
|
||||||
|
adaptive_floor_max_warm_writers_global: u32,
|
||||||
|
me_health_interval_ms_unhealthy: u64,
|
||||||
|
me_health_interval_ms_healthy: u64,
|
||||||
|
me_warn_rate_limit_ms: u64,
|
||||||
) {
|
) {
|
||||||
self.hardswap.store(hardswap, Ordering::Relaxed);
|
self.hardswap.store(hardswap, Ordering::Relaxed);
|
||||||
self.me_pool_drain_ttl_secs
|
self.me_pool_drain_ttl_secs
|
||||||
@@ -467,6 +546,14 @@ impl MePool {
|
|||||||
.store(secret_atomic_snapshot, Ordering::Relaxed);
|
.store(secret_atomic_snapshot, Ordering::Relaxed);
|
||||||
self.me_deterministic_writer_sort
|
self.me_deterministic_writer_sort
|
||||||
.store(deterministic_writer_sort, Ordering::Relaxed);
|
.store(deterministic_writer_sort, Ordering::Relaxed);
|
||||||
|
let previous_writer_pick_mode = self.writer_pick_mode();
|
||||||
|
self.me_writer_pick_mode
|
||||||
|
.store(writer_pick_mode.as_u8(), Ordering::Relaxed);
|
||||||
|
self.me_writer_pick_sample_size
|
||||||
|
.store(writer_pick_sample_size.clamp(2, 4), Ordering::Relaxed);
|
||||||
|
if previous_writer_pick_mode != writer_pick_mode {
|
||||||
|
self.stats.increment_me_writer_pick_mode_switch_total();
|
||||||
|
}
|
||||||
self.me_single_endpoint_shadow_writers
|
self.me_single_endpoint_shadow_writers
|
||||||
.store(single_endpoint_shadow_writers, Ordering::Relaxed);
|
.store(single_endpoint_shadow_writers, Ordering::Relaxed);
|
||||||
self.me_single_endpoint_outage_mode_enabled
|
self.me_single_endpoint_outage_mode_enabled
|
||||||
@@ -504,6 +591,26 @@ impl MePool {
|
|||||||
adaptive_floor_max_extra_writers_multi_per_core as u32,
|
adaptive_floor_max_extra_writers_multi_per_core as u32,
|
||||||
Ordering::Relaxed,
|
Ordering::Relaxed,
|
||||||
);
|
);
|
||||||
|
self.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.store(
|
||||||
|
adaptive_floor_max_active_writers_per_core as u32,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
);
|
||||||
|
self.me_adaptive_floor_max_warm_writers_per_core
|
||||||
|
.store(
|
||||||
|
adaptive_floor_max_warm_writers_per_core as u32,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
);
|
||||||
|
self.me_adaptive_floor_max_active_writers_global
|
||||||
|
.store(adaptive_floor_max_active_writers_global, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_max_warm_writers_global
|
||||||
|
.store(adaptive_floor_max_warm_writers_global, Ordering::Relaxed);
|
||||||
|
self.me_health_interval_ms_unhealthy
|
||||||
|
.store(me_health_interval_ms_unhealthy.max(1), Ordering::Relaxed);
|
||||||
|
self.me_health_interval_ms_healthy
|
||||||
|
.store(me_health_interval_ms_healthy.max(1), Ordering::Relaxed);
|
||||||
|
self.me_warn_rate_limit_ms
|
||||||
|
.store(me_warn_rate_limit_ms.max(1), Ordering::Relaxed);
|
||||||
if previous_floor_mode != floor_mode {
|
if previous_floor_mode != floor_mode {
|
||||||
self.stats.increment_me_floor_mode_switch_total();
|
self.stats.increment_me_floor_mode_switch_total();
|
||||||
match (previous_floor_mode, floor_mode) {
|
match (previous_floor_mode, floor_mode) {
|
||||||
@@ -574,11 +681,26 @@ impl MePool {
|
|||||||
self.proxy_secret.read().await.key_selector
|
self.proxy_secret.read().await.key_selector
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn active_writer_count_total(&self) -> usize {
|
pub(super) async fn non_draining_writer_counts_by_contour(&self) -> (usize, usize, usize) {
|
||||||
let ws = self.writers.read().await;
|
let ws = self.writers.read().await;
|
||||||
ws.iter()
|
let mut active = 0usize;
|
||||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
let mut warm = 0usize;
|
||||||
.count()
|
for writer in ws.iter() {
|
||||||
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
|
||||||
|
WriterContour::Active => active = active.saturating_add(1),
|
||||||
|
WriterContour::Warm => warm = warm.saturating_add(1),
|
||||||
|
WriterContour::Draining => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(active, warm, active.saturating_add(warm))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn active_contour_writer_count_total(&self) -> usize {
|
||||||
|
let (active, _, _) = self.non_draining_writer_counts_by_contour().await;
|
||||||
|
active
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn secret_snapshot(&self) -> SecretSnapshot {
|
pub(super) async fn secret_snapshot(&self) -> SecretSnapshot {
|
||||||
@@ -589,6 +711,16 @@ impl MePool {
|
|||||||
MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed))
|
MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn writer_pick_mode(&self) -> MeWriterPickMode {
|
||||||
|
MeWriterPickMode::from_u8(self.me_writer_pick_mode.load(Ordering::Relaxed))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn writer_pick_sample_size(&self) -> usize {
|
||||||
|
self.me_writer_pick_sample_size
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
.clamp(2, 4) as usize
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize {
|
pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize {
|
||||||
if endpoint_count == 0 {
|
if endpoint_count == 0 {
|
||||||
return 0;
|
return 0;
|
||||||
@@ -624,13 +756,6 @@ impl MePool {
|
|||||||
.max(1)
|
.max(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_writers_per_core_total(&self) -> usize {
|
|
||||||
(self
|
|
||||||
.me_adaptive_floor_writers_per_core_total
|
|
||||||
.load(Ordering::Relaxed) as usize)
|
|
||||||
.max(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_max_extra_single_per_core(&self) -> usize {
|
pub(super) fn adaptive_floor_max_extra_single_per_core(&self) -> usize {
|
||||||
self.me_adaptive_floor_max_extra_writers_single_per_core
|
self.me_adaptive_floor_max_extra_writers_single_per_core
|
||||||
.load(Ordering::Relaxed) as usize
|
.load(Ordering::Relaxed) as usize
|
||||||
@@ -641,6 +766,34 @@ impl MePool {
|
|||||||
.load(Ordering::Relaxed) as usize
|
.load(Ordering::Relaxed) as usize
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn adaptive_floor_max_active_writers_per_core(&self) -> usize {
|
||||||
|
(self
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.load(Ordering::Relaxed) as usize)
|
||||||
|
.max(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn adaptive_floor_max_warm_writers_per_core(&self) -> usize {
|
||||||
|
(self
|
||||||
|
.me_adaptive_floor_max_warm_writers_per_core
|
||||||
|
.load(Ordering::Relaxed) as usize)
|
||||||
|
.max(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn adaptive_floor_max_active_writers_global(&self) -> usize {
|
||||||
|
(self
|
||||||
|
.me_adaptive_floor_max_active_writers_global
|
||||||
|
.load(Ordering::Relaxed) as usize)
|
||||||
|
.max(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn adaptive_floor_max_warm_writers_global(&self) -> usize {
|
||||||
|
(self
|
||||||
|
.me_adaptive_floor_max_warm_writers_global
|
||||||
|
.load(Ordering::Relaxed) as usize)
|
||||||
|
.max(1)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_detected_cpu_cores(&self) -> usize {
|
pub(super) fn adaptive_floor_detected_cpu_cores(&self) -> usize {
|
||||||
std::thread::available_parallelism()
|
std::thread::available_parallelism()
|
||||||
.map(|value| value.get())
|
.map(|value| value.get())
|
||||||
@@ -669,28 +822,126 @@ impl MePool {
|
|||||||
effective
|
effective
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn adaptive_floor_global_cap_raw(&self) -> usize {
|
pub(super) fn adaptive_floor_active_cap_configured_total(&self) -> usize {
|
||||||
let cores = self.adaptive_floor_effective_cpu_cores();
|
let cores = self.adaptive_floor_effective_cpu_cores();
|
||||||
let cap = cores.saturating_mul(self.adaptive_floor_writers_per_core_total());
|
let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_active_writers_per_core());
|
||||||
self.me_adaptive_floor_global_cap_raw
|
let configured = per_core_cap.min(self.adaptive_floor_max_active_writers_global());
|
||||||
.store(cap as u64, Ordering::Relaxed);
|
self.me_adaptive_floor_active_cap_configured
|
||||||
self.stats.set_me_floor_global_cap_raw_gauge(cap as u64);
|
.store(configured as u64, Ordering::Relaxed);
|
||||||
cap
|
self.stats
|
||||||
|
.set_me_floor_active_cap_configured_gauge(configured as u64);
|
||||||
|
configured
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn adaptive_floor_warm_cap_configured_total(&self) -> usize {
|
||||||
|
let cores = self.adaptive_floor_effective_cpu_cores();
|
||||||
|
let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_warm_writers_per_core());
|
||||||
|
let configured = per_core_cap.min(self.adaptive_floor_max_warm_writers_global());
|
||||||
|
self.me_adaptive_floor_warm_cap_configured
|
||||||
|
.store(configured as u64, Ordering::Relaxed);
|
||||||
|
self.stats
|
||||||
|
.set_me_floor_warm_cap_configured_gauge(configured as u64);
|
||||||
|
configured
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn set_adaptive_floor_runtime_caps(
|
pub(super) fn set_adaptive_floor_runtime_caps(
|
||||||
&self,
|
&self,
|
||||||
global_cap_effective: usize,
|
active_cap_configured: usize,
|
||||||
|
active_cap_effective: usize,
|
||||||
|
warm_cap_configured: usize,
|
||||||
|
warm_cap_effective: usize,
|
||||||
target_writers_total: usize,
|
target_writers_total: usize,
|
||||||
|
active_writers_current: usize,
|
||||||
|
warm_writers_current: usize,
|
||||||
) {
|
) {
|
||||||
|
self.me_adaptive_floor_global_cap_raw
|
||||||
|
.store(active_cap_configured as u64, Ordering::Relaxed);
|
||||||
self.me_adaptive_floor_global_cap_effective
|
self.me_adaptive_floor_global_cap_effective
|
||||||
.store(global_cap_effective as u64, Ordering::Relaxed);
|
.store(active_cap_effective as u64, Ordering::Relaxed);
|
||||||
self.me_adaptive_floor_target_writers_total
|
self.me_adaptive_floor_target_writers_total
|
||||||
.store(target_writers_total as u64, Ordering::Relaxed);
|
.store(target_writers_total as u64, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_active_cap_configured
|
||||||
|
.store(active_cap_configured as u64, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_active_cap_effective
|
||||||
|
.store(active_cap_effective as u64, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_warm_cap_configured
|
||||||
|
.store(warm_cap_configured as u64, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_warm_cap_effective
|
||||||
|
.store(warm_cap_effective as u64, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_active_writers_current
|
||||||
|
.store(active_writers_current as u64, Ordering::Relaxed);
|
||||||
|
self.me_adaptive_floor_warm_writers_current
|
||||||
|
.store(warm_writers_current as u64, Ordering::Relaxed);
|
||||||
self.stats
|
self.stats
|
||||||
.set_me_floor_global_cap_effective_gauge(global_cap_effective as u64);
|
.set_me_floor_global_cap_raw_gauge(active_cap_configured as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_floor_global_cap_effective_gauge(active_cap_effective as u64);
|
||||||
self.stats
|
self.stats
|
||||||
.set_me_floor_target_writers_total_gauge(target_writers_total as u64);
|
.set_me_floor_target_writers_total_gauge(target_writers_total as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_floor_active_cap_configured_gauge(active_cap_configured as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_floor_active_cap_effective_gauge(active_cap_effective as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_floor_warm_cap_configured_gauge(warm_cap_configured as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_floor_warm_cap_effective_gauge(warm_cap_effective as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_writers_active_current_gauge(active_writers_current as u64);
|
||||||
|
self.stats
|
||||||
|
.set_me_writers_warm_current_gauge(warm_writers_current as u64);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn active_coverage_required_total(&self) -> usize {
|
||||||
|
let mut endpoints_by_dc = HashMap::<i32, HashSet<SocketAddr>>::new();
|
||||||
|
|
||||||
|
if self.decision.ipv4_me {
|
||||||
|
let map = self.proxy_map_v4.read().await;
|
||||||
|
for (dc, addrs) in map.iter() {
|
||||||
|
let entry = endpoints_by_dc.entry(*dc).or_default();
|
||||||
|
for (ip, port) in addrs.iter().copied() {
|
||||||
|
entry.insert(SocketAddr::new(ip, port));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.decision.ipv6_me {
|
||||||
|
let map = self.proxy_map_v6.read().await;
|
||||||
|
for (dc, addrs) in map.iter() {
|
||||||
|
let entry = endpoints_by_dc.entry(*dc).or_default();
|
||||||
|
for (ip, port) in addrs.iter().copied() {
|
||||||
|
entry.insert(SocketAddr::new(ip, port));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoints_by_dc
|
||||||
|
.values()
|
||||||
|
.map(|endpoints| self.required_writers_for_dc_with_floor_mode(endpoints.len(), false))
|
||||||
|
.sum()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn can_open_writer_for_contour(
|
||||||
|
&self,
|
||||||
|
contour: WriterContour,
|
||||||
|
allow_coverage_override: bool,
|
||||||
|
) -> bool {
|
||||||
|
let (active_writers, warm_writers, _) = self.non_draining_writer_counts_by_contour().await;
|
||||||
|
match contour {
|
||||||
|
WriterContour::Active => {
|
||||||
|
let active_cap = self.adaptive_floor_active_cap_configured_total();
|
||||||
|
if active_writers < active_cap {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if !allow_coverage_override {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let coverage_required = self.active_coverage_required_total().await;
|
||||||
|
active_writers < coverage_required
|
||||||
|
}
|
||||||
|
WriterContour::Warm => warm_writers < self.adaptive_floor_warm_cap_configured_total(),
|
||||||
|
WriterContour::Draining => true,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn required_writers_for_dc_with_floor_mode(
|
pub(super) fn required_writers_for_dc_with_floor_mode(
|
||||||
@@ -779,16 +1030,34 @@ impl MePool {
|
|||||||
if dc == 0 { 2 } else { dc }
|
if dc == 0 { 2 } else { dc }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn dc_lookup_chain_for_target(&self, target_dc: i32) -> Vec<i32> {
|
pub(super) async fn has_configured_endpoints_for_dc(&self, dc: i32) -> bool {
|
||||||
let mut out = Vec::with_capacity(1);
|
if self.decision.ipv4_me {
|
||||||
if target_dc != 0 {
|
let map = self.proxy_map_v4.read().await;
|
||||||
out.push(target_dc);
|
if map.get(&dc).is_some_and(|endpoints| !endpoints.is_empty()) {
|
||||||
} else {
|
return true;
|
||||||
// Use default DC only when target DC is unknown and pinning is not established.
|
}
|
||||||
let fallback_dc = self.default_dc_for_routing();
|
|
||||||
out.push(fallback_dc);
|
|
||||||
}
|
}
|
||||||
out
|
|
||||||
|
if self.decision.ipv6_me {
|
||||||
|
let map = self.proxy_map_v6.read().await;
|
||||||
|
if map.get(&dc).is_some_and(|endpoints| !endpoints.is_empty()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn resolve_target_dc_for_routing(&self, target_dc: i32) -> (i32, bool) {
|
||||||
|
if target_dc == 0 {
|
||||||
|
return (self.default_dc_for_routing(), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.has_configured_endpoints_for_dc(target_dc).await {
|
||||||
|
return (target_dc, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
(self.default_dc_for_routing(), true)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn resolve_dc_for_endpoint(&self, addr: SocketAddr) -> i32 {
|
pub(super) async fn resolve_dc_for_endpoint(&self, addr: SocketAddr) -> i32 {
|
||||||
@@ -830,6 +1099,62 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn build_preferred_endpoints_by_dc(
|
||||||
|
decision: &NetworkDecision,
|
||||||
|
map_v4: &HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||||
|
map_v6: &HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||||
|
) -> HashMap<i32, Vec<SocketAddr>> {
|
||||||
|
let mut out = HashMap::<i32, Vec<SocketAddr>>::new();
|
||||||
|
let mut dcs = HashSet::<i32>::new();
|
||||||
|
dcs.extend(map_v4.keys().copied());
|
||||||
|
dcs.extend(map_v6.keys().copied());
|
||||||
|
|
||||||
|
for dc in dcs {
|
||||||
|
let v4 = map_v4
|
||||||
|
.get(&dc)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let v6 = map_v6
|
||||||
|
.get(&dc)
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let mut selected = if decision.effective_multipath {
|
||||||
|
let mut both = Vec::<SocketAddr>::with_capacity(v4.len().saturating_add(v6.len()));
|
||||||
|
if decision.prefer_ipv6() {
|
||||||
|
both.extend(v6.iter().copied());
|
||||||
|
both.extend(v4.iter().copied());
|
||||||
|
} else {
|
||||||
|
both.extend(v4.iter().copied());
|
||||||
|
both.extend(v6.iter().copied());
|
||||||
|
}
|
||||||
|
both
|
||||||
|
} else if decision.prefer_ipv6() {
|
||||||
|
if !v6.is_empty() { v6 } else { v4 }
|
||||||
|
} else if !v4.is_empty() {
|
||||||
|
v4
|
||||||
|
} else {
|
||||||
|
v6
|
||||||
|
};
|
||||||
|
|
||||||
|
selected.sort_unstable();
|
||||||
|
selected.dedup();
|
||||||
|
out.insert(dc, selected);
|
||||||
|
}
|
||||||
|
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
fn build_endpoint_dc_map_from_maps(
|
fn build_endpoint_dc_map_from_maps(
|
||||||
map_v4: &HashMap<i32, Vec<(IpAddr, u16)>>,
|
map_v4: &HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||||
map_v6: &HashMap<i32, Vec<(IpAddr, u16)>>,
|
map_v6: &HashMap<i32, Vec<(IpAddr, u16)>>,
|
||||||
@@ -852,6 +1177,25 @@ impl MePool {
|
|||||||
let map_v4 = self.proxy_map_v4.read().await.clone();
|
let map_v4 = self.proxy_map_v4.read().await.clone();
|
||||||
let map_v6 = self.proxy_map_v6.read().await.clone();
|
let map_v6 = self.proxy_map_v6.read().await.clone();
|
||||||
let rebuilt = Self::build_endpoint_dc_map_from_maps(&map_v4, &map_v6);
|
let rebuilt = Self::build_endpoint_dc_map_from_maps(&map_v4, &map_v6);
|
||||||
|
let preferred = Self::build_preferred_endpoints_by_dc(&self.decision, &map_v4, &map_v6);
|
||||||
*self.endpoint_dc_map.write().await = rebuilt;
|
*self.endpoint_dc_map.write().await = rebuilt;
|
||||||
|
*self.preferred_endpoints_by_dc.write().await = preferred;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn preferred_endpoints_for_dc(&self, dc: i32) -> Vec<SocketAddr> {
|
||||||
|
let guard = self.preferred_endpoints_by_dc.read().await;
|
||||||
|
guard.get(&dc).cloned().unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn health_interval_unhealthy(&self) -> Duration {
|
||||||
|
Duration::from_millis(self.me_health_interval_ms_unhealthy.load(Ordering::Relaxed).max(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn health_interval_healthy(&self) -> Duration {
|
||||||
|
Duration::from_millis(self.me_health_interval_ms_healthy.load(Ordering::Relaxed).max(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn warn_rate_limit_duration(&self) -> Duration {
|
||||||
|
Duration::from_millis(self.me_warn_rate_limit_ms.load(Ordering::Relaxed).max(1))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -110,7 +110,10 @@ impl MePool {
|
|||||||
pub async fn reconnect_all(self: &Arc<Self>) {
|
pub async fn reconnect_all(self: &Arc<Self>) {
|
||||||
let ws = self.writers.read().await.clone();
|
let ws = self.writers.read().await.clone();
|
||||||
for w in ws {
|
for w in ws {
|
||||||
if let Ok(()) = self.connect_one(w.addr, self.rng.as_ref()).await {
|
if let Ok(()) = self
|
||||||
|
.connect_one_for_dc(w.addr, w.writer_dc, self.rng.as_ref())
|
||||||
|
.await
|
||||||
|
{
|
||||||
self.mark_writer_draining(w.id).await;
|
self.mark_writer_draining(w.id).await;
|
||||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,7 +55,11 @@ impl MePool {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
.collect();
|
.collect();
|
||||||
if self.active_writer_count_for_endpoints(&endpoints).await >= target_writers {
|
if self
|
||||||
|
.active_writer_count_for_dc_endpoints(dc, &endpoints)
|
||||||
|
.await
|
||||||
|
>= target_writers
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let pool = Arc::clone(self);
|
let pool = Arc::clone(self);
|
||||||
@@ -67,6 +71,7 @@ impl MePool {
|
|||||||
target_writers,
|
target_writers,
|
||||||
rng_clone,
|
rng_clone,
|
||||||
connect_concurrency,
|
connect_concurrency,
|
||||||
|
true,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
});
|
});
|
||||||
@@ -79,7 +84,7 @@ impl MePool {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
.collect();
|
.collect();
|
||||||
if self.active_writer_count_for_endpoints(&endpoints).await == 0 {
|
if self.active_writer_count_for_dc_endpoints(*dc, &endpoints).await == 0 {
|
||||||
missing_dcs.push(*dc);
|
missing_dcs.push(*dc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -110,6 +115,7 @@ impl MePool {
|
|||||||
target_writers,
|
target_writers,
|
||||||
rng_clone_local,
|
rng_clone_local,
|
||||||
connect_concurrency,
|
connect_concurrency,
|
||||||
|
false,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
});
|
});
|
||||||
@@ -143,6 +149,7 @@ impl MePool {
|
|||||||
target_writers: usize,
|
target_writers: usize,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
connect_concurrency: usize,
|
connect_concurrency: usize,
|
||||||
|
allow_coverage_override: bool,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
if addrs.is_empty() {
|
if addrs.is_empty() {
|
||||||
return false;
|
return false;
|
||||||
@@ -156,7 +163,9 @@ impl MePool {
|
|||||||
let endpoint_set: HashSet<SocketAddr> = endpoints.iter().copied().collect();
|
let endpoint_set: HashSet<SocketAddr> = endpoints.iter().copied().collect();
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let alive = self.active_writer_count_for_endpoints(&endpoint_set).await;
|
let alive = self
|
||||||
|
.active_writer_count_for_dc_endpoints(dc, &endpoint_set)
|
||||||
|
.await;
|
||||||
if alive >= target_writers {
|
if alive >= target_writers {
|
||||||
info!(
|
info!(
|
||||||
dc = %dc,
|
dc = %dc,
|
||||||
@@ -174,9 +183,17 @@ impl MePool {
|
|||||||
let pool = Arc::clone(&self);
|
let pool = Arc::clone(&self);
|
||||||
let rng_clone = Arc::clone(&rng);
|
let rng_clone = Arc::clone(&rng);
|
||||||
let endpoints_clone = endpoints.clone();
|
let endpoints_clone = endpoints.clone();
|
||||||
|
let generation = self.current_generation();
|
||||||
join.spawn(async move {
|
join.spawn(async move {
|
||||||
pool.connect_endpoints_round_robin(&endpoints_clone, rng_clone.as_ref())
|
pool.connect_endpoints_round_robin_with_generation_contour(
|
||||||
.await
|
dc,
|
||||||
|
&endpoints_clone,
|
||||||
|
rng_clone.as_ref(),
|
||||||
|
generation,
|
||||||
|
super::pool::WriterContour::Active,
|
||||||
|
allow_coverage_override,
|
||||||
|
)
|
||||||
|
.await
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,7 +210,9 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let alive_after = self.active_writer_count_for_endpoints(&endpoint_set).await;
|
let alive_after = self
|
||||||
|
.active_writer_count_for_dc_endpoints(dc, &endpoint_set)
|
||||||
|
.await;
|
||||||
if alive_after >= target_writers {
|
if alive_after >= target_writers {
|
||||||
info!(
|
info!(
|
||||||
dc = %dc,
|
dc = %dc,
|
||||||
@@ -204,12 +223,25 @@ impl MePool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if !progress {
|
if !progress {
|
||||||
warn!(
|
let active_writers_current = self.active_contour_writer_count_total().await;
|
||||||
dc = %dc,
|
let active_cap_configured = self.adaptive_floor_active_cap_configured_total();
|
||||||
alive = alive_after,
|
if !allow_coverage_override && active_writers_current >= active_cap_configured {
|
||||||
target_writers,
|
info!(
|
||||||
"All ME servers for DC failed at init"
|
dc = %dc,
|
||||||
);
|
alive = alive_after,
|
||||||
|
target_writers,
|
||||||
|
active_writers_current,
|
||||||
|
active_cap_configured,
|
||||||
|
"ME init saturation stopped by active writer cap"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
warn!(
|
||||||
|
dc = %dc,
|
||||||
|
alive = alive_after,
|
||||||
|
target_writers,
|
||||||
|
"All ME servers for DC failed at init"
|
||||||
|
);
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use std::collections::HashSet;
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
@@ -9,7 +9,7 @@ use tracing::{debug, info, warn};
|
|||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
use crate::network::IpFamily;
|
use crate::network::IpFamily;
|
||||||
|
|
||||||
use super::pool::{MePool, RefillDcKey, WriterContour};
|
use super::pool::{MePool, RefillDcKey, RefillEndpointKey, WriterContour};
|
||||||
|
|
||||||
const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20;
|
const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20;
|
||||||
const ME_FLAP_QUARANTINE_SECS: u64 = 25;
|
const ME_FLAP_QUARANTINE_SECS: u64 = 25;
|
||||||
@@ -82,82 +82,79 @@ impl MePool {
|
|||||||
Vec::new()
|
Vec::new()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn has_refill_inflight_for_endpoints(&self, endpoints: &[SocketAddr]) -> bool {
|
pub(super) async fn has_refill_inflight_for_dc_key(&self, key: RefillDcKey) -> bool {
|
||||||
if endpoints.is_empty() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
let guard = self.refill_inflight.lock().await;
|
|
||||||
if endpoints.iter().any(|addr| guard.contains(addr)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let dc_keys = self.resolve_refill_dc_keys_for_endpoints(endpoints).await;
|
|
||||||
if dc_keys.is_empty() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let guard = self.refill_inflight_dc.lock().await;
|
let guard = self.refill_inflight_dc.lock().await;
|
||||||
dc_keys.iter().any(|key| guard.contains(key))
|
guard.contains(&key)
|
||||||
}
|
|
||||||
|
|
||||||
async fn resolve_refill_dc_key_for_addr(&self, addr: SocketAddr) -> Option<RefillDcKey> {
|
|
||||||
let family = if addr.is_ipv4() {
|
|
||||||
IpFamily::V4
|
|
||||||
} else {
|
|
||||||
IpFamily::V6
|
|
||||||
};
|
|
||||||
Some(RefillDcKey {
|
|
||||||
dc: self.resolve_dc_for_endpoint(addr).await,
|
|
||||||
family,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn resolve_refill_dc_keys_for_endpoints(
|
|
||||||
&self,
|
|
||||||
endpoints: &[SocketAddr],
|
|
||||||
) -> HashSet<RefillDcKey> {
|
|
||||||
let mut out = HashSet::<RefillDcKey>::new();
|
|
||||||
for addr in endpoints {
|
|
||||||
if let Some(key) = self.resolve_refill_dc_key_for_addr(*addr).await {
|
|
||||||
out.insert(key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn connect_endpoints_round_robin(
|
pub(super) async fn connect_endpoints_round_robin(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
|
dc: i32,
|
||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
rng: &SecureRandom,
|
rng: &SecureRandom,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
self.connect_endpoints_round_robin_with_generation_contour(
|
self.connect_endpoints_round_robin_with_generation_contour(
|
||||||
|
dc,
|
||||||
endpoints,
|
endpoints,
|
||||||
rng,
|
rng,
|
||||||
self.current_generation(),
|
self.current_generation(),
|
||||||
WriterContour::Active,
|
WriterContour::Active,
|
||||||
|
false,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn connect_endpoints_round_robin_with_generation_contour(
|
pub(super) async fn connect_endpoints_round_robin_with_generation_contour(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
|
dc: i32,
|
||||||
endpoints: &[SocketAddr],
|
endpoints: &[SocketAddr],
|
||||||
rng: &SecureRandom,
|
rng: &SecureRandom,
|
||||||
generation: u64,
|
generation: u64,
|
||||||
contour: WriterContour,
|
contour: WriterContour,
|
||||||
|
allow_coverage_override: bool,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let candidates = self.connectable_endpoints(endpoints).await;
|
let mut candidates = self.connectable_endpoints(endpoints).await;
|
||||||
if candidates.is_empty() {
|
if candidates.is_empty() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if candidates.len() > 1 {
|
||||||
|
let mut active_by_endpoint = HashMap::<SocketAddr, usize>::new();
|
||||||
|
let ws = self.writers.read().await;
|
||||||
|
for writer in ws.iter() {
|
||||||
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if writer.writer_dc != dc {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
super::pool::WriterContour::from_u8(
|
||||||
|
writer.contour.load(Ordering::Relaxed),
|
||||||
|
),
|
||||||
|
super::pool::WriterContour::Active
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if candidates.contains(&writer.addr) {
|
||||||
|
*active_by_endpoint.entry(writer.addr).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(ws);
|
||||||
|
candidates.sort_by_key(|addr| (active_by_endpoint.get(addr).copied().unwrap_or(0), *addr));
|
||||||
|
}
|
||||||
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len();
|
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len();
|
||||||
for offset in 0..candidates.len() {
|
for offset in 0..candidates.len() {
|
||||||
let idx = (start + offset) % candidates.len();
|
let idx = (start + offset) % candidates.len();
|
||||||
let addr = candidates[idx];
|
let addr = candidates[idx];
|
||||||
match self
|
match self
|
||||||
.connect_one_with_generation_contour(addr, rng, generation, contour)
|
.connect_one_with_generation_contour_for_dc_with_cap_policy(
|
||||||
|
addr,
|
||||||
|
rng,
|
||||||
|
generation,
|
||||||
|
contour,
|
||||||
|
dc,
|
||||||
|
allow_coverage_override,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(()) => return true,
|
Ok(()) => return true,
|
||||||
@@ -167,9 +164,8 @@ impl MePool {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn endpoints_for_same_dc(&self, addr: SocketAddr) -> Vec<SocketAddr> {
|
async fn endpoints_for_dc(&self, target_dc: i32) -> Vec<SocketAddr> {
|
||||||
let mut endpoints = HashSet::<SocketAddr>::new();
|
let mut endpoints = HashSet::<SocketAddr>::new();
|
||||||
let target_dc = self.resolve_dc_for_endpoint(addr).await;
|
|
||||||
|
|
||||||
if self.decision.ipv4_me {
|
if self.decision.ipv4_me {
|
||||||
let map = self.proxy_map_v4.read().await;
|
let map = self.proxy_map_v4.read().await;
|
||||||
@@ -194,14 +190,14 @@ impl MePool {
|
|||||||
sorted
|
sorted
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn refill_writer_after_loss(self: &Arc<Self>, addr: SocketAddr) -> bool {
|
async fn refill_writer_after_loss(self: &Arc<Self>, addr: SocketAddr, writer_dc: i32) -> bool {
|
||||||
let fast_retries = self.me_reconnect_fast_retry_count.max(1);
|
let fast_retries = self.me_reconnect_fast_retry_count.max(1);
|
||||||
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
||||||
|
|
||||||
if !same_endpoint_quarantined {
|
if !same_endpoint_quarantined {
|
||||||
for attempt in 0..fast_retries {
|
for attempt in 0..fast_retries {
|
||||||
self.stats.increment_me_reconnect_attempt();
|
self.stats.increment_me_reconnect_attempt();
|
||||||
match self.connect_one(addr, self.rng.as_ref()).await {
|
match self.connect_one_for_dc(addr, writer_dc, self.rng.as_ref()).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
self.stats.increment_me_reconnect_success();
|
self.stats.increment_me_reconnect_success();
|
||||||
self.stats.increment_me_writer_restored_same_endpoint_total();
|
self.stats.increment_me_writer_restored_same_endpoint_total();
|
||||||
@@ -229,7 +225,7 @@ impl MePool {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let dc_endpoints = self.endpoints_for_same_dc(addr).await;
|
let dc_endpoints = self.endpoints_for_dc(writer_dc).await;
|
||||||
if dc_endpoints.is_empty() {
|
if dc_endpoints.is_empty() {
|
||||||
self.stats.increment_me_refill_failed_total();
|
self.stats.increment_me_refill_failed_total();
|
||||||
return false;
|
return false;
|
||||||
@@ -238,7 +234,7 @@ impl MePool {
|
|||||||
for attempt in 0..fast_retries {
|
for attempt in 0..fast_retries {
|
||||||
self.stats.increment_me_reconnect_attempt();
|
self.stats.increment_me_reconnect_attempt();
|
||||||
if self
|
if self
|
||||||
.connect_endpoints_round_robin(&dc_endpoints, self.rng.as_ref())
|
.connect_endpoints_round_robin(writer_dc, &dc_endpoints, self.rng.as_ref())
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
self.stats.increment_me_reconnect_success();
|
self.stats.increment_me_reconnect_success();
|
||||||
@@ -256,48 +252,63 @@ impl MePool {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn trigger_immediate_refill(self: &Arc<Self>, addr: SocketAddr) {
|
pub(crate) fn trigger_immediate_refill_for_dc(self: &Arc<Self>, addr: SocketAddr, writer_dc: i32) {
|
||||||
|
let endpoint_key = RefillEndpointKey {
|
||||||
|
dc: writer_dc,
|
||||||
|
addr,
|
||||||
|
};
|
||||||
|
let pre_inserted = if let Ok(mut guard) = self.refill_inflight.try_lock() {
|
||||||
|
if !guard.insert(endpoint_key) {
|
||||||
|
self.stats.increment_me_refill_skipped_inflight_total();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
|
||||||
let pool = Arc::clone(self);
|
let pool = Arc::clone(self);
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let dc_endpoints = pool.endpoints_for_same_dc(addr).await;
|
let dc_key = RefillDcKey {
|
||||||
let dc_keys = pool.resolve_refill_dc_keys_for_endpoints(&dc_endpoints).await;
|
dc: writer_dc,
|
||||||
|
family: if addr.is_ipv4() {
|
||||||
|
IpFamily::V4
|
||||||
|
} else {
|
||||||
|
IpFamily::V6
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
{
|
if !pre_inserted {
|
||||||
let mut guard = pool.refill_inflight.lock().await;
|
let mut guard = pool.refill_inflight.lock().await;
|
||||||
if !guard.insert(addr) {
|
if !guard.insert(endpoint_key) {
|
||||||
pool.stats.increment_me_refill_skipped_inflight_total();
|
pool.stats.increment_me_refill_skipped_inflight_total();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !dc_keys.is_empty() {
|
{
|
||||||
let mut dc_guard = pool.refill_inflight_dc.lock().await;
|
let mut dc_guard = pool.refill_inflight_dc.lock().await;
|
||||||
if dc_keys.iter().any(|key| dc_guard.contains(key)) {
|
if dc_guard.contains(&dc_key) {
|
||||||
pool.stats.increment_me_refill_skipped_inflight_total();
|
pool.stats.increment_me_refill_skipped_inflight_total();
|
||||||
drop(dc_guard);
|
drop(dc_guard);
|
||||||
let mut guard = pool.refill_inflight.lock().await;
|
let mut guard = pool.refill_inflight.lock().await;
|
||||||
guard.remove(&addr);
|
guard.remove(&endpoint_key);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
dc_guard.extend(dc_keys.iter().copied());
|
dc_guard.insert(dc_key);
|
||||||
}
|
}
|
||||||
|
|
||||||
pool.stats.increment_me_refill_triggered_total();
|
pool.stats.increment_me_refill_triggered_total();
|
||||||
|
let restored = pool.refill_writer_after_loss(addr, writer_dc).await;
|
||||||
let restored = pool.refill_writer_after_loss(addr).await;
|
|
||||||
if !restored {
|
if !restored {
|
||||||
warn!(%addr, "ME immediate refill failed");
|
warn!(%addr, dc = writer_dc, "ME immediate refill failed");
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut guard = pool.refill_inflight.lock().await;
|
let mut guard = pool.refill_inflight.lock().await;
|
||||||
guard.remove(&addr);
|
guard.remove(&endpoint_key);
|
||||||
drop(guard);
|
drop(guard);
|
||||||
if !dc_keys.is_empty() {
|
let mut dc_guard = pool.refill_inflight_dc.lock().await;
|
||||||
let mut dc_guard = pool.refill_inflight_dc.lock().await;
|
dc_guard.remove(&dc_key);
|
||||||
for key in &dc_keys {
|
|
||||||
dc_guard.remove(key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ impl MePool {
|
|||||||
|
|
||||||
fn coverage_ratio(
|
fn coverage_ratio(
|
||||||
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
||||||
active_writer_addrs: &HashSet<SocketAddr>,
|
active_writer_addrs: &HashSet<(i32, SocketAddr)>,
|
||||||
) -> (f32, Vec<i32>) {
|
) -> (f32, Vec<i32>) {
|
||||||
if desired_by_dc.is_empty() {
|
if desired_by_dc.is_empty() {
|
||||||
return (1.0, Vec::new());
|
return (1.0, Vec::new());
|
||||||
@@ -76,7 +76,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
if endpoints
|
if endpoints
|
||||||
.iter()
|
.iter()
|
||||||
.any(|addr| active_writer_addrs.contains(addr))
|
.any(|addr| active_writer_addrs.contains(&(*dc, *addr)))
|
||||||
{
|
{
|
||||||
covered += 1;
|
covered += 1;
|
||||||
} else {
|
} else {
|
||||||
@@ -91,32 +91,25 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn reconcile_connections(self: &Arc<Self>, rng: &SecureRandom) {
|
pub async fn reconcile_connections(self: &Arc<Self>, rng: &SecureRandom) {
|
||||||
let writers = self.writers.read().await;
|
|
||||||
let current: HashSet<SocketAddr> = writers
|
|
||||||
.iter()
|
|
||||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
|
||||||
.map(|w| w.addr)
|
|
||||||
.collect();
|
|
||||||
drop(writers);
|
|
||||||
|
|
||||||
for family in self.family_order() {
|
for family in self.family_order() {
|
||||||
let map = self.proxy_map_for_family(family).await;
|
let map = self.proxy_map_for_family(family).await;
|
||||||
for (_dc, addrs) in &map {
|
for (dc, addrs) in &map {
|
||||||
let dc_addrs: Vec<SocketAddr> = addrs
|
let dc_addrs: Vec<SocketAddr> = addrs
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
.map(|(ip, port)| SocketAddr::new(*ip, *port))
|
||||||
.collect();
|
.collect();
|
||||||
if !dc_addrs.iter().any(|a| current.contains(a)) {
|
let dc_endpoints: HashSet<SocketAddr> = dc_addrs.iter().copied().collect();
|
||||||
|
if self.active_writer_count_for_dc_endpoints(*dc, &dc_endpoints).await == 0 {
|
||||||
let mut shuffled = dc_addrs.clone();
|
let mut shuffled = dc_addrs.clone();
|
||||||
shuffled.shuffle(&mut rand::rng());
|
shuffled.shuffle(&mut rand::rng());
|
||||||
for addr in shuffled {
|
for addr in shuffled {
|
||||||
if self.connect_one(addr, rng).await.is_ok() {
|
if self.connect_one_for_dc(addr, *dc, rng).await.is_ok() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !self.decision.effective_multipath && !current.is_empty() {
|
if !self.decision.effective_multipath && self.connection_count() > 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -174,26 +167,30 @@ impl MePool {
|
|||||||
core.saturating_add(rand::rng().random_range(0..=jitter))
|
core.saturating_add(rand::rng().random_range(0..=jitter))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fresh_writer_count_for_endpoints(
|
async fn fresh_writer_count_for_dc_endpoints(
|
||||||
&self,
|
&self,
|
||||||
generation: u64,
|
generation: u64,
|
||||||
|
dc: i32,
|
||||||
endpoints: &HashSet<SocketAddr>,
|
endpoints: &HashSet<SocketAddr>,
|
||||||
) -> usize {
|
) -> usize {
|
||||||
let ws = self.writers.read().await;
|
let ws = self.writers.read().await;
|
||||||
ws.iter()
|
ws.iter()
|
||||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||||
.filter(|w| w.generation == generation)
|
.filter(|w| w.generation == generation)
|
||||||
|
.filter(|w| w.writer_dc == dc)
|
||||||
.filter(|w| endpoints.contains(&w.addr))
|
.filter(|w| endpoints.contains(&w.addr))
|
||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn active_writer_count_for_endpoints(
|
pub(super) async fn active_writer_count_for_dc_endpoints(
|
||||||
&self,
|
&self,
|
||||||
|
dc: i32,
|
||||||
endpoints: &HashSet<SocketAddr>,
|
endpoints: &HashSet<SocketAddr>,
|
||||||
) -> usize {
|
) -> usize {
|
||||||
let ws = self.writers.read().await;
|
let ws = self.writers.read().await;
|
||||||
ws.iter()
|
ws.iter()
|
||||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||||
|
.filter(|w| w.writer_dc == dc)
|
||||||
.filter(|w| endpoints.contains(&w.addr))
|
.filter(|w| endpoints.contains(&w.addr))
|
||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
@@ -220,7 +217,7 @@ impl MePool {
|
|||||||
let required = self.required_writers_for_dc(endpoint_list.len());
|
let required = self.required_writers_for_dc(endpoint_list.len());
|
||||||
let mut completed = false;
|
let mut completed = false;
|
||||||
let mut last_fresh_count = self
|
let mut last_fresh_count = self
|
||||||
.fresh_writer_count_for_endpoints(generation, endpoints)
|
.fresh_writer_count_for_dc_endpoints(generation, *dc, endpoints)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
for pass_idx in 0..total_passes {
|
for pass_idx in 0..total_passes {
|
||||||
@@ -247,10 +244,12 @@ impl MePool {
|
|||||||
|
|
||||||
let connected = self
|
let connected = self
|
||||||
.connect_endpoints_round_robin_with_generation_contour(
|
.connect_endpoints_round_robin_with_generation_contour(
|
||||||
|
*dc,
|
||||||
&endpoint_list,
|
&endpoint_list,
|
||||||
rng,
|
rng,
|
||||||
generation,
|
generation,
|
||||||
WriterContour::Warm,
|
WriterContour::Warm,
|
||||||
|
false,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
debug!(
|
debug!(
|
||||||
@@ -265,7 +264,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
last_fresh_count = self
|
last_fresh_count = self
|
||||||
.fresh_writer_count_for_endpoints(generation, endpoints)
|
.fresh_writer_count_for_dc_endpoints(generation, *dc, endpoints)
|
||||||
.await;
|
.await;
|
||||||
if last_fresh_count >= required {
|
if last_fresh_count >= required {
|
||||||
completed = true;
|
completed = true;
|
||||||
@@ -377,10 +376,10 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let writers = self.writers.read().await;
|
let writers = self.writers.read().await;
|
||||||
let active_writer_addrs: HashSet<SocketAddr> = writers
|
let active_writer_addrs: HashSet<(i32, SocketAddr)> = writers
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||||
.map(|w| w.addr)
|
.map(|w| (w.writer_dc, w.addr))
|
||||||
.collect();
|
.collect();
|
||||||
let min_ratio = Self::permille_to_ratio(
|
let min_ratio = Self::permille_to_ratio(
|
||||||
self.me_pool_min_fresh_ratio_permille
|
self.me_pool_min_fresh_ratio_permille
|
||||||
@@ -410,6 +409,7 @@ impl MePool {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||||
.filter(|w| w.generation == generation)
|
.filter(|w| w.generation == generation)
|
||||||
|
.filter(|w| w.writer_dc == *dc)
|
||||||
.filter(|w| endpoints.contains(&w.addr))
|
.filter(|w| endpoints.contains(&w.addr))
|
||||||
.count();
|
.count();
|
||||||
if fresh_count < required {
|
if fresh_count < required {
|
||||||
@@ -438,9 +438,9 @@ impl MePool {
|
|||||||
self.promote_warm_generation_to_active(generation).await;
|
self.promote_warm_generation_to_active(generation).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let desired_addrs: HashSet<SocketAddr> = desired_by_dc
|
let desired_addrs: HashSet<(i32, SocketAddr)> = desired_by_dc
|
||||||
.values()
|
.iter()
|
||||||
.flat_map(|set| set.iter().copied())
|
.flat_map(|(dc, set)| set.iter().copied().map(|addr| (*dc, addr)))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let stale_writer_ids: Vec<u64> = writers
|
let stale_writer_ids: Vec<u64> = writers
|
||||||
@@ -450,7 +450,7 @@ impl MePool {
|
|||||||
if hardswap {
|
if hardswap {
|
||||||
w.generation < generation
|
w.generation < generation
|
||||||
} else {
|
} else {
|
||||||
!desired_addrs.contains(&w.addr)
|
!desired_addrs.contains(&(w.writer_dc, w.addr))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.map(|w| w.id)
|
.map(|w| w.id)
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ pub(crate) struct MeApiWriterStatusSnapshot {
|
|||||||
pub(crate) struct MeApiDcStatusSnapshot {
|
pub(crate) struct MeApiDcStatusSnapshot {
|
||||||
pub dc: i16,
|
pub dc: i16,
|
||||||
pub endpoints: Vec<SocketAddr>,
|
pub endpoints: Vec<SocketAddr>,
|
||||||
|
pub endpoint_writers: Vec<MeApiDcEndpointWriterSnapshot>,
|
||||||
pub available_endpoints: usize,
|
pub available_endpoints: usize,
|
||||||
pub available_pct: f64,
|
pub available_pct: f64,
|
||||||
pub required_writers: usize,
|
pub required_writers: usize,
|
||||||
@@ -38,6 +39,12 @@ pub(crate) struct MeApiDcStatusSnapshot {
|
|||||||
pub load: usize,
|
pub load: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub(crate) struct MeApiDcEndpointWriterSnapshot {
|
||||||
|
pub endpoint: SocketAddr,
|
||||||
|
pub active_writers: usize,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub(crate) struct MeApiStatusSnapshot {
|
pub(crate) struct MeApiStatusSnapshot {
|
||||||
pub generated_at_epoch_secs: u64,
|
pub generated_at_epoch_secs: u64,
|
||||||
@@ -82,11 +89,21 @@ pub(crate) struct MeApiRuntimeSnapshot {
|
|||||||
pub adaptive_floor_cpu_cores_override: u16,
|
pub adaptive_floor_cpu_cores_override: u16,
|
||||||
pub adaptive_floor_max_extra_writers_single_per_core: u16,
|
pub adaptive_floor_max_extra_writers_single_per_core: u16,
|
||||||
pub adaptive_floor_max_extra_writers_multi_per_core: u16,
|
pub adaptive_floor_max_extra_writers_multi_per_core: u16,
|
||||||
|
pub adaptive_floor_max_active_writers_per_core: u16,
|
||||||
|
pub adaptive_floor_max_warm_writers_per_core: u16,
|
||||||
|
pub adaptive_floor_max_active_writers_global: u32,
|
||||||
|
pub adaptive_floor_max_warm_writers_global: u32,
|
||||||
pub adaptive_floor_cpu_cores_detected: u32,
|
pub adaptive_floor_cpu_cores_detected: u32,
|
||||||
pub adaptive_floor_cpu_cores_effective: u32,
|
pub adaptive_floor_cpu_cores_effective: u32,
|
||||||
pub adaptive_floor_global_cap_raw: u64,
|
pub adaptive_floor_global_cap_raw: u64,
|
||||||
pub adaptive_floor_global_cap_effective: u64,
|
pub adaptive_floor_global_cap_effective: u64,
|
||||||
pub adaptive_floor_target_writers_total: u64,
|
pub adaptive_floor_target_writers_total: u64,
|
||||||
|
pub adaptive_floor_active_cap_configured: u64,
|
||||||
|
pub adaptive_floor_active_cap_effective: u64,
|
||||||
|
pub adaptive_floor_warm_cap_configured: u64,
|
||||||
|
pub adaptive_floor_warm_cap_effective: u64,
|
||||||
|
pub adaptive_floor_active_writers_current: u64,
|
||||||
|
pub adaptive_floor_warm_writers_current: u64,
|
||||||
pub me_keepalive_enabled: bool,
|
pub me_keepalive_enabled: bool,
|
||||||
pub me_keepalive_interval_secs: u64,
|
pub me_keepalive_interval_secs: u64,
|
||||||
pub me_keepalive_jitter_secs: u64,
|
pub me_keepalive_jitter_secs: u64,
|
||||||
@@ -108,6 +125,8 @@ pub(crate) struct MeApiRuntimeSnapshot {
|
|||||||
pub me_single_endpoint_outage_backoff_max_ms: u64,
|
pub me_single_endpoint_outage_backoff_max_ms: u64,
|
||||||
pub me_single_endpoint_shadow_rotate_every_secs: u64,
|
pub me_single_endpoint_shadow_rotate_every_secs: u64,
|
||||||
pub me_deterministic_writer_sort: bool,
|
pub me_deterministic_writer_sort: bool,
|
||||||
|
pub me_writer_pick_mode: &'static str,
|
||||||
|
pub me_writer_pick_sample_size: u8,
|
||||||
pub me_socks_kdf_policy: &'static str,
|
pub me_socks_kdf_policy: &'static str,
|
||||||
pub quarantined_endpoints: Vec<MeApiQuarantinedEndpointSnapshot>,
|
pub quarantined_endpoints: Vec<MeApiQuarantinedEndpointSnapshot>,
|
||||||
pub network_path: Vec<MeApiDcPathSnapshot>,
|
pub network_path: Vec<MeApiDcPathSnapshot>,
|
||||||
@@ -130,19 +149,18 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let writers = self.writers.read().await.clone();
|
let writers = self.writers.read().await.clone();
|
||||||
let mut live_writers_by_endpoint = HashMap::<SocketAddr, usize>::new();
|
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
||||||
for writer in writers {
|
for writer in writers {
|
||||||
if writer.draining.load(Ordering::Relaxed) {
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
*live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1;
|
if let Ok(dc) = i16::try_from(writer.writer_dc) {
|
||||||
|
*live_writers_by_dc.entry(dc).or_insert(0) += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for endpoints in endpoints_by_dc.values() {
|
for dc in endpoints_by_dc.keys() {
|
||||||
let alive: usize = endpoints
|
let alive = live_writers_by_dc.get(dc).copied().unwrap_or(0);
|
||||||
.iter()
|
|
||||||
.map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0))
|
|
||||||
.sum();
|
|
||||||
if alive == 0 {
|
if alive == 0 {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -168,24 +186,23 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let writers = self.writers.read().await.clone();
|
let writers = self.writers.read().await.clone();
|
||||||
let mut live_writers_by_endpoint = HashMap::<SocketAddr, usize>::new();
|
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
||||||
for writer in writers {
|
for writer in writers {
|
||||||
if writer.draining.load(Ordering::Relaxed) {
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
*live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1;
|
if let Ok(dc) = i16::try_from(writer.writer_dc) {
|
||||||
|
*live_writers_by_dc.entry(dc).or_insert(0) += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for endpoints in endpoints_by_dc.values() {
|
for (dc, endpoints) in endpoints_by_dc {
|
||||||
let endpoint_count = endpoints.len();
|
let endpoint_count = endpoints.len();
|
||||||
if endpoint_count == 0 {
|
if endpoint_count == 0 {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
|
let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
|
||||||
let alive: usize = endpoints
|
let alive = live_writers_by_dc.get(&dc).copied().unwrap_or(0);
|
||||||
.iter()
|
|
||||||
.map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0))
|
|
||||||
.sum();
|
|
||||||
if alive < required {
|
if alive < required {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -207,13 +224,6 @@ impl MePool {
|
|||||||
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
extend_signed_endpoints(&mut endpoints_by_dc, map);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut endpoint_to_dc = HashMap::<SocketAddr, BTreeSet<i16>>::new();
|
|
||||||
for (dc, endpoints) in &endpoints_by_dc {
|
|
||||||
for endpoint in endpoints {
|
|
||||||
endpoint_to_dc.entry(*endpoint).or_default().insert(*dc);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let configured_dc_groups = endpoints_by_dc.len();
|
let configured_dc_groups = endpoints_by_dc.len();
|
||||||
let configured_endpoints = endpoints_by_dc.values().map(BTreeSet::len).sum();
|
let configured_endpoints = endpoints_by_dc.values().map(BTreeSet::len).sum();
|
||||||
|
|
||||||
@@ -227,20 +237,14 @@ impl MePool {
|
|||||||
let rtt = self.rtt_stats.lock().await.clone();
|
let rtt = self.rtt_stats.lock().await.clone();
|
||||||
let writers = self.writers.read().await.clone();
|
let writers = self.writers.read().await.clone();
|
||||||
|
|
||||||
let mut live_writers_by_endpoint = HashMap::<SocketAddr, usize>::new();
|
let mut live_writers_by_dc_endpoint = HashMap::<(i16, SocketAddr), usize>::new();
|
||||||
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
||||||
let mut dc_rtt_agg = HashMap::<i16, (f64, u64)>::new();
|
let mut dc_rtt_agg = HashMap::<i16, (f64, u64)>::new();
|
||||||
let mut writer_rows = Vec::<MeApiWriterStatusSnapshot>::with_capacity(writers.len());
|
let mut writer_rows = Vec::<MeApiWriterStatusSnapshot>::with_capacity(writers.len());
|
||||||
|
|
||||||
for writer in writers {
|
for writer in writers {
|
||||||
let endpoint = writer.addr;
|
let endpoint = writer.addr;
|
||||||
let dc = endpoint_to_dc.get(&endpoint).and_then(|dcs| {
|
let dc = i16::try_from(writer.writer_dc).ok();
|
||||||
if dcs.len() == 1 {
|
|
||||||
dcs.iter().next().copied()
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let draining = writer.draining.load(Ordering::Relaxed);
|
let draining = writer.draining.load(Ordering::Relaxed);
|
||||||
let degraded = writer.degraded.load(Ordering::Relaxed);
|
let degraded = writer.degraded.load(Ordering::Relaxed);
|
||||||
let bound_clients = activity
|
let bound_clients = activity
|
||||||
@@ -259,8 +263,10 @@ impl MePool {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if !draining {
|
if !draining {
|
||||||
*live_writers_by_endpoint.entry(endpoint).or_insert(0) += 1;
|
|
||||||
if let Some(dc_idx) = dc {
|
if let Some(dc_idx) = dc {
|
||||||
|
*live_writers_by_dc_endpoint
|
||||||
|
.entry((dc_idx, endpoint))
|
||||||
|
.or_insert(0) += 1;
|
||||||
*live_writers_by_dc.entry(dc_idx).or_insert(0) += 1;
|
*live_writers_by_dc.entry(dc_idx).or_insert(0) += 1;
|
||||||
if let Some(ema_ms) = rtt_ema_ms {
|
if let Some(ema_ms) = rtt_ema_ms {
|
||||||
let entry = dc_rtt_agg.entry(dc_idx).or_insert((0.0, 0));
|
let entry = dc_rtt_agg.entry(dc_idx).or_insert((0.0, 0));
|
||||||
@@ -298,7 +304,7 @@ impl MePool {
|
|||||||
let endpoint_count = endpoints.len();
|
let endpoint_count = endpoints.len();
|
||||||
let dc_available_endpoints = endpoints
|
let dc_available_endpoints = endpoints
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|endpoint| live_writers_by_endpoint.contains_key(endpoint))
|
.filter(|endpoint| live_writers_by_dc_endpoint.contains_key(&(dc, **endpoint)))
|
||||||
.count();
|
.count();
|
||||||
let base_required = self.required_writers_for_dc(endpoint_count);
|
let base_required = self.required_writers_for_dc(endpoint_count);
|
||||||
let dc_required_writers =
|
let dc_required_writers =
|
||||||
@@ -341,6 +347,16 @@ impl MePool {
|
|||||||
|
|
||||||
dcs.push(MeApiDcStatusSnapshot {
|
dcs.push(MeApiDcStatusSnapshot {
|
||||||
dc,
|
dc,
|
||||||
|
endpoint_writers: endpoints
|
||||||
|
.iter()
|
||||||
|
.map(|endpoint| MeApiDcEndpointWriterSnapshot {
|
||||||
|
endpoint: *endpoint,
|
||||||
|
active_writers: live_writers_by_dc_endpoint
|
||||||
|
.get(&(dc, *endpoint))
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(0),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
endpoints: endpoints.into_iter().collect(),
|
endpoints: endpoints.into_iter().collect(),
|
||||||
available_endpoints: dc_available_endpoints,
|
available_endpoints: dc_available_endpoints,
|
||||||
available_pct: ratio_pct(dc_available_endpoints, endpoint_count),
|
available_pct: ratio_pct(dc_available_endpoints, endpoint_count),
|
||||||
@@ -443,6 +459,18 @@ impl MePool {
|
|||||||
adaptive_floor_max_extra_writers_multi_per_core: self
|
adaptive_floor_max_extra_writers_multi_per_core: self
|
||||||
.me_adaptive_floor_max_extra_writers_multi_per_core
|
.me_adaptive_floor_max_extra_writers_multi_per_core
|
||||||
.load(Ordering::Relaxed) as u16,
|
.load(Ordering::Relaxed) as u16,
|
||||||
|
adaptive_floor_max_active_writers_per_core: self
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.load(Ordering::Relaxed) as u16,
|
||||||
|
adaptive_floor_max_warm_writers_per_core: self
|
||||||
|
.me_adaptive_floor_max_warm_writers_per_core
|
||||||
|
.load(Ordering::Relaxed) as u16,
|
||||||
|
adaptive_floor_max_active_writers_global: self
|
||||||
|
.me_adaptive_floor_max_active_writers_global
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_max_warm_writers_global: self
|
||||||
|
.me_adaptive_floor_max_warm_writers_global
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_cpu_cores_detected: self
|
adaptive_floor_cpu_cores_detected: self
|
||||||
.me_adaptive_floor_cpu_cores_detected
|
.me_adaptive_floor_cpu_cores_detected
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
@@ -458,6 +486,24 @@ impl MePool {
|
|||||||
adaptive_floor_target_writers_total: self
|
adaptive_floor_target_writers_total: self
|
||||||
.me_adaptive_floor_target_writers_total
|
.me_adaptive_floor_target_writers_total
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_active_cap_configured: self
|
||||||
|
.me_adaptive_floor_active_cap_configured
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_active_cap_effective: self
|
||||||
|
.me_adaptive_floor_active_cap_effective
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_warm_cap_configured: self
|
||||||
|
.me_adaptive_floor_warm_cap_configured
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_warm_cap_effective: self
|
||||||
|
.me_adaptive_floor_warm_cap_effective
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_active_writers_current: self
|
||||||
|
.me_adaptive_floor_active_writers_current
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
|
adaptive_floor_warm_writers_current: self
|
||||||
|
.me_adaptive_floor_warm_writers_current
|
||||||
|
.load(Ordering::Relaxed),
|
||||||
me_keepalive_enabled: self.me_keepalive_enabled,
|
me_keepalive_enabled: self.me_keepalive_enabled,
|
||||||
me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(),
|
me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(),
|
||||||
me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(),
|
me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(),
|
||||||
@@ -495,6 +541,8 @@ impl MePool {
|
|||||||
me_deterministic_writer_sort: self
|
me_deterministic_writer_sort: self
|
||||||
.me_deterministic_writer_sort
|
.me_deterministic_writer_sort
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
|
me_writer_pick_mode: writer_pick_mode_label(self.writer_pick_mode()),
|
||||||
|
me_writer_pick_sample_size: self.writer_pick_sample_size() as u8,
|
||||||
me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()),
|
me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()),
|
||||||
quarantined_endpoints,
|
quarantined_endpoints,
|
||||||
network_path,
|
network_path,
|
||||||
@@ -543,6 +591,13 @@ fn bind_stale_mode_label(mode: MeBindStaleMode) -> &'static str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn writer_pick_mode_label(mode: crate::config::MeWriterPickMode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
crate::config::MeWriterPickMode::SortedRr => "sorted_rr",
|
||||||
|
crate::config::MeWriterPickMode::P2c => "p2c",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn socks_kdf_policy_label(policy: MeSocksKdfPolicy) -> &'static str {
|
fn socks_kdf_policy_label(policy: MeSocksKdfPolicy) -> &'static str {
|
||||||
match policy {
|
match policy {
|
||||||
MeSocksKdfPolicy::Strict => "strict",
|
MeSocksKdfPolicy::Strict => "strict",
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
use bytes::BytesMut;
|
use bytes::BytesMut;
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
@@ -49,12 +50,18 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn connect_one(self: &Arc<Self>, addr: SocketAddr, rng: &SecureRandom) -> Result<()> {
|
pub(crate) async fn connect_one_for_dc(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
addr: SocketAddr,
|
||||||
|
writer_dc: i32,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
) -> Result<()> {
|
||||||
self.connect_one_with_generation_contour(
|
self.connect_one_with_generation_contour(
|
||||||
addr,
|
addr,
|
||||||
rng,
|
rng,
|
||||||
self.current_generation(),
|
self.current_generation(),
|
||||||
WriterContour::Active,
|
WriterContour::Active,
|
||||||
|
writer_dc,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -65,23 +72,68 @@ impl MePool {
|
|||||||
rng: &SecureRandom,
|
rng: &SecureRandom,
|
||||||
generation: u64,
|
generation: u64,
|
||||||
contour: WriterContour,
|
contour: WriterContour,
|
||||||
|
writer_dc: i32,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
self.connect_one_with_generation_contour_for_dc(addr, rng, generation, contour, writer_dc)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn connect_one_with_generation_contour_for_dc(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
addr: SocketAddr,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
generation: u64,
|
||||||
|
contour: WriterContour,
|
||||||
|
writer_dc: i32,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.connect_one_with_generation_contour_for_dc_with_cap_policy(
|
||||||
|
addr,
|
||||||
|
rng,
|
||||||
|
generation,
|
||||||
|
contour,
|
||||||
|
writer_dc,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn connect_one_with_generation_contour_for_dc_with_cap_policy(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
addr: SocketAddr,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
generation: u64,
|
||||||
|
contour: WriterContour,
|
||||||
|
writer_dc: i32,
|
||||||
|
allow_coverage_override: bool,
|
||||||
|
) -> Result<()> {
|
||||||
|
if !self
|
||||||
|
.can_open_writer_for_contour(contour, allow_coverage_override)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
return Err(ProxyError::Proxy(format!(
|
||||||
|
"ME {contour:?} writer cap reached"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
let secret_len = self.proxy_secret.read().await.secret.len();
|
let secret_len = self.proxy_secret.read().await.secret.len();
|
||||||
if secret_len < 32 {
|
if secret_len < 32 {
|
||||||
return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into()));
|
return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let (stream, _connect_ms, upstream_egress) = self.connect_tcp(addr).await?;
|
let dc_idx = i16::try_from(writer_dc).ok();
|
||||||
|
let (stream, _connect_ms, upstream_egress) = self.connect_tcp(addr, dc_idx).await?;
|
||||||
let hs = self.handshake_only(stream, addr, upstream_egress, rng).await?;
|
let hs = self.handshake_only(stream, addr, upstream_egress, rng).await?;
|
||||||
|
|
||||||
let writer_id = self.next_writer_id.fetch_add(1, Ordering::Relaxed);
|
let writer_id = self.next_writer_id.fetch_add(1, Ordering::Relaxed);
|
||||||
let contour = Arc::new(AtomicU8::new(contour.as_u8()));
|
let contour = Arc::new(AtomicU8::new(contour.as_u8()));
|
||||||
let cancel = CancellationToken::new();
|
let cancel = CancellationToken::new();
|
||||||
let degraded = Arc::new(AtomicBool::new(false));
|
let degraded = Arc::new(AtomicBool::new(false));
|
||||||
|
let rtt_ema_ms_x10 = Arc::new(AtomicU32::new(0));
|
||||||
let draining = Arc::new(AtomicBool::new(false));
|
let draining = Arc::new(AtomicBool::new(false));
|
||||||
let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0));
|
let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0));
|
||||||
|
let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0));
|
||||||
let allow_drain_fallback = Arc::new(AtomicBool::new(false));
|
let allow_drain_fallback = Arc::new(AtomicBool::new(false));
|
||||||
let (tx, mut rx) = mpsc::channel::<WriterCommand>(4096);
|
let (tx, mut rx) = mpsc::channel::<WriterCommand>(self.writer_cmd_channel_capacity);
|
||||||
let mut rpc_writer = RpcWriter {
|
let mut rpc_writer = RpcWriter {
|
||||||
writer: hs.wr,
|
writer: hs.wr,
|
||||||
key: hs.write_key,
|
key: hs.write_key,
|
||||||
@@ -111,14 +163,17 @@ impl MePool {
|
|||||||
let writer = MeWriter {
|
let writer = MeWriter {
|
||||||
id: writer_id,
|
id: writer_id,
|
||||||
addr,
|
addr,
|
||||||
|
writer_dc,
|
||||||
generation,
|
generation,
|
||||||
contour: contour.clone(),
|
contour: contour.clone(),
|
||||||
created_at: Instant::now(),
|
created_at: Instant::now(),
|
||||||
tx: tx.clone(),
|
tx: tx.clone(),
|
||||||
cancel: cancel.clone(),
|
cancel: cancel.clone(),
|
||||||
degraded: degraded.clone(),
|
degraded: degraded.clone(),
|
||||||
|
rtt_ema_ms_x10: rtt_ema_ms_x10.clone(),
|
||||||
draining: draining.clone(),
|
draining: draining.clone(),
|
||||||
draining_started_at_epoch_secs: draining_started_at_epoch_secs.clone(),
|
draining_started_at_epoch_secs: draining_started_at_epoch_secs.clone(),
|
||||||
|
drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(),
|
||||||
allow_drain_fallback: allow_drain_fallback.clone(),
|
allow_drain_fallback: allow_drain_fallback.clone(),
|
||||||
};
|
};
|
||||||
self.writers.write().await.push(writer.clone());
|
self.writers.write().await.push(writer.clone());
|
||||||
@@ -169,6 +224,7 @@ impl MePool {
|
|||||||
stats_reader,
|
stats_reader,
|
||||||
writer_id,
|
writer_id,
|
||||||
degraded.clone(),
|
degraded.clone(),
|
||||||
|
rtt_ema_ms_x10.clone(),
|
||||||
cancel_reader_token.clone(),
|
cancel_reader_token.clone(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -254,17 +310,47 @@ impl MePool {
|
|||||||
p.extend_from_slice(&sent_id.to_le_bytes());
|
p.extend_from_slice(&sent_id.to_le_bytes());
|
||||||
{
|
{
|
||||||
let mut tracker = ping_tracker_ping.lock().await;
|
let mut tracker = ping_tracker_ping.lock().await;
|
||||||
let before = tracker.len();
|
let now_epoch_ms = std::time::SystemTime::now()
|
||||||
tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120));
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
let expired = before.saturating_sub(tracker.len());
|
.unwrap_or_default()
|
||||||
if expired > 0 {
|
.as_millis() as u64;
|
||||||
stats_ping.increment_me_keepalive_timeout_by(expired as u64);
|
let mut run_cleanup = false;
|
||||||
|
if let Some(pool) = pool_ping.upgrade() {
|
||||||
|
let last_cleanup_ms = pool
|
||||||
|
.ping_tracker_last_cleanup_epoch_ms
|
||||||
|
.load(Ordering::Relaxed);
|
||||||
|
if now_epoch_ms.saturating_sub(last_cleanup_ms) >= 30_000
|
||||||
|
&& pool
|
||||||
|
.ping_tracker_last_cleanup_epoch_ms
|
||||||
|
.compare_exchange(
|
||||||
|
last_cleanup_ms,
|
||||||
|
now_epoch_ms,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
run_cleanup = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if run_cleanup {
|
||||||
|
let before = tracker.len();
|
||||||
|
tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120));
|
||||||
|
let expired = before.saturating_sub(tracker.len());
|
||||||
|
if expired > 0 {
|
||||||
|
stats_ping.increment_me_keepalive_timeout_by(expired as u64);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
tracker.insert(sent_id, (std::time::Instant::now(), writer_id));
|
tracker.insert(sent_id, (std::time::Instant::now(), writer_id));
|
||||||
}
|
}
|
||||||
ping_id = ping_id.wrapping_add(1);
|
ping_id = ping_id.wrapping_add(1);
|
||||||
stats_ping.increment_me_keepalive_sent();
|
stats_ping.increment_me_keepalive_sent();
|
||||||
if tx_ping.send(WriterCommand::DataAndFlush(p)).await.is_err() {
|
if tx_ping
|
||||||
|
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
stats_ping.increment_me_keepalive_failed();
|
stats_ping.increment_me_keepalive_failed();
|
||||||
debug!("ME ping failed, removing dead writer");
|
debug!("ME ping failed, removing dead writer");
|
||||||
cancel_ping.cancel();
|
cancel_ping.cancel();
|
||||||
@@ -338,7 +424,11 @@ impl MePool {
|
|||||||
meta.proto_flags,
|
meta.proto_flags,
|
||||||
);
|
);
|
||||||
|
|
||||||
if tx_signal.send(WriterCommand::DataAndFlush(payload)).await.is_err() {
|
if tx_signal
|
||||||
|
.send(WriterCommand::DataAndFlush(payload))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_failed_total();
|
stats_signal.increment_me_rpc_proxy_req_signal_failed_total();
|
||||||
let _ = pool.registry.unregister(conn_id).await;
|
let _ = pool.registry.unregister(conn_id).await;
|
||||||
cancel_signal.cancel();
|
cancel_signal.cancel();
|
||||||
@@ -369,7 +459,7 @@ impl MePool {
|
|||||||
close_payload.extend_from_slice(&conn_id.to_le_bytes());
|
close_payload.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
|
|
||||||
if tx_signal
|
if tx_signal
|
||||||
.send(WriterCommand::DataAndFlush(close_payload))
|
.send(WriterCommand::DataAndFlush(Bytes::from(close_payload)))
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
@@ -404,6 +494,7 @@ impl MePool {
|
|||||||
async fn remove_writer_only(self: &Arc<Self>, writer_id: u64) -> Vec<BoundConn> {
|
async fn remove_writer_only(self: &Arc<Self>, writer_id: u64) -> Vec<BoundConn> {
|
||||||
let mut close_tx: Option<mpsc::Sender<WriterCommand>> = None;
|
let mut close_tx: Option<mpsc::Sender<WriterCommand>> = None;
|
||||||
let mut removed_addr: Option<SocketAddr> = None;
|
let mut removed_addr: Option<SocketAddr> = None;
|
||||||
|
let mut removed_dc: Option<i32> = None;
|
||||||
let mut removed_uptime: Option<Duration> = None;
|
let mut removed_uptime: Option<Duration> = None;
|
||||||
let mut trigger_refill = false;
|
let mut trigger_refill = false;
|
||||||
{
|
{
|
||||||
@@ -417,6 +508,7 @@ impl MePool {
|
|||||||
self.stats.increment_me_writer_removed_total();
|
self.stats.increment_me_writer_removed_total();
|
||||||
w.cancel.cancel();
|
w.cancel.cancel();
|
||||||
removed_addr = Some(w.addr);
|
removed_addr = Some(w.addr);
|
||||||
|
removed_dc = Some(w.writer_dc);
|
||||||
removed_uptime = Some(w.created_at.elapsed());
|
removed_uptime = Some(w.created_at.elapsed());
|
||||||
trigger_refill = !was_draining;
|
trigger_refill = !was_draining;
|
||||||
if trigger_refill {
|
if trigger_refill {
|
||||||
@@ -431,11 +523,12 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
if trigger_refill
|
if trigger_refill
|
||||||
&& let Some(addr) = removed_addr
|
&& let Some(addr) = removed_addr
|
||||||
|
&& let Some(writer_dc) = removed_dc
|
||||||
{
|
{
|
||||||
if let Some(uptime) = removed_uptime {
|
if let Some(uptime) = removed_uptime {
|
||||||
self.maybe_quarantine_flapping_endpoint(addr, uptime).await;
|
self.maybe_quarantine_flapping_endpoint(addr, uptime).await;
|
||||||
}
|
}
|
||||||
self.trigger_immediate_refill(addr);
|
self.trigger_immediate_refill_for_dc(addr, writer_dc);
|
||||||
}
|
}
|
||||||
self.rtt_stats.lock().await.remove(&writer_id);
|
self.rtt_stats.lock().await.remove(&writer_id);
|
||||||
self.registry.writer_lost(writer_id).await
|
self.registry.writer_lost(writer_id).await
|
||||||
@@ -454,8 +547,14 @@ impl MePool {
|
|||||||
let already_draining = w.draining.swap(true, Ordering::Relaxed);
|
let already_draining = w.draining.swap(true, Ordering::Relaxed);
|
||||||
w.allow_drain_fallback
|
w.allow_drain_fallback
|
||||||
.store(allow_drain_fallback, Ordering::Relaxed);
|
.store(allow_drain_fallback, Ordering::Relaxed);
|
||||||
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
w.draining_started_at_epoch_secs
|
w.draining_started_at_epoch_secs
|
||||||
.store(Self::now_epoch_secs(), Ordering::Relaxed);
|
.store(now_epoch_secs, Ordering::Relaxed);
|
||||||
|
let drain_deadline_epoch_secs = timeout
|
||||||
|
.map(|duration| now_epoch_secs.saturating_add(duration.as_secs()))
|
||||||
|
.unwrap_or(0);
|
||||||
|
w.drain_deadline_epoch_secs
|
||||||
|
.store(drain_deadline_epoch_secs, Ordering::Relaxed);
|
||||||
if !already_draining {
|
if !already_draining {
|
||||||
self.stats.increment_pool_drain_active();
|
self.stats.increment_pool_drain_active();
|
||||||
}
|
}
|
||||||
@@ -479,26 +578,6 @@ impl MePool {
|
|||||||
allow_drain_fallback,
|
allow_drain_fallback,
|
||||||
"ME writer marked draining"
|
"ME writer marked draining"
|
||||||
);
|
);
|
||||||
|
|
||||||
let pool = Arc::downgrade(self);
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let deadline = timeout.map(|t| Instant::now() + t);
|
|
||||||
while let Some(p) = pool.upgrade() {
|
|
||||||
if let Some(deadline_at) = deadline
|
|
||||||
&& Instant::now() >= deadline_at
|
|
||||||
{
|
|
||||||
warn!(writer_id, "Drain timeout, force-closing");
|
|
||||||
p.stats.increment_pool_force_close_total();
|
|
||||||
let _ = p.remove_writer_and_close_clients(writer_id).await;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if p.registry.is_writer_empty(writer_id).await {
|
|
||||||
let _ = p.remove_writer_only(writer_id).await;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn mark_writer_draining(self: &Arc<Self>, writer_id: u64) {
|
pub(crate) async fn mark_writer_draining(self: &Arc<Self>, writer_id: u64) {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use bytes::{Bytes, BytesMut};
|
use bytes::{Bytes, BytesMut};
|
||||||
@@ -34,6 +34,7 @@ pub(crate) async fn reader_loop(
|
|||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
_writer_id: u64,
|
_writer_id: u64,
|
||||||
degraded: Arc<AtomicBool>,
|
degraded: Arc<AtomicBool>,
|
||||||
|
writer_rtt_ema_ms_x10: Arc<AtomicU32>,
|
||||||
cancel: CancellationToken,
|
cancel: CancellationToken,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut raw = enc_leftover;
|
let mut raw = enc_leftover;
|
||||||
@@ -181,7 +182,11 @@ pub(crate) async fn reader_loop(
|
|||||||
let mut pong = Vec::with_capacity(12);
|
let mut pong = Vec::with_capacity(12);
|
||||||
pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes());
|
pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes());
|
||||||
pong.extend_from_slice(&ping_id.to_le_bytes());
|
pong.extend_from_slice(&ping_id.to_le_bytes());
|
||||||
if tx.send(WriterCommand::DataAndFlush(pong)).await.is_err() {
|
if tx
|
||||||
|
.send(WriterCommand::DataAndFlush(Bytes::from(pong)))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
warn!("PONG send failed");
|
warn!("PONG send failed");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -204,6 +209,8 @@ pub(crate) async fn reader_loop(
|
|||||||
}
|
}
|
||||||
let degraded_now = entry.1 > entry.0 * 2.0;
|
let degraded_now = entry.1 > entry.0 * 2.0;
|
||||||
degraded.store(degraded_now, Ordering::Relaxed);
|
degraded.store(degraded_now, Ordering::Relaxed);
|
||||||
|
writer_rtt_ema_ms_x10
|
||||||
|
.store((entry.1 * 10.0).clamp(0.0, u32::MAX as f64) as u32, Ordering::Relaxed);
|
||||||
trace!(writer_id = wid, rtt_ms = rtt, ema_ms = entry.1, base_ms = entry.0, degraded = degraded_now, "ME RTT sample");
|
trace!(writer_id = wid, rtt_ms = rtt, ema_ms = entry.1, base_ms = entry.0, degraded = degraded_now, "ME RTT sample");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -222,5 +229,5 @@ async fn send_close_conn(tx: &mpsc::Sender<WriterCommand>, conn_id: u64) {
|
|||||||
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
p.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
|
|
||||||
let _ = tx.send(WriterCommand::DataAndFlush(p)).await;
|
let _ = tx.send(WriterCommand::DataAndFlush(Bytes::from(p))).await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ use tokio::sync::mpsc::error::TrySendError;
|
|||||||
use super::codec::WriterCommand;
|
use super::codec::WriterCommand;
|
||||||
use super::MeResponse;
|
use super::MeResponse;
|
||||||
|
|
||||||
const ROUTE_CHANNEL_CAPACITY: usize = 4096;
|
|
||||||
const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25;
|
const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25;
|
||||||
const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120;
|
const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120;
|
||||||
const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80;
|
const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80;
|
||||||
@@ -78,6 +77,7 @@ impl RegistryInner {
|
|||||||
pub struct ConnRegistry {
|
pub struct ConnRegistry {
|
||||||
inner: RwLock<RegistryInner>,
|
inner: RwLock<RegistryInner>,
|
||||||
next_id: AtomicU64,
|
next_id: AtomicU64,
|
||||||
|
route_channel_capacity: usize,
|
||||||
route_backpressure_base_timeout_ms: AtomicU64,
|
route_backpressure_base_timeout_ms: AtomicU64,
|
||||||
route_backpressure_high_timeout_ms: AtomicU64,
|
route_backpressure_high_timeout_ms: AtomicU64,
|
||||||
route_backpressure_high_watermark_pct: AtomicU8,
|
route_backpressure_high_watermark_pct: AtomicU8,
|
||||||
@@ -91,11 +91,12 @@ impl ConnRegistry {
|
|||||||
.as_secs()
|
.as_secs()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new() -> Self {
|
pub fn with_route_channel_capacity(route_channel_capacity: usize) -> Self {
|
||||||
let start = rand::random::<u64>() | 1;
|
let start = rand::random::<u64>() | 1;
|
||||||
Self {
|
Self {
|
||||||
inner: RwLock::new(RegistryInner::new()),
|
inner: RwLock::new(RegistryInner::new()),
|
||||||
next_id: AtomicU64::new(start),
|
next_id: AtomicU64::new(start),
|
||||||
|
route_channel_capacity: route_channel_capacity.max(1),
|
||||||
route_backpressure_base_timeout_ms: AtomicU64::new(
|
route_backpressure_base_timeout_ms: AtomicU64::new(
|
||||||
ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS,
|
ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS,
|
||||||
),
|
),
|
||||||
@@ -108,6 +109,11 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::with_route_channel_capacity(4096)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn update_route_backpressure_policy(
|
pub fn update_route_backpressure_policy(
|
||||||
&self,
|
&self,
|
||||||
base_timeout_ms: u64,
|
base_timeout_ms: u64,
|
||||||
@@ -127,7 +133,7 @@ impl ConnRegistry {
|
|||||||
|
|
||||||
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
|
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
|
||||||
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
|
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
|
||||||
let (tx, rx) = mpsc::channel(ROUTE_CHANNEL_CAPACITY);
|
let (tx, rx) = mpsc::channel(self.route_channel_capacity);
|
||||||
self.inner.write().await.map.insert(id, tx);
|
self.inner.write().await.map.insert(id, tx);
|
||||||
(id, rx)
|
(id, rx)
|
||||||
}
|
}
|
||||||
@@ -179,11 +185,11 @@ impl ConnRegistry {
|
|||||||
.route_backpressure_high_watermark_pct
|
.route_backpressure_high_watermark_pct
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
.clamp(1, 100);
|
.clamp(1, 100);
|
||||||
let used = ROUTE_CHANNEL_CAPACITY.saturating_sub(tx.capacity());
|
let used = self.route_channel_capacity.saturating_sub(tx.capacity());
|
||||||
let used_pct = if ROUTE_CHANNEL_CAPACITY == 0 {
|
let used_pct = if self.route_channel_capacity == 0 {
|
||||||
100
|
100
|
||||||
} else {
|
} else {
|
||||||
(used.saturating_mul(100) / ROUTE_CHANNEL_CAPACITY) as u8
|
(used.saturating_mul(100) / self.route_channel_capacity) as u8
|
||||||
};
|
};
|
||||||
let high_profile = used_pct >= high_watermark_pct;
|
let high_profile = used_pct >= high_watermark_pct;
|
||||||
let timeout_ms = if high_profile {
|
let timeout_ms = if high_profile {
|
||||||
@@ -264,6 +270,20 @@ impl ConnRegistry {
|
|||||||
inner.writer_idle_since_epoch_secs.clone()
|
inner.writer_idle_since_epoch_secs.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn writer_idle_since_for_writer_ids(
|
||||||
|
&self,
|
||||||
|
writer_ids: &[u64],
|
||||||
|
) -> HashMap<u64, u64> {
|
||||||
|
let inner = self.inner.read().await;
|
||||||
|
let mut out = HashMap::<u64, u64>::with_capacity(writer_ids.len());
|
||||||
|
for writer_id in writer_ids {
|
||||||
|
if let Some(idle_since) = inner.writer_idle_since_epoch_secs.get(writer_id).copied() {
|
||||||
|
out.insert(*writer_id, idle_since);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn writer_activity_snapshot(&self) -> WriterActivitySnapshot {
|
pub(super) async fn writer_activity_snapshot(&self) -> WriterActivitySnapshot {
|
||||||
let inner = self.inner.read().await;
|
let inner = self.inner.read().await;
|
||||||
let mut bound_clients_by_writer = HashMap::<u64, usize>::new();
|
let mut bound_clients_by_writer = HashMap::<u64, usize>::new();
|
||||||
|
|||||||
@@ -5,10 +5,11 @@ use std::sync::Arc;
|
|||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
use tokio::sync::mpsc::error::TrySendError;
|
use tokio::sync::mpsc::error::TrySendError;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use crate::config::MeRouteNoWriterMode;
|
use crate::config::{MeRouteNoWriterMode, MeWriterPickMode};
|
||||||
use crate::error::{ProxyError, Result};
|
use crate::error::{ProxyError, Result};
|
||||||
use crate::network::IpFamily;
|
use crate::network::IpFamily;
|
||||||
use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32};
|
use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32};
|
||||||
@@ -23,6 +24,10 @@ use super::registry::ConnMeta;
|
|||||||
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
|
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
|
||||||
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
|
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
|
||||||
const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
|
const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
|
||||||
|
const PICK_PENALTY_WARM: u64 = 200;
|
||||||
|
const PICK_PENALTY_DRAINING: u64 = 600;
|
||||||
|
const PICK_PENALTY_STALE: u64 = 300;
|
||||||
|
const PICK_PENALTY_DEGRADED: u64 = 250;
|
||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
|
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
|
||||||
@@ -53,12 +58,16 @@ impl MePool {
|
|||||||
};
|
};
|
||||||
let no_writer_mode =
|
let no_writer_mode =
|
||||||
MeRouteNoWriterMode::from_u8(self.me_route_no_writer_mode.load(Ordering::Relaxed));
|
MeRouteNoWriterMode::from_u8(self.me_route_no_writer_mode.load(Ordering::Relaxed));
|
||||||
|
let (routed_dc, unknown_target_dc) = self
|
||||||
|
.resolve_target_dc_for_routing(target_dc as i32)
|
||||||
|
.await;
|
||||||
let mut no_writer_deadline: Option<Instant> = None;
|
let mut no_writer_deadline: Option<Instant> = None;
|
||||||
let mut emergency_attempts = 0u32;
|
let mut emergency_attempts = 0u32;
|
||||||
let mut async_recovery_triggered = false;
|
let mut async_recovery_triggered = false;
|
||||||
let mut hybrid_recovery_round = 0u32;
|
let mut hybrid_recovery_round = 0u32;
|
||||||
let mut hybrid_last_recovery_at: Option<Instant> = None;
|
let mut hybrid_last_recovery_at: Option<Instant> = None;
|
||||||
let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50));
|
let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50));
|
||||||
|
let mut hybrid_wait_current = hybrid_wait_step;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
if let Some(current) = self.registry.get_writer(conn_id).await {
|
if let Some(current) = self.registry.get_writer(conn_id).await {
|
||||||
@@ -89,9 +98,9 @@ impl MePool {
|
|||||||
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
||||||
Instant::now() + self.me_route_no_writer_wait
|
Instant::now() + self.me_route_no_writer_wait
|
||||||
});
|
});
|
||||||
if !async_recovery_triggered {
|
if !async_recovery_triggered && !unknown_target_dc {
|
||||||
let triggered =
|
let triggered =
|
||||||
self.trigger_async_recovery_for_target_dc(target_dc).await;
|
self.trigger_async_recovery_for_target_dc(routed_dc).await;
|
||||||
if !triggered {
|
if !triggered {
|
||||||
self.trigger_async_recovery_global().await;
|
self.trigger_async_recovery_global().await;
|
||||||
}
|
}
|
||||||
@@ -107,31 +116,34 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
MeRouteNoWriterMode::InlineRecoveryLegacy => {
|
MeRouteNoWriterMode::InlineRecoveryLegacy => {
|
||||||
self.stats.increment_me_inline_recovery_total();
|
self.stats.increment_me_inline_recovery_total();
|
||||||
for _ in 0..self.me_route_inline_recovery_attempts.max(1) {
|
if !unknown_target_dc {
|
||||||
for family in self.family_order() {
|
for _ in 0..self.me_route_inline_recovery_attempts.max(1) {
|
||||||
let map = match family {
|
for family in self.family_order() {
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await.clone(),
|
let map = match family {
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await.clone(),
|
IpFamily::V4 => self.proxy_map_v4.read().await.clone(),
|
||||||
};
|
IpFamily::V6 => self.proxy_map_v6.read().await.clone(),
|
||||||
for (_dc, addrs) in &map {
|
};
|
||||||
for (ip, port) in addrs {
|
for (dc, addrs) in &map {
|
||||||
let addr = SocketAddr::new(*ip, *port);
|
for (ip, port) in addrs {
|
||||||
let _ = self.connect_one(addr, self.rng.as_ref()).await;
|
let addr = SocketAddr::new(*ip, *port);
|
||||||
|
let _ = self
|
||||||
|
.connect_one_for_dc(addr, *dc, self.rng.as_ref())
|
||||||
|
.await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
if !self.writers.read().await.is_empty() {
|
||||||
if !self.writers.read().await.is_empty() {
|
break;
|
||||||
break;
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !self.writers.read().await.is_empty() {
|
if !self.writers.read().await.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let waiter = self.writer_available.notified();
|
let deadline = *no_writer_deadline
|
||||||
if tokio::time::timeout(self.me_route_inline_recovery_wait, waiter)
|
.get_or_insert_with(|| Instant::now() + self.me_route_inline_recovery_wait);
|
||||||
.await
|
if !self.wait_for_writer_until(deadline).await {
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
if !self.writers.read().await.is_empty() {
|
if !self.writers.read().await.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -143,15 +155,20 @@ impl MePool {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
||||||
self.maybe_trigger_hybrid_recovery(
|
if !unknown_target_dc {
|
||||||
target_dc,
|
self.maybe_trigger_hybrid_recovery(
|
||||||
&mut hybrid_recovery_round,
|
routed_dc,
|
||||||
&mut hybrid_last_recovery_at,
|
&mut hybrid_recovery_round,
|
||||||
hybrid_wait_step,
|
&mut hybrid_last_recovery_at,
|
||||||
)
|
hybrid_wait_current,
|
||||||
.await;
|
)
|
||||||
let deadline = Instant::now() + hybrid_wait_step;
|
.await;
|
||||||
|
}
|
||||||
|
let deadline = Instant::now() + hybrid_wait_current;
|
||||||
let _ = self.wait_for_writer_until(deadline).await;
|
let _ = self.wait_for_writer_until(deadline).await;
|
||||||
|
hybrid_wait_current =
|
||||||
|
(hybrid_wait_current.saturating_mul(2))
|
||||||
|
.min(Duration::from_millis(400));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -160,29 +177,31 @@ impl MePool {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let mut candidate_indices = self
|
let mut candidate_indices = self
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
.candidate_indices_for_dc(&writers_snapshot, routed_dc, false)
|
||||||
.await;
|
.await;
|
||||||
if candidate_indices.is_empty() {
|
if candidate_indices.is_empty() {
|
||||||
candidate_indices = self
|
candidate_indices = self
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
.candidate_indices_for_dc(&writers_snapshot, routed_dc, true)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
if candidate_indices.is_empty() {
|
if candidate_indices.is_empty() {
|
||||||
|
let pick_mode = self.writer_pick_mode();
|
||||||
match no_writer_mode {
|
match no_writer_mode {
|
||||||
MeRouteNoWriterMode::AsyncRecoveryFailfast => {
|
MeRouteNoWriterMode::AsyncRecoveryFailfast => {
|
||||||
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
||||||
Instant::now() + self.me_route_no_writer_wait
|
Instant::now() + self.me_route_no_writer_wait
|
||||||
});
|
});
|
||||||
if !async_recovery_triggered {
|
if !async_recovery_triggered && !unknown_target_dc {
|
||||||
let triggered = self.trigger_async_recovery_for_target_dc(target_dc).await;
|
let triggered = self.trigger_async_recovery_for_target_dc(routed_dc).await;
|
||||||
if !triggered {
|
if !triggered {
|
||||||
self.trigger_async_recovery_global().await;
|
self.trigger_async_recovery_global().await;
|
||||||
}
|
}
|
||||||
async_recovery_triggered = true;
|
async_recovery_triggered = true;
|
||||||
}
|
}
|
||||||
if self.wait_for_candidate_until(target_dc, deadline).await {
|
if self.wait_for_candidate_until(routed_dc, deadline).await {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
|
||||||
self.stats.increment_me_no_writer_failfast_total();
|
self.stats.increment_me_no_writer_failfast_total();
|
||||||
return Err(ProxyError::Proxy(
|
return Err(ProxyError::Proxy(
|
||||||
"No ME writers available for target DC in failfast window".into(),
|
"No ME writers available for target DC in failfast window".into(),
|
||||||
@@ -190,15 +209,26 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
MeRouteNoWriterMode::InlineRecoveryLegacy => {
|
MeRouteNoWriterMode::InlineRecoveryLegacy => {
|
||||||
self.stats.increment_me_inline_recovery_total();
|
self.stats.increment_me_inline_recovery_total();
|
||||||
|
if unknown_target_dc {
|
||||||
|
let deadline = *no_writer_deadline
|
||||||
|
.get_or_insert_with(|| Instant::now() + self.me_route_inline_recovery_wait);
|
||||||
|
if self.wait_for_candidate_until(routed_dc, deadline).await {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
|
||||||
|
self.stats.increment_me_no_writer_failfast_total();
|
||||||
|
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
||||||
|
}
|
||||||
if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) {
|
if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) {
|
||||||
|
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
|
||||||
self.stats.increment_me_no_writer_failfast_total();
|
self.stats.increment_me_no_writer_failfast_total();
|
||||||
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
||||||
}
|
}
|
||||||
emergency_attempts += 1;
|
emergency_attempts += 1;
|
||||||
let mut endpoints = self.endpoint_candidates_for_target_dc(target_dc).await;
|
let mut endpoints = self.endpoint_candidates_for_target_dc(routed_dc).await;
|
||||||
endpoints.shuffle(&mut rand::rng());
|
endpoints.shuffle(&mut rand::rng());
|
||||||
for addr in endpoints {
|
for addr in endpoints {
|
||||||
if self.connect_one(addr, self.rng.as_ref()).await.is_ok() {
|
if self.connect_one_for_dc(addr, routed_dc, self.rng.as_ref()).await.is_ok() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -207,96 +237,126 @@ impl MePool {
|
|||||||
writers_snapshot = ws2.clone();
|
writers_snapshot = ws2.clone();
|
||||||
drop(ws2);
|
drop(ws2);
|
||||||
candidate_indices = self
|
candidate_indices = self
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
.candidate_indices_for_dc(&writers_snapshot, routed_dc, false)
|
||||||
.await;
|
.await;
|
||||||
if candidate_indices.is_empty() {
|
if candidate_indices.is_empty() {
|
||||||
candidate_indices = self
|
candidate_indices = self
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
.candidate_indices_for_dc(&writers_snapshot, routed_dc, true)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
if candidate_indices.is_empty() {
|
if candidate_indices.is_empty() {
|
||||||
|
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
|
||||||
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
||||||
self.maybe_trigger_hybrid_recovery(
|
if !unknown_target_dc {
|
||||||
target_dc,
|
self.maybe_trigger_hybrid_recovery(
|
||||||
&mut hybrid_recovery_round,
|
routed_dc,
|
||||||
&mut hybrid_last_recovery_at,
|
&mut hybrid_recovery_round,
|
||||||
hybrid_wait_step,
|
&mut hybrid_last_recovery_at,
|
||||||
)
|
hybrid_wait_current,
|
||||||
.await;
|
)
|
||||||
let deadline = Instant::now() + hybrid_wait_step;
|
.await;
|
||||||
let _ = self.wait_for_candidate_until(target_dc, deadline).await;
|
}
|
||||||
|
let deadline = Instant::now() + hybrid_wait_current;
|
||||||
|
let _ = self.wait_for_candidate_until(routed_dc, deadline).await;
|
||||||
|
hybrid_wait_current = (hybrid_wait_current.saturating_mul(2))
|
||||||
|
.min(Duration::from_millis(400));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let writer_idle_since = self.registry.writer_idle_since_snapshot().await;
|
hybrid_wait_current = hybrid_wait_step;
|
||||||
|
let pick_mode = self.writer_pick_mode();
|
||||||
|
let pick_sample_size = self.writer_pick_sample_size();
|
||||||
|
let writer_ids: Vec<u64> = candidate_indices
|
||||||
|
.iter()
|
||||||
|
.map(|idx| writers_snapshot[*idx].id)
|
||||||
|
.collect();
|
||||||
|
let writer_idle_since = self
|
||||||
|
.registry
|
||||||
|
.writer_idle_since_for_writer_ids(&writer_ids)
|
||||||
|
.await;
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
|
|
||||||
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
|
|
||||||
candidate_indices.sort_by(|lhs, rhs| {
|
|
||||||
let left = &writers_snapshot[*lhs];
|
|
||||||
let right = &writers_snapshot[*rhs];
|
|
||||||
let left_key = (
|
|
||||||
self.writer_contour_rank_for_selection(left),
|
|
||||||
(left.generation < self.current_generation()) as usize,
|
|
||||||
left.degraded.load(Ordering::Relaxed) as usize,
|
|
||||||
self.writer_idle_rank_for_selection(
|
|
||||||
left,
|
|
||||||
&writer_idle_since,
|
|
||||||
now_epoch_secs,
|
|
||||||
),
|
|
||||||
Reverse(left.tx.capacity()),
|
|
||||||
left.addr,
|
|
||||||
left.id,
|
|
||||||
);
|
|
||||||
let right_key = (
|
|
||||||
self.writer_contour_rank_for_selection(right),
|
|
||||||
(right.generation < self.current_generation()) as usize,
|
|
||||||
right.degraded.load(Ordering::Relaxed) as usize,
|
|
||||||
self.writer_idle_rank_for_selection(
|
|
||||||
right,
|
|
||||||
&writer_idle_since,
|
|
||||||
now_epoch_secs,
|
|
||||||
),
|
|
||||||
Reverse(right.tx.capacity()),
|
|
||||||
right.addr,
|
|
||||||
right.id,
|
|
||||||
);
|
|
||||||
left_key.cmp(&right_key)
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
candidate_indices.sort_by_key(|idx| {
|
|
||||||
let w = &writers_snapshot[*idx];
|
|
||||||
let degraded = w.degraded.load(Ordering::Relaxed);
|
|
||||||
let stale = (w.generation < self.current_generation()) as usize;
|
|
||||||
(
|
|
||||||
self.writer_contour_rank_for_selection(w),
|
|
||||||
stale,
|
|
||||||
degraded as usize,
|
|
||||||
self.writer_idle_rank_for_selection(
|
|
||||||
w,
|
|
||||||
&writer_idle_since,
|
|
||||||
now_epoch_secs,
|
|
||||||
),
|
|
||||||
Reverse(w.tx.capacity()),
|
|
||||||
)
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len();
|
let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len();
|
||||||
|
let ordered_candidate_indices = if pick_mode == MeWriterPickMode::P2c {
|
||||||
|
self.p2c_ordered_candidate_indices(
|
||||||
|
&candidate_indices,
|
||||||
|
&writers_snapshot,
|
||||||
|
&writer_idle_since,
|
||||||
|
now_epoch_secs,
|
||||||
|
start,
|
||||||
|
pick_sample_size,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
|
||||||
|
candidate_indices.sort_by(|lhs, rhs| {
|
||||||
|
let left = &writers_snapshot[*lhs];
|
||||||
|
let right = &writers_snapshot[*rhs];
|
||||||
|
let left_key = (
|
||||||
|
self.writer_contour_rank_for_selection(left),
|
||||||
|
(left.generation < self.current_generation()) as usize,
|
||||||
|
left.degraded.load(Ordering::Relaxed) as usize,
|
||||||
|
self.writer_idle_rank_for_selection(
|
||||||
|
left,
|
||||||
|
&writer_idle_since,
|
||||||
|
now_epoch_secs,
|
||||||
|
),
|
||||||
|
Reverse(left.tx.capacity()),
|
||||||
|
left.addr,
|
||||||
|
left.id,
|
||||||
|
);
|
||||||
|
let right_key = (
|
||||||
|
self.writer_contour_rank_for_selection(right),
|
||||||
|
(right.generation < self.current_generation()) as usize,
|
||||||
|
right.degraded.load(Ordering::Relaxed) as usize,
|
||||||
|
self.writer_idle_rank_for_selection(
|
||||||
|
right,
|
||||||
|
&writer_idle_since,
|
||||||
|
now_epoch_secs,
|
||||||
|
),
|
||||||
|
Reverse(right.tx.capacity()),
|
||||||
|
right.addr,
|
||||||
|
right.id,
|
||||||
|
);
|
||||||
|
left_key.cmp(&right_key)
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
candidate_indices.sort_by_key(|idx| {
|
||||||
|
let w = &writers_snapshot[*idx];
|
||||||
|
let degraded = w.degraded.load(Ordering::Relaxed);
|
||||||
|
let stale = (w.generation < self.current_generation()) as usize;
|
||||||
|
(
|
||||||
|
self.writer_contour_rank_for_selection(w),
|
||||||
|
stale,
|
||||||
|
degraded as usize,
|
||||||
|
self.writer_idle_rank_for_selection(
|
||||||
|
w,
|
||||||
|
&writer_idle_since,
|
||||||
|
now_epoch_secs,
|
||||||
|
),
|
||||||
|
Reverse(w.tx.capacity()),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut ordered = Vec::<usize>::with_capacity(candidate_indices.len());
|
||||||
|
for offset in 0..candidate_indices.len() {
|
||||||
|
ordered.push(candidate_indices[(start + offset) % candidate_indices.len()]);
|
||||||
|
}
|
||||||
|
ordered
|
||||||
|
};
|
||||||
let mut fallback_blocking_idx: Option<usize> = None;
|
let mut fallback_blocking_idx: Option<usize> = None;
|
||||||
|
|
||||||
for offset in 0..candidate_indices.len() {
|
for idx in ordered_candidate_indices {
|
||||||
let idx = candidate_indices[(start + offset) % candidate_indices.len()];
|
|
||||||
let w = &writers_snapshot[idx];
|
let w = &writers_snapshot[idx];
|
||||||
if !self.writer_accepts_new_binding(w) {
|
if !self.writer_accepts_new_binding(w) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
match w.tx.try_send(WriterCommand::Data(payload.clone())) {
|
match w.tx.try_send(WriterCommand::Data(payload.clone())) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
|
self.stats.increment_me_writer_pick_success_try_total(pick_mode);
|
||||||
self.registry
|
self.registry
|
||||||
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
|
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
|
||||||
.await;
|
.await;
|
||||||
@@ -318,6 +378,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(TrySendError::Closed(_)) => {
|
Err(TrySendError::Closed(_)) => {
|
||||||
|
self.stats.increment_me_writer_pick_closed_total(pick_mode);
|
||||||
warn!(writer_id = w.id, "ME writer channel closed");
|
warn!(writer_id = w.id, "ME writer channel closed");
|
||||||
self.remove_writer_and_close_clients(w.id).await;
|
self.remove_writer_and_close_clients(w.id).await;
|
||||||
continue;
|
continue;
|
||||||
@@ -326,15 +387,20 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let Some(blocking_idx) = fallback_blocking_idx else {
|
let Some(blocking_idx) = fallback_blocking_idx else {
|
||||||
|
self.stats.increment_me_writer_pick_full_total(pick_mode);
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
||||||
let w = writers_snapshot[blocking_idx].clone();
|
let w = writers_snapshot[blocking_idx].clone();
|
||||||
if !self.writer_accepts_new_binding(&w) {
|
if !self.writer_accepts_new_binding(&w) {
|
||||||
|
self.stats.increment_me_writer_pick_full_total(pick_mode);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
self.stats.increment_me_writer_pick_blocking_fallback_total();
|
||||||
match w.tx.send(WriterCommand::Data(payload.clone())).await {
|
match w.tx.send(WriterCommand::Data(payload.clone())).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
|
self.stats
|
||||||
|
.increment_me_writer_pick_success_fallback_total(pick_mode);
|
||||||
self.registry
|
self.registry
|
||||||
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
|
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
|
||||||
.await;
|
.await;
|
||||||
@@ -344,6 +410,7 @@ impl MePool {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
|
self.stats.increment_me_writer_pick_closed_total(pick_mode);
|
||||||
warn!(writer_id = w.id, "ME writer channel closed (blocking)");
|
warn!(writer_id = w.id, "ME writer channel closed (blocking)");
|
||||||
self.remove_writer_and_close_clients(w.id).await;
|
self.remove_writer_and_close_clients(w.id).await;
|
||||||
}
|
}
|
||||||
@@ -367,32 +434,32 @@ impl MePool {
|
|||||||
!self.writers.read().await.is_empty()
|
!self.writers.read().await.is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_candidate_until(&self, target_dc: i16, deadline: Instant) -> bool {
|
async fn wait_for_candidate_until(&self, routed_dc: i32, deadline: Instant) -> bool {
|
||||||
loop {
|
loop {
|
||||||
if self.has_candidate_for_target_dc(target_dc).await {
|
if self.has_candidate_for_target_dc(routed_dc).await {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
if now >= deadline {
|
if now >= deadline {
|
||||||
return self.has_candidate_for_target_dc(target_dc).await;
|
return self.has_candidate_for_target_dc(routed_dc).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let waiter = self.writer_available.notified();
|
let waiter = self.writer_available.notified();
|
||||||
if self.has_candidate_for_target_dc(target_dc).await {
|
if self.has_candidate_for_target_dc(routed_dc).await {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||||
if remaining.is_zero() {
|
if remaining.is_zero() {
|
||||||
return self.has_candidate_for_target_dc(target_dc).await;
|
return self.has_candidate_for_target_dc(routed_dc).await;
|
||||||
}
|
}
|
||||||
if tokio::time::timeout(remaining, waiter).await.is_err() {
|
if tokio::time::timeout(remaining, waiter).await.is_err() {
|
||||||
return self.has_candidate_for_target_dc(target_dc).await;
|
return self.has_candidate_for_target_dc(routed_dc).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn has_candidate_for_target_dc(&self, target_dc: i16) -> bool {
|
async fn has_candidate_for_target_dc(&self, routed_dc: i32) -> bool {
|
||||||
let writers_snapshot = {
|
let writers_snapshot = {
|
||||||
let ws = self.writers.read().await;
|
let ws = self.writers.read().await;
|
||||||
if ws.is_empty() {
|
if ws.is_empty() {
|
||||||
@@ -401,41 +468,41 @@ impl MePool {
|
|||||||
ws.clone()
|
ws.clone()
|
||||||
};
|
};
|
||||||
let mut candidate_indices = self
|
let mut candidate_indices = self
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, false)
|
.candidate_indices_for_dc(&writers_snapshot, routed_dc, false)
|
||||||
.await;
|
.await;
|
||||||
if candidate_indices.is_empty() {
|
if candidate_indices.is_empty() {
|
||||||
candidate_indices = self
|
candidate_indices = self
|
||||||
.candidate_indices_for_dc(&writers_snapshot, target_dc, true)
|
.candidate_indices_for_dc(&writers_snapshot, routed_dc, true)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
!candidate_indices.is_empty()
|
!candidate_indices.is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn trigger_async_recovery_for_target_dc(self: &Arc<Self>, target_dc: i16) -> bool {
|
async fn trigger_async_recovery_for_target_dc(self: &Arc<Self>, routed_dc: i32) -> bool {
|
||||||
let endpoints = self.endpoint_candidates_for_target_dc(target_dc).await;
|
let endpoints = self.endpoint_candidates_for_target_dc(routed_dc).await;
|
||||||
if endpoints.is_empty() {
|
if endpoints.is_empty() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
self.stats.increment_me_async_recovery_trigger_total();
|
self.stats.increment_me_async_recovery_trigger_total();
|
||||||
for addr in endpoints.into_iter().take(8) {
|
for addr in endpoints.into_iter().take(8) {
|
||||||
self.trigger_immediate_refill(addr);
|
self.trigger_immediate_refill_for_dc(addr, routed_dc);
|
||||||
}
|
}
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn trigger_async_recovery_global(self: &Arc<Self>) {
|
async fn trigger_async_recovery_global(self: &Arc<Self>) {
|
||||||
self.stats.increment_me_async_recovery_trigger_total();
|
self.stats.increment_me_async_recovery_trigger_total();
|
||||||
let mut seen = HashSet::<SocketAddr>::new();
|
let mut seen = HashSet::<(i32, SocketAddr)>::new();
|
||||||
for family in self.family_order() {
|
for family in self.family_order() {
|
||||||
let map_guard = match family {
|
let map_guard = match family {
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await,
|
IpFamily::V4 => self.proxy_map_v4.read().await,
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await,
|
IpFamily::V6 => self.proxy_map_v6.read().await,
|
||||||
};
|
};
|
||||||
for addrs in map_guard.values() {
|
for (dc, addrs) in map_guard.iter() {
|
||||||
for (ip, port) in addrs {
|
for (ip, port) in addrs {
|
||||||
let addr = SocketAddr::new(*ip, *port);
|
let addr = SocketAddr::new(*ip, *port);
|
||||||
if seen.insert(addr) {
|
if seen.insert((*dc, addr)) {
|
||||||
self.trigger_immediate_refill(addr);
|
self.trigger_immediate_refill_for_dc(addr, *dc);
|
||||||
}
|
}
|
||||||
if seen.len() >= 8 {
|
if seen.len() >= 8 {
|
||||||
return;
|
return;
|
||||||
@@ -445,44 +512,13 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn endpoint_candidates_for_target_dc(&self, target_dc: i16) -> Vec<SocketAddr> {
|
async fn endpoint_candidates_for_target_dc(&self, routed_dc: i32) -> Vec<SocketAddr> {
|
||||||
let key = target_dc as i32;
|
self.preferred_endpoints_for_dc(routed_dc).await
|
||||||
let mut preferred = Vec::<SocketAddr>::new();
|
|
||||||
let mut seen = HashSet::<SocketAddr>::new();
|
|
||||||
let lookup_keys = self.dc_lookup_chain_for_target(key);
|
|
||||||
|
|
||||||
for family in self.family_order() {
|
|
||||||
let map_guard = match family {
|
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await,
|
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await,
|
|
||||||
};
|
|
||||||
let mut family_selected = Vec::<SocketAddr>::new();
|
|
||||||
for lookup in lookup_keys.iter().copied() {
|
|
||||||
if let Some(addrs) = map_guard.get(&lookup) {
|
|
||||||
for (ip, port) in addrs {
|
|
||||||
family_selected.push(SocketAddr::new(*ip, *port));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !family_selected.is_empty() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for addr in family_selected {
|
|
||||||
if seen.insert(addr) {
|
|
||||||
preferred.push(addr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !preferred.is_empty() && !self.decision.effective_multipath {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
preferred
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn maybe_trigger_hybrid_recovery(
|
async fn maybe_trigger_hybrid_recovery(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
target_dc: i16,
|
routed_dc: i32,
|
||||||
hybrid_recovery_round: &mut u32,
|
hybrid_recovery_round: &mut u32,
|
||||||
hybrid_last_recovery_at: &mut Option<Instant>,
|
hybrid_last_recovery_at: &mut Option<Instant>,
|
||||||
hybrid_wait_step: Duration,
|
hybrid_wait_step: Duration,
|
||||||
@@ -494,7 +530,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let round = *hybrid_recovery_round;
|
let round = *hybrid_recovery_round;
|
||||||
let target_triggered = self.trigger_async_recovery_for_target_dc(target_dc).await;
|
let target_triggered = self.trigger_async_recovery_for_target_dc(routed_dc).await;
|
||||||
if !target_triggered || round % HYBRID_GLOBAL_BURST_PERIOD_ROUNDS == 0 {
|
if !target_triggered || round % HYBRID_GLOBAL_BURST_PERIOD_ROUNDS == 0 {
|
||||||
self.trigger_async_recovery_global().await;
|
self.trigger_async_recovery_global().await;
|
||||||
}
|
}
|
||||||
@@ -507,7 +543,11 @@ impl MePool {
|
|||||||
let mut p = Vec::with_capacity(12);
|
let mut p = Vec::with_capacity(12);
|
||||||
p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
p.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
if w.tx.send(WriterCommand::DataAndFlush(p)).await.is_err() {
|
if w.tx
|
||||||
|
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
debug!("ME close write failed");
|
debug!("ME close write failed");
|
||||||
self.remove_writer_and_close_clients(w.writer_id).await;
|
self.remove_writer_and_close_clients(w.writer_id).await;
|
||||||
}
|
}
|
||||||
@@ -524,7 +564,7 @@ impl MePool {
|
|||||||
let mut p = Vec::with_capacity(12);
|
let mut p = Vec::with_capacity(12);
|
||||||
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
p.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
match w.tx.try_send(WriterCommand::DataAndFlush(p)) {
|
match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(TrySendError::Full(cmd)) => {
|
Err(TrySendError::Full(cmd)) => {
|
||||||
let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await;
|
let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await;
|
||||||
@@ -557,38 +597,10 @@ impl MePool {
|
|||||||
pub(super) async fn candidate_indices_for_dc(
|
pub(super) async fn candidate_indices_for_dc(
|
||||||
&self,
|
&self,
|
||||||
writers: &[super::pool::MeWriter],
|
writers: &[super::pool::MeWriter],
|
||||||
target_dc: i16,
|
routed_dc: i32,
|
||||||
include_warm: bool,
|
include_warm: bool,
|
||||||
) -> Vec<usize> {
|
) -> Vec<usize> {
|
||||||
let key = target_dc as i32;
|
let preferred = self.preferred_endpoints_for_dc(routed_dc).await;
|
||||||
let mut preferred = HashSet::<SocketAddr>::new();
|
|
||||||
let lookup_keys = self.dc_lookup_chain_for_target(key);
|
|
||||||
|
|
||||||
for family in self.family_order() {
|
|
||||||
let map_guard = match family {
|
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await,
|
|
||||||
IpFamily::V6 => self.proxy_map_v6.read().await,
|
|
||||||
};
|
|
||||||
let mut family_selected = Vec::<SocketAddr>::new();
|
|
||||||
for lookup in lookup_keys.iter().copied() {
|
|
||||||
if let Some(v) = map_guard.get(&lookup) {
|
|
||||||
family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
|
|
||||||
}
|
|
||||||
if !family_selected.is_empty() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for endpoint in family_selected {
|
|
||||||
preferred.insert(endpoint);
|
|
||||||
}
|
|
||||||
|
|
||||||
drop(map_guard);
|
|
||||||
|
|
||||||
if !preferred.is_empty() && !self.decision.effective_multipath {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if preferred.is_empty() {
|
if preferred.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
@@ -598,7 +610,7 @@ impl MePool {
|
|||||||
if !self.writer_eligible_for_selection(w, include_warm) {
|
if !self.writer_eligible_for_selection(w, include_warm) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if preferred.contains(&w.addr) {
|
if w.writer_dc == routed_dc && preferred.iter().any(|endpoint| *endpoint == w.addr) {
|
||||||
out.push(idx);
|
out.push(idx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -647,4 +659,87 @@ impl MePool {
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn writer_pick_score(
|
||||||
|
&self,
|
||||||
|
writer: &super::pool::MeWriter,
|
||||||
|
idle_since_by_writer: &HashMap<u64, u64>,
|
||||||
|
now_epoch_secs: u64,
|
||||||
|
) -> u64 {
|
||||||
|
let contour_penalty = match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
|
||||||
|
WriterContour::Active => 0,
|
||||||
|
WriterContour::Warm => PICK_PENALTY_WARM,
|
||||||
|
WriterContour::Draining => PICK_PENALTY_DRAINING,
|
||||||
|
};
|
||||||
|
let stale_penalty = if writer.generation < self.current_generation() {
|
||||||
|
PICK_PENALTY_STALE
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let degraded_penalty = if writer.degraded.load(Ordering::Relaxed) {
|
||||||
|
PICK_PENALTY_DEGRADED
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let idle_penalty =
|
||||||
|
(self.writer_idle_rank_for_selection(writer, idle_since_by_writer, now_epoch_secs) as u64)
|
||||||
|
* 100;
|
||||||
|
let queue_cap = self.writer_cmd_channel_capacity.max(1) as u64;
|
||||||
|
let queue_remaining = writer.tx.capacity() as u64;
|
||||||
|
let queue_used = queue_cap.saturating_sub(queue_remaining.min(queue_cap));
|
||||||
|
let queue_util_pct = queue_used.saturating_mul(100) / queue_cap;
|
||||||
|
let queue_penalty = queue_util_pct.saturating_mul(4);
|
||||||
|
let rtt_penalty = ((writer.rtt_ema_ms_x10.load(Ordering::Relaxed) as u64).saturating_add(5) / 10)
|
||||||
|
.min(400);
|
||||||
|
|
||||||
|
contour_penalty
|
||||||
|
.saturating_add(stale_penalty)
|
||||||
|
.saturating_add(degraded_penalty)
|
||||||
|
.saturating_add(idle_penalty)
|
||||||
|
.saturating_add(queue_penalty)
|
||||||
|
.saturating_add(rtt_penalty)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn p2c_ordered_candidate_indices(
|
||||||
|
&self,
|
||||||
|
candidate_indices: &[usize],
|
||||||
|
writers_snapshot: &[super::pool::MeWriter],
|
||||||
|
idle_since_by_writer: &HashMap<u64, u64>,
|
||||||
|
now_epoch_secs: u64,
|
||||||
|
start: usize,
|
||||||
|
sample_size: usize,
|
||||||
|
) -> Vec<usize> {
|
||||||
|
let total = candidate_indices.len();
|
||||||
|
if total == 0 {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut sampled = Vec::<usize>::with_capacity(sample_size.min(total));
|
||||||
|
let mut seen = HashSet::<usize>::with_capacity(total);
|
||||||
|
for offset in 0..sample_size.min(total) {
|
||||||
|
let idx = candidate_indices[(start + offset) % total];
|
||||||
|
if seen.insert(idx) {
|
||||||
|
sampled.push(idx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sampled.sort_by_key(|idx| {
|
||||||
|
let writer = &writers_snapshot[*idx];
|
||||||
|
(
|
||||||
|
self.writer_pick_score(writer, idle_since_by_writer, now_epoch_secs),
|
||||||
|
writer.addr,
|
||||||
|
writer.id,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut ordered = Vec::<usize>::with_capacity(total);
|
||||||
|
ordered.extend(sampled.iter().copied());
|
||||||
|
for offset in 0..total {
|
||||||
|
let idx = candidate_indices[(start + offset) % total];
|
||||||
|
if seen.insert(idx) {
|
||||||
|
ordered.push(idx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ordered
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
use crate::protocol::constants::*;
|
use crate::protocol::constants::*;
|
||||||
|
|
||||||
@@ -48,7 +49,7 @@ pub(crate) fn build_proxy_req_payload(
|
|||||||
data: &[u8],
|
data: &[u8],
|
||||||
proxy_tag: Option<&[u8]>,
|
proxy_tag: Option<&[u8]>,
|
||||||
proto_flags: u32,
|
proto_flags: u32,
|
||||||
) -> Vec<u8> {
|
) -> Bytes {
|
||||||
let mut b = Vec::with_capacity(128 + data.len());
|
let mut b = Vec::with_capacity(128 + data.len());
|
||||||
|
|
||||||
b.extend_from_slice(&RPC_PROXY_REQ_U32.to_le_bytes());
|
b.extend_from_slice(&RPC_PROXY_REQ_U32.to_le_bytes());
|
||||||
@@ -85,7 +86,7 @@ pub(crate) fn build_proxy_req_payload(
|
|||||||
}
|
}
|
||||||
|
|
||||||
b.extend_from_slice(data);
|
b.extend_from_slice(data);
|
||||||
b
|
Bytes::from(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn proto_flags_for_tag(tag: crate::protocol::constants::ProtoTag, has_proxy_tag: bool) -> u32 {
|
pub fn proto_flags_for_tag(tag: crate::protocol::constants::ProtoTag, has_proxy_tag: bool) -> u32 {
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
use std::collections::{BTreeSet, HashMap};
|
use std::collections::{BTreeSet, HashMap};
|
||||||
use std::net::{SocketAddr, IpAddr};
|
use std::net::{SocketAddr, IpAddr};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::net::TcpStream;
|
use tokio::net::TcpStream;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
@@ -237,6 +237,8 @@ pub struct UpstreamManager {
|
|||||||
connect_budget: Duration,
|
connect_budget: Duration,
|
||||||
unhealthy_fail_threshold: u32,
|
unhealthy_fail_threshold: u32,
|
||||||
connect_failfast_hard_errors: bool,
|
connect_failfast_hard_errors: bool,
|
||||||
|
no_upstreams_warn_epoch_ms: Arc<AtomicU64>,
|
||||||
|
no_healthy_warn_epoch_ms: Arc<AtomicU64>,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,10 +264,35 @@ impl UpstreamManager {
|
|||||||
connect_budget: Duration::from_millis(connect_budget_ms.max(1)),
|
connect_budget: Duration::from_millis(connect_budget_ms.max(1)),
|
||||||
unhealthy_fail_threshold: unhealthy_fail_threshold.max(1),
|
unhealthy_fail_threshold: unhealthy_fail_threshold.max(1),
|
||||||
connect_failfast_hard_errors,
|
connect_failfast_hard_errors,
|
||||||
|
no_upstreams_warn_epoch_ms: Arc::new(AtomicU64::new(0)),
|
||||||
|
no_healthy_warn_epoch_ms: Arc::new(AtomicU64::new(0)),
|
||||||
stats,
|
stats,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn now_epoch_ms() -> u64 {
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis() as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
fn should_emit_warn(last_epoch_ms: &AtomicU64, cooldown_ms: u64) -> bool {
|
||||||
|
let now_epoch_ms = Self::now_epoch_ms();
|
||||||
|
let previous_epoch_ms = last_epoch_ms.load(Ordering::Relaxed);
|
||||||
|
if now_epoch_ms.saturating_sub(previous_epoch_ms) < cooldown_ms {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
last_epoch_ms
|
||||||
|
.compare_exchange(
|
||||||
|
previous_epoch_ms,
|
||||||
|
now_epoch_ms,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
)
|
||||||
|
.is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn try_api_snapshot(&self) -> Option<UpstreamApiSnapshot> {
|
pub fn try_api_snapshot(&self) -> Option<UpstreamApiSnapshot> {
|
||||||
let guard = self.upstreams.try_read().ok()?;
|
let guard = self.upstreams.try_read().ok()?;
|
||||||
let now = std::time::Instant::now();
|
let now = std::time::Instant::now();
|
||||||
@@ -533,12 +560,22 @@ impl UpstreamManager {
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if filtered_upstreams.is_empty() {
|
if filtered_upstreams.is_empty() {
|
||||||
warn!(scope = scope, "No upstreams available! Using first (direct?)");
|
if Self::should_emit_warn(
|
||||||
|
self.no_upstreams_warn_epoch_ms.as_ref(),
|
||||||
|
5_000,
|
||||||
|
) {
|
||||||
|
warn!(scope = scope, "No upstreams available! Using first (direct?)");
|
||||||
|
}
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
if healthy.is_empty() {
|
if healthy.is_empty() {
|
||||||
warn!(scope = scope, "No healthy upstreams available! Using random.");
|
if Self::should_emit_warn(
|
||||||
|
self.no_healthy_warn_epoch_ms.as_ref(),
|
||||||
|
5_000,
|
||||||
|
) {
|
||||||
|
warn!(scope = scope, "No healthy upstreams available! Using random.");
|
||||||
|
}
|
||||||
return Some(filtered_upstreams[rand::rng().gen_range(0..filtered_upstreams.len())]);
|
return Some(filtered_upstreams[rand::rng().gen_range(0..filtered_upstreams.len())]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
[Unit]
|
[Unit]
|
||||||
Description=Telemt
|
Description=Telemt
|
||||||
After=network.target
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
WorkingDirectory=/bin
|
WorkingDirectory=/etc/telemt
|
||||||
ExecStart=/bin/telemt /etc/telemt.toml
|
ExecStart=/bin/telemt /etc/telemt.toml
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
LimitNOFILE=65536
|
LimitNOFILE=262144
|
||||||
|
TasksMax=8192
|
||||||
|
MemoryAccounting=yes
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
|
|||||||
Reference in New Issue
Block a user