Compare commits

...

35 Commits

Author SHA1 Message Date
Alexey
b41257f54e Merge pull request #372 from telemt/bump
Update Cargo.toml
2026-03-08 03:46:01 +03:00
Alexey
76b28aea74 Update Cargo.toml 2026-03-08 03:45:46 +03:00
Alexey
aa315f5d72 Merge pull request #371 from telemt/flow-defaults
Update defaults.rs
2026-03-08 03:45:28 +03:00
Alexey
c28b82a618 Update defaults.rs 2026-03-08 03:45:01 +03:00
Alexey
e7bdc80956 Merge pull request #370 from telemt/bump
Update Cargo.toml
2026-03-08 03:09:45 +03:00
Alexey
d641137537 Update Cargo.toml 2026-03-08 03:09:33 +03:00
Alexey
4fd22b3219 ME Writer Pick + Active-by-Endpoint: merge pull request #369 from telemt/flow-pick
ME Writer Pick + Active-by-Endpoint
2026-03-08 03:07:38 +03:00
Alexey
fca0e3f619 ME Writer Pick in Metrics+API 2026-03-08 03:06:45 +03:00
Alexey
9401c46727 ME Writer Pick 2026-03-08 03:05:47 +03:00
Alexey
6b3697ee87 ME Active-by-Endpoint 2026-03-08 03:04:27 +03:00
Alexey
c08160600e Update pool_writer.rs 2026-03-08 03:03:41 +03:00
Alexey
cd5c60ce1e Update reader.rs 2026-03-08 03:03:35 +03:00
Alexey
ae1c97e27a Merge pull request #360 from Shulyaka/patch-1
Update telemt.service
2026-03-07 19:55:43 +03:00
Alexey
cfee7de66b Update telemt.service 2026-03-07 19:55:28 +03:00
Denis Shulyaka
c942c492ad Apply suggestions from code review
Co-authored-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-03-07 19:51:37 +03:00
Alexey
0e4be43b2b Merge pull request #365 from amirotin/improve-install-script
improve install script
2026-03-07 19:49:56 +03:00
Alexey
7eb2b60855 Update install.sh 2026-03-07 19:49:45 +03:00
Mirotin Artem
373ae3281e Update install.sh 2026-03-07 19:43:55 +03:00
Mirotin Artem
178630e3bf Merge branch 'main' into improve-install-script 2026-03-07 19:40:09 +03:00
Alexey
67f307cd43 Merge pull request #367 from telemt/bump
Update Cargo.toml
2026-03-07 19:37:50 +03:00
Alexey
ca2eaa9ead Update Cargo.toml 2026-03-07 19:37:40 +03:00
Alexey
3c78daea0c CPU/RAM improvements + removing hot-path obstacles: merge pull request #366 from telemt/flow-perf
CPU/RAM improvements + removing hot-path obstacles
2026-03-07 19:37:09 +03:00
Alexey
d2baa8e721 CPU/RAM improvements + removing hot-path obstacles 2026-03-07 19:33:48 +03:00
Mirotin Artem
a0cf4b4713 improve install script 2026-03-07 19:07:30 +03:00
Alexey
1bd249b0a9 Merge pull request #363 from telemt/me-true
Update config.toml
2026-03-07 18:43:59 +03:00
Alexey
2f47ec5797 Update config.toml 2026-03-07 18:43:48 +03:00
Denis Shulyaka
80f3661b8e Modify telemt.service for network dependencies
Updated service dependencies and added SELinux context.

`network-online.target` is required to get the ip address and check telegram servers
2026-03-07 17:36:44 +03:00
Alexey
32eeb4a98c Merge pull request #358 from hookzof/patch-1
Fix typo in QUICK_START_GUIDE.ru.md
2026-03-07 17:31:23 +03:00
Alexey
a74cc14ed9 Init in API + ME Adaptive Floor Upper-Limit: merge pull request #359 from telemt/flow-api
Init in API + ME Adaptive Floor Upper-Limit
2026-03-07 17:30:10 +03:00
Alexey
5f77f83b48 ME Adaptive Floor Upper-Limit 2026-03-07 17:27:56 +03:00
Talya
d543dbca92 Fix typo in QUICK_START_GUIDE.ru.md 2026-03-07 14:48:02 +01:00
Alexey
02f9d59f5a Merge pull request #357 from telemt/bump
Update Cargo.toml
2026-03-07 16:34:43 +03:00
Alexey
7b745bc7bc Update Cargo.toml 2026-03-07 16:34:32 +03:00
Alexey
5ac0ef1ffd Init in API 2026-03-07 16:18:09 +03:00
Alexey
e1f3efb619 API from main 2026-03-07 15:37:49 +03:00
31 changed files with 2868 additions and 325 deletions

View File

@@ -1,6 +1,6 @@
[package] [package]
name = "telemt" name = "telemt"
version = "3.3.8" version = "3.3.12"
edition = "2024" edition = "2024"
[dependencies] [dependencies]

View File

@@ -4,7 +4,7 @@
# === General Settings === # === General Settings ===
[general] [general]
use_middle_proxy = false use_middle_proxy = true
# Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags] # Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags]
# ad_tag = "00000000000000000000000000000000" # ad_tag = "00000000000000000000000000000000"
# Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot) # Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot)

View File

@@ -126,7 +126,7 @@ WantedBy=multi-user.target
```bash ```bash
curl -s http://127.0.0.1:9091/v1/users | jq curl -s http://127.0.0.1:9091/v1/users | jq
``` ```
> Одной ссылкой модет пользоваться сколько угодно человек. > Одной ссылкой может пользоваться сколько угодно человек.
> [!WARNING] > [!WARNING]
> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете! > Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете!

View File

@@ -1,73 +1,93 @@
sudo bash -c ' #!/bin/sh
set -e set -eu
# --- Проверка на существующую установку --- REPO="${REPO:-telemt/telemt}"
if systemctl list-unit-files | grep -q telemt.service; then BIN_NAME="${BIN_NAME:-telemt}"
# --- РЕЖИМ ОБНОВЛЕНИЯ --- VERSION="${1:-${VERSION:-latest}}"
echo "--- Обнаружена существующая установка Telemt. Запускаю обновление... ---" INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
echo "[*] Остановка службы telemt..." say() {
systemctl stop telemt || true # Игнорируем ошибку, если служба уже остановлена printf '%s\n' "$*"
}
echo "[1/2] Скачивание последней версии Telemt..." die() {
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz printf 'Error: %s\n' "$*" >&2
exit 1
}
echo "[1/2] Замена исполняемого файла в /usr/local/bin..." need_cmd() {
mv telemt /usr/local/bin/telemt command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
chmod +x /usr/local/bin/telemt }
echo "[2/2] Запуск службы..." detect_arch() {
systemctl start telemt arch="$(uname -m)"
case "$arch" in
x86_64|amd64) printf 'x86_64\n' ;;
aarch64|arm64) printf 'aarch64\n' ;;
*) die "unsupported architecture: $arch" ;;
esac
}
echo "--- Обновление Telemt успешно завершено! ---" detect_libc() {
echo case "$(ldd --version 2>&1 || true)" in
echo "Для проверки статуса службы выполните:" *musl*) printf 'musl\n' ;;
echo " systemctl status telemt" *) printf 'gnu\n' ;;
esac
}
else fetch_to_stdout() {
# --- РЕЖИМ НОВОЙ УСТАНОВКИ --- url="$1"
echo "--- Начало автоматической установки Telemt ---" if command -v curl >/dev/null 2>&1; then
curl -fsSL "$url"
elif command -v wget >/dev/null 2>&1; then
wget -qO- "$url"
else
die "neither curl nor wget is installed"
fi
}
# Шаг 1: Скачивание и установка бинарного файла install_binary() {
echo "[1/5] Скачивание последней версии Telemt..." src="$1"
wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz dst="$2"
echo "[1/5] Перемещение исполняемого файла в /usr/local/bin и установка прав..." if [ -w "$INSTALL_DIR" ] || { [ ! -e "$INSTALL_DIR" ] && [ -w "$(dirname "$INSTALL_DIR")" ]; }; then
mv telemt /usr/local/bin/telemt mkdir -p "$INSTALL_DIR"
chmod +x /usr/local/bin/telemt install -m 0755 "$src" "$dst"
elif command -v sudo >/dev/null 2>&1; then
sudo mkdir -p "$INSTALL_DIR"
sudo install -m 0755 "$src" "$dst"
else
die "cannot write to $INSTALL_DIR and sudo is not available"
fi
}
# Шаг 2: Генерация секрета need_cmd uname
echo "[2/5] Генерация секретного ключа..." need_cmd tar
SECRET=$(openssl rand -hex 16) need_cmd mktemp
need_cmd grep
need_cmd install
# Шаг 3: Создание файла конфигурации ARCH="$(detect_arch)"
echo "[3/5] Создание файла конфигурации /etc/telemt.toml..." LIBC="$(detect_libc)"
printf "# === General Settings ===\n[general]\n[general.modes]\nclassic = false\nsecure = false\ntls = true\n\n# === Anti-Censorship & Masking ===\n[censorship]\n# !!! ВАЖНО: Замените на ваш домен или домен, который вы хотите использовать для маскировки !!!\ntls_domain = \"petrovich.ru\"\n\n[access.users]\nhello = \"%s\"\n" "$SECRET" > /etc/telemt.toml
# Шаг 4: Создание службы Systemd case "$VERSION" in
echo "[4/5] Создание службы systemd..." latest)
printf "[Unit]\nDescription=Telemt Proxy\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/telemt /etc/telemt.toml\nRestart=on-failure\nRestartSec=5\nLimitNOFILE=65536\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/telemt.service URL="https://github.com/$REPO/releases/latest/download/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz"
;;
*)
URL="https://github.com/$REPO/releases/download/${VERSION}/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz"
;;
esac
# Шаг 5: Запуск службы TMPDIR="$(mktemp -d)"
echo "[5/5] Перезагрузка systemd, запуск и включение службы telemt..." trap 'rm -rf "$TMPDIR"' EXIT INT TERM
systemctl daemon-reload
systemctl start telemt
systemctl enable telemt
echo "--- Установка и запуск Telemt успешно завершены! ---" say "Installing $BIN_NAME ($VERSION) for $ARCH-linux-$LIBC..."
echo fetch_to_stdout "$URL" | tar -xzf - -C "$TMPDIR"
echo "ВАЖНАЯ ИНФОРМАЦИЯ:"
echo "===================" [ -f "$TMPDIR/$BIN_NAME" ] || die "archive did not contain $BIN_NAME"
echo "1. Вам НЕОБХОДИМО отредактировать файл /etc/telemt.toml и заменить '\''petrovich.ru'\'' на другой домен"
echo " с помощью команды:" install_binary "$TMPDIR/$BIN_NAME" "$INSTALL_DIR/$BIN_NAME"
echo " nano /etc/telemt.toml"
echo " После редактирования файла перезапустите службу командой:" say "Installed: $INSTALL_DIR/$BIN_NAME"
echo " sudo systemctl restart telemt" "$INSTALL_DIR/$BIN_NAME" --version 2>/dev/null || true
echo
echo "2. Для проверки статуса службы выполните команду:"
echo " systemctl status telemt"
echo
echo "3. Для получения ссылок на подключение выполните команду:"
echo " journalctl -u telemt -n -g '\''links'\'' --no-pager -o cat | tac"
fi
'

View File

@@ -11,11 +11,12 @@ use hyper::server::conn::http1;
use hyper::service::service_fn; use hyper::service::service_fn;
use hyper::{Method, Request, Response, StatusCode}; use hyper::{Method, Request, Response, StatusCode};
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tokio::sync::{Mutex, watch}; use tokio::sync::{Mutex, RwLock, watch};
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use crate::config::ProxyConfig; use crate::config::ProxyConfig;
use crate::ip_tracker::UserIpTracker; use crate::ip_tracker::UserIpTracker;
use crate::startup::StartupTracker;
use crate::stats::Stats; use crate::stats::Stats;
use crate::transport::middle_proxy::MePool; use crate::transport::middle_proxy::MePool;
use crate::transport::UpstreamManager; use crate::transport::UpstreamManager;
@@ -25,6 +26,7 @@ mod events;
mod http_utils; mod http_utils;
mod model; mod model;
mod runtime_edge; mod runtime_edge;
mod runtime_init;
mod runtime_min; mod runtime_min;
mod runtime_stats; mod runtime_stats;
mod runtime_watch; mod runtime_watch;
@@ -41,6 +43,7 @@ use runtime_edge::{
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data, EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
build_runtime_events_recent_data, build_runtime_events_recent_data,
}; };
use runtime_init::build_runtime_initialization_data;
use runtime_min::{ use runtime_min::{
build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data, build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data,
build_runtime_upstream_quality_data, build_security_whitelist_data, build_runtime_upstream_quality_data, build_security_whitelist_data,
@@ -67,7 +70,7 @@ pub(super) struct ApiRuntimeState {
pub(super) struct ApiShared { pub(super) struct ApiShared {
pub(super) stats: Arc<Stats>, pub(super) stats: Arc<Stats>,
pub(super) ip_tracker: Arc<UserIpTracker>, pub(super) ip_tracker: Arc<UserIpTracker>,
pub(super) me_pool: Option<Arc<MePool>>, pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
pub(super) upstream_manager: Arc<UpstreamManager>, pub(super) upstream_manager: Arc<UpstreamManager>,
pub(super) config_path: PathBuf, pub(super) config_path: PathBuf,
pub(super) startup_detected_ip_v4: Option<IpAddr>, pub(super) startup_detected_ip_v4: Option<IpAddr>,
@@ -79,6 +82,7 @@ pub(super) struct ApiShared {
pub(super) runtime_events: Arc<ApiEventStore>, pub(super) runtime_events: Arc<ApiEventStore>,
pub(super) request_id: Arc<AtomicU64>, pub(super) request_id: Arc<AtomicU64>,
pub(super) runtime_state: Arc<ApiRuntimeState>, pub(super) runtime_state: Arc<ApiRuntimeState>,
pub(super) startup_tracker: Arc<StartupTracker>,
} }
impl ApiShared { impl ApiShared {
@@ -91,7 +95,7 @@ pub async fn serve(
listen: SocketAddr, listen: SocketAddr,
stats: Arc<Stats>, stats: Arc<Stats>,
ip_tracker: Arc<UserIpTracker>, ip_tracker: Arc<UserIpTracker>,
me_pool: Option<Arc<MePool>>, me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
upstream_manager: Arc<UpstreamManager>, upstream_manager: Arc<UpstreamManager>,
config_rx: watch::Receiver<Arc<ProxyConfig>>, config_rx: watch::Receiver<Arc<ProxyConfig>>,
admission_rx: watch::Receiver<bool>, admission_rx: watch::Receiver<bool>,
@@ -99,6 +103,7 @@ pub async fn serve(
startup_detected_ip_v4: Option<IpAddr>, startup_detected_ip_v4: Option<IpAddr>,
startup_detected_ip_v6: Option<IpAddr>, startup_detected_ip_v6: Option<IpAddr>,
process_started_at_epoch_secs: u64, process_started_at_epoch_secs: u64,
startup_tracker: Arc<StartupTracker>,
) { ) {
let listener = match TcpListener::bind(listen).await { let listener = match TcpListener::bind(listen).await {
Ok(listener) => listener, Ok(listener) => listener,
@@ -138,6 +143,7 @@ pub async fn serve(
)), )),
request_id: Arc::new(AtomicU64::new(1)), request_id: Arc::new(AtomicU64::new(1)),
runtime_state: runtime_state.clone(), runtime_state: runtime_state.clone(),
startup_tracker,
}); });
spawn_runtime_watchers( spawn_runtime_watchers(
@@ -248,7 +254,12 @@ async fn handle(
} }
("GET", "/v1/runtime/gates") => { ("GET", "/v1/runtime/gates") => {
let revision = current_revision(&shared.config_path).await?; let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()); let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()).await;
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/runtime/initialization") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_runtime_initialization_data(shared.as_ref()).await;
Ok(success_response(StatusCode::OK, data, revision)) Ok(success_response(StatusCode::OK, data, revision))
} }
("GET", "/v1/limits/effective") => { ("GET", "/v1/limits/effective") => {

View File

@@ -266,6 +266,7 @@ pub(super) struct MeWritersData {
pub(super) struct DcStatus { pub(super) struct DcStatus {
pub(super) dc: i16, pub(super) dc: i16,
pub(super) endpoints: Vec<String>, pub(super) endpoints: Vec<String>,
pub(super) endpoint_writers: Vec<DcEndpointWriters>,
pub(super) available_endpoints: usize, pub(super) available_endpoints: usize,
pub(super) available_pct: f64, pub(super) available_pct: f64,
pub(super) required_writers: usize, pub(super) required_writers: usize,
@@ -279,6 +280,12 @@ pub(super) struct DcStatus {
pub(super) load: usize, pub(super) load: usize,
} }
#[derive(Serialize, Clone)]
pub(super) struct DcEndpointWriters {
pub(super) endpoint: String,
pub(super) active_writers: usize,
}
#[derive(Serialize, Clone)] #[derive(Serialize, Clone)]
pub(super) struct DcStatusData { pub(super) struct DcStatusData {
pub(super) middle_proxy_enabled: bool, pub(super) middle_proxy_enabled: bool,
@@ -318,11 +325,21 @@ pub(super) struct MinimalMeRuntimeData {
pub(super) adaptive_floor_cpu_cores_override: u16, pub(super) adaptive_floor_cpu_cores_override: u16,
pub(super) adaptive_floor_max_extra_writers_single_per_core: u16, pub(super) adaptive_floor_max_extra_writers_single_per_core: u16,
pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16, pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16,
pub(super) adaptive_floor_max_active_writers_per_core: u16,
pub(super) adaptive_floor_max_warm_writers_per_core: u16,
pub(super) adaptive_floor_max_active_writers_global: u32,
pub(super) adaptive_floor_max_warm_writers_global: u32,
pub(super) adaptive_floor_cpu_cores_detected: u32, pub(super) adaptive_floor_cpu_cores_detected: u32,
pub(super) adaptive_floor_cpu_cores_effective: u32, pub(super) adaptive_floor_cpu_cores_effective: u32,
pub(super) adaptive_floor_global_cap_raw: u64, pub(super) adaptive_floor_global_cap_raw: u64,
pub(super) adaptive_floor_global_cap_effective: u64, pub(super) adaptive_floor_global_cap_effective: u64,
pub(super) adaptive_floor_target_writers_total: u64, pub(super) adaptive_floor_target_writers_total: u64,
pub(super) adaptive_floor_active_cap_configured: u64,
pub(super) adaptive_floor_active_cap_effective: u64,
pub(super) adaptive_floor_warm_cap_configured: u64,
pub(super) adaptive_floor_warm_cap_effective: u64,
pub(super) adaptive_floor_active_writers_current: u64,
pub(super) adaptive_floor_warm_writers_current: u64,
pub(super) me_keepalive_enabled: bool, pub(super) me_keepalive_enabled: bool,
pub(super) me_keepalive_interval_secs: u64, pub(super) me_keepalive_interval_secs: u64,
pub(super) me_keepalive_jitter_secs: u64, pub(super) me_keepalive_jitter_secs: u64,
@@ -344,6 +361,8 @@ pub(super) struct MinimalMeRuntimeData {
pub(super) me_single_endpoint_outage_backoff_max_ms: u64, pub(super) me_single_endpoint_outage_backoff_max_ms: u64,
pub(super) me_single_endpoint_shadow_rotate_every_secs: u64, pub(super) me_single_endpoint_shadow_rotate_every_secs: u64,
pub(super) me_deterministic_writer_sort: bool, pub(super) me_deterministic_writer_sort: bool,
pub(super) me_writer_pick_mode: &'static str,
pub(super) me_writer_pick_sample_size: u8,
pub(super) me_socks_kdf_policy: &'static str, pub(super) me_socks_kdf_policy: &'static str,
pub(super) quarantined_endpoints_total: usize, pub(super) quarantined_endpoints_total: usize,
pub(super) quarantined_endpoints: Vec<MinimalQuarantineData>, pub(super) quarantined_endpoints: Vec<MinimalQuarantineData>,

186
src/api/runtime_init.rs Normal file
View File

@@ -0,0 +1,186 @@
use serde::Serialize;
use crate::startup::{
COMPONENT_ME_CONNECTIVITY_PING, COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1,
COMPONENT_ME_PROXY_CONFIG_V4, COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH,
StartupComponentStatus, StartupMeStatus, compute_progress_pct,
};
use super::ApiShared;
#[derive(Serialize)]
pub(super) struct RuntimeInitializationComponentData {
pub(super) id: &'static str,
pub(super) title: &'static str,
pub(super) status: &'static str,
pub(super) started_at_epoch_ms: Option<u64>,
pub(super) finished_at_epoch_ms: Option<u64>,
pub(super) duration_ms: Option<u64>,
pub(super) attempts: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) details: Option<String>,
}
#[derive(Serialize)]
pub(super) struct RuntimeInitializationMeData {
pub(super) status: &'static str,
pub(super) current_stage: String,
pub(super) progress_pct: f64,
pub(super) init_attempt: u32,
pub(super) retry_limit: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) last_error: Option<String>,
}
#[derive(Serialize)]
pub(super) struct RuntimeInitializationData {
pub(super) status: &'static str,
pub(super) degraded: bool,
pub(super) current_stage: String,
pub(super) progress_pct: f64,
pub(super) started_at_epoch_secs: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) ready_at_epoch_secs: Option<u64>,
pub(super) total_elapsed_ms: u64,
pub(super) transport_mode: String,
pub(super) me: RuntimeInitializationMeData,
pub(super) components: Vec<RuntimeInitializationComponentData>,
}
#[derive(Clone)]
pub(super) struct RuntimeStartupSummaryData {
pub(super) status: &'static str,
pub(super) stage: String,
pub(super) progress_pct: f64,
}
pub(super) async fn build_runtime_startup_summary(shared: &ApiShared) -> RuntimeStartupSummaryData {
let snapshot = shared.startup_tracker.snapshot().await;
let me_pool_progress = current_me_pool_stage_progress(shared).await;
let progress_pct = compute_progress_pct(&snapshot, me_pool_progress);
RuntimeStartupSummaryData {
status: snapshot.status.as_str(),
stage: snapshot.current_stage,
progress_pct,
}
}
pub(super) async fn build_runtime_initialization_data(
shared: &ApiShared,
) -> RuntimeInitializationData {
let snapshot = shared.startup_tracker.snapshot().await;
let me_pool_progress = current_me_pool_stage_progress(shared).await;
let progress_pct = compute_progress_pct(&snapshot, me_pool_progress);
let me_progress_pct = compute_me_progress_pct(&snapshot, me_pool_progress);
RuntimeInitializationData {
status: snapshot.status.as_str(),
degraded: snapshot.degraded,
current_stage: snapshot.current_stage,
progress_pct,
started_at_epoch_secs: snapshot.started_at_epoch_secs,
ready_at_epoch_secs: snapshot.ready_at_epoch_secs,
total_elapsed_ms: snapshot.total_elapsed_ms,
transport_mode: snapshot.transport_mode,
me: RuntimeInitializationMeData {
status: snapshot.me.status.as_str(),
current_stage: snapshot.me.current_stage,
progress_pct: me_progress_pct,
init_attempt: snapshot.me.init_attempt,
retry_limit: snapshot.me.retry_limit,
last_error: snapshot.me.last_error,
},
components: snapshot
.components
.into_iter()
.map(|component| RuntimeInitializationComponentData {
id: component.id,
title: component.title,
status: component.status.as_str(),
started_at_epoch_ms: component.started_at_epoch_ms,
finished_at_epoch_ms: component.finished_at_epoch_ms,
duration_ms: component.duration_ms,
attempts: component.attempts,
details: component.details,
})
.collect(),
}
}
fn compute_me_progress_pct(
snapshot: &crate::startup::StartupSnapshot,
me_pool_progress: Option<f64>,
) -> f64 {
match snapshot.me.status {
StartupMeStatus::Pending => 0.0,
StartupMeStatus::Ready | StartupMeStatus::Failed | StartupMeStatus::Skipped => 100.0,
StartupMeStatus::Initializing => {
let mut total_weight = 0.0f64;
let mut completed_weight = 0.0f64;
for component in &snapshot.components {
if !is_me_component(component.id) {
continue;
}
total_weight += component.weight;
let unit_progress = match component.status {
StartupComponentStatus::Pending => 0.0,
StartupComponentStatus::Running => {
if component.id == COMPONENT_ME_POOL_INIT_STAGE1 {
me_pool_progress.unwrap_or(0.0).clamp(0.0, 1.0)
} else {
0.0
}
}
StartupComponentStatus::Ready
| StartupComponentStatus::Failed
| StartupComponentStatus::Skipped => 1.0,
};
completed_weight += component.weight * unit_progress;
}
if total_weight <= f64::EPSILON {
0.0
} else {
((completed_weight / total_weight) * 100.0).clamp(0.0, 100.0)
}
}
}
}
fn is_me_component(component_id: &str) -> bool {
matches!(
component_id,
COMPONENT_ME_SECRET_FETCH
| COMPONENT_ME_PROXY_CONFIG_V4
| COMPONENT_ME_PROXY_CONFIG_V6
| COMPONENT_ME_POOL_CONSTRUCT
| COMPONENT_ME_POOL_INIT_STAGE1
| COMPONENT_ME_CONNECTIVITY_PING
)
}
async fn current_me_pool_stage_progress(shared: &ApiShared) -> Option<f64> {
let snapshot = shared.startup_tracker.snapshot().await;
if snapshot.me.status != StartupMeStatus::Initializing {
return None;
}
let pool = shared.me_pool.read().await.clone()?;
let status = pool.api_status_snapshot().await;
let configured_dc_groups = status.configured_dc_groups;
let covered_dc_groups = status
.dcs
.iter()
.filter(|dc| dc.alive_writers > 0)
.count();
let dc_coverage = ratio_01(covered_dc_groups, configured_dc_groups);
let writer_coverage = ratio_01(status.alive_writers, status.required_writers);
Some((0.7 * dc_coverage + 0.3 * writer_coverage).clamp(0.0, 1.0))
}
fn ratio_01(part: usize, total: usize) -> f64 {
if total == 0 {
return 0.0;
}
((part as f64) / (total as f64)).clamp(0.0, 1.0)
}

View File

@@ -260,7 +260,7 @@ pub(super) fn build_security_whitelist_data(cfg: &ProxyConfig) -> SecurityWhitel
pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData { pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData {
let now_epoch_secs = now_epoch_secs(); let now_epoch_secs = now_epoch_secs();
let Some(pool) = &shared.me_pool else { let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeMePoolStateData { return RuntimeMePoolStateData {
enabled: false, enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON), reason: Some(SOURCE_UNAVAILABLE_REASON),
@@ -350,7 +350,7 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData { pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData {
let now_epoch_secs = now_epoch_secs(); let now_epoch_secs = now_epoch_secs();
let Some(pool) = &shared.me_pool else { let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeMeQualityData { return RuntimeMeQualityData {
enabled: false, enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON), reason: Some(SOURCE_UNAVAILABLE_REASON),
@@ -486,7 +486,7 @@ pub(super) async fn build_runtime_upstream_quality_data(
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData { pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
let now_epoch_secs = now_epoch_secs(); let now_epoch_secs = now_epoch_secs();
let Some(pool) = &shared.me_pool else { let Some(pool) = shared.me_pool.read().await.clone() else {
return RuntimeNatStunData { return RuntimeNatStunData {
enabled: false, enabled: false,
reason: Some(SOURCE_UNAVAILABLE_REASON), reason: Some(SOURCE_UNAVAILABLE_REASON),

View File

@@ -7,10 +7,10 @@ use crate::transport::UpstreamRouteKind;
use super::ApiShared; use super::ApiShared;
use super::model::{ use super::model::{
DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary, MinimalAllData, DcEndpointWriters, DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary,
MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData, MinimalQuarantineData, MinimalAllData, MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData,
UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData, ZeroAllData, MinimalQuarantineData, UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData,
ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData, ZeroAllData, ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData,
ZeroUpstreamData, ZeroUpstreamData,
}; };
@@ -297,7 +297,7 @@ async fn get_minimal_payload_cached(
} }
} }
let pool = shared.me_pool.as_ref()?; let pool = shared.me_pool.read().await.clone()?;
let status = pool.api_status_snapshot().await; let status = pool.api_status_snapshot().await;
let runtime = pool.api_runtime_snapshot().await; let runtime = pool.api_runtime_snapshot().await;
let generated_at_epoch_secs = status.generated_at_epoch_secs; let generated_at_epoch_secs = status.generated_at_epoch_secs;
@@ -346,6 +346,14 @@ async fn get_minimal_payload_cached(
.into_iter() .into_iter()
.map(|value| value.to_string()) .map(|value| value.to_string())
.collect(), .collect(),
endpoint_writers: entry
.endpoint_writers
.into_iter()
.map(|coverage| DcEndpointWriters {
endpoint: coverage.endpoint.to_string(),
active_writers: coverage.active_writers,
})
.collect(),
available_endpoints: entry.available_endpoints, available_endpoints: entry.available_endpoints,
available_pct: entry.available_pct, available_pct: entry.available_pct,
required_writers: entry.required_writers, required_writers: entry.required_writers,
@@ -380,11 +388,25 @@ async fn get_minimal_payload_cached(
.adaptive_floor_max_extra_writers_single_per_core, .adaptive_floor_max_extra_writers_single_per_core,
adaptive_floor_max_extra_writers_multi_per_core: runtime adaptive_floor_max_extra_writers_multi_per_core: runtime
.adaptive_floor_max_extra_writers_multi_per_core, .adaptive_floor_max_extra_writers_multi_per_core,
adaptive_floor_max_active_writers_per_core: runtime
.adaptive_floor_max_active_writers_per_core,
adaptive_floor_max_warm_writers_per_core: runtime
.adaptive_floor_max_warm_writers_per_core,
adaptive_floor_max_active_writers_global: runtime
.adaptive_floor_max_active_writers_global,
adaptive_floor_max_warm_writers_global: runtime
.adaptive_floor_max_warm_writers_global,
adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected, adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected,
adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective, adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective,
adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw, adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw,
adaptive_floor_global_cap_effective: runtime.adaptive_floor_global_cap_effective, adaptive_floor_global_cap_effective: runtime.adaptive_floor_global_cap_effective,
adaptive_floor_target_writers_total: runtime.adaptive_floor_target_writers_total, adaptive_floor_target_writers_total: runtime.adaptive_floor_target_writers_total,
adaptive_floor_active_cap_configured: runtime.adaptive_floor_active_cap_configured,
adaptive_floor_active_cap_effective: runtime.adaptive_floor_active_cap_effective,
adaptive_floor_warm_cap_configured: runtime.adaptive_floor_warm_cap_configured,
adaptive_floor_warm_cap_effective: runtime.adaptive_floor_warm_cap_effective,
adaptive_floor_active_writers_current: runtime.adaptive_floor_active_writers_current,
adaptive_floor_warm_writers_current: runtime.adaptive_floor_warm_writers_current,
me_keepalive_enabled: runtime.me_keepalive_enabled, me_keepalive_enabled: runtime.me_keepalive_enabled,
me_keepalive_interval_secs: runtime.me_keepalive_interval_secs, me_keepalive_interval_secs: runtime.me_keepalive_interval_secs,
me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs, me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs,
@@ -408,6 +430,8 @@ async fn get_minimal_payload_cached(
me_single_endpoint_shadow_rotate_every_secs: runtime me_single_endpoint_shadow_rotate_every_secs: runtime
.me_single_endpoint_shadow_rotate_every_secs, .me_single_endpoint_shadow_rotate_every_secs,
me_deterministic_writer_sort: runtime.me_deterministic_writer_sort, me_deterministic_writer_sort: runtime.me_deterministic_writer_sort,
me_writer_pick_mode: runtime.me_writer_pick_mode,
me_writer_pick_sample_size: runtime.me_writer_pick_sample_size,
me_socks_kdf_policy: runtime.me_socks_kdf_policy, me_socks_kdf_policy: runtime.me_socks_kdf_policy,
quarantined_endpoints_total: runtime.quarantined_endpoints.len(), quarantined_endpoints_total: runtime.quarantined_endpoints.len(),
quarantined_endpoints: runtime quarantined_endpoints: runtime

View File

@@ -2,9 +2,10 @@ use std::sync::atomic::Ordering;
use serde::Serialize; use serde::Serialize;
use crate::config::{MeFloorMode, ProxyConfig, UserMaxUniqueIpsMode}; use crate::config::{MeFloorMode, MeWriterPickMode, ProxyConfig, UserMaxUniqueIpsMode};
use super::ApiShared; use super::ApiShared;
use super::runtime_init::build_runtime_startup_summary;
#[derive(Serialize)] #[derive(Serialize)]
pub(super) struct SystemInfoData { pub(super) struct SystemInfoData {
@@ -34,6 +35,9 @@ pub(super) struct RuntimeGatesData {
pub(super) me_runtime_ready: bool, pub(super) me_runtime_ready: bool,
pub(super) me2dc_fallback_enabled: bool, pub(super) me2dc_fallback_enabled: bool,
pub(super) use_middle_proxy: bool, pub(super) use_middle_proxy: bool,
pub(super) startup_status: &'static str,
pub(super) startup_stage: String,
pub(super) startup_progress_pct: f64,
} }
#[derive(Serialize)] #[derive(Serialize)]
@@ -66,10 +70,16 @@ pub(super) struct EffectiveMiddleProxyLimits {
pub(super) adaptive_floor_cpu_cores_override: u16, pub(super) adaptive_floor_cpu_cores_override: u16,
pub(super) adaptive_floor_max_extra_writers_single_per_core: u16, pub(super) adaptive_floor_max_extra_writers_single_per_core: u16,
pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16, pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16,
pub(super) adaptive_floor_max_active_writers_per_core: u16,
pub(super) adaptive_floor_max_warm_writers_per_core: u16,
pub(super) adaptive_floor_max_active_writers_global: u32,
pub(super) adaptive_floor_max_warm_writers_global: u32,
pub(super) reconnect_max_concurrent_per_dc: u32, pub(super) reconnect_max_concurrent_per_dc: u32,
pub(super) reconnect_backoff_base_ms: u64, pub(super) reconnect_backoff_base_ms: u64,
pub(super) reconnect_backoff_cap_ms: u64, pub(super) reconnect_backoff_cap_ms: u64,
pub(super) reconnect_fast_retry_count: u32, pub(super) reconnect_fast_retry_count: u32,
pub(super) writer_pick_mode: &'static str,
pub(super) writer_pick_sample_size: u8,
pub(super) me2dc_fallback: bool, pub(super) me2dc_fallback: bool,
} }
@@ -142,12 +152,18 @@ pub(super) fn build_system_info_data(
} }
} }
pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) -> RuntimeGatesData { pub(super) async fn build_runtime_gates_data(
shared: &ApiShared,
cfg: &ProxyConfig,
) -> RuntimeGatesData {
let startup_summary = build_runtime_startup_summary(shared).await;
let me_runtime_ready = if !cfg.general.use_middle_proxy { let me_runtime_ready = if !cfg.general.use_middle_proxy {
true true
} else { } else {
shared shared
.me_pool .me_pool
.read()
.await
.as_ref() .as_ref()
.map(|pool| pool.is_runtime_ready()) .map(|pool| pool.is_runtime_ready())
.unwrap_or(false) .unwrap_or(false)
@@ -159,6 +175,9 @@ pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) ->
me_runtime_ready, me_runtime_ready,
me2dc_fallback_enabled: cfg.general.me2dc_fallback, me2dc_fallback_enabled: cfg.general.me2dc_fallback,
use_middle_proxy: cfg.general.use_middle_proxy, use_middle_proxy: cfg.general.use_middle_proxy,
startup_status: startup_summary.status,
startup_stage: startup_summary.stage,
startup_progress_pct: startup_summary.progress_pct,
} }
} }
@@ -204,10 +223,24 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD
adaptive_floor_max_extra_writers_multi_per_core: cfg adaptive_floor_max_extra_writers_multi_per_core: cfg
.general .general
.me_adaptive_floor_max_extra_writers_multi_per_core, .me_adaptive_floor_max_extra_writers_multi_per_core,
adaptive_floor_max_active_writers_per_core: cfg
.general
.me_adaptive_floor_max_active_writers_per_core,
adaptive_floor_max_warm_writers_per_core: cfg
.general
.me_adaptive_floor_max_warm_writers_per_core,
adaptive_floor_max_active_writers_global: cfg
.general
.me_adaptive_floor_max_active_writers_global,
adaptive_floor_max_warm_writers_global: cfg
.general
.me_adaptive_floor_max_warm_writers_global,
reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc, reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc,
reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms, reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms,
reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms, reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms,
reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count, reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count,
writer_pick_mode: me_writer_pick_mode_label(cfg.general.me_writer_pick_mode),
writer_pick_sample_size: cfg.general.me_writer_pick_sample_size,
me2dc_fallback: cfg.general.me2dc_fallback, me2dc_fallback: cfg.general.me2dc_fallback,
}, },
user_ip_policy: EffectiveUserIpPolicyLimits { user_ip_policy: EffectiveUserIpPolicyLimits {
@@ -245,3 +278,10 @@ fn me_floor_mode_label(mode: MeFloorMode) -> &'static str {
MeFloorMode::Adaptive => "adaptive", MeFloorMode::Adaptive => "adaptive",
} }
} }
fn me_writer_pick_mode_label(mode: MeWriterPickMode) -> &'static str {
match mode {
MeWriterPickMode::SortedRr => "sorted_rr",
MeWriterPickMode::P2c => "p2c",
}
}

View File

@@ -17,6 +17,18 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_WRITERS_PER_CORE_TOTAL: u16 = 48;
const DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE: u16 = 0; const DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE: u16 = 0;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE: u16 = 1; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE: u16 = 1;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE: u16 = 2; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE: u16 = 2;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256;
const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 4096;
const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 768;
const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024;
const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3;
const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000;
const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000;
const DEFAULT_ME_ADMISSION_POLL_MS: u64 = 1000;
const DEFAULT_ME_WARN_RATE_LIMIT_MS: u64 = 5000;
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
@@ -276,6 +288,54 @@ pub(crate) fn default_me_adaptive_floor_max_extra_writers_multi_per_core() -> u1
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE
} }
pub(crate) fn default_me_adaptive_floor_max_active_writers_per_core() -> u16 {
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE
}
pub(crate) fn default_me_adaptive_floor_max_warm_writers_per_core() -> u16 {
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE
}
pub(crate) fn default_me_adaptive_floor_max_active_writers_global() -> u32 {
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL
}
pub(crate) fn default_me_adaptive_floor_max_warm_writers_global() -> u32 {
DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL
}
pub(crate) fn default_me_writer_cmd_channel_capacity() -> usize {
DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY
}
pub(crate) fn default_me_route_channel_capacity() -> usize {
DEFAULT_ME_ROUTE_CHANNEL_CAPACITY
}
pub(crate) fn default_me_c2me_channel_capacity() -> usize {
DEFAULT_ME_C2ME_CHANNEL_CAPACITY
}
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
}
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
}
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
}
pub(crate) fn default_me_admission_poll_ms() -> u64 {
DEFAULT_ME_ADMISSION_POLL_MS
}
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
DEFAULT_ME_WARN_RATE_LIMIT_MS
}
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
} }

View File

@@ -29,7 +29,10 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher};
use tokio::sync::{mpsc, watch}; use tokio::sync::{mpsc, watch};
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use crate::config::{LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel}; use crate::config::{
LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
MeWriterPickMode,
};
use super::load::ProxyConfig; use super::load::ProxyConfig;
// ── Hot fields ──────────────────────────────────────────────────────────────── // ── Hot fields ────────────────────────────────────────────────────────────────
@@ -57,6 +60,8 @@ pub struct HotFields {
pub me_bind_stale_ttl_secs: u64, pub me_bind_stale_ttl_secs: u64,
pub me_secret_atomic_snapshot: bool, pub me_secret_atomic_snapshot: bool,
pub me_deterministic_writer_sort: bool, pub me_deterministic_writer_sort: bool,
pub me_writer_pick_mode: MeWriterPickMode,
pub me_writer_pick_sample_size: u8,
pub me_single_endpoint_shadow_writers: u8, pub me_single_endpoint_shadow_writers: u8,
pub me_single_endpoint_outage_mode_enabled: bool, pub me_single_endpoint_outage_mode_enabled: bool,
pub me_single_endpoint_outage_disable_quarantine: bool, pub me_single_endpoint_outage_disable_quarantine: bool,
@@ -84,9 +89,17 @@ pub struct HotFields {
pub me_adaptive_floor_cpu_cores_override: u16, pub me_adaptive_floor_cpu_cores_override: u16,
pub me_adaptive_floor_max_extra_writers_single_per_core: u16, pub me_adaptive_floor_max_extra_writers_single_per_core: u16,
pub me_adaptive_floor_max_extra_writers_multi_per_core: u16, pub me_adaptive_floor_max_extra_writers_multi_per_core: u16,
pub me_adaptive_floor_max_active_writers_per_core: u16,
pub me_adaptive_floor_max_warm_writers_per_core: u16,
pub me_adaptive_floor_max_active_writers_global: u32,
pub me_adaptive_floor_max_warm_writers_global: u32,
pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_base_timeout_ms: u64,
pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64,
pub me_route_backpressure_high_watermark_pct: u8, pub me_route_backpressure_high_watermark_pct: u8,
pub me_health_interval_ms_unhealthy: u64,
pub me_health_interval_ms_healthy: u64,
pub me_admission_poll_ms: u64,
pub me_warn_rate_limit_ms: u64,
pub users: std::collections::HashMap<String, String>, pub users: std::collections::HashMap<String, String>,
pub user_ad_tags: std::collections::HashMap<String, String>, pub user_ad_tags: std::collections::HashMap<String, String>,
pub user_max_tcp_conns: std::collections::HashMap<String, usize>, pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
@@ -122,6 +135,8 @@ impl HotFields {
me_bind_stale_ttl_secs: cfg.general.me_bind_stale_ttl_secs, me_bind_stale_ttl_secs: cfg.general.me_bind_stale_ttl_secs,
me_secret_atomic_snapshot: cfg.general.me_secret_atomic_snapshot, me_secret_atomic_snapshot: cfg.general.me_secret_atomic_snapshot,
me_deterministic_writer_sort: cfg.general.me_deterministic_writer_sort, me_deterministic_writer_sort: cfg.general.me_deterministic_writer_sort,
me_writer_pick_mode: cfg.general.me_writer_pick_mode,
me_writer_pick_sample_size: cfg.general.me_writer_pick_sample_size,
me_single_endpoint_shadow_writers: cfg.general.me_single_endpoint_shadow_writers, me_single_endpoint_shadow_writers: cfg.general.me_single_endpoint_shadow_writers,
me_single_endpoint_outage_mode_enabled: cfg me_single_endpoint_outage_mode_enabled: cfg
.general .general
@@ -173,9 +188,25 @@ impl HotFields {
me_adaptive_floor_max_extra_writers_multi_per_core: cfg me_adaptive_floor_max_extra_writers_multi_per_core: cfg
.general .general
.me_adaptive_floor_max_extra_writers_multi_per_core, .me_adaptive_floor_max_extra_writers_multi_per_core,
me_adaptive_floor_max_active_writers_per_core: cfg
.general
.me_adaptive_floor_max_active_writers_per_core,
me_adaptive_floor_max_warm_writers_per_core: cfg
.general
.me_adaptive_floor_max_warm_writers_per_core,
me_adaptive_floor_max_active_writers_global: cfg
.general
.me_adaptive_floor_max_active_writers_global,
me_adaptive_floor_max_warm_writers_global: cfg
.general
.me_adaptive_floor_max_warm_writers_global,
me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms,
me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms,
me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct,
me_health_interval_ms_unhealthy: cfg.general.me_health_interval_ms_unhealthy,
me_health_interval_ms_healthy: cfg.general.me_health_interval_ms_healthy,
me_admission_poll_ms: cfg.general.me_admission_poll_ms,
me_warn_rate_limit_ms: cfg.general.me_warn_rate_limit_ms,
users: cfg.access.users.clone(), users: cfg.access.users.clone(),
user_ad_tags: cfg.access.user_ad_tags.clone(), user_ad_tags: cfg.access.user_ad_tags.clone(),
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(), user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
@@ -268,6 +299,8 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
cfg.general.me_bind_stale_ttl_secs = new.general.me_bind_stale_ttl_secs; cfg.general.me_bind_stale_ttl_secs = new.general.me_bind_stale_ttl_secs;
cfg.general.me_secret_atomic_snapshot = new.general.me_secret_atomic_snapshot; cfg.general.me_secret_atomic_snapshot = new.general.me_secret_atomic_snapshot;
cfg.general.me_deterministic_writer_sort = new.general.me_deterministic_writer_sort; cfg.general.me_deterministic_writer_sort = new.general.me_deterministic_writer_sort;
cfg.general.me_writer_pick_mode = new.general.me_writer_pick_mode;
cfg.general.me_writer_pick_sample_size = new.general.me_writer_pick_sample_size;
cfg.general.me_single_endpoint_shadow_writers = new.general.me_single_endpoint_shadow_writers; cfg.general.me_single_endpoint_shadow_writers = new.general.me_single_endpoint_shadow_writers;
cfg.general.me_single_endpoint_outage_mode_enabled = cfg.general.me_single_endpoint_outage_mode_enabled =
new.general.me_single_endpoint_outage_mode_enabled; new.general.me_single_endpoint_outage_mode_enabled;
@@ -305,12 +338,24 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
new.general.me_adaptive_floor_max_extra_writers_single_per_core; new.general.me_adaptive_floor_max_extra_writers_single_per_core;
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core = cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core =
new.general.me_adaptive_floor_max_extra_writers_multi_per_core; new.general.me_adaptive_floor_max_extra_writers_multi_per_core;
cfg.general.me_adaptive_floor_max_active_writers_per_core =
new.general.me_adaptive_floor_max_active_writers_per_core;
cfg.general.me_adaptive_floor_max_warm_writers_per_core =
new.general.me_adaptive_floor_max_warm_writers_per_core;
cfg.general.me_adaptive_floor_max_active_writers_global =
new.general.me_adaptive_floor_max_active_writers_global;
cfg.general.me_adaptive_floor_max_warm_writers_global =
new.general.me_adaptive_floor_max_warm_writers_global;
cfg.general.me_route_backpressure_base_timeout_ms = cfg.general.me_route_backpressure_base_timeout_ms =
new.general.me_route_backpressure_base_timeout_ms; new.general.me_route_backpressure_base_timeout_ms;
cfg.general.me_route_backpressure_high_timeout_ms = cfg.general.me_route_backpressure_high_timeout_ms =
new.general.me_route_backpressure_high_timeout_ms; new.general.me_route_backpressure_high_timeout_ms;
cfg.general.me_route_backpressure_high_watermark_pct = cfg.general.me_route_backpressure_high_watermark_pct =
new.general.me_route_backpressure_high_watermark_pct; new.general.me_route_backpressure_high_watermark_pct;
cfg.general.me_health_interval_ms_unhealthy = new.general.me_health_interval_ms_unhealthy;
cfg.general.me_health_interval_ms_healthy = new.general.me_health_interval_ms_healthy;
cfg.general.me_admission_poll_ms = new.general.me_admission_poll_ms;
cfg.general.me_warn_rate_limit_ms = new.general.me_warn_rate_limit_ms;
cfg.access.users = new.access.users.clone(); cfg.access.users = new.access.users.clone();
cfg.access.user_ad_tags = new.access.user_ad_tags.clone(); cfg.access.user_ad_tags = new.access.user_ad_tags.clone();
@@ -647,11 +692,15 @@ fn log_changes(
} }
if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot
|| old_hot.me_deterministic_writer_sort != new_hot.me_deterministic_writer_sort || old_hot.me_deterministic_writer_sort != new_hot.me_deterministic_writer_sort
|| old_hot.me_writer_pick_mode != new_hot.me_writer_pick_mode
|| old_hot.me_writer_pick_sample_size != new_hot.me_writer_pick_sample_size
{ {
info!( info!(
"config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={}", "config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={} writer_pick_mode={:?} writer_pick_sample_size={}",
new_hot.me_secret_atomic_snapshot, new_hot.me_secret_atomic_snapshot,
new_hot.me_deterministic_writer_sort new_hot.me_deterministic_writer_sort,
new_hot.me_writer_pick_mode,
new_hot.me_writer_pick_sample_size,
); );
} }
if old_hot.me_single_endpoint_shadow_writers != new_hot.me_single_endpoint_shadow_writers if old_hot.me_single_endpoint_shadow_writers != new_hot.me_single_endpoint_shadow_writers
@@ -739,9 +788,17 @@ fn log_changes(
!= new_hot.me_adaptive_floor_max_extra_writers_single_per_core != new_hot.me_adaptive_floor_max_extra_writers_single_per_core
|| old_hot.me_adaptive_floor_max_extra_writers_multi_per_core || old_hot.me_adaptive_floor_max_extra_writers_multi_per_core
!= new_hot.me_adaptive_floor_max_extra_writers_multi_per_core != new_hot.me_adaptive_floor_max_extra_writers_multi_per_core
|| old_hot.me_adaptive_floor_max_active_writers_per_core
!= new_hot.me_adaptive_floor_max_active_writers_per_core
|| old_hot.me_adaptive_floor_max_warm_writers_per_core
!= new_hot.me_adaptive_floor_max_warm_writers_per_core
|| old_hot.me_adaptive_floor_max_active_writers_global
!= new_hot.me_adaptive_floor_max_active_writers_global
|| old_hot.me_adaptive_floor_max_warm_writers_global
!= new_hot.me_adaptive_floor_max_warm_writers_global
{ {
info!( info!(
"config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={}", "config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={} max_active_per_core={} max_warm_per_core={} max_active_global={} max_warm_global={}",
new_hot.me_floor_mode, new_hot.me_floor_mode,
new_hot.me_adaptive_floor_idle_secs, new_hot.me_adaptive_floor_idle_secs,
new_hot.me_adaptive_floor_min_writers_single_endpoint, new_hot.me_adaptive_floor_min_writers_single_endpoint,
@@ -751,6 +808,10 @@ fn log_changes(
new_hot.me_adaptive_floor_cpu_cores_override, new_hot.me_adaptive_floor_cpu_cores_override,
new_hot.me_adaptive_floor_max_extra_writers_single_per_core, new_hot.me_adaptive_floor_max_extra_writers_single_per_core,
new_hot.me_adaptive_floor_max_extra_writers_multi_per_core, new_hot.me_adaptive_floor_max_extra_writers_multi_per_core,
new_hot.me_adaptive_floor_max_active_writers_per_core,
new_hot.me_adaptive_floor_max_warm_writers_per_core,
new_hot.me_adaptive_floor_max_active_writers_global,
new_hot.me_adaptive_floor_max_warm_writers_global,
); );
} }
@@ -760,12 +821,21 @@ fn log_changes(
!= new_hot.me_route_backpressure_high_timeout_ms != new_hot.me_route_backpressure_high_timeout_ms
|| old_hot.me_route_backpressure_high_watermark_pct || old_hot.me_route_backpressure_high_watermark_pct
!= new_hot.me_route_backpressure_high_watermark_pct != new_hot.me_route_backpressure_high_watermark_pct
|| old_hot.me_health_interval_ms_unhealthy
!= new_hot.me_health_interval_ms_unhealthy
|| old_hot.me_health_interval_ms_healthy != new_hot.me_health_interval_ms_healthy
|| old_hot.me_admission_poll_ms != new_hot.me_admission_poll_ms
|| old_hot.me_warn_rate_limit_ms != new_hot.me_warn_rate_limit_ms
{ {
info!( info!(
"config reload: me_route_backpressure: base={}ms high={}ms watermark={}%", "config reload: me_route_backpressure: base={}ms high={}ms watermark={}%; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms",
new_hot.me_route_backpressure_base_timeout_ms, new_hot.me_route_backpressure_base_timeout_ms,
new_hot.me_route_backpressure_high_timeout_ms, new_hot.me_route_backpressure_high_timeout_ms,
new_hot.me_route_backpressure_high_watermark_pct, new_hot.me_route_backpressure_high_watermark_pct,
new_hot.me_health_interval_ms_unhealthy,
new_hot.me_health_interval_ms_healthy,
new_hot.me_admission_poll_ms,
new_hot.me_warn_rate_limit_ms,
); );
} }

View File

@@ -285,6 +285,48 @@ impl ProxyConfig {
)); ));
} }
if config.general.me_writer_cmd_channel_capacity == 0 {
return Err(ProxyError::Config(
"general.me_writer_cmd_channel_capacity must be > 0".to_string(),
));
}
if config.general.me_route_channel_capacity == 0 {
return Err(ProxyError::Config(
"general.me_route_channel_capacity must be > 0".to_string(),
));
}
if config.general.me_c2me_channel_capacity == 0 {
return Err(ProxyError::Config(
"general.me_c2me_channel_capacity must be > 0".to_string(),
));
}
if config.general.me_health_interval_ms_unhealthy == 0 {
return Err(ProxyError::Config(
"general.me_health_interval_ms_unhealthy must be > 0".to_string(),
));
}
if config.general.me_health_interval_ms_healthy == 0 {
return Err(ProxyError::Config(
"general.me_health_interval_ms_healthy must be > 0".to_string(),
));
}
if config.general.me_admission_poll_ms == 0 {
return Err(ProxyError::Config(
"general.me_admission_poll_ms must be > 0".to_string(),
));
}
if config.general.me_warn_rate_limit_ms == 0 {
return Err(ProxyError::Config(
"general.me_warn_rate_limit_ms must be > 0".to_string(),
));
}
if config.access.user_max_unique_ips_window_secs == 0 { if config.access.user_max_unique_ips_window_secs == 0 {
return Err(ProxyError::Config( return Err(ProxyError::Config(
"access.user_max_unique_ips_window_secs must be > 0".to_string(), "access.user_max_unique_ips_window_secs must be > 0".to_string(),
@@ -327,6 +369,30 @@ impl ProxyConfig {
)); ));
} }
if config.general.me_adaptive_floor_max_active_writers_per_core == 0 {
return Err(ProxyError::Config(
"general.me_adaptive_floor_max_active_writers_per_core must be > 0".to_string(),
));
}
if config.general.me_adaptive_floor_max_warm_writers_per_core == 0 {
return Err(ProxyError::Config(
"general.me_adaptive_floor_max_warm_writers_per_core must be > 0".to_string(),
));
}
if config.general.me_adaptive_floor_max_active_writers_global == 0 {
return Err(ProxyError::Config(
"general.me_adaptive_floor_max_active_writers_global must be > 0".to_string(),
));
}
if config.general.me_adaptive_floor_max_warm_writers_global == 0 {
return Err(ProxyError::Config(
"general.me_adaptive_floor_max_warm_writers_global must be > 0".to_string(),
));
}
if config.general.me_single_endpoint_outage_backoff_min_ms == 0 { if config.general.me_single_endpoint_outage_backoff_min_ms == 0 {
return Err(ProxyError::Config( return Err(ProxyError::Config(
"general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(), "general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(),
@@ -453,6 +519,12 @@ impl ProxyConfig {
)); ));
} }
if !(2..=4).contains(&config.general.me_writer_pick_sample_size) {
return Err(ProxyError::Config(
"general.me_writer_pick_sample_size must be within [2, 4]".to_string(),
));
}
if config.general.me_route_inline_recovery_attempts == 0 { if config.general.me_route_inline_recovery_attempts == 0 {
return Err(ProxyError::Config( return Err(ProxyError::Config(
"general.me_route_inline_recovery_attempts must be > 0".to_string(), "general.me_route_inline_recovery_attempts must be > 0".to_string(),
@@ -1238,6 +1310,46 @@ mod tests {
let _ = std::fs::remove_file(path); let _ = std::fs::remove_file(path);
} }
#[test]
fn me_adaptive_floor_max_active_writers_per_core_zero_is_rejected() {
let toml = r#"
[general]
me_adaptive_floor_max_active_writers_per_core = 0
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_me_adaptive_floor_max_active_per_core_zero_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.me_adaptive_floor_max_active_writers_per_core must be > 0"));
let _ = std::fs::remove_file(path);
}
#[test]
fn me_adaptive_floor_max_warm_writers_global_zero_is_rejected() {
let toml = r#"
[general]
me_adaptive_floor_max_warm_writers_global = 0
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_me_adaptive_floor_max_warm_global_zero_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.me_adaptive_floor_max_warm_writers_global must be > 0"));
let _ = std::fs::remove_file(path);
}
#[test] #[test]
fn upstream_connect_retry_attempts_zero_is_rejected() { fn upstream_connect_retry_attempts_zero_is_rejected() {
let toml = r#" let toml = r#"

View File

@@ -212,6 +212,32 @@ impl MeRouteNoWriterMode {
} }
} }
/// Middle-End writer selection mode for new client bindings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum MeWriterPickMode {
SortedRr,
#[default]
P2c,
}
impl MeWriterPickMode {
pub fn as_u8(self) -> u8 {
match self {
MeWriterPickMode::SortedRr => 0,
MeWriterPickMode::P2c => 1,
}
}
pub fn from_u8(raw: u8) -> Self {
match raw {
0 => MeWriterPickMode::SortedRr,
1 => MeWriterPickMode::P2c,
_ => MeWriterPickMode::P2c,
}
}
}
/// Per-user unique source IP limit mode. /// Per-user unique source IP limit mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
@@ -420,6 +446,18 @@ pub struct GeneralConfig {
#[serde(default = "default_rpc_proxy_req_every")] #[serde(default = "default_rpc_proxy_req_every")]
pub rpc_proxy_req_every: u64, pub rpc_proxy_req_every: u64,
/// Capacity of per-ME writer command channel.
#[serde(default = "default_me_writer_cmd_channel_capacity")]
pub me_writer_cmd_channel_capacity: usize,
/// Capacity of per-connection ME response route channel.
#[serde(default = "default_me_route_channel_capacity")]
pub me_route_channel_capacity: usize,
/// Capacity of per-client command queue from client reader to ME sender task.
#[serde(default = "default_me_c2me_channel_capacity")]
pub me_c2me_channel_capacity: usize,
/// Max pending ciphertext buffer per client writer (bytes). /// Max pending ciphertext buffer per client writer (bytes).
/// Controls FakeTLS backpressure vs throughput. /// Controls FakeTLS backpressure vs throughput.
#[serde(default = "default_crypto_pending_buffer")] #[serde(default = "default_crypto_pending_buffer")]
@@ -545,6 +583,22 @@ pub struct GeneralConfig {
#[serde(default = "default_me_adaptive_floor_max_extra_writers_multi_per_core")] #[serde(default = "default_me_adaptive_floor_max_extra_writers_multi_per_core")]
pub me_adaptive_floor_max_extra_writers_multi_per_core: u16, pub me_adaptive_floor_max_extra_writers_multi_per_core: u16,
/// Hard cap for active ME writers per logical CPU core.
#[serde(default = "default_me_adaptive_floor_max_active_writers_per_core")]
pub me_adaptive_floor_max_active_writers_per_core: u16,
/// Hard cap for warm ME writers per logical CPU core.
#[serde(default = "default_me_adaptive_floor_max_warm_writers_per_core")]
pub me_adaptive_floor_max_warm_writers_per_core: u16,
/// Hard global cap for active ME writers.
#[serde(default = "default_me_adaptive_floor_max_active_writers_global")]
pub me_adaptive_floor_max_active_writers_global: u32,
/// Hard global cap for warm ME writers.
#[serde(default = "default_me_adaptive_floor_max_warm_writers_global")]
pub me_adaptive_floor_max_warm_writers_global: u32,
/// Connect attempts for the selected upstream before returning error/fallback. /// Connect attempts for the selected upstream before returning error/fallback.
#[serde(default = "default_upstream_connect_retry_attempts")] #[serde(default = "default_upstream_connect_retry_attempts")]
pub upstream_connect_retry_attempts: u32, pub upstream_connect_retry_attempts: u32,
@@ -604,6 +658,22 @@ pub struct GeneralConfig {
#[serde(default = "default_me_route_backpressure_high_watermark_pct")] #[serde(default = "default_me_route_backpressure_high_watermark_pct")]
pub me_route_backpressure_high_watermark_pct: u8, pub me_route_backpressure_high_watermark_pct: u8,
/// Health monitor interval in milliseconds while writer coverage is degraded.
#[serde(default = "default_me_health_interval_ms_unhealthy")]
pub me_health_interval_ms_unhealthy: u64,
/// Health monitor interval in milliseconds while writer coverage is stable.
#[serde(default = "default_me_health_interval_ms_healthy")]
pub me_health_interval_ms_healthy: u64,
/// Poll interval in milliseconds for conditional-admission state checks.
#[serde(default = "default_me_admission_poll_ms")]
pub me_admission_poll_ms: u64,
/// Cooldown for repetitive ME warning logs in milliseconds.
#[serde(default = "default_me_warn_rate_limit_ms")]
pub me_warn_rate_limit_ms: u64,
/// ME route behavior when no writer is immediately available. /// ME route behavior when no writer is immediately available.
#[serde(default)] #[serde(default)]
pub me_route_no_writer_mode: MeRouteNoWriterMode, pub me_route_no_writer_mode: MeRouteNoWriterMode,
@@ -738,6 +808,14 @@ pub struct GeneralConfig {
#[serde(default = "default_me_deterministic_writer_sort")] #[serde(default = "default_me_deterministic_writer_sort")]
pub me_deterministic_writer_sort: bool, pub me_deterministic_writer_sort: bool,
/// Writer selection mode for ME route bind path.
#[serde(default)]
pub me_writer_pick_mode: MeWriterPickMode,
/// Number of candidates sampled by writer picker in `p2c` mode.
#[serde(default = "default_me_writer_pick_sample_size")]
pub me_writer_pick_sample_size: u8,
/// Enable NTP drift check at startup. /// Enable NTP drift check at startup.
#[serde(default = "default_ntp_check")] #[serde(default = "default_ntp_check")]
pub ntp_check: bool, pub ntp_check: bool,
@@ -780,6 +858,9 @@ impl Default for GeneralConfig {
me_keepalive_jitter_secs: default_keepalive_jitter(), me_keepalive_jitter_secs: default_keepalive_jitter(),
me_keepalive_payload_random: default_true(), me_keepalive_payload_random: default_true(),
rpc_proxy_req_every: default_rpc_proxy_req_every(), rpc_proxy_req_every: default_rpc_proxy_req_every(),
me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(),
me_route_channel_capacity: default_me_route_channel_capacity(),
me_c2me_channel_capacity: default_me_c2me_channel_capacity(),
me_warmup_stagger_enabled: default_true(), me_warmup_stagger_enabled: default_true(),
me_warmup_step_delay_ms: default_warmup_step_delay_ms(), me_warmup_step_delay_ms: default_warmup_step_delay_ms(),
me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(), me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(),
@@ -802,6 +883,10 @@ impl Default for GeneralConfig {
me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(), me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(),
me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(), me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(),
me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(), me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(),
me_adaptive_floor_max_active_writers_per_core: default_me_adaptive_floor_max_active_writers_per_core(),
me_adaptive_floor_max_warm_writers_per_core: default_me_adaptive_floor_max_warm_writers_per_core(),
me_adaptive_floor_max_active_writers_global: default_me_adaptive_floor_max_active_writers_global(),
me_adaptive_floor_max_warm_writers_global: default_me_adaptive_floor_max_warm_writers_global(),
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
upstream_connect_budget_ms: default_upstream_connect_budget_ms(), upstream_connect_budget_ms: default_upstream_connect_budget_ms(),
@@ -817,6 +902,10 @@ impl Default for GeneralConfig {
me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(), me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(),
me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(), me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(),
me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(), me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(),
me_health_interval_ms_unhealthy: default_me_health_interval_ms_unhealthy(),
me_health_interval_ms_healthy: default_me_health_interval_ms_healthy(),
me_admission_poll_ms: default_me_admission_poll_ms(),
me_warn_rate_limit_ms: default_me_warn_rate_limit_ms(),
me_route_no_writer_mode: MeRouteNoWriterMode::default(), me_route_no_writer_mode: MeRouteNoWriterMode::default(),
me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(), me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(),
me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(), me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(),
@@ -857,6 +946,8 @@ impl Default for GeneralConfig {
me_reinit_trigger_channel: default_me_reinit_trigger_channel(), me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(), me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
me_deterministic_writer_sort: default_me_deterministic_writer_sort(), me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
me_writer_pick_mode: MeWriterPickMode::default(),
me_writer_pick_sample_size: default_me_writer_pick_sample_size(),
ntp_check: default_ntp_check(), ntp_check: default_ntp_check(),
ntp_servers: default_ntp_servers(), ntp_servers: default_ntp_servers(),
auto_degradation_enabled: default_true(), auto_degradation_enabled: default_true(),

View File

@@ -8,7 +8,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use rand::Rng; use rand::Rng;
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tokio::signal; use tokio::signal;
use tokio::sync::{Semaphore, mpsc, watch}; use tokio::sync::{RwLock, Semaphore, mpsc, watch};
use tracing::{debug, error, info, warn}; use tracing::{debug, error, info, warn};
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
#[cfg(unix)] #[cfg(unix)]
@@ -26,6 +26,7 @@ mod protocol;
mod proxy; mod proxy;
mod stats; mod stats;
mod stream; mod stream;
mod startup;
mod transport; mod transport;
mod tls_front; mod tls_front;
mod util; mod util;
@@ -39,6 +40,14 @@ use crate::proxy::ClientHandler;
use crate::stats::beobachten::BeobachtenStore; use crate::stats::beobachten::BeobachtenStore;
use crate::stats::telemetry::TelemetryPolicy; use crate::stats::telemetry::TelemetryPolicy;
use crate::stats::{ReplayChecker, Stats}; use crate::stats::{ReplayChecker, Stats};
use crate::startup::{
COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, COMPONENT_CONFIG_WATCHER_START,
COMPONENT_DC_CONNECTIVITY_PING, COMPONENT_LISTENERS_BIND, COMPONENT_ME_CONNECTIVITY_PING,
COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, COMPONENT_ME_PROXY_CONFIG_V4,
COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, COMPONENT_METRICS_START,
COMPONENT_NETWORK_PROBE, COMPONENT_RUNTIME_READY, COMPONENT_TLS_FRONT_BOOTSTRAP,
COMPONENT_TRACING_INIT, StartupMeStatus, StartupTracker,
};
use crate::stream::BufferPool; use crate::stream::BufferPool;
use crate::transport::middle_proxy::{ use crate::transport::middle_proxy::{
MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line, MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line,
@@ -373,6 +382,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
.unwrap_or_default() .unwrap_or_default()
.as_secs(); .as_secs();
let startup_tracker = Arc::new(StartupTracker::new(process_started_at_epoch_secs));
startup_tracker
.start_component(COMPONENT_CONFIG_LOAD, Some("load and validate config".to_string()))
.await;
let (config_path, cli_silent, cli_log_level) = parse_cli(); let (config_path, cli_silent, cli_log_level) = parse_cli();
let mut config = match ProxyConfig::load(&config_path) { let mut config = match ProxyConfig::load(&config_path) {
@@ -399,6 +412,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
eprintln!("[telemt] Invalid network.dns_overrides: {}", e); eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
std::process::exit(1); std::process::exit(1);
} }
startup_tracker
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
.await;
let has_rust_log = std::env::var("RUST_LOG").is_ok(); let has_rust_log = std::env::var("RUST_LOG").is_ok();
let effective_log_level = if cli_silent { let effective_log_level = if cli_silent {
@@ -410,6 +426,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
}; };
let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info")); let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info"));
startup_tracker
.start_component(COMPONENT_TRACING_INIT, Some("initialize tracing subscriber".to_string()))
.await;
// Configure color output based on config // Configure color output based on config
let fmt_layer = if config.general.disable_colors { let fmt_layer = if config.general.disable_colors {
@@ -422,6 +441,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
.with(filter_layer) .with(filter_layer)
.with(fmt_layer) .with(fmt_layer)
.init(); .init();
startup_tracker
.complete_component(COMPONENT_TRACING_INIT, Some("tracing initialized".to_string()))
.await;
info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION")); info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION"));
info!("Log level: {}", effective_log_level); info!("Log level: {}", effective_log_level);
@@ -473,6 +495,95 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
config.general.upstream_connect_failfast_hard_errors, config.general.upstream_connect_failfast_hard_errors,
stats.clone(), stats.clone(),
)); ));
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.load_limits(&config.access.user_max_unique_ips).await;
ip_tracker
.set_limit_policy(
config.access.user_max_unique_ips_mode,
config.access.user_max_unique_ips_window_secs,
)
.await;
if !config.access.user_max_unique_ips.is_empty() {
info!(
"IP limits configured for {} users",
config.access.user_max_unique_ips.len()
);
}
if !config.network.dns_overrides.is_empty() {
info!(
"Runtime DNS overrides configured: {} entries",
config.network.dns_overrides.len()
);
}
let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone()));
let initial_admission_open = !config.general.use_middle_proxy;
let (admission_tx, admission_rx) = watch::channel(initial_admission_open);
let api_me_pool = Arc::new(RwLock::new(None::<Arc<MePool>>));
startup_tracker
.start_component(COMPONENT_API_BOOTSTRAP, Some("spawn API listener task".to_string()))
.await;
if config.server.api.enabled {
let listen = match config.server.api.listen.parse::<SocketAddr>() {
Ok(listen) => listen,
Err(error) => {
warn!(
error = %error,
listen = %config.server.api.listen,
"Invalid server.api.listen; API is disabled"
);
SocketAddr::from(([127, 0, 0, 1], 0))
}
};
if listen.port() != 0 {
let stats_api = stats.clone();
let ip_tracker_api = ip_tracker.clone();
let me_pool_api = api_me_pool.clone();
let upstream_manager_api = upstream_manager.clone();
let config_rx_api = api_config_rx.clone();
let admission_rx_api = admission_rx.clone();
let config_path_api = std::path::PathBuf::from(&config_path);
let startup_tracker_api = startup_tracker.clone();
tokio::spawn(async move {
api::serve(
listen,
stats_api,
ip_tracker_api,
me_pool_api,
upstream_manager_api,
config_rx_api,
admission_rx_api,
config_path_api,
None,
None,
process_started_at_epoch_secs,
startup_tracker_api,
)
.await;
});
startup_tracker
.complete_component(
COMPONENT_API_BOOTSTRAP,
Some(format!("api task spawned on {}", listen)),
)
.await;
} else {
startup_tracker
.skip_component(
COMPONENT_API_BOOTSTRAP,
Some("server.api.listen has zero port".to_string()),
)
.await;
}
} else {
startup_tracker
.skip_component(
COMPONENT_API_BOOTSTRAP,
Some("server.api.enabled is false".to_string()),
)
.await;
}
let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
tls_domains.push(config.censorship.tls_domain.clone()); tls_domains.push(config.censorship.tls_domain.clone());
@@ -483,6 +594,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
} }
// Start TLS front fetching in background immediately, in parallel with STUN probing. // Start TLS front fetching in background immediately, in parallel with STUN probing.
startup_tracker
.start_component(
COMPONENT_TLS_FRONT_BOOTSTRAP,
Some("initialize TLS front cache/bootstrap tasks".to_string()),
)
.await;
let tls_cache: Option<Arc<TlsFrontCache>> = if config.censorship.tls_emulation { let tls_cache: Option<Arc<TlsFrontCache>> = if config.censorship.tls_emulation {
let cache = Arc::new(TlsFrontCache::new( let cache = Arc::new(TlsFrontCache::new(
&tls_domains, &tls_domains,
@@ -603,9 +720,26 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
Some(cache) Some(cache)
} else { } else {
startup_tracker
.skip_component(
COMPONENT_TLS_FRONT_BOOTSTRAP,
Some("censorship.tls_emulation is false".to_string()),
)
.await;
None None
}; };
if tls_cache.is_some() {
startup_tracker
.complete_component(
COMPONENT_TLS_FRONT_BOOTSTRAP,
Some("tls front cache is initialized".to_string()),
)
.await;
}
startup_tracker
.start_component(COMPONENT_NETWORK_PROBE, Some("probe network capabilities".to_string()))
.await;
let probe = run_probe( let probe = run_probe(
&config.network, &config.network,
config.general.middle_proxy_nat_probe, config.general.middle_proxy_nat_probe,
@@ -614,32 +748,18 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
.await?; .await?;
let decision = decide_network_capabilities(&config.network, &probe); let decision = decide_network_capabilities(&config.network, &probe);
log_probe_result(&probe, &decision); log_probe_result(&probe, &decision);
startup_tracker
.complete_component(
COMPONENT_NETWORK_PROBE,
Some("network capabilities determined".to_string()),
)
.await;
let prefer_ipv6 = decision.prefer_ipv6(); let prefer_ipv6 = decision.prefer_ipv6();
let mut use_middle_proxy = config.general.use_middle_proxy; let mut use_middle_proxy = config.general.use_middle_proxy;
let beobachten = Arc::new(BeobachtenStore::new()); let beobachten = Arc::new(BeobachtenStore::new());
let rng = Arc::new(SecureRandom::new()); let rng = Arc::new(SecureRandom::new());
// IP Tracker initialization
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.load_limits(&config.access.user_max_unique_ips).await;
ip_tracker
.set_limit_policy(
config.access.user_max_unique_ips_mode,
config.access.user_max_unique_ips_window_secs,
)
.await;
if !config.access.user_max_unique_ips.is_empty() {
info!("IP limits configured for {} users", config.access.user_max_unique_ips.len());
}
if !config.network.dns_overrides.is_empty() {
info!(
"Runtime DNS overrides configured: {} entries",
config.network.dns_overrides.len()
);
}
// Connection concurrency limit // Connection concurrency limit
let max_connections = Arc::new(Semaphore::new(10_000)); let max_connections = Arc::new(Semaphore::new(10_000));
@@ -657,6 +777,59 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
} }
} }
if use_middle_proxy {
startup_tracker
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_SECRET_FETCH)
.await;
startup_tracker
.start_component(
COMPONENT_ME_SECRET_FETCH,
Some("fetch proxy-secret from source/cache".to_string()),
)
.await;
startup_tracker
.set_me_retry_limit(if !me2dc_fallback || me_init_retry_attempts == 0 {
"unlimited".to_string()
} else {
me_init_retry_attempts.to_string()
})
.await;
} else {
startup_tracker
.set_me_status(StartupMeStatus::Skipped, "skipped")
.await;
startup_tracker
.skip_component(
COMPONENT_ME_SECRET_FETCH,
Some("middle proxy mode disabled".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_PROXY_CONFIG_V4,
Some("middle proxy mode disabled".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_PROXY_CONFIG_V6,
Some("middle proxy mode disabled".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_POOL_CONSTRUCT,
Some("middle proxy mode disabled".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_POOL_INIT_STAGE1,
Some("middle proxy mode disabled".to_string()),
)
.await;
}
// ===================================================================== // =====================================================================
// Middle Proxy initialization (if enabled) // Middle Proxy initialization (if enabled)
// ===================================================================== // =====================================================================
@@ -694,6 +867,9 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
{ {
Ok(proxy_secret) => break Some(proxy_secret), Ok(proxy_secret) => break Some(proxy_secret),
Err(e) => { Err(e) => {
startup_tracker
.set_me_last_error(Some(e.to_string()))
.await;
if me2dc_fallback { if me2dc_fallback {
error!( error!(
error = %e, error = %e,
@@ -713,6 +889,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
}; };
match proxy_secret { match proxy_secret {
Some(proxy_secret) => { Some(proxy_secret) => {
startup_tracker
.complete_component(
COMPONENT_ME_SECRET_FETCH,
Some("proxy-secret loaded".to_string()),
)
.await;
info!( info!(
secret_len = proxy_secret.len(), secret_len = proxy_secret.len(),
key_sig = format_args!( key_sig = format_args!(
@@ -731,6 +913,15 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
"Proxy-secret loaded" "Proxy-secret loaded"
); );
startup_tracker
.start_component(
COMPONENT_ME_PROXY_CONFIG_V4,
Some("load startup proxy-config v4".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4)
.await;
let cfg_v4 = load_startup_proxy_config_snapshot( let cfg_v4 = load_startup_proxy_config_snapshot(
"https://core.telegram.org/getProxyConfig", "https://core.telegram.org/getProxyConfig",
config.general.proxy_config_v4_cache_path.as_deref(), config.general.proxy_config_v4_cache_path.as_deref(),
@@ -738,6 +929,30 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
"getProxyConfig", "getProxyConfig",
) )
.await; .await;
if cfg_v4.is_some() {
startup_tracker
.complete_component(
COMPONENT_ME_PROXY_CONFIG_V4,
Some("proxy-config v4 loaded".to_string()),
)
.await;
} else {
startup_tracker
.fail_component(
COMPONENT_ME_PROXY_CONFIG_V4,
Some("proxy-config v4 unavailable".to_string()),
)
.await;
}
startup_tracker
.start_component(
COMPONENT_ME_PROXY_CONFIG_V6,
Some("load startup proxy-config v6".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6)
.await;
let cfg_v6 = load_startup_proxy_config_snapshot( let cfg_v6 = load_startup_proxy_config_snapshot(
"https://core.telegram.org/getProxyConfigV6", "https://core.telegram.org/getProxyConfigV6",
config.general.proxy_config_v6_cache_path.as_deref(), config.general.proxy_config_v6_cache_path.as_deref(),
@@ -745,8 +960,32 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
"getProxyConfigV6", "getProxyConfigV6",
) )
.await; .await;
if cfg_v6.is_some() {
startup_tracker
.complete_component(
COMPONENT_ME_PROXY_CONFIG_V6,
Some("proxy-config v6 loaded".to_string()),
)
.await;
} else {
startup_tracker
.fail_component(
COMPONENT_ME_PROXY_CONFIG_V6,
Some("proxy-config v6 unavailable".to_string()),
)
.await;
}
if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) { if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) {
startup_tracker
.start_component(
COMPONENT_ME_POOL_CONSTRUCT,
Some("construct ME pool".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_CONSTRUCT)
.await;
let pool = MePool::new( let pool = MePool::new(
proxy_tag.clone(), proxy_tag.clone(),
proxy_secret, proxy_secret,
@@ -792,6 +1031,10 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
config.general.me_adaptive_floor_cpu_cores_override, config.general.me_adaptive_floor_cpu_cores_override,
config.general.me_adaptive_floor_max_extra_writers_single_per_core, config.general.me_adaptive_floor_max_extra_writers_single_per_core,
config.general.me_adaptive_floor_max_extra_writers_multi_per_core, config.general.me_adaptive_floor_max_extra_writers_multi_per_core,
config.general.me_adaptive_floor_max_active_writers_per_core,
config.general.me_adaptive_floor_max_warm_writers_per_core,
config.general.me_adaptive_floor_max_active_writers_global,
config.general.me_adaptive_floor_max_warm_writers_global,
config.general.hardswap, config.general.hardswap,
config.general.me_pool_drain_ttl_secs, config.general.me_pool_drain_ttl_secs,
config.general.effective_me_pool_force_close_secs(), config.general.effective_me_pool_force_close_secs(),
@@ -804,21 +1047,60 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
config.general.me_bind_stale_ttl_secs, config.general.me_bind_stale_ttl_secs,
config.general.me_secret_atomic_snapshot, config.general.me_secret_atomic_snapshot,
config.general.me_deterministic_writer_sort, config.general.me_deterministic_writer_sort,
config.general.me_writer_pick_mode,
config.general.me_writer_pick_sample_size,
config.general.me_socks_kdf_policy, config.general.me_socks_kdf_policy,
config.general.me_writer_cmd_channel_capacity,
config.general.me_route_channel_capacity,
config.general.me_route_backpressure_base_timeout_ms, config.general.me_route_backpressure_base_timeout_ms,
config.general.me_route_backpressure_high_timeout_ms, config.general.me_route_backpressure_high_timeout_ms,
config.general.me_route_backpressure_high_watermark_pct, config.general.me_route_backpressure_high_watermark_pct,
config.general.me_health_interval_ms_unhealthy,
config.general.me_health_interval_ms_healthy,
config.general.me_warn_rate_limit_ms,
config.general.me_route_no_writer_mode, config.general.me_route_no_writer_mode,
config.general.me_route_no_writer_wait_ms, config.general.me_route_no_writer_wait_ms,
config.general.me_route_inline_recovery_attempts, config.general.me_route_inline_recovery_attempts,
config.general.me_route_inline_recovery_wait_ms, config.general.me_route_inline_recovery_wait_ms,
); );
startup_tracker
.complete_component(
COMPONENT_ME_POOL_CONSTRUCT,
Some("ME pool object created".to_string()),
)
.await;
*api_me_pool.write().await = Some(pool.clone());
startup_tracker
.start_component(
COMPONENT_ME_POOL_INIT_STAGE1,
Some("initialize ME pool writers".to_string()),
)
.await;
startup_tracker
.set_me_status(
StartupMeStatus::Initializing,
COMPONENT_ME_POOL_INIT_STAGE1,
)
.await;
let mut init_attempt: u32 = 0; let mut init_attempt: u32 = 0;
loop { loop {
init_attempt = init_attempt.saturating_add(1); init_attempt = init_attempt.saturating_add(1);
startup_tracker.set_me_init_attempt(init_attempt).await;
match pool.init(pool_size, &rng).await { match pool.init(pool_size, &rng).await {
Ok(()) => { Ok(()) => {
startup_tracker
.set_me_last_error(None)
.await;
startup_tracker
.complete_component(
COMPONENT_ME_POOL_INIT_STAGE1,
Some("ME pool initialized".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Ready, "ready")
.await;
info!( info!(
attempt = init_attempt, attempt = init_attempt,
"Middle-End pool initialized successfully" "Middle-End pool initialized successfully"
@@ -838,8 +1120,20 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
break Some(pool); break Some(pool);
} }
Err(e) => { Err(e) => {
startup_tracker
.set_me_last_error(Some(e.to_string()))
.await;
let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; let retries_limited = me2dc_fallback && me_init_retry_attempts > 0;
if retries_limited && init_attempt >= me_init_retry_attempts { if retries_limited && init_attempt >= me_init_retry_attempts {
startup_tracker
.fail_component(
COMPONENT_ME_POOL_INIT_STAGE1,
Some("ME init retry budget exhausted".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Failed, "failed")
.await;
error!( error!(
error = %e, error = %e,
attempt = init_attempt, attempt = init_attempt,
@@ -879,10 +1173,60 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
} }
} }
} else { } else {
startup_tracker
.skip_component(
COMPONENT_ME_POOL_CONSTRUCT,
Some("ME configs are incomplete".to_string()),
)
.await;
startup_tracker
.fail_component(
COMPONENT_ME_POOL_INIT_STAGE1,
Some("ME configs are incomplete".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Failed, "failed")
.await;
None None
} }
} }
None => None, None => {
startup_tracker
.fail_component(
COMPONENT_ME_SECRET_FETCH,
Some("proxy-secret unavailable".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_PROXY_CONFIG_V4,
Some("proxy-secret unavailable".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_PROXY_CONFIG_V6,
Some("proxy-secret unavailable".to_string()),
)
.await;
startup_tracker
.skip_component(
COMPONENT_ME_POOL_CONSTRUCT,
Some("proxy-secret unavailable".to_string()),
)
.await;
startup_tracker
.fail_component(
COMPONENT_ME_POOL_INIT_STAGE1,
Some("proxy-secret unavailable".to_string()),
)
.await;
startup_tracker
.set_me_status(StartupMeStatus::Failed, "failed")
.await;
None
}
} }
} else { } else {
None None
@@ -890,12 +1234,33 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
// If ME failed to initialize, force direct-only mode. // If ME failed to initialize, force direct-only mode.
if me_pool.is_some() { if me_pool.is_some() {
startup_tracker
.set_transport_mode("middle_proxy")
.await;
startup_tracker
.set_degraded(false)
.await;
info!("Transport: Middle-End Proxy - all DC-over-RPC"); info!("Transport: Middle-End Proxy - all DC-over-RPC");
} else { } else {
let _ = use_middle_proxy; let _ = use_middle_proxy;
use_middle_proxy = false; use_middle_proxy = false;
// Make runtime config reflect direct-only mode for handlers. // Make runtime config reflect direct-only mode for handlers.
config.general.use_middle_proxy = false; config.general.use_middle_proxy = false;
startup_tracker
.set_transport_mode("direct")
.await;
startup_tracker
.set_degraded(true)
.await;
if me2dc_fallback {
startup_tracker
.set_me_status(StartupMeStatus::Failed, "fallback_to_direct")
.await;
} else {
startup_tracker
.set_me_status(StartupMeStatus::Skipped, "skipped")
.await;
}
info!("Transport: Direct DC - TCP - standard DC-over-TCP"); info!("Transport: Direct DC - TCP - standard DC-over-TCP");
} }
@@ -910,6 +1275,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096)); let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096));
// Middle-End ping before DC connectivity // Middle-End ping before DC connectivity
if me_pool.is_some() {
startup_tracker
.start_component(
COMPONENT_ME_CONNECTIVITY_PING,
Some("run startup ME connectivity check".to_string()),
)
.await;
} else {
startup_tracker
.skip_component(
COMPONENT_ME_CONNECTIVITY_PING,
Some("ME pool is not available".to_string()),
)
.await;
}
if let Some(ref pool) = me_pool { if let Some(ref pool) = me_pool {
let me_results = run_me_ping(pool, &rng).await; let me_results = run_me_ping(pool, &rng).await;
@@ -979,9 +1359,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
} }
} }
info!("============================================================"); info!("============================================================");
startup_tracker
.complete_component(
COMPONENT_ME_CONNECTIVITY_PING,
Some("startup ME connectivity check completed".to_string()),
)
.await;
} }
info!("================= Telegram DC Connectivity ================="); info!("================= Telegram DC Connectivity =================");
startup_tracker
.start_component(
COMPONENT_DC_CONNECTIVITY_PING,
Some("run startup DC connectivity check".to_string()),
)
.await;
let ping_results = upstream_manager let ping_results = upstream_manager
.ping_all_dcs( .ping_all_dcs(
@@ -1061,9 +1453,21 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
info!("============================================================"); info!("============================================================");
} }
} }
startup_tracker
.complete_component(
COMPONENT_DC_CONNECTIVITY_PING,
Some("startup DC connectivity check completed".to_string()),
)
.await;
let initialized_secs = process_started_at.elapsed().as_secs(); let initialized_secs = process_started_at.elapsed().as_secs();
let second_suffix = if initialized_secs == 1 { "" } else { "s" }; let second_suffix = if initialized_secs == 1 { "" } else { "s" };
startup_tracker
.start_component(
COMPONENT_RUNTIME_READY,
Some("finalize startup runtime state".to_string()),
)
.await;
info!("===================== Telegram Startup ====================="); info!("===================== Telegram Startup =====================");
info!( info!(
" DC/ME Initialized in {} second{}", " DC/ME Initialized in {} second{}",
@@ -1074,6 +1478,7 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
if let Some(ref pool) = me_pool { if let Some(ref pool) = me_pool {
pool.set_runtime_ready(true); pool.set_runtime_ready(true);
} }
*api_me_pool.write().await = me_pool.clone();
// Background tasks // Background tasks
let um_clone = upstream_manager.clone(); let um_clone = upstream_manager.clone();
@@ -1105,6 +1510,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
// ── Hot-reload watcher ──────────────────────────────────────────────── // ── Hot-reload watcher ────────────────────────────────────────────────
// Uses inotify to detect file changes instantly (SIGHUP also works). // Uses inotify to detect file changes instantly (SIGHUP also works).
// detected_ip_v4/v6 are passed so newly added users get correct TG links. // detected_ip_v4/v6 are passed so newly added users get correct TG links.
startup_tracker
.start_component(
COMPONENT_CONFIG_WATCHER_START,
Some("spawn config hot-reload watcher".to_string()),
)
.await;
let (config_rx, mut log_level_rx): ( let (config_rx, mut log_level_rx): (
tokio::sync::watch::Receiver<Arc<ProxyConfig>>, tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
tokio::sync::watch::Receiver<LogLevel>, tokio::sync::watch::Receiver<LogLevel>,
@@ -1114,6 +1525,23 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
detected_ip_v4, detected_ip_v4,
detected_ip_v6, detected_ip_v6,
); );
startup_tracker
.complete_component(
COMPONENT_CONFIG_WATCHER_START,
Some("config hot-reload watcher started".to_string()),
)
.await;
let mut config_rx_api_bridge = config_rx.clone();
let api_config_tx_bridge = api_config_tx.clone();
tokio::spawn(async move {
loop {
if config_rx_api_bridge.changed().await.is_err() {
break;
}
let cfg = config_rx_api_bridge.borrow_and_update().clone();
api_config_tx_bridge.send_replace(cfg);
}
});
let stats_policy = stats.clone(); let stats_policy = stats.clone();
let mut config_rx_policy = config_rx.clone(); let mut config_rx_policy = config_rx.clone();
@@ -1244,6 +1672,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
}); });
} }
startup_tracker
.start_component(
COMPONENT_LISTENERS_BIND,
Some("bind TCP/Unix listeners".to_string()),
)
.await;
let mut listeners = Vec::new(); let mut listeners = Vec::new();
for listener_conf in &config.server.listeners { for listener_conf in &config.server.listeners {
@@ -1345,7 +1779,6 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
print_proxy_links(&host, port, &config); print_proxy_links(&host, port, &config);
} }
let (admission_tx, admission_rx) = watch::channel(true);
if config.general.use_middle_proxy { if config.general.use_middle_proxy {
if let Some(pool) = me_pool.as_ref() { if let Some(pool) = me_pool.as_ref() {
let initial_open = pool.admission_ready_conditional_cast().await; let initial_open = pool.admission_ready_conditional_cast().await;
@@ -1358,11 +1791,24 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
let pool_for_gate = pool.clone(); let pool_for_gate = pool.clone();
let admission_tx_gate = admission_tx.clone(); let admission_tx_gate = admission_tx.clone();
let mut config_rx_gate = config_rx.clone();
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
tokio::spawn(async move { tokio::spawn(async move {
let mut gate_open = initial_open; let mut gate_open = initial_open;
let mut open_streak = if initial_open { 1u32 } else { 0u32 }; let mut open_streak = if initial_open { 1u32 } else { 0u32 };
let mut close_streak = if initial_open { 0u32 } else { 1u32 }; let mut close_streak = if initial_open { 0u32 } else { 1u32 };
loop { loop {
tokio::select! {
changed = config_rx_gate.changed() => {
if changed.is_err() {
break;
}
let cfg = config_rx_gate.borrow_and_update().clone();
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
continue;
}
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
}
let ready = pool_for_gate.admission_ready_conditional_cast().await; let ready = pool_for_gate.admission_ready_conditional_cast().await;
if ready { if ready {
open_streak = open_streak.saturating_add(1); open_streak = open_streak.saturating_add(1);
@@ -1387,7 +1833,6 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
); );
} }
} }
tokio::time::sleep(Duration::from_millis(250)).await;
} }
}); });
} else { } else {
@@ -1495,6 +1940,16 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
} }
}); });
} }
startup_tracker
.complete_component(
COMPONENT_LISTENERS_BIND,
Some(format!(
"listeners configured tcp={} unix={}",
listeners.len(),
has_unix_listener
)),
)
.await;
if listeners.is_empty() && !has_unix_listener { if listeners.is_empty() && !has_unix_listener {
error!("No listeners. Exiting."); error!("No listeners. Exiting.");
@@ -1528,6 +1983,12 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
}); });
if let Some(port) = config.server.metrics_port { if let Some(port) = config.server.metrics_port {
startup_tracker
.start_component(
COMPONENT_METRICS_START,
Some(format!("spawn metrics endpoint on {}", port)),
)
.await;
let stats = stats.clone(); let stats = stats.clone();
let beobachten = beobachten.clone(); let beobachten = beobachten.clone();
let config_rx_metrics = config_rx.clone(); let config_rx_metrics = config_rx.clone();
@@ -1544,48 +2005,28 @@ async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
) )
.await; .await;
}); });
startup_tracker
.complete_component(
COMPONENT_METRICS_START,
Some("metrics task spawned".to_string()),
)
.await;
} else {
startup_tracker
.skip_component(
COMPONENT_METRICS_START,
Some("server.metrics_port is not configured".to_string()),
)
.await;
} }
if config.server.api.enabled { startup_tracker
let listen = match config.server.api.listen.parse::<SocketAddr>() { .complete_component(
Ok(listen) => listen, COMPONENT_RUNTIME_READY,
Err(error) => { Some("startup pipeline is fully initialized".to_string()),
warn!( )
error = %error, .await;
listen = %config.server.api.listen, startup_tracker.mark_ready().await;
"Invalid server.api.listen; API is disabled"
);
SocketAddr::from(([127, 0, 0, 1], 0))
}
};
if listen.port() != 0 {
let stats = stats.clone();
let ip_tracker_api = ip_tracker.clone();
let me_pool_api = me_pool.clone();
let upstream_manager_api = upstream_manager.clone();
let config_rx_api = config_rx.clone();
let admission_rx_api = admission_rx.clone();
let config_path_api = std::path::PathBuf::from(&config_path);
let startup_detected_ip_v4 = detected_ip_v4;
let startup_detected_ip_v6 = detected_ip_v6;
tokio::spawn(async move {
api::serve(
listen,
stats,
ip_tracker_api,
me_pool_api,
upstream_manager_api,
config_rx_api,
admission_rx_api,
config_path_api,
startup_detected_ip_v4,
startup_detected_ip_v6,
process_started_at_epoch_secs,
)
.await;
});
}
}
for (listener, listener_proxy_protocol) in listeners { for (listener, listener_proxy_protocol) in listeners {
let mut config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone(); let mut config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();

View File

@@ -689,6 +689,135 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
} }
); );
let _ = writeln!(
out,
"# HELP telemt_me_writer_pick_total ME writer-pick outcomes by mode and result"
);
let _ = writeln!(out, "# TYPE telemt_me_writer_pick_total counter");
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_try\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_sorted_rr_success_try_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_fallback\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_sorted_rr_success_fallback_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"full\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_sorted_rr_full_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"closed\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_sorted_rr_closed_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"no_candidate\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_sorted_rr_no_candidate_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_try\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_p2c_success_try_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_fallback\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_p2c_success_fallback_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"full\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_p2c_full_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"closed\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_p2c_closed_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_me_writer_pick_total{{mode=\"p2c\",result=\"no_candidate\"}} {}",
if me_allows_normal {
stats.get_me_writer_pick_p2c_no_candidate_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_writer_pick_blocking_fallback_total ME writer-pick blocking fallback attempts"
);
let _ = writeln!(
out,
"# TYPE telemt_me_writer_pick_blocking_fallback_total counter"
);
let _ = writeln!(
out,
"telemt_me_writer_pick_blocking_fallback_total {}",
if me_allows_normal {
stats.get_me_writer_pick_blocking_fallback_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_writer_pick_mode_switch_total Writer-pick mode switches via runtime updates"
);
let _ = writeln!(out, "# TYPE telemt_me_writer_pick_mode_switch_total counter");
let _ = writeln!(
out,
"telemt_me_writer_pick_mode_switch_total {}",
if me_allows_normal {
stats.get_me_writer_pick_mode_switch_total()
} else {
0
}
);
let _ = writeln!( let _ = writeln!(
out, out,
"# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes" "# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes"
@@ -1053,6 +1182,102 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
0 0
} }
); );
let _ = writeln!(
out,
"# HELP telemt_me_adaptive_floor_active_cap_configured Runtime configured active writer cap"
);
let _ = writeln!(
out,
"# TYPE telemt_me_adaptive_floor_active_cap_configured gauge"
);
let _ = writeln!(
out,
"telemt_me_adaptive_floor_active_cap_configured {}",
if me_allows_normal {
stats.get_me_floor_active_cap_configured_gauge()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_adaptive_floor_active_cap_effective Runtime effective active writer cap"
);
let _ = writeln!(
out,
"# TYPE telemt_me_adaptive_floor_active_cap_effective gauge"
);
let _ = writeln!(
out,
"telemt_me_adaptive_floor_active_cap_effective {}",
if me_allows_normal {
stats.get_me_floor_active_cap_effective_gauge()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_adaptive_floor_warm_cap_configured Runtime configured warm writer cap"
);
let _ = writeln!(
out,
"# TYPE telemt_me_adaptive_floor_warm_cap_configured gauge"
);
let _ = writeln!(
out,
"telemt_me_adaptive_floor_warm_cap_configured {}",
if me_allows_normal {
stats.get_me_floor_warm_cap_configured_gauge()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_adaptive_floor_warm_cap_effective Runtime effective warm writer cap"
);
let _ = writeln!(
out,
"# TYPE telemt_me_adaptive_floor_warm_cap_effective gauge"
);
let _ = writeln!(
out,
"telemt_me_adaptive_floor_warm_cap_effective {}",
if me_allows_normal {
stats.get_me_floor_warm_cap_effective_gauge()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_writers_active_current Current non-draining active ME writers"
);
let _ = writeln!(out, "# TYPE telemt_me_writers_active_current gauge");
let _ = writeln!(
out,
"telemt_me_writers_active_current {}",
if me_allows_normal {
stats.get_me_writers_active_current_gauge()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_writers_warm_current Current non-draining warm ME writers"
);
let _ = writeln!(out, "# TYPE telemt_me_writers_warm_current gauge");
let _ = writeln!(
out,
"telemt_me_writers_warm_current {}",
if me_allows_normal {
stats.get_me_writers_warm_current_gauge()
} else {
0
}
);
let _ = writeln!( let _ = writeln!(
out, out,
"# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps" "# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps"

View File

@@ -27,7 +27,7 @@ enum C2MeCommand {
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60); const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync"; const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
const C2ME_CHANNEL_CAPACITY: usize = 1024; const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128;
const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64; const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64;
const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32; const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32;
static DESYNC_DEDUP: OnceLock<Mutex<HashMap<u64, Instant>>> = OnceLock::new(); static DESYNC_DEDUP: OnceLock<Mutex<HashMap<u64, Instant>>> = OnceLock::new();
@@ -271,7 +271,11 @@ where
let frame_limit = config.general.max_client_frame; let frame_limit = config.general.max_client_frame;
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(C2ME_CHANNEL_CAPACITY); let c2me_channel_capacity = config
.general
.me_c2me_channel_capacity
.max(C2ME_CHANNEL_CAPACITY_FALLBACK);
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
let me_pool_c2me = me_pool.clone(); let me_pool_c2me = me_pool.clone();
let effective_tag = effective_tag; let effective_tag = effective_tag;
let c2me_sender = tokio::spawn(async move { let c2me_sender = tokio::spawn(async move {

373
src/startup.rs Normal file
View File

@@ -0,0 +1,373 @@
use std::time::{Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
pub const COMPONENT_CONFIG_LOAD: &str = "config_load";
pub const COMPONENT_TRACING_INIT: &str = "tracing_init";
pub const COMPONENT_API_BOOTSTRAP: &str = "api_bootstrap";
pub const COMPONENT_TLS_FRONT_BOOTSTRAP: &str = "tls_front_bootstrap";
pub const COMPONENT_NETWORK_PROBE: &str = "network_probe";
pub const COMPONENT_ME_SECRET_FETCH: &str = "me_secret_fetch";
pub const COMPONENT_ME_PROXY_CONFIG_V4: &str = "me_proxy_config_fetch_v4";
pub const COMPONENT_ME_PROXY_CONFIG_V6: &str = "me_proxy_config_fetch_v6";
pub const COMPONENT_ME_POOL_CONSTRUCT: &str = "me_pool_construct";
pub const COMPONENT_ME_POOL_INIT_STAGE1: &str = "me_pool_init_stage1";
pub const COMPONENT_ME_CONNECTIVITY_PING: &str = "me_connectivity_ping";
pub const COMPONENT_DC_CONNECTIVITY_PING: &str = "dc_connectivity_ping";
pub const COMPONENT_LISTENERS_BIND: &str = "listeners_bind";
pub const COMPONENT_CONFIG_WATCHER_START: &str = "config_watcher_start";
pub const COMPONENT_METRICS_START: &str = "metrics_start";
pub const COMPONENT_RUNTIME_READY: &str = "runtime_ready";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StartupStatus {
Initializing,
Ready,
}
impl StartupStatus {
pub fn as_str(self) -> &'static str {
match self {
Self::Initializing => "initializing",
Self::Ready => "ready",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StartupComponentStatus {
Pending,
Running,
Ready,
Failed,
Skipped,
}
impl StartupComponentStatus {
pub fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Running => "running",
Self::Ready => "ready",
Self::Failed => "failed",
Self::Skipped => "skipped",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StartupMeStatus {
Pending,
Initializing,
Ready,
Failed,
Skipped,
}
impl StartupMeStatus {
pub fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Initializing => "initializing",
Self::Ready => "ready",
Self::Failed => "failed",
Self::Skipped => "skipped",
}
}
}
#[derive(Clone, Debug)]
pub struct StartupComponentSnapshot {
pub id: &'static str,
pub title: &'static str,
pub weight: f64,
pub status: StartupComponentStatus,
pub started_at_epoch_ms: Option<u64>,
pub finished_at_epoch_ms: Option<u64>,
pub duration_ms: Option<u64>,
pub attempts: u32,
pub details: Option<String>,
}
#[derive(Clone, Debug)]
pub struct StartupMeSnapshot {
pub status: StartupMeStatus,
pub current_stage: String,
pub init_attempt: u32,
pub retry_limit: String,
pub last_error: Option<String>,
}
#[derive(Clone, Debug)]
pub struct StartupSnapshot {
pub status: StartupStatus,
pub degraded: bool,
pub current_stage: String,
pub started_at_epoch_secs: u64,
pub ready_at_epoch_secs: Option<u64>,
pub total_elapsed_ms: u64,
pub transport_mode: String,
pub me: StartupMeSnapshot,
pub components: Vec<StartupComponentSnapshot>,
}
#[derive(Clone, Debug)]
struct StartupComponent {
id: &'static str,
title: &'static str,
weight: f64,
status: StartupComponentStatus,
started_at_epoch_ms: Option<u64>,
finished_at_epoch_ms: Option<u64>,
duration_ms: Option<u64>,
attempts: u32,
details: Option<String>,
}
#[derive(Clone, Debug)]
struct StartupState {
status: StartupStatus,
degraded: bool,
current_stage: String,
started_at_epoch_secs: u64,
ready_at_epoch_secs: Option<u64>,
transport_mode: String,
me: StartupMeSnapshot,
components: Vec<StartupComponent>,
}
pub struct StartupTracker {
started_at_instant: Instant,
state: RwLock<StartupState>,
}
impl StartupTracker {
pub fn new(started_at_epoch_secs: u64) -> Self {
Self {
started_at_instant: Instant::now(),
state: RwLock::new(StartupState {
status: StartupStatus::Initializing,
degraded: false,
current_stage: COMPONENT_CONFIG_LOAD.to_string(),
started_at_epoch_secs,
ready_at_epoch_secs: None,
transport_mode: "unknown".to_string(),
me: StartupMeSnapshot {
status: StartupMeStatus::Pending,
current_stage: "pending".to_string(),
init_attempt: 0,
retry_limit: "unlimited".to_string(),
last_error: None,
},
components: component_blueprint(),
}),
}
}
pub async fn set_transport_mode(&self, mode: &'static str) {
self.state.write().await.transport_mode = mode.to_string();
}
pub async fn set_degraded(&self, degraded: bool) {
self.state.write().await.degraded = degraded;
}
pub async fn start_component(&self, id: &'static str, details: Option<String>) {
let mut guard = self.state.write().await;
guard.current_stage = id.to_string();
if let Some(component) = guard.components.iter_mut().find(|component| component.id == id) {
if component.started_at_epoch_ms.is_none() {
component.started_at_epoch_ms = Some(now_epoch_ms());
}
component.attempts = component.attempts.saturating_add(1);
component.status = StartupComponentStatus::Running;
component.details = normalize_details(details);
}
}
pub async fn complete_component(&self, id: &'static str, details: Option<String>) {
self.finish_component(id, StartupComponentStatus::Ready, details)
.await;
}
pub async fn fail_component(&self, id: &'static str, details: Option<String>) {
self.finish_component(id, StartupComponentStatus::Failed, details)
.await;
}
pub async fn skip_component(&self, id: &'static str, details: Option<String>) {
self.finish_component(id, StartupComponentStatus::Skipped, details)
.await;
}
async fn finish_component(
&self,
id: &'static str,
status: StartupComponentStatus,
details: Option<String>,
) {
let mut guard = self.state.write().await;
let finished_at = now_epoch_ms();
if let Some(component) = guard.components.iter_mut().find(|component| component.id == id) {
if component.started_at_epoch_ms.is_none() {
component.started_at_epoch_ms = Some(finished_at);
component.attempts = component.attempts.saturating_add(1);
}
component.finished_at_epoch_ms = Some(finished_at);
component.duration_ms = component
.started_at_epoch_ms
.map(|started_at| finished_at.saturating_sub(started_at));
component.status = status;
component.details = normalize_details(details);
}
}
pub async fn set_me_status(&self, status: StartupMeStatus, stage: &'static str) {
let mut guard = self.state.write().await;
guard.me.status = status;
guard.me.current_stage = stage.to_string();
}
pub async fn set_me_retry_limit(&self, retry_limit: String) {
self.state.write().await.me.retry_limit = retry_limit;
}
pub async fn set_me_init_attempt(&self, attempt: u32) {
self.state.write().await.me.init_attempt = attempt;
}
pub async fn set_me_last_error(&self, error: Option<String>) {
self.state.write().await.me.last_error = normalize_details(error);
}
pub async fn mark_ready(&self) {
let mut guard = self.state.write().await;
if guard.status == StartupStatus::Ready {
return;
}
guard.status = StartupStatus::Ready;
guard.current_stage = "ready".to_string();
guard.ready_at_epoch_secs = Some(now_epoch_secs());
}
pub async fn snapshot(&self) -> StartupSnapshot {
let guard = self.state.read().await;
StartupSnapshot {
status: guard.status,
degraded: guard.degraded,
current_stage: guard.current_stage.clone(),
started_at_epoch_secs: guard.started_at_epoch_secs,
ready_at_epoch_secs: guard.ready_at_epoch_secs,
total_elapsed_ms: self.started_at_instant.elapsed().as_millis() as u64,
transport_mode: guard.transport_mode.clone(),
me: guard.me.clone(),
components: guard
.components
.iter()
.map(|component| StartupComponentSnapshot {
id: component.id,
title: component.title,
weight: component.weight,
status: component.status,
started_at_epoch_ms: component.started_at_epoch_ms,
finished_at_epoch_ms: component.finished_at_epoch_ms,
duration_ms: component.duration_ms,
attempts: component.attempts,
details: component.details.clone(),
})
.collect(),
}
}
}
pub fn compute_progress_pct(snapshot: &StartupSnapshot, me_stage_progress: Option<f64>) -> f64 {
if snapshot.status == StartupStatus::Ready {
return 100.0;
}
let mut total_weight = 0.0f64;
let mut completed_weight = 0.0f64;
for component in &snapshot.components {
total_weight += component.weight;
let unit_progress = match component.status {
StartupComponentStatus::Pending => 0.0,
StartupComponentStatus::Running => {
if component.id == COMPONENT_ME_POOL_INIT_STAGE1 {
me_stage_progress.unwrap_or(0.0).clamp(0.0, 1.0)
} else {
0.0
}
}
StartupComponentStatus::Ready
| StartupComponentStatus::Failed
| StartupComponentStatus::Skipped => 1.0,
};
completed_weight += component.weight * unit_progress;
}
if total_weight <= f64::EPSILON {
0.0
} else {
((completed_weight / total_weight) * 100.0).clamp(0.0, 100.0)
}
}
fn component_blueprint() -> Vec<StartupComponent> {
vec![
component(COMPONENT_CONFIG_LOAD, "Config load", 5.0),
component(COMPONENT_TRACING_INIT, "Tracing init", 3.0),
component(COMPONENT_API_BOOTSTRAP, "API bootstrap", 5.0),
component(COMPONENT_TLS_FRONT_BOOTSTRAP, "TLS front bootstrap", 5.0),
component(COMPONENT_NETWORK_PROBE, "Network probe", 10.0),
component(COMPONENT_ME_SECRET_FETCH, "ME secret fetch", 8.0),
component(COMPONENT_ME_PROXY_CONFIG_V4, "ME config v4 fetch", 4.0),
component(COMPONENT_ME_PROXY_CONFIG_V6, "ME config v6 fetch", 4.0),
component(COMPONENT_ME_POOL_CONSTRUCT, "ME pool construct", 6.0),
component(COMPONENT_ME_POOL_INIT_STAGE1, "ME pool init stage1", 24.0),
component(COMPONENT_ME_CONNECTIVITY_PING, "ME connectivity ping", 6.0),
component(COMPONENT_DC_CONNECTIVITY_PING, "DC connectivity ping", 8.0),
component(COMPONENT_LISTENERS_BIND, "Listener bind", 8.0),
component(COMPONENT_CONFIG_WATCHER_START, "Config watcher start", 2.0),
component(COMPONENT_METRICS_START, "Metrics start", 1.0),
component(COMPONENT_RUNTIME_READY, "Runtime ready", 1.0),
]
}
fn component(id: &'static str, title: &'static str, weight: f64) -> StartupComponent {
StartupComponent {
id,
title,
weight,
status: StartupComponentStatus::Pending,
started_at_epoch_ms: None,
finished_at_epoch_ms: None,
duration_ms: None,
attempts: 0,
details: None,
}
}
fn normalize_details(details: Option<String>) -> Option<String> {
details.map(|detail| {
if detail.len() <= 256 {
detail
} else {
detail[..256].to_string()
}
})
}
fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn now_epoch_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}

View File

@@ -16,7 +16,7 @@ use std::collections::hash_map::DefaultHasher;
use std::collections::VecDeque; use std::collections::VecDeque;
use tracing::debug; use tracing::debug;
use crate::config::MeTelemetryLevel; use crate::config::{MeTelemetryLevel, MeWriterPickMode};
use self::telemetry::TelemetryPolicy; use self::telemetry::TelemetryPolicy;
// ============= Stats ============= // ============= Stats =============
@@ -80,6 +80,12 @@ pub struct Stats {
me_floor_global_cap_raw_gauge: AtomicU64, me_floor_global_cap_raw_gauge: AtomicU64,
me_floor_global_cap_effective_gauge: AtomicU64, me_floor_global_cap_effective_gauge: AtomicU64,
me_floor_target_writers_total_gauge: AtomicU64, me_floor_target_writers_total_gauge: AtomicU64,
me_floor_active_cap_configured_gauge: AtomicU64,
me_floor_active_cap_effective_gauge: AtomicU64,
me_floor_warm_cap_configured_gauge: AtomicU64,
me_floor_warm_cap_effective_gauge: AtomicU64,
me_writers_active_current_gauge: AtomicU64,
me_writers_warm_current_gauge: AtomicU64,
me_floor_cap_block_total: AtomicU64, me_floor_cap_block_total: AtomicU64,
me_floor_swap_idle_total: AtomicU64, me_floor_swap_idle_total: AtomicU64,
me_floor_swap_idle_failed_total: AtomicU64, me_floor_swap_idle_failed_total: AtomicU64,
@@ -89,6 +95,18 @@ pub struct Stats {
me_route_drop_queue_full: AtomicU64, me_route_drop_queue_full: AtomicU64,
me_route_drop_queue_full_base: AtomicU64, me_route_drop_queue_full_base: AtomicU64,
me_route_drop_queue_full_high: AtomicU64, me_route_drop_queue_full_high: AtomicU64,
me_writer_pick_sorted_rr_success_try_total: AtomicU64,
me_writer_pick_sorted_rr_success_fallback_total: AtomicU64,
me_writer_pick_sorted_rr_full_total: AtomicU64,
me_writer_pick_sorted_rr_closed_total: AtomicU64,
me_writer_pick_sorted_rr_no_candidate_total: AtomicU64,
me_writer_pick_p2c_success_try_total: AtomicU64,
me_writer_pick_p2c_success_fallback_total: AtomicU64,
me_writer_pick_p2c_full_total: AtomicU64,
me_writer_pick_p2c_closed_total: AtomicU64,
me_writer_pick_p2c_no_candidate_total: AtomicU64,
me_writer_pick_blocking_fallback_total: AtomicU64,
me_writer_pick_mode_switch_total: AtomicU64,
me_socks_kdf_strict_reject: AtomicU64, me_socks_kdf_strict_reject: AtomicU64,
me_socks_kdf_compat_fallback: AtomicU64, me_socks_kdf_compat_fallback: AtomicU64,
secure_padding_invalid: AtomicU64, secure_padding_invalid: AtomicU64,
@@ -491,6 +509,93 @@ impl Stats {
self.me_route_drop_queue_full_high.fetch_add(1, Ordering::Relaxed); self.me_route_drop_queue_full_high.fetch_add(1, Ordering::Relaxed);
} }
} }
pub fn increment_me_writer_pick_success_try_total(&self, mode: MeWriterPickMode) {
if !self.telemetry_me_allows_normal() {
return;
}
match mode {
MeWriterPickMode::SortedRr => {
self.me_writer_pick_sorted_rr_success_try_total
.fetch_add(1, Ordering::Relaxed);
}
MeWriterPickMode::P2c => {
self.me_writer_pick_p2c_success_try_total
.fetch_add(1, Ordering::Relaxed);
}
}
}
pub fn increment_me_writer_pick_success_fallback_total(&self, mode: MeWriterPickMode) {
if !self.telemetry_me_allows_normal() {
return;
}
match mode {
MeWriterPickMode::SortedRr => {
self.me_writer_pick_sorted_rr_success_fallback_total
.fetch_add(1, Ordering::Relaxed);
}
MeWriterPickMode::P2c => {
self.me_writer_pick_p2c_success_fallback_total
.fetch_add(1, Ordering::Relaxed);
}
}
}
pub fn increment_me_writer_pick_full_total(&self, mode: MeWriterPickMode) {
if !self.telemetry_me_allows_normal() {
return;
}
match mode {
MeWriterPickMode::SortedRr => {
self.me_writer_pick_sorted_rr_full_total
.fetch_add(1, Ordering::Relaxed);
}
MeWriterPickMode::P2c => {
self.me_writer_pick_p2c_full_total
.fetch_add(1, Ordering::Relaxed);
}
}
}
pub fn increment_me_writer_pick_closed_total(&self, mode: MeWriterPickMode) {
if !self.telemetry_me_allows_normal() {
return;
}
match mode {
MeWriterPickMode::SortedRr => {
self.me_writer_pick_sorted_rr_closed_total
.fetch_add(1, Ordering::Relaxed);
}
MeWriterPickMode::P2c => {
self.me_writer_pick_p2c_closed_total
.fetch_add(1, Ordering::Relaxed);
}
}
}
pub fn increment_me_writer_pick_no_candidate_total(&self, mode: MeWriterPickMode) {
if !self.telemetry_me_allows_normal() {
return;
}
match mode {
MeWriterPickMode::SortedRr => {
self.me_writer_pick_sorted_rr_no_candidate_total
.fetch_add(1, Ordering::Relaxed);
}
MeWriterPickMode::P2c => {
self.me_writer_pick_p2c_no_candidate_total
.fetch_add(1, Ordering::Relaxed);
}
}
}
pub fn increment_me_writer_pick_blocking_fallback_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_writer_pick_blocking_fallback_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_writer_pick_mode_switch_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_writer_pick_mode_switch_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_socks_kdf_strict_reject(&self) { pub fn increment_me_socks_kdf_strict_reject(&self) {
if self.telemetry_me_allows_normal() { if self.telemetry_me_allows_normal() {
self.me_socks_kdf_strict_reject.fetch_add(1, Ordering::Relaxed); self.me_socks_kdf_strict_reject.fetch_add(1, Ordering::Relaxed);
@@ -764,6 +869,42 @@ impl Stats {
.store(value, Ordering::Relaxed); .store(value, Ordering::Relaxed);
} }
} }
pub fn set_me_floor_active_cap_configured_gauge(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_floor_active_cap_configured_gauge
.store(value, Ordering::Relaxed);
}
}
pub fn set_me_floor_active_cap_effective_gauge(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_floor_active_cap_effective_gauge
.store(value, Ordering::Relaxed);
}
}
pub fn set_me_floor_warm_cap_configured_gauge(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_floor_warm_cap_configured_gauge
.store(value, Ordering::Relaxed);
}
}
pub fn set_me_floor_warm_cap_effective_gauge(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_floor_warm_cap_effective_gauge
.store(value, Ordering::Relaxed);
}
}
pub fn set_me_writers_active_current_gauge(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_writers_active_current_gauge
.store(value, Ordering::Relaxed);
}
}
pub fn set_me_writers_warm_current_gauge(&self, value: u64) {
if self.telemetry_me_allows_normal() {
self.me_writers_warm_current_gauge
.store(value, Ordering::Relaxed);
}
}
pub fn increment_me_floor_cap_block_total(&self) { pub fn increment_me_floor_cap_block_total(&self) {
if self.telemetry_me_allows_normal() { if self.telemetry_me_allows_normal() {
self.me_floor_cap_block_total.fetch_add(1, Ordering::Relaxed); self.me_floor_cap_block_total.fetch_add(1, Ordering::Relaxed);
@@ -904,6 +1045,30 @@ impl Stats {
self.me_floor_target_writers_total_gauge self.me_floor_target_writers_total_gauge
.load(Ordering::Relaxed) .load(Ordering::Relaxed)
} }
pub fn get_me_floor_active_cap_configured_gauge(&self) -> u64 {
self.me_floor_active_cap_configured_gauge
.load(Ordering::Relaxed)
}
pub fn get_me_floor_active_cap_effective_gauge(&self) -> u64 {
self.me_floor_active_cap_effective_gauge
.load(Ordering::Relaxed)
}
pub fn get_me_floor_warm_cap_configured_gauge(&self) -> u64 {
self.me_floor_warm_cap_configured_gauge
.load(Ordering::Relaxed)
}
pub fn get_me_floor_warm_cap_effective_gauge(&self) -> u64 {
self.me_floor_warm_cap_effective_gauge
.load(Ordering::Relaxed)
}
pub fn get_me_writers_active_current_gauge(&self) -> u64 {
self.me_writers_active_current_gauge
.load(Ordering::Relaxed)
}
pub fn get_me_writers_warm_current_gauge(&self) -> u64 {
self.me_writers_warm_current_gauge
.load(Ordering::Relaxed)
}
pub fn get_me_floor_cap_block_total(&self) -> u64 { pub fn get_me_floor_cap_block_total(&self) -> u64 {
self.me_floor_cap_block_total.load(Ordering::Relaxed) self.me_floor_cap_block_total.load(Ordering::Relaxed)
} }
@@ -935,6 +1100,52 @@ impl Stats {
pub fn get_me_route_drop_queue_full_high(&self) -> u64 { pub fn get_me_route_drop_queue_full_high(&self) -> u64 {
self.me_route_drop_queue_full_high.load(Ordering::Relaxed) self.me_route_drop_queue_full_high.load(Ordering::Relaxed)
} }
pub fn get_me_writer_pick_sorted_rr_success_try_total(&self) -> u64 {
self.me_writer_pick_sorted_rr_success_try_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_sorted_rr_success_fallback_total(&self) -> u64 {
self.me_writer_pick_sorted_rr_success_fallback_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_sorted_rr_full_total(&self) -> u64 {
self.me_writer_pick_sorted_rr_full_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_sorted_rr_closed_total(&self) -> u64 {
self.me_writer_pick_sorted_rr_closed_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_sorted_rr_no_candidate_total(&self) -> u64 {
self.me_writer_pick_sorted_rr_no_candidate_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_p2c_success_try_total(&self) -> u64 {
self.me_writer_pick_p2c_success_try_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_p2c_success_fallback_total(&self) -> u64 {
self.me_writer_pick_p2c_success_fallback_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_p2c_full_total(&self) -> u64 {
self.me_writer_pick_p2c_full_total.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_p2c_closed_total(&self) -> u64 {
self.me_writer_pick_p2c_closed_total.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_p2c_no_candidate_total(&self) -> u64 {
self.me_writer_pick_p2c_no_candidate_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_blocking_fallback_total(&self) -> u64 {
self.me_writer_pick_blocking_fallback_total
.load(Ordering::Relaxed)
}
pub fn get_me_writer_pick_mode_switch_total(&self) -> u64 {
self.me_writer_pick_mode_switch_total
.load(Ordering::Relaxed)
}
pub fn get_me_socks_kdf_strict_reject(&self) -> u64 { pub fn get_me_socks_kdf_strict_reject(&self) -> u64 {
self.me_socks_kdf_strict_reject.load(Ordering::Relaxed) self.me_socks_kdf_strict_reject.load(Ordering::Relaxed)
} }

View File

@@ -306,6 +306,8 @@ async fn run_update_cycle(
cfg.general.me_bind_stale_ttl_secs, cfg.general.me_bind_stale_ttl_secs,
cfg.general.me_secret_atomic_snapshot, cfg.general.me_secret_atomic_snapshot,
cfg.general.me_deterministic_writer_sort, cfg.general.me_deterministic_writer_sort,
cfg.general.me_writer_pick_mode,
cfg.general.me_writer_pick_sample_size,
cfg.general.me_single_endpoint_shadow_writers, cfg.general.me_single_endpoint_shadow_writers,
cfg.general.me_single_endpoint_outage_mode_enabled, cfg.general.me_single_endpoint_outage_mode_enabled,
cfg.general.me_single_endpoint_outage_disable_quarantine, cfg.general.me_single_endpoint_outage_disable_quarantine,
@@ -321,6 +323,13 @@ async fn run_update_cycle(
cfg.general.me_adaptive_floor_cpu_cores_override, cfg.general.me_adaptive_floor_cpu_cores_override,
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core, cfg.general.me_adaptive_floor_max_extra_writers_single_per_core,
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core, cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core,
cfg.general.me_adaptive_floor_max_active_writers_per_core,
cfg.general.me_adaptive_floor_max_warm_writers_per_core,
cfg.general.me_adaptive_floor_max_active_writers_global,
cfg.general.me_adaptive_floor_max_warm_writers_global,
cfg.general.me_health_interval_ms_unhealthy,
cfg.general.me_health_interval_ms_healthy,
cfg.general.me_warn_rate_limit_ms,
); );
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
@@ -523,6 +532,8 @@ pub async fn me_config_updater(
cfg.general.me_bind_stale_ttl_secs, cfg.general.me_bind_stale_ttl_secs,
cfg.general.me_secret_atomic_snapshot, cfg.general.me_secret_atomic_snapshot,
cfg.general.me_deterministic_writer_sort, cfg.general.me_deterministic_writer_sort,
cfg.general.me_writer_pick_mode,
cfg.general.me_writer_pick_sample_size,
cfg.general.me_single_endpoint_shadow_writers, cfg.general.me_single_endpoint_shadow_writers,
cfg.general.me_single_endpoint_outage_mode_enabled, cfg.general.me_single_endpoint_outage_mode_enabled,
cfg.general.me_single_endpoint_outage_disable_quarantine, cfg.general.me_single_endpoint_outage_disable_quarantine,
@@ -538,6 +549,13 @@ pub async fn me_config_updater(
cfg.general.me_adaptive_floor_cpu_cores_override, cfg.general.me_adaptive_floor_cpu_cores_override,
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core, cfg.general.me_adaptive_floor_max_extra_writers_single_per_core,
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core, cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core,
cfg.general.me_adaptive_floor_max_active_writers_per_core,
cfg.general.me_adaptive_floor_max_warm_writers_per_core,
cfg.general.me_adaptive_floor_max_active_writers_global,
cfg.general.me_adaptive_floor_max_warm_writers_global,
cfg.general.me_health_interval_ms_unhealthy,
cfg.general.me_health_interval_ms_healthy,
cfg.general.me_warn_rate_limit_ms,
); );
let new_secs = cfg.general.effective_update_every_secs().max(1); let new_secs = cfg.general.effective_update_every_secs().max(1);
if new_secs == update_every_secs { if new_secs == update_every_secs {

View File

@@ -13,7 +13,6 @@ use crate::network::IpFamily;
use super::MePool; use super::MePool;
const HEALTH_INTERVAL_SECS: u64 = 1;
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
#[allow(dead_code)] #[allow(dead_code)]
const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1; const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1;
@@ -42,7 +41,12 @@ struct DcFloorPlanEntry {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct FamilyFloorPlan { struct FamilyFloorPlan {
by_dc: HashMap<i32, DcFloorPlanEntry>, by_dc: HashMap<i32, DcFloorPlanEntry>,
global_cap_effective_total: usize, active_cap_configured_total: usize,
active_cap_effective_total: usize,
warm_cap_configured_total: usize,
warm_cap_effective_total: usize,
active_writers_current: usize,
warm_writers_current: usize,
target_writers_total: usize, target_writers_total: usize,
} }
@@ -57,11 +61,18 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new();
let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new();
let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new();
let mut degraded_interval = true;
loop { loop {
tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await; let interval = if degraded_interval {
pool.health_interval_unhealthy()
} else {
pool.health_interval_healthy()
};
tokio::time::sleep(interval).await;
pool.prune_closed_writers().await; pool.prune_closed_writers().await;
reap_draining_writers(&pool).await; reap_draining_writers(&pool).await;
check_family( let v4_degraded = check_family(
IpFamily::V4, IpFamily::V4,
&pool, &pool,
&rng, &rng,
@@ -75,9 +86,10 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
&mut idle_refresh_next_attempt, &mut idle_refresh_next_attempt,
&mut adaptive_idle_since, &mut adaptive_idle_since,
&mut adaptive_recover_until, &mut adaptive_recover_until,
&mut floor_warn_next_allowed,
) )
.await; .await;
check_family( let v6_degraded = check_family(
IpFamily::V6, IpFamily::V6,
&pool, &pool,
&rng, &rng,
@@ -91,8 +103,10 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
&mut idle_refresh_next_attempt, &mut idle_refresh_next_attempt,
&mut adaptive_idle_since, &mut adaptive_idle_since,
&mut adaptive_recover_until, &mut adaptive_recover_until,
&mut floor_warn_next_allowed,
) )
.await; .await;
degraded_interval = v4_degraded || v6_degraded;
} }
} }
@@ -132,15 +146,18 @@ async fn check_family(
idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>,
adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>,
) { floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
) -> bool {
let enabled = match family { let enabled = match family {
IpFamily::V4 => pool.decision.ipv4_me, IpFamily::V4 => pool.decision.ipv4_me,
IpFamily::V6 => pool.decision.ipv6_me, IpFamily::V6 => pool.decision.ipv6_me,
}; };
if !enabled { if !enabled {
return; return false;
} }
let mut family_degraded = false;
let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new(); let mut dc_endpoints = HashMap::<i32, Vec<SocketAddr>>::new();
let map_guard = match family { let map_guard = match family {
IpFamily::V4 => pool.proxy_map_v4.read().await, IpFamily::V4 => pool.proxy_map_v4.read().await,
@@ -169,6 +186,14 @@ async fn check_family(
for writer in pool.writers.read().await.iter().filter(|w| { for writer in pool.writers.read().await.iter().filter(|w| {
!w.draining.load(std::sync::atomic::Ordering::Relaxed) !w.draining.load(std::sync::atomic::Ordering::Relaxed)
}) { }) {
if !matches!(
super::pool::WriterContour::from_u8(
writer.contour.load(std::sync::atomic::Ordering::Relaxed),
),
super::pool::WriterContour::Active
) {
continue;
}
let key = (writer.writer_dc, writer.addr); let key = (writer.writer_dc, writer.addr);
*live_addr_counts.entry(key).or_insert(0) += 1; *live_addr_counts.entry(key).or_insert(0) += 1;
live_writer_ids_by_addr live_writer_ids_by_addr
@@ -194,8 +219,13 @@ async fn check_family(
) )
.await; .await;
pool.set_adaptive_floor_runtime_caps( pool.set_adaptive_floor_runtime_caps(
floor_plan.global_cap_effective_total, floor_plan.active_cap_configured_total,
floor_plan.active_cap_effective_total,
floor_plan.warm_cap_configured_total,
floor_plan.warm_cap_effective_total,
floor_plan.target_writers_total, floor_plan.target_writers_total,
floor_plan.active_writers_current,
floor_plan.warm_writers_current,
); );
for (dc, endpoints) in dc_endpoints { for (dc, endpoints) in dc_endpoints {
@@ -216,6 +246,7 @@ async fn check_family(
.sum::<usize>(); .sum::<usize>();
if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 { if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 {
family_degraded = true;
if single_endpoint_outage.insert(key) { if single_endpoint_outage.insert(key) {
pool.stats.increment_me_single_endpoint_outage_enter_total(); pool.stats.increment_me_single_endpoint_outage_enter_total();
warn!( warn!(
@@ -292,6 +323,7 @@ async fn check_family(
continue; continue;
} }
let missing = required - alive; let missing = required - alive;
family_degraded = true;
let now = Instant::now(); let now = Instant::now();
if reconnect_budget == 0 { if reconnect_budget == 0 {
@@ -344,8 +376,8 @@ async fn check_family(
break; break;
} }
reconnect_budget = reconnect_budget.saturating_sub(1); reconnect_budget = reconnect_budget.saturating_sub(1);
if pool.floor_mode() == MeFloorMode::Adaptive if pool.active_contour_writer_count_total().await
&& pool.active_writer_count_total().await >= floor_plan.global_cap_effective_total >= floor_plan.active_cap_effective_total
{ {
let swapped = maybe_swap_idle_writer_for_cap( let swapped = maybe_swap_idle_writer_for_cap(
pool, pool,
@@ -370,7 +402,7 @@ async fn check_family(
?family, ?family,
alive, alive,
required, required,
global_cap_effective_total = floor_plan.global_cap_effective_total, active_cap_effective_total = floor_plan.active_cap_effective_total,
"Adaptive floor cap reached, reconnect attempt blocked" "Adaptive floor cap reached, reconnect attempt blocked"
); );
break; break;
@@ -420,15 +452,23 @@ async fn check_family(
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1))); + Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
next_attempt.insert(key, now + wait); next_attempt.insert(key, now + wait);
if pool.is_runtime_ready() { if pool.is_runtime_ready() {
warn!( let warn_cooldown = pool.warn_rate_limit_duration();
dc = %dc, if should_emit_rate_limited_warn(
?family, floor_warn_next_allowed,
alive = now_alive, key,
required, now,
endpoint_count = endpoints.len(), warn_cooldown,
backoff_ms = next_ms, ) {
"DC writer floor is below required level, scheduled reconnect" warn!(
); dc = %dc,
?family,
alive = now_alive,
required,
endpoint_count = endpoints.len(),
backoff_ms = next_ms,
"DC writer floor is below required level, scheduled reconnect"
);
}
} else { } else {
info!( info!(
dc = %dc, dc = %dc,
@@ -445,6 +485,8 @@ async fn check_family(
*v = v.saturating_sub(1); *v = v.saturating_sub(1);
} }
} }
family_degraded
} }
fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize { fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
@@ -456,6 +498,23 @@ fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
.clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX) .clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX)
} }
fn should_emit_rate_limited_warn(
next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
key: (i32, IpFamily),
now: Instant,
cooldown: Duration,
) -> bool {
let Some(ready_at) = next_allowed.get(&key).copied() else {
next_allowed.insert(key, now + cooldown);
return true;
};
if now >= ready_at {
next_allowed.insert(key, now + cooldown);
return true;
}
false
}
fn adaptive_floor_class_min( fn adaptive_floor_class_min(
pool: &Arc<MePool>, pool: &Arc<MePool>,
endpoint_count: usize, endpoint_count: usize,
@@ -518,6 +577,8 @@ async fn build_family_floor_plan(
let floor_mode = pool.floor_mode(); let floor_mode = pool.floor_mode();
let is_adaptive = floor_mode == MeFloorMode::Adaptive; let is_adaptive = floor_mode == MeFloorMode::Adaptive;
let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1); let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1);
let (active_writers_current, warm_writers_current, _) =
pool.non_draining_writer_counts_by_contour().await;
for (dc, endpoints) in dc_endpoints { for (dc, endpoints) in dc_endpoints {
if endpoints.is_empty() { if endpoints.is_empty() {
@@ -576,9 +637,16 @@ async fn build_family_floor_plan(
} }
if entries.is_empty() { if entries.is_empty() {
let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total();
let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total();
return FamilyFloorPlan { return FamilyFloorPlan {
by_dc, by_dc,
global_cap_effective_total: 0, active_cap_configured_total,
active_cap_effective_total: active_cap_configured_total,
warm_cap_configured_total,
warm_cap_effective_total: warm_cap_configured_total,
active_writers_current,
warm_writers_current,
target_writers_total: 0, target_writers_total: 0,
}; };
} }
@@ -588,20 +656,26 @@ async fn build_family_floor_plan(
.iter() .iter()
.map(|entry| entry.target_required) .map(|entry| entry.target_required)
.sum::<usize>(); .sum::<usize>();
let active_total = pool.active_writer_count_total().await; let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total();
let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total();
for entry in entries { for entry in entries {
by_dc.insert(entry.dc, entry); by_dc.insert(entry.dc, entry);
} }
return FamilyFloorPlan { return FamilyFloorPlan {
by_dc, by_dc,
global_cap_effective_total: active_total.max(target_total), active_cap_configured_total,
active_cap_effective_total: active_cap_configured_total.max(target_total),
warm_cap_configured_total,
warm_cap_effective_total: warm_cap_configured_total,
active_writers_current,
warm_writers_current,
target_writers_total: target_total, target_writers_total: target_total,
}; };
} }
let global_cap_raw = pool.adaptive_floor_global_cap_raw(); let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total();
let total_active = pool.active_writer_count_total().await; let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total();
let other_active = total_active.saturating_sub(family_active_total); let other_active = active_writers_current.saturating_sub(family_active_total);
let min_sum = entries let min_sum = entries
.iter() .iter()
.map(|entry| entry.min_required) .map(|entry| entry.min_required)
@@ -610,7 +684,7 @@ async fn build_family_floor_plan(
.iter() .iter()
.map(|entry| entry.target_required) .map(|entry| entry.target_required)
.sum::<usize>(); .sum::<usize>();
let family_cap = global_cap_raw let family_cap = active_cap_configured_total
.saturating_sub(other_active) .saturating_sub(other_active)
.max(min_sum); .max(min_sum);
if target_sum > family_cap { if target_sum > family_cap {
@@ -645,11 +719,17 @@ async fn build_family_floor_plan(
for entry in entries { for entry in entries {
by_dc.insert(entry.dc, entry); by_dc.insert(entry.dc, entry);
} }
let global_cap_effective_total = global_cap_raw.max(other_active.saturating_add(min_sum)); let active_cap_effective_total =
active_cap_configured_total.max(other_active.saturating_add(min_sum));
let target_writers_total = other_active.saturating_add(target_sum); let target_writers_total = other_active.saturating_add(target_sum);
FamilyFloorPlan { FamilyFloorPlan {
by_dc, by_dc,
global_cap_effective_total, active_cap_configured_total,
active_cap_effective_total,
warm_cap_configured_total,
warm_cap_effective_total: warm_cap_configured_total,
active_writers_current,
warm_writers_current,
target_writers_total, target_writers_total,
} }
} }

View File

@@ -7,7 +7,9 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::{Mutex, Notify, RwLock, mpsc}; use tokio::sync::{Mutex, Notify, RwLock, mpsc};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use crate::config::{MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy}; use crate::config::{
MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode,
};
use crate::crypto::SecureRandom; use crate::crypto::SecureRandom;
use crate::network::IpFamily; use crate::network::IpFamily;
use crate::network::probe::NetworkDecision; use crate::network::probe::NetworkDecision;
@@ -39,6 +41,7 @@ pub struct MeWriter {
pub tx: mpsc::Sender<WriterCommand>, pub tx: mpsc::Sender<WriterCommand>,
pub cancel: CancellationToken, pub cancel: CancellationToken,
pub degraded: Arc<AtomicBool>, pub degraded: Arc<AtomicBool>,
pub rtt_ema_ms_x10: Arc<AtomicU32>,
pub draining: Arc<AtomicBool>, pub draining: Arc<AtomicBool>,
pub draining_started_at_epoch_secs: Arc<AtomicU64>, pub draining_started_at_epoch_secs: Arc<AtomicU64>,
pub drain_deadline_epoch_secs: Arc<AtomicU64>, pub drain_deadline_epoch_secs: Arc<AtomicU64>,
@@ -103,6 +106,7 @@ pub struct MePool {
pub(super) me_keepalive_jitter: Duration, pub(super) me_keepalive_jitter: Duration,
pub(super) me_keepalive_payload_random: bool, pub(super) me_keepalive_payload_random: bool,
pub(super) rpc_proxy_req_every_secs: AtomicU64, pub(super) rpc_proxy_req_every_secs: AtomicU64,
pub(super) writer_cmd_channel_capacity: usize,
pub(super) me_warmup_stagger_enabled: bool, pub(super) me_warmup_stagger_enabled: bool,
pub(super) me_warmup_step_delay: Duration, pub(super) me_warmup_step_delay: Duration,
pub(super) me_warmup_step_jitter: Duration, pub(super) me_warmup_step_jitter: Duration,
@@ -125,11 +129,21 @@ pub struct MePool {
pub(super) me_adaptive_floor_cpu_cores_override: AtomicU32, pub(super) me_adaptive_floor_cpu_cores_override: AtomicU32,
pub(super) me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32, pub(super) me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32,
pub(super) me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32, pub(super) me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32,
pub(super) me_adaptive_floor_max_active_writers_per_core: AtomicU32,
pub(super) me_adaptive_floor_max_warm_writers_per_core: AtomicU32,
pub(super) me_adaptive_floor_max_active_writers_global: AtomicU32,
pub(super) me_adaptive_floor_max_warm_writers_global: AtomicU32,
pub(super) me_adaptive_floor_cpu_cores_detected: AtomicU32, pub(super) me_adaptive_floor_cpu_cores_detected: AtomicU32,
pub(super) me_adaptive_floor_cpu_cores_effective: AtomicU32, pub(super) me_adaptive_floor_cpu_cores_effective: AtomicU32,
pub(super) me_adaptive_floor_global_cap_raw: AtomicU64, pub(super) me_adaptive_floor_global_cap_raw: AtomicU64,
pub(super) me_adaptive_floor_global_cap_effective: AtomicU64, pub(super) me_adaptive_floor_global_cap_effective: AtomicU64,
pub(super) me_adaptive_floor_target_writers_total: AtomicU64, pub(super) me_adaptive_floor_target_writers_total: AtomicU64,
pub(super) me_adaptive_floor_active_cap_configured: AtomicU64,
pub(super) me_adaptive_floor_active_cap_effective: AtomicU64,
pub(super) me_adaptive_floor_warm_cap_configured: AtomicU64,
pub(super) me_adaptive_floor_warm_cap_effective: AtomicU64,
pub(super) me_adaptive_floor_active_writers_current: AtomicU64,
pub(super) me_adaptive_floor_warm_writers_current: AtomicU64,
pub(super) proxy_map_v4: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>, pub(super) proxy_map_v4: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
pub(super) proxy_map_v6: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>, pub(super) proxy_map_v6: Arc<RwLock<HashMap<i32, Vec<(IpAddr, u16)>>>>,
pub(super) endpoint_dc_map: Arc<RwLock<HashMap<SocketAddr, Option<i32>>>>, pub(super) endpoint_dc_map: Arc<RwLock<HashMap<SocketAddr, Option<i32>>>>,
@@ -166,13 +180,19 @@ pub struct MePool {
pub(super) me_bind_stale_ttl_secs: AtomicU64, pub(super) me_bind_stale_ttl_secs: AtomicU64,
pub(super) secret_atomic_snapshot: AtomicBool, pub(super) secret_atomic_snapshot: AtomicBool,
pub(super) me_deterministic_writer_sort: AtomicBool, pub(super) me_deterministic_writer_sort: AtomicBool,
pub(super) me_writer_pick_mode: AtomicU8,
pub(super) me_writer_pick_sample_size: AtomicU8,
pub(super) me_socks_kdf_policy: AtomicU8, pub(super) me_socks_kdf_policy: AtomicU8,
pub(super) me_route_no_writer_mode: AtomicU8, pub(super) me_route_no_writer_mode: AtomicU8,
pub(super) me_route_no_writer_wait: Duration, pub(super) me_route_no_writer_wait: Duration,
pub(super) me_route_inline_recovery_attempts: u32, pub(super) me_route_inline_recovery_attempts: u32,
pub(super) me_route_inline_recovery_wait: Duration, pub(super) me_route_inline_recovery_wait: Duration,
pub(super) me_health_interval_ms_unhealthy: AtomicU64,
pub(super) me_health_interval_ms_healthy: AtomicU64,
pub(super) me_warn_rate_limit_ms: AtomicU64,
pub(super) runtime_ready: AtomicBool, pub(super) runtime_ready: AtomicBool,
pool_size: usize, pool_size: usize,
pub(super) preferred_endpoints_by_dc: Arc<RwLock<HashMap<i32, Vec<SocketAddr>>>>,
} }
#[derive(Debug, Default)] #[derive(Debug, Default)]
@@ -243,6 +263,10 @@ impl MePool {
me_adaptive_floor_cpu_cores_override: u16, me_adaptive_floor_cpu_cores_override: u16,
me_adaptive_floor_max_extra_writers_single_per_core: u16, me_adaptive_floor_max_extra_writers_single_per_core: u16,
me_adaptive_floor_max_extra_writers_multi_per_core: u16, me_adaptive_floor_max_extra_writers_multi_per_core: u16,
me_adaptive_floor_max_active_writers_per_core: u16,
me_adaptive_floor_max_warm_writers_per_core: u16,
me_adaptive_floor_max_active_writers_global: u32,
me_adaptive_floor_max_warm_writers_global: u32,
hardswap: bool, hardswap: bool,
me_pool_drain_ttl_secs: u64, me_pool_drain_ttl_secs: u64,
me_pool_force_close_secs: u64, me_pool_force_close_secs: u64,
@@ -255,17 +279,28 @@ impl MePool {
me_bind_stale_ttl_secs: u64, me_bind_stale_ttl_secs: u64,
me_secret_atomic_snapshot: bool, me_secret_atomic_snapshot: bool,
me_deterministic_writer_sort: bool, me_deterministic_writer_sort: bool,
me_writer_pick_mode: MeWriterPickMode,
me_writer_pick_sample_size: u8,
me_socks_kdf_policy: MeSocksKdfPolicy, me_socks_kdf_policy: MeSocksKdfPolicy,
me_writer_cmd_channel_capacity: usize,
me_route_channel_capacity: usize,
me_route_backpressure_base_timeout_ms: u64, me_route_backpressure_base_timeout_ms: u64,
me_route_backpressure_high_timeout_ms: u64, me_route_backpressure_high_timeout_ms: u64,
me_route_backpressure_high_watermark_pct: u8, me_route_backpressure_high_watermark_pct: u8,
me_health_interval_ms_unhealthy: u64,
me_health_interval_ms_healthy: u64,
me_warn_rate_limit_ms: u64,
me_route_no_writer_mode: MeRouteNoWriterMode, me_route_no_writer_mode: MeRouteNoWriterMode,
me_route_no_writer_wait_ms: u64, me_route_no_writer_wait_ms: u64,
me_route_inline_recovery_attempts: u32, me_route_inline_recovery_attempts: u32,
me_route_inline_recovery_wait_ms: u64, me_route_inline_recovery_wait_ms: u64,
) -> Arc<Self> { ) -> Arc<Self> {
let endpoint_dc_map = Self::build_endpoint_dc_map_from_maps(&proxy_map_v4, &proxy_map_v6); let endpoint_dc_map = Self::build_endpoint_dc_map_from_maps(&proxy_map_v4, &proxy_map_v6);
let registry = Arc::new(ConnRegistry::new()); let preferred_endpoints_by_dc =
Self::build_preferred_endpoints_by_dc(&decision, &proxy_map_v4, &proxy_map_v6);
let registry = Arc::new(ConnRegistry::with_route_channel_capacity(
me_route_channel_capacity,
));
registry.update_route_backpressure_policy( registry.update_route_backpressure_policy(
me_route_backpressure_base_timeout_ms, me_route_backpressure_base_timeout_ms,
me_route_backpressure_high_timeout_ms, me_route_backpressure_high_timeout_ms,
@@ -312,6 +347,7 @@ impl MePool {
me_keepalive_jitter: Duration::from_secs(me_keepalive_jitter_secs), me_keepalive_jitter: Duration::from_secs(me_keepalive_jitter_secs),
me_keepalive_payload_random, me_keepalive_payload_random,
rpc_proxy_req_every_secs: AtomicU64::new(rpc_proxy_req_every_secs), rpc_proxy_req_every_secs: AtomicU64::new(rpc_proxy_req_every_secs),
writer_cmd_channel_capacity: me_writer_cmd_channel_capacity.max(1),
me_warmup_stagger_enabled, me_warmup_stagger_enabled,
me_warmup_step_delay: Duration::from_millis(me_warmup_step_delay_ms), me_warmup_step_delay: Duration::from_millis(me_warmup_step_delay_ms),
me_warmup_step_jitter: Duration::from_millis(me_warmup_step_jitter_ms), me_warmup_step_jitter: Duration::from_millis(me_warmup_step_jitter_ms),
@@ -358,11 +394,29 @@ impl MePool {
me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32::new( me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32::new(
me_adaptive_floor_max_extra_writers_multi_per_core as u32, me_adaptive_floor_max_extra_writers_multi_per_core as u32,
), ),
me_adaptive_floor_max_active_writers_per_core: AtomicU32::new(
me_adaptive_floor_max_active_writers_per_core as u32,
),
me_adaptive_floor_max_warm_writers_per_core: AtomicU32::new(
me_adaptive_floor_max_warm_writers_per_core as u32,
),
me_adaptive_floor_max_active_writers_global: AtomicU32::new(
me_adaptive_floor_max_active_writers_global,
),
me_adaptive_floor_max_warm_writers_global: AtomicU32::new(
me_adaptive_floor_max_warm_writers_global,
),
me_adaptive_floor_cpu_cores_detected: AtomicU32::new(1), me_adaptive_floor_cpu_cores_detected: AtomicU32::new(1),
me_adaptive_floor_cpu_cores_effective: AtomicU32::new(1), me_adaptive_floor_cpu_cores_effective: AtomicU32::new(1),
me_adaptive_floor_global_cap_raw: AtomicU64::new(0), me_adaptive_floor_global_cap_raw: AtomicU64::new(0),
me_adaptive_floor_global_cap_effective: AtomicU64::new(0), me_adaptive_floor_global_cap_effective: AtomicU64::new(0),
me_adaptive_floor_target_writers_total: AtomicU64::new(0), me_adaptive_floor_target_writers_total: AtomicU64::new(0),
me_adaptive_floor_active_cap_configured: AtomicU64::new(0),
me_adaptive_floor_active_cap_effective: AtomicU64::new(0),
me_adaptive_floor_warm_cap_configured: AtomicU64::new(0),
me_adaptive_floor_warm_cap_effective: AtomicU64::new(0),
me_adaptive_floor_active_writers_current: AtomicU64::new(0),
me_adaptive_floor_warm_writers_current: AtomicU64::new(0),
pool_size: 2, pool_size: 2,
proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)),
proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)),
@@ -403,12 +457,18 @@ impl MePool {
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs), me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot), secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot),
me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort), me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort),
me_writer_pick_mode: AtomicU8::new(me_writer_pick_mode.as_u8()),
me_writer_pick_sample_size: AtomicU8::new(me_writer_pick_sample_size.clamp(2, 4)),
me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()), me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()),
me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()), me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()),
me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms),
me_route_inline_recovery_attempts, me_route_inline_recovery_attempts,
me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms), me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms),
me_health_interval_ms_unhealthy: AtomicU64::new(me_health_interval_ms_unhealthy.max(1)),
me_health_interval_ms_healthy: AtomicU64::new(me_health_interval_ms_healthy.max(1)),
me_warn_rate_limit_ms: AtomicU64::new(me_warn_rate_limit_ms.max(1)),
runtime_ready: AtomicBool::new(false), runtime_ready: AtomicBool::new(false),
preferred_endpoints_by_dc: Arc::new(RwLock::new(preferred_endpoints_by_dc)),
}) })
} }
@@ -438,6 +498,8 @@ impl MePool {
bind_stale_ttl_secs: u64, bind_stale_ttl_secs: u64,
secret_atomic_snapshot: bool, secret_atomic_snapshot: bool,
deterministic_writer_sort: bool, deterministic_writer_sort: bool,
writer_pick_mode: MeWriterPickMode,
writer_pick_sample_size: u8,
single_endpoint_shadow_writers: u8, single_endpoint_shadow_writers: u8,
single_endpoint_outage_mode_enabled: bool, single_endpoint_outage_mode_enabled: bool,
single_endpoint_outage_disable_quarantine: bool, single_endpoint_outage_disable_quarantine: bool,
@@ -453,6 +515,13 @@ impl MePool {
adaptive_floor_cpu_cores_override: u16, adaptive_floor_cpu_cores_override: u16,
adaptive_floor_max_extra_writers_single_per_core: u16, adaptive_floor_max_extra_writers_single_per_core: u16,
adaptive_floor_max_extra_writers_multi_per_core: u16, adaptive_floor_max_extra_writers_multi_per_core: u16,
adaptive_floor_max_active_writers_per_core: u16,
adaptive_floor_max_warm_writers_per_core: u16,
adaptive_floor_max_active_writers_global: u32,
adaptive_floor_max_warm_writers_global: u32,
me_health_interval_ms_unhealthy: u64,
me_health_interval_ms_healthy: u64,
me_warn_rate_limit_ms: u64,
) { ) {
self.hardswap.store(hardswap, Ordering::Relaxed); self.hardswap.store(hardswap, Ordering::Relaxed);
self.me_pool_drain_ttl_secs self.me_pool_drain_ttl_secs
@@ -477,6 +546,14 @@ impl MePool {
.store(secret_atomic_snapshot, Ordering::Relaxed); .store(secret_atomic_snapshot, Ordering::Relaxed);
self.me_deterministic_writer_sort self.me_deterministic_writer_sort
.store(deterministic_writer_sort, Ordering::Relaxed); .store(deterministic_writer_sort, Ordering::Relaxed);
let previous_writer_pick_mode = self.writer_pick_mode();
self.me_writer_pick_mode
.store(writer_pick_mode.as_u8(), Ordering::Relaxed);
self.me_writer_pick_sample_size
.store(writer_pick_sample_size.clamp(2, 4), Ordering::Relaxed);
if previous_writer_pick_mode != writer_pick_mode {
self.stats.increment_me_writer_pick_mode_switch_total();
}
self.me_single_endpoint_shadow_writers self.me_single_endpoint_shadow_writers
.store(single_endpoint_shadow_writers, Ordering::Relaxed); .store(single_endpoint_shadow_writers, Ordering::Relaxed);
self.me_single_endpoint_outage_mode_enabled self.me_single_endpoint_outage_mode_enabled
@@ -514,6 +591,26 @@ impl MePool {
adaptive_floor_max_extra_writers_multi_per_core as u32, adaptive_floor_max_extra_writers_multi_per_core as u32,
Ordering::Relaxed, Ordering::Relaxed,
); );
self.me_adaptive_floor_max_active_writers_per_core
.store(
adaptive_floor_max_active_writers_per_core as u32,
Ordering::Relaxed,
);
self.me_adaptive_floor_max_warm_writers_per_core
.store(
adaptive_floor_max_warm_writers_per_core as u32,
Ordering::Relaxed,
);
self.me_adaptive_floor_max_active_writers_global
.store(adaptive_floor_max_active_writers_global, Ordering::Relaxed);
self.me_adaptive_floor_max_warm_writers_global
.store(adaptive_floor_max_warm_writers_global, Ordering::Relaxed);
self.me_health_interval_ms_unhealthy
.store(me_health_interval_ms_unhealthy.max(1), Ordering::Relaxed);
self.me_health_interval_ms_healthy
.store(me_health_interval_ms_healthy.max(1), Ordering::Relaxed);
self.me_warn_rate_limit_ms
.store(me_warn_rate_limit_ms.max(1), Ordering::Relaxed);
if previous_floor_mode != floor_mode { if previous_floor_mode != floor_mode {
self.stats.increment_me_floor_mode_switch_total(); self.stats.increment_me_floor_mode_switch_total();
match (previous_floor_mode, floor_mode) { match (previous_floor_mode, floor_mode) {
@@ -584,11 +681,26 @@ impl MePool {
self.proxy_secret.read().await.key_selector self.proxy_secret.read().await.key_selector
} }
pub(super) async fn active_writer_count_total(&self) -> usize { pub(super) async fn non_draining_writer_counts_by_contour(&self) -> (usize, usize, usize) {
let ws = self.writers.read().await; let ws = self.writers.read().await;
ws.iter() let mut active = 0usize;
.filter(|w| !w.draining.load(Ordering::Relaxed)) let mut warm = 0usize;
.count() for writer in ws.iter() {
if writer.draining.load(Ordering::Relaxed) {
continue;
}
match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
WriterContour::Active => active = active.saturating_add(1),
WriterContour::Warm => warm = warm.saturating_add(1),
WriterContour::Draining => {}
}
}
(active, warm, active.saturating_add(warm))
}
pub(super) async fn active_contour_writer_count_total(&self) -> usize {
let (active, _, _) = self.non_draining_writer_counts_by_contour().await;
active
} }
pub(super) async fn secret_snapshot(&self) -> SecretSnapshot { pub(super) async fn secret_snapshot(&self) -> SecretSnapshot {
@@ -599,6 +711,16 @@ impl MePool {
MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed)) MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed))
} }
pub(super) fn writer_pick_mode(&self) -> MeWriterPickMode {
MeWriterPickMode::from_u8(self.me_writer_pick_mode.load(Ordering::Relaxed))
}
pub(super) fn writer_pick_sample_size(&self) -> usize {
self.me_writer_pick_sample_size
.load(Ordering::Relaxed)
.clamp(2, 4) as usize
}
pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize { pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize {
if endpoint_count == 0 { if endpoint_count == 0 {
return 0; return 0;
@@ -634,13 +756,6 @@ impl MePool {
.max(1) .max(1)
} }
pub(super) fn adaptive_floor_writers_per_core_total(&self) -> usize {
(self
.me_adaptive_floor_writers_per_core_total
.load(Ordering::Relaxed) as usize)
.max(1)
}
pub(super) fn adaptive_floor_max_extra_single_per_core(&self) -> usize { pub(super) fn adaptive_floor_max_extra_single_per_core(&self) -> usize {
self.me_adaptive_floor_max_extra_writers_single_per_core self.me_adaptive_floor_max_extra_writers_single_per_core
.load(Ordering::Relaxed) as usize .load(Ordering::Relaxed) as usize
@@ -651,6 +766,34 @@ impl MePool {
.load(Ordering::Relaxed) as usize .load(Ordering::Relaxed) as usize
} }
pub(super) fn adaptive_floor_max_active_writers_per_core(&self) -> usize {
(self
.me_adaptive_floor_max_active_writers_per_core
.load(Ordering::Relaxed) as usize)
.max(1)
}
pub(super) fn adaptive_floor_max_warm_writers_per_core(&self) -> usize {
(self
.me_adaptive_floor_max_warm_writers_per_core
.load(Ordering::Relaxed) as usize)
.max(1)
}
pub(super) fn adaptive_floor_max_active_writers_global(&self) -> usize {
(self
.me_adaptive_floor_max_active_writers_global
.load(Ordering::Relaxed) as usize)
.max(1)
}
pub(super) fn adaptive_floor_max_warm_writers_global(&self) -> usize {
(self
.me_adaptive_floor_max_warm_writers_global
.load(Ordering::Relaxed) as usize)
.max(1)
}
pub(super) fn adaptive_floor_detected_cpu_cores(&self) -> usize { pub(super) fn adaptive_floor_detected_cpu_cores(&self) -> usize {
std::thread::available_parallelism() std::thread::available_parallelism()
.map(|value| value.get()) .map(|value| value.get())
@@ -679,28 +822,126 @@ impl MePool {
effective effective
} }
pub(super) fn adaptive_floor_global_cap_raw(&self) -> usize { pub(super) fn adaptive_floor_active_cap_configured_total(&self) -> usize {
let cores = self.adaptive_floor_effective_cpu_cores(); let cores = self.adaptive_floor_effective_cpu_cores();
let cap = cores.saturating_mul(self.adaptive_floor_writers_per_core_total()); let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_active_writers_per_core());
self.me_adaptive_floor_global_cap_raw let configured = per_core_cap.min(self.adaptive_floor_max_active_writers_global());
.store(cap as u64, Ordering::Relaxed); self.me_adaptive_floor_active_cap_configured
self.stats.set_me_floor_global_cap_raw_gauge(cap as u64); .store(configured as u64, Ordering::Relaxed);
cap self.stats
.set_me_floor_active_cap_configured_gauge(configured as u64);
configured
}
pub(super) fn adaptive_floor_warm_cap_configured_total(&self) -> usize {
let cores = self.adaptive_floor_effective_cpu_cores();
let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_warm_writers_per_core());
let configured = per_core_cap.min(self.adaptive_floor_max_warm_writers_global());
self.me_adaptive_floor_warm_cap_configured
.store(configured as u64, Ordering::Relaxed);
self.stats
.set_me_floor_warm_cap_configured_gauge(configured as u64);
configured
} }
pub(super) fn set_adaptive_floor_runtime_caps( pub(super) fn set_adaptive_floor_runtime_caps(
&self, &self,
global_cap_effective: usize, active_cap_configured: usize,
active_cap_effective: usize,
warm_cap_configured: usize,
warm_cap_effective: usize,
target_writers_total: usize, target_writers_total: usize,
active_writers_current: usize,
warm_writers_current: usize,
) { ) {
self.me_adaptive_floor_global_cap_raw
.store(active_cap_configured as u64, Ordering::Relaxed);
self.me_adaptive_floor_global_cap_effective self.me_adaptive_floor_global_cap_effective
.store(global_cap_effective as u64, Ordering::Relaxed); .store(active_cap_effective as u64, Ordering::Relaxed);
self.me_adaptive_floor_target_writers_total self.me_adaptive_floor_target_writers_total
.store(target_writers_total as u64, Ordering::Relaxed); .store(target_writers_total as u64, Ordering::Relaxed);
self.me_adaptive_floor_active_cap_configured
.store(active_cap_configured as u64, Ordering::Relaxed);
self.me_adaptive_floor_active_cap_effective
.store(active_cap_effective as u64, Ordering::Relaxed);
self.me_adaptive_floor_warm_cap_configured
.store(warm_cap_configured as u64, Ordering::Relaxed);
self.me_adaptive_floor_warm_cap_effective
.store(warm_cap_effective as u64, Ordering::Relaxed);
self.me_adaptive_floor_active_writers_current
.store(active_writers_current as u64, Ordering::Relaxed);
self.me_adaptive_floor_warm_writers_current
.store(warm_writers_current as u64, Ordering::Relaxed);
self.stats self.stats
.set_me_floor_global_cap_effective_gauge(global_cap_effective as u64); .set_me_floor_global_cap_raw_gauge(active_cap_configured as u64);
self.stats
.set_me_floor_global_cap_effective_gauge(active_cap_effective as u64);
self.stats self.stats
.set_me_floor_target_writers_total_gauge(target_writers_total as u64); .set_me_floor_target_writers_total_gauge(target_writers_total as u64);
self.stats
.set_me_floor_active_cap_configured_gauge(active_cap_configured as u64);
self.stats
.set_me_floor_active_cap_effective_gauge(active_cap_effective as u64);
self.stats
.set_me_floor_warm_cap_configured_gauge(warm_cap_configured as u64);
self.stats
.set_me_floor_warm_cap_effective_gauge(warm_cap_effective as u64);
self.stats
.set_me_writers_active_current_gauge(active_writers_current as u64);
self.stats
.set_me_writers_warm_current_gauge(warm_writers_current as u64);
}
pub(super) async fn active_coverage_required_total(&self) -> usize {
let mut endpoints_by_dc = HashMap::<i32, HashSet<SocketAddr>>::new();
if self.decision.ipv4_me {
let map = self.proxy_map_v4.read().await;
for (dc, addrs) in map.iter() {
let entry = endpoints_by_dc.entry(*dc).or_default();
for (ip, port) in addrs.iter().copied() {
entry.insert(SocketAddr::new(ip, port));
}
}
}
if self.decision.ipv6_me {
let map = self.proxy_map_v6.read().await;
for (dc, addrs) in map.iter() {
let entry = endpoints_by_dc.entry(*dc).or_default();
for (ip, port) in addrs.iter().copied() {
entry.insert(SocketAddr::new(ip, port));
}
}
}
endpoints_by_dc
.values()
.map(|endpoints| self.required_writers_for_dc_with_floor_mode(endpoints.len(), false))
.sum()
}
pub(super) async fn can_open_writer_for_contour(
&self,
contour: WriterContour,
allow_coverage_override: bool,
) -> bool {
let (active_writers, warm_writers, _) = self.non_draining_writer_counts_by_contour().await;
match contour {
WriterContour::Active => {
let active_cap = self.adaptive_floor_active_cap_configured_total();
if active_writers < active_cap {
return true;
}
if !allow_coverage_override {
return false;
}
let coverage_required = self.active_coverage_required_total().await;
active_writers < coverage_required
}
WriterContour::Warm => warm_writers < self.adaptive_floor_warm_cap_configured_total(),
WriterContour::Draining => true,
}
} }
pub(super) fn required_writers_for_dc_with_floor_mode( pub(super) fn required_writers_for_dc_with_floor_mode(
@@ -858,6 +1099,62 @@ impl MePool {
} }
} }
fn build_preferred_endpoints_by_dc(
decision: &NetworkDecision,
map_v4: &HashMap<i32, Vec<(IpAddr, u16)>>,
map_v6: &HashMap<i32, Vec<(IpAddr, u16)>>,
) -> HashMap<i32, Vec<SocketAddr>> {
let mut out = HashMap::<i32, Vec<SocketAddr>>::new();
let mut dcs = HashSet::<i32>::new();
dcs.extend(map_v4.keys().copied());
dcs.extend(map_v6.keys().copied());
for dc in dcs {
let v4 = map_v4
.get(&dc)
.map(|items| {
items
.iter()
.map(|(ip, port)| SocketAddr::new(*ip, *port))
.collect::<Vec<_>>()
})
.unwrap_or_default();
let v6 = map_v6
.get(&dc)
.map(|items| {
items
.iter()
.map(|(ip, port)| SocketAddr::new(*ip, *port))
.collect::<Vec<_>>()
})
.unwrap_or_default();
let mut selected = if decision.effective_multipath {
let mut both = Vec::<SocketAddr>::with_capacity(v4.len().saturating_add(v6.len()));
if decision.prefer_ipv6() {
both.extend(v6.iter().copied());
both.extend(v4.iter().copied());
} else {
both.extend(v4.iter().copied());
both.extend(v6.iter().copied());
}
both
} else if decision.prefer_ipv6() {
if !v6.is_empty() { v6 } else { v4 }
} else if !v4.is_empty() {
v4
} else {
v6
};
selected.sort_unstable();
selected.dedup();
out.insert(dc, selected);
}
out
}
fn build_endpoint_dc_map_from_maps( fn build_endpoint_dc_map_from_maps(
map_v4: &HashMap<i32, Vec<(IpAddr, u16)>>, map_v4: &HashMap<i32, Vec<(IpAddr, u16)>>,
map_v6: &HashMap<i32, Vec<(IpAddr, u16)>>, map_v6: &HashMap<i32, Vec<(IpAddr, u16)>>,
@@ -880,6 +1177,25 @@ impl MePool {
let map_v4 = self.proxy_map_v4.read().await.clone(); let map_v4 = self.proxy_map_v4.read().await.clone();
let map_v6 = self.proxy_map_v6.read().await.clone(); let map_v6 = self.proxy_map_v6.read().await.clone();
let rebuilt = Self::build_endpoint_dc_map_from_maps(&map_v4, &map_v6); let rebuilt = Self::build_endpoint_dc_map_from_maps(&map_v4, &map_v6);
let preferred = Self::build_preferred_endpoints_by_dc(&self.decision, &map_v4, &map_v6);
*self.endpoint_dc_map.write().await = rebuilt; *self.endpoint_dc_map.write().await = rebuilt;
*self.preferred_endpoints_by_dc.write().await = preferred;
}
pub(super) async fn preferred_endpoints_for_dc(&self, dc: i32) -> Vec<SocketAddr> {
let guard = self.preferred_endpoints_by_dc.read().await;
guard.get(&dc).cloned().unwrap_or_default()
}
pub(super) fn health_interval_unhealthy(&self) -> Duration {
Duration::from_millis(self.me_health_interval_ms_unhealthy.load(Ordering::Relaxed).max(1))
}
pub(super) fn health_interval_healthy(&self) -> Duration {
Duration::from_millis(self.me_health_interval_ms_healthy.load(Ordering::Relaxed).max(1))
}
pub(super) fn warn_rate_limit_duration(&self) -> Duration {
Duration::from_millis(self.me_warn_rate_limit_ms.load(Ordering::Relaxed).max(1))
} }
} }

View File

@@ -71,6 +71,7 @@ impl MePool {
target_writers, target_writers,
rng_clone, rng_clone,
connect_concurrency, connect_concurrency,
true,
) )
.await .await
}); });
@@ -114,6 +115,7 @@ impl MePool {
target_writers, target_writers,
rng_clone_local, rng_clone_local,
connect_concurrency, connect_concurrency,
false,
) )
.await .await
}); });
@@ -147,6 +149,7 @@ impl MePool {
target_writers: usize, target_writers: usize,
rng: Arc<SecureRandom>, rng: Arc<SecureRandom>,
connect_concurrency: usize, connect_concurrency: usize,
allow_coverage_override: bool,
) -> bool { ) -> bool {
if addrs.is_empty() { if addrs.is_empty() {
return false; return false;
@@ -180,9 +183,17 @@ impl MePool {
let pool = Arc::clone(&self); let pool = Arc::clone(&self);
let rng_clone = Arc::clone(&rng); let rng_clone = Arc::clone(&rng);
let endpoints_clone = endpoints.clone(); let endpoints_clone = endpoints.clone();
let generation = self.current_generation();
join.spawn(async move { join.spawn(async move {
pool.connect_endpoints_round_robin(dc, &endpoints_clone, rng_clone.as_ref()) pool.connect_endpoints_round_robin_with_generation_contour(
.await dc,
&endpoints_clone,
rng_clone.as_ref(),
generation,
super::pool::WriterContour::Active,
allow_coverage_override,
)
.await
}); });
} }
@@ -212,12 +223,25 @@ impl MePool {
return true; return true;
} }
if !progress { if !progress {
warn!( let active_writers_current = self.active_contour_writer_count_total().await;
dc = %dc, let active_cap_configured = self.adaptive_floor_active_cap_configured_total();
alive = alive_after, if !allow_coverage_override && active_writers_current >= active_cap_configured {
target_writers, info!(
"All ME servers for DC failed at init" dc = %dc,
); alive = alive_after,
target_writers,
active_writers_current,
active_cap_configured,
"ME init saturation stopped by active writer cap"
);
} else {
warn!(
dc = %dc,
alive = alive_after,
target_writers,
"All ME servers for DC failed at init"
);
}
return false; return false;
} }

View File

@@ -1,4 +1,4 @@
use std::collections::HashSet; use std::collections::{HashMap, HashSet};
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
@@ -99,6 +99,7 @@ impl MePool {
rng, rng,
self.current_generation(), self.current_generation(),
WriterContour::Active, WriterContour::Active,
false,
) )
.await .await
} }
@@ -110,17 +111,50 @@ impl MePool {
rng: &SecureRandom, rng: &SecureRandom,
generation: u64, generation: u64,
contour: WriterContour, contour: WriterContour,
allow_coverage_override: bool,
) -> bool { ) -> bool {
let candidates = self.connectable_endpoints(endpoints).await; let mut candidates = self.connectable_endpoints(endpoints).await;
if candidates.is_empty() { if candidates.is_empty() {
return false; return false;
} }
if candidates.len() > 1 {
let mut active_by_endpoint = HashMap::<SocketAddr, usize>::new();
let ws = self.writers.read().await;
for writer in ws.iter() {
if writer.draining.load(Ordering::Relaxed) {
continue;
}
if writer.writer_dc != dc {
continue;
}
if !matches!(
super::pool::WriterContour::from_u8(
writer.contour.load(Ordering::Relaxed),
),
super::pool::WriterContour::Active
) {
continue;
}
if candidates.contains(&writer.addr) {
*active_by_endpoint.entry(writer.addr).or_insert(0) += 1;
}
}
drop(ws);
candidates.sort_by_key(|addr| (active_by_endpoint.get(addr).copied().unwrap_or(0), *addr));
}
let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len(); let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len();
for offset in 0..candidates.len() { for offset in 0..candidates.len() {
let idx = (start + offset) % candidates.len(); let idx = (start + offset) % candidates.len();
let addr = candidates[idx]; let addr = candidates[idx];
match self match self
.connect_one_with_generation_contour_for_dc(addr, rng, generation, contour, dc) .connect_one_with_generation_contour_for_dc_with_cap_policy(
addr,
rng,
generation,
contour,
dc,
allow_coverage_override,
)
.await .await
{ {
Ok(()) => return true, Ok(()) => return true,

View File

@@ -249,6 +249,7 @@ impl MePool {
rng, rng,
generation, generation,
WriterContour::Warm, WriterContour::Warm,
false,
) )
.await; .await;
debug!( debug!(

View File

@@ -25,6 +25,7 @@ pub(crate) struct MeApiWriterStatusSnapshot {
pub(crate) struct MeApiDcStatusSnapshot { pub(crate) struct MeApiDcStatusSnapshot {
pub dc: i16, pub dc: i16,
pub endpoints: Vec<SocketAddr>, pub endpoints: Vec<SocketAddr>,
pub endpoint_writers: Vec<MeApiDcEndpointWriterSnapshot>,
pub available_endpoints: usize, pub available_endpoints: usize,
pub available_pct: f64, pub available_pct: f64,
pub required_writers: usize, pub required_writers: usize,
@@ -38,6 +39,12 @@ pub(crate) struct MeApiDcStatusSnapshot {
pub load: usize, pub load: usize,
} }
#[derive(Clone, Debug)]
pub(crate) struct MeApiDcEndpointWriterSnapshot {
pub endpoint: SocketAddr,
pub active_writers: usize,
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) struct MeApiStatusSnapshot { pub(crate) struct MeApiStatusSnapshot {
pub generated_at_epoch_secs: u64, pub generated_at_epoch_secs: u64,
@@ -82,11 +89,21 @@ pub(crate) struct MeApiRuntimeSnapshot {
pub adaptive_floor_cpu_cores_override: u16, pub adaptive_floor_cpu_cores_override: u16,
pub adaptive_floor_max_extra_writers_single_per_core: u16, pub adaptive_floor_max_extra_writers_single_per_core: u16,
pub adaptive_floor_max_extra_writers_multi_per_core: u16, pub adaptive_floor_max_extra_writers_multi_per_core: u16,
pub adaptive_floor_max_active_writers_per_core: u16,
pub adaptive_floor_max_warm_writers_per_core: u16,
pub adaptive_floor_max_active_writers_global: u32,
pub adaptive_floor_max_warm_writers_global: u32,
pub adaptive_floor_cpu_cores_detected: u32, pub adaptive_floor_cpu_cores_detected: u32,
pub adaptive_floor_cpu_cores_effective: u32, pub adaptive_floor_cpu_cores_effective: u32,
pub adaptive_floor_global_cap_raw: u64, pub adaptive_floor_global_cap_raw: u64,
pub adaptive_floor_global_cap_effective: u64, pub adaptive_floor_global_cap_effective: u64,
pub adaptive_floor_target_writers_total: u64, pub adaptive_floor_target_writers_total: u64,
pub adaptive_floor_active_cap_configured: u64,
pub adaptive_floor_active_cap_effective: u64,
pub adaptive_floor_warm_cap_configured: u64,
pub adaptive_floor_warm_cap_effective: u64,
pub adaptive_floor_active_writers_current: u64,
pub adaptive_floor_warm_writers_current: u64,
pub me_keepalive_enabled: bool, pub me_keepalive_enabled: bool,
pub me_keepalive_interval_secs: u64, pub me_keepalive_interval_secs: u64,
pub me_keepalive_jitter_secs: u64, pub me_keepalive_jitter_secs: u64,
@@ -108,6 +125,8 @@ pub(crate) struct MeApiRuntimeSnapshot {
pub me_single_endpoint_outage_backoff_max_ms: u64, pub me_single_endpoint_outage_backoff_max_ms: u64,
pub me_single_endpoint_shadow_rotate_every_secs: u64, pub me_single_endpoint_shadow_rotate_every_secs: u64,
pub me_deterministic_writer_sort: bool, pub me_deterministic_writer_sort: bool,
pub me_writer_pick_mode: &'static str,
pub me_writer_pick_sample_size: u8,
pub me_socks_kdf_policy: &'static str, pub me_socks_kdf_policy: &'static str,
pub quarantined_endpoints: Vec<MeApiQuarantinedEndpointSnapshot>, pub quarantined_endpoints: Vec<MeApiQuarantinedEndpointSnapshot>,
pub network_path: Vec<MeApiDcPathSnapshot>, pub network_path: Vec<MeApiDcPathSnapshot>,
@@ -328,6 +347,16 @@ impl MePool {
dcs.push(MeApiDcStatusSnapshot { dcs.push(MeApiDcStatusSnapshot {
dc, dc,
endpoint_writers: endpoints
.iter()
.map(|endpoint| MeApiDcEndpointWriterSnapshot {
endpoint: *endpoint,
active_writers: live_writers_by_dc_endpoint
.get(&(dc, *endpoint))
.copied()
.unwrap_or(0),
})
.collect(),
endpoints: endpoints.into_iter().collect(), endpoints: endpoints.into_iter().collect(),
available_endpoints: dc_available_endpoints, available_endpoints: dc_available_endpoints,
available_pct: ratio_pct(dc_available_endpoints, endpoint_count), available_pct: ratio_pct(dc_available_endpoints, endpoint_count),
@@ -430,6 +459,18 @@ impl MePool {
adaptive_floor_max_extra_writers_multi_per_core: self adaptive_floor_max_extra_writers_multi_per_core: self
.me_adaptive_floor_max_extra_writers_multi_per_core .me_adaptive_floor_max_extra_writers_multi_per_core
.load(Ordering::Relaxed) as u16, .load(Ordering::Relaxed) as u16,
adaptive_floor_max_active_writers_per_core: self
.me_adaptive_floor_max_active_writers_per_core
.load(Ordering::Relaxed) as u16,
adaptive_floor_max_warm_writers_per_core: self
.me_adaptive_floor_max_warm_writers_per_core
.load(Ordering::Relaxed) as u16,
adaptive_floor_max_active_writers_global: self
.me_adaptive_floor_max_active_writers_global
.load(Ordering::Relaxed),
adaptive_floor_max_warm_writers_global: self
.me_adaptive_floor_max_warm_writers_global
.load(Ordering::Relaxed),
adaptive_floor_cpu_cores_detected: self adaptive_floor_cpu_cores_detected: self
.me_adaptive_floor_cpu_cores_detected .me_adaptive_floor_cpu_cores_detected
.load(Ordering::Relaxed), .load(Ordering::Relaxed),
@@ -445,6 +486,24 @@ impl MePool {
adaptive_floor_target_writers_total: self adaptive_floor_target_writers_total: self
.me_adaptive_floor_target_writers_total .me_adaptive_floor_target_writers_total
.load(Ordering::Relaxed), .load(Ordering::Relaxed),
adaptive_floor_active_cap_configured: self
.me_adaptive_floor_active_cap_configured
.load(Ordering::Relaxed),
adaptive_floor_active_cap_effective: self
.me_adaptive_floor_active_cap_effective
.load(Ordering::Relaxed),
adaptive_floor_warm_cap_configured: self
.me_adaptive_floor_warm_cap_configured
.load(Ordering::Relaxed),
adaptive_floor_warm_cap_effective: self
.me_adaptive_floor_warm_cap_effective
.load(Ordering::Relaxed),
adaptive_floor_active_writers_current: self
.me_adaptive_floor_active_writers_current
.load(Ordering::Relaxed),
adaptive_floor_warm_writers_current: self
.me_adaptive_floor_warm_writers_current
.load(Ordering::Relaxed),
me_keepalive_enabled: self.me_keepalive_enabled, me_keepalive_enabled: self.me_keepalive_enabled,
me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(), me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(),
me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(), me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(),
@@ -482,6 +541,8 @@ impl MePool {
me_deterministic_writer_sort: self me_deterministic_writer_sort: self
.me_deterministic_writer_sort .me_deterministic_writer_sort
.load(Ordering::Relaxed), .load(Ordering::Relaxed),
me_writer_pick_mode: writer_pick_mode_label(self.writer_pick_mode()),
me_writer_pick_sample_size: self.writer_pick_sample_size() as u8,
me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()), me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()),
quarantined_endpoints, quarantined_endpoints,
network_path, network_path,
@@ -530,6 +591,13 @@ fn bind_stale_mode_label(mode: MeBindStaleMode) -> &'static str {
} }
} }
fn writer_pick_mode_label(mode: crate::config::MeWriterPickMode) -> &'static str {
match mode {
crate::config::MeWriterPickMode::SortedRr => "sorted_rr",
crate::config::MeWriterPickMode::P2c => "p2c",
}
}
fn socks_kdf_policy_label(policy: MeSocksKdfPolicy) -> &'static str { fn socks_kdf_policy_label(policy: MeSocksKdfPolicy) -> &'static str {
match policy { match policy {
MeSocksKdfPolicy::Strict => "strict", MeSocksKdfPolicy::Strict => "strict",

View File

@@ -1,6 +1,6 @@
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use std::io::ErrorKind; use std::io::ErrorKind;
@@ -86,6 +86,35 @@ impl MePool {
contour: WriterContour, contour: WriterContour,
writer_dc: i32, writer_dc: i32,
) -> Result<()> { ) -> Result<()> {
self.connect_one_with_generation_contour_for_dc_with_cap_policy(
addr,
rng,
generation,
contour,
writer_dc,
false,
)
.await
}
pub(super) async fn connect_one_with_generation_contour_for_dc_with_cap_policy(
self: &Arc<Self>,
addr: SocketAddr,
rng: &SecureRandom,
generation: u64,
contour: WriterContour,
writer_dc: i32,
allow_coverage_override: bool,
) -> Result<()> {
if !self
.can_open_writer_for_contour(contour, allow_coverage_override)
.await
{
return Err(ProxyError::Proxy(format!(
"ME {contour:?} writer cap reached"
)));
}
let secret_len = self.proxy_secret.read().await.secret.len(); let secret_len = self.proxy_secret.read().await.secret.len();
if secret_len < 32 { if secret_len < 32 {
return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into())); return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into()));
@@ -99,11 +128,12 @@ impl MePool {
let contour = Arc::new(AtomicU8::new(contour.as_u8())); let contour = Arc::new(AtomicU8::new(contour.as_u8()));
let cancel = CancellationToken::new(); let cancel = CancellationToken::new();
let degraded = Arc::new(AtomicBool::new(false)); let degraded = Arc::new(AtomicBool::new(false));
let rtt_ema_ms_x10 = Arc::new(AtomicU32::new(0));
let draining = Arc::new(AtomicBool::new(false)); let draining = Arc::new(AtomicBool::new(false));
let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0)); let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0));
let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0)); let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0));
let allow_drain_fallback = Arc::new(AtomicBool::new(false)); let allow_drain_fallback = Arc::new(AtomicBool::new(false));
let (tx, mut rx) = mpsc::channel::<WriterCommand>(4096); let (tx, mut rx) = mpsc::channel::<WriterCommand>(self.writer_cmd_channel_capacity);
let mut rpc_writer = RpcWriter { let mut rpc_writer = RpcWriter {
writer: hs.wr, writer: hs.wr,
key: hs.write_key, key: hs.write_key,
@@ -140,6 +170,7 @@ impl MePool {
tx: tx.clone(), tx: tx.clone(),
cancel: cancel.clone(), cancel: cancel.clone(),
degraded: degraded.clone(), degraded: degraded.clone(),
rtt_ema_ms_x10: rtt_ema_ms_x10.clone(),
draining: draining.clone(), draining: draining.clone(),
draining_started_at_epoch_secs: draining_started_at_epoch_secs.clone(), draining_started_at_epoch_secs: draining_started_at_epoch_secs.clone(),
drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(), drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(),
@@ -193,6 +224,7 @@ impl MePool {
stats_reader, stats_reader,
writer_id, writer_id,
degraded.clone(), degraded.clone(),
rtt_ema_ms_x10.clone(),
cancel_reader_token.clone(), cancel_reader_token.clone(),
) )
.await; .await;

View File

@@ -1,7 +1,7 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::io::ErrorKind; use std::io::ErrorKind;
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::time::Instant; use std::time::Instant;
use bytes::{Bytes, BytesMut}; use bytes::{Bytes, BytesMut};
@@ -34,6 +34,7 @@ pub(crate) async fn reader_loop(
stats: Arc<Stats>, stats: Arc<Stats>,
_writer_id: u64, _writer_id: u64,
degraded: Arc<AtomicBool>, degraded: Arc<AtomicBool>,
writer_rtt_ema_ms_x10: Arc<AtomicU32>,
cancel: CancellationToken, cancel: CancellationToken,
) -> Result<()> { ) -> Result<()> {
let mut raw = enc_leftover; let mut raw = enc_leftover;
@@ -208,6 +209,8 @@ pub(crate) async fn reader_loop(
} }
let degraded_now = entry.1 > entry.0 * 2.0; let degraded_now = entry.1 > entry.0 * 2.0;
degraded.store(degraded_now, Ordering::Relaxed); degraded.store(degraded_now, Ordering::Relaxed);
writer_rtt_ema_ms_x10
.store((entry.1 * 10.0).clamp(0.0, u32::MAX as f64) as u32, Ordering::Relaxed);
trace!(writer_id = wid, rtt_ms = rtt, ema_ms = entry.1, base_ms = entry.0, degraded = degraded_now, "ME RTT sample"); trace!(writer_id = wid, rtt_ms = rtt, ema_ms = entry.1, base_ms = entry.0, degraded = degraded_now, "ME RTT sample");
} }
} else { } else {

View File

@@ -9,7 +9,6 @@ use tokio::sync::mpsc::error::TrySendError;
use super::codec::WriterCommand; use super::codec::WriterCommand;
use super::MeResponse; use super::MeResponse;
const ROUTE_CHANNEL_CAPACITY: usize = 4096;
const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25; const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25;
const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120; const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120;
const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80; const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80;
@@ -78,6 +77,7 @@ impl RegistryInner {
pub struct ConnRegistry { pub struct ConnRegistry {
inner: RwLock<RegistryInner>, inner: RwLock<RegistryInner>,
next_id: AtomicU64, next_id: AtomicU64,
route_channel_capacity: usize,
route_backpressure_base_timeout_ms: AtomicU64, route_backpressure_base_timeout_ms: AtomicU64,
route_backpressure_high_timeout_ms: AtomicU64, route_backpressure_high_timeout_ms: AtomicU64,
route_backpressure_high_watermark_pct: AtomicU8, route_backpressure_high_watermark_pct: AtomicU8,
@@ -91,11 +91,12 @@ impl ConnRegistry {
.as_secs() .as_secs()
} }
pub fn new() -> Self { pub fn with_route_channel_capacity(route_channel_capacity: usize) -> Self {
let start = rand::random::<u64>() | 1; let start = rand::random::<u64>() | 1;
Self { Self {
inner: RwLock::new(RegistryInner::new()), inner: RwLock::new(RegistryInner::new()),
next_id: AtomicU64::new(start), next_id: AtomicU64::new(start),
route_channel_capacity: route_channel_capacity.max(1),
route_backpressure_base_timeout_ms: AtomicU64::new( route_backpressure_base_timeout_ms: AtomicU64::new(
ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS, ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS,
), ),
@@ -108,6 +109,11 @@ impl ConnRegistry {
} }
} }
#[cfg(test)]
pub fn new() -> Self {
Self::with_route_channel_capacity(4096)
}
pub fn update_route_backpressure_policy( pub fn update_route_backpressure_policy(
&self, &self,
base_timeout_ms: u64, base_timeout_ms: u64,
@@ -127,7 +133,7 @@ impl ConnRegistry {
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) { pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
let id = self.next_id.fetch_add(1, Ordering::Relaxed); let id = self.next_id.fetch_add(1, Ordering::Relaxed);
let (tx, rx) = mpsc::channel(ROUTE_CHANNEL_CAPACITY); let (tx, rx) = mpsc::channel(self.route_channel_capacity);
self.inner.write().await.map.insert(id, tx); self.inner.write().await.map.insert(id, tx);
(id, rx) (id, rx)
} }
@@ -179,11 +185,11 @@ impl ConnRegistry {
.route_backpressure_high_watermark_pct .route_backpressure_high_watermark_pct
.load(Ordering::Relaxed) .load(Ordering::Relaxed)
.clamp(1, 100); .clamp(1, 100);
let used = ROUTE_CHANNEL_CAPACITY.saturating_sub(tx.capacity()); let used = self.route_channel_capacity.saturating_sub(tx.capacity());
let used_pct = if ROUTE_CHANNEL_CAPACITY == 0 { let used_pct = if self.route_channel_capacity == 0 {
100 100
} else { } else {
(used.saturating_mul(100) / ROUTE_CHANNEL_CAPACITY) as u8 (used.saturating_mul(100) / self.route_channel_capacity) as u8
}; };
let high_profile = used_pct >= high_watermark_pct; let high_profile = used_pct >= high_watermark_pct;
let timeout_ms = if high_profile { let timeout_ms = if high_profile {

View File

@@ -9,7 +9,7 @@ use bytes::Bytes;
use tokio::sync::mpsc::error::TrySendError; use tokio::sync::mpsc::error::TrySendError;
use tracing::{debug, warn}; use tracing::{debug, warn};
use crate::config::MeRouteNoWriterMode; use crate::config::{MeRouteNoWriterMode, MeWriterPickMode};
use crate::error::{ProxyError, Result}; use crate::error::{ProxyError, Result};
use crate::network::IpFamily; use crate::network::IpFamily;
use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32}; use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32};
@@ -24,6 +24,10 @@ use super::registry::ConnMeta;
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45; const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55; const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4; const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
const PICK_PENALTY_WARM: u64 = 200;
const PICK_PENALTY_DRAINING: u64 = 600;
const PICK_PENALTY_STALE: u64 = 300;
const PICK_PENALTY_DEGRADED: u64 = 250;
impl MePool { impl MePool {
/// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default. /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
@@ -181,6 +185,7 @@ impl MePool {
.await; .await;
} }
if candidate_indices.is_empty() { if candidate_indices.is_empty() {
let pick_mode = self.writer_pick_mode();
match no_writer_mode { match no_writer_mode {
MeRouteNoWriterMode::AsyncRecoveryFailfast => { MeRouteNoWriterMode::AsyncRecoveryFailfast => {
let deadline = *no_writer_deadline.get_or_insert_with(|| { let deadline = *no_writer_deadline.get_or_insert_with(|| {
@@ -196,6 +201,7 @@ impl MePool {
if self.wait_for_candidate_until(routed_dc, deadline).await { if self.wait_for_candidate_until(routed_dc, deadline).await {
continue; continue;
} }
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
self.stats.increment_me_no_writer_failfast_total(); self.stats.increment_me_no_writer_failfast_total();
return Err(ProxyError::Proxy( return Err(ProxyError::Proxy(
"No ME writers available for target DC in failfast window".into(), "No ME writers available for target DC in failfast window".into(),
@@ -209,10 +215,12 @@ impl MePool {
if self.wait_for_candidate_until(routed_dc, deadline).await { if self.wait_for_candidate_until(routed_dc, deadline).await {
continue; continue;
} }
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
self.stats.increment_me_no_writer_failfast_total(); self.stats.increment_me_no_writer_failfast_total();
return Err(ProxyError::Proxy("No ME writers available for target DC".into())); return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
} }
if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) { if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) {
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
self.stats.increment_me_no_writer_failfast_total(); self.stats.increment_me_no_writer_failfast_total();
return Err(ProxyError::Proxy("No ME writers available for target DC".into())); return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
} }
@@ -237,6 +245,7 @@ impl MePool {
.await; .await;
} }
if candidate_indices.is_empty() { if candidate_indices.is_empty() {
self.stats.increment_me_writer_pick_no_candidate_total(pick_mode);
return Err(ProxyError::Proxy("No ME writers available for target DC".into())); return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
} }
} }
@@ -259,6 +268,8 @@ impl MePool {
} }
} }
hybrid_wait_current = hybrid_wait_step; hybrid_wait_current = hybrid_wait_step;
let pick_mode = self.writer_pick_mode();
let pick_sample_size = self.writer_pick_sample_size();
let writer_ids: Vec<u64> = candidate_indices let writer_ids: Vec<u64> = candidate_indices
.iter() .iter()
.map(|idx| writers_snapshot[*idx].id) .map(|idx| writers_snapshot[*idx].id)
@@ -268,69 +279,84 @@ impl MePool {
.writer_idle_since_for_writer_ids(&writer_ids) .writer_idle_since_for_writer_ids(&writer_ids)
.await; .await;
let now_epoch_secs = Self::now_epoch_secs(); let now_epoch_secs = Self::now_epoch_secs();
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
candidate_indices.sort_by(|lhs, rhs| {
let left = &writers_snapshot[*lhs];
let right = &writers_snapshot[*rhs];
let left_key = (
self.writer_contour_rank_for_selection(left),
(left.generation < self.current_generation()) as usize,
left.degraded.load(Ordering::Relaxed) as usize,
self.writer_idle_rank_for_selection(
left,
&writer_idle_since,
now_epoch_secs,
),
Reverse(left.tx.capacity()),
left.addr,
left.id,
);
let right_key = (
self.writer_contour_rank_for_selection(right),
(right.generation < self.current_generation()) as usize,
right.degraded.load(Ordering::Relaxed) as usize,
self.writer_idle_rank_for_selection(
right,
&writer_idle_since,
now_epoch_secs,
),
Reverse(right.tx.capacity()),
right.addr,
right.id,
);
left_key.cmp(&right_key)
});
} else {
candidate_indices.sort_by_key(|idx| {
let w = &writers_snapshot[*idx];
let degraded = w.degraded.load(Ordering::Relaxed);
let stale = (w.generation < self.current_generation()) as usize;
(
self.writer_contour_rank_for_selection(w),
stale,
degraded as usize,
self.writer_idle_rank_for_selection(
w,
&writer_idle_since,
now_epoch_secs,
),
Reverse(w.tx.capacity()),
)
});
}
let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len(); let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len();
let ordered_candidate_indices = if pick_mode == MeWriterPickMode::P2c {
self.p2c_ordered_candidate_indices(
&candidate_indices,
&writers_snapshot,
&writer_idle_since,
now_epoch_secs,
start,
pick_sample_size,
)
} else {
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
candidate_indices.sort_by(|lhs, rhs| {
let left = &writers_snapshot[*lhs];
let right = &writers_snapshot[*rhs];
let left_key = (
self.writer_contour_rank_for_selection(left),
(left.generation < self.current_generation()) as usize,
left.degraded.load(Ordering::Relaxed) as usize,
self.writer_idle_rank_for_selection(
left,
&writer_idle_since,
now_epoch_secs,
),
Reverse(left.tx.capacity()),
left.addr,
left.id,
);
let right_key = (
self.writer_contour_rank_for_selection(right),
(right.generation < self.current_generation()) as usize,
right.degraded.load(Ordering::Relaxed) as usize,
self.writer_idle_rank_for_selection(
right,
&writer_idle_since,
now_epoch_secs,
),
Reverse(right.tx.capacity()),
right.addr,
right.id,
);
left_key.cmp(&right_key)
});
} else {
candidate_indices.sort_by_key(|idx| {
let w = &writers_snapshot[*idx];
let degraded = w.degraded.load(Ordering::Relaxed);
let stale = (w.generation < self.current_generation()) as usize;
(
self.writer_contour_rank_for_selection(w),
stale,
degraded as usize,
self.writer_idle_rank_for_selection(
w,
&writer_idle_since,
now_epoch_secs,
),
Reverse(w.tx.capacity()),
)
});
}
let mut ordered = Vec::<usize>::with_capacity(candidate_indices.len());
for offset in 0..candidate_indices.len() {
ordered.push(candidate_indices[(start + offset) % candidate_indices.len()]);
}
ordered
};
let mut fallback_blocking_idx: Option<usize> = None; let mut fallback_blocking_idx: Option<usize> = None;
for offset in 0..candidate_indices.len() { for idx in ordered_candidate_indices {
let idx = candidate_indices[(start + offset) % candidate_indices.len()];
let w = &writers_snapshot[idx]; let w = &writers_snapshot[idx];
if !self.writer_accepts_new_binding(w) { if !self.writer_accepts_new_binding(w) {
continue; continue;
} }
match w.tx.try_send(WriterCommand::Data(payload.clone())) { match w.tx.try_send(WriterCommand::Data(payload.clone())) {
Ok(()) => { Ok(()) => {
self.stats.increment_me_writer_pick_success_try_total(pick_mode);
self.registry self.registry
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone()) .bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
.await; .await;
@@ -352,6 +378,7 @@ impl MePool {
} }
} }
Err(TrySendError::Closed(_)) => { Err(TrySendError::Closed(_)) => {
self.stats.increment_me_writer_pick_closed_total(pick_mode);
warn!(writer_id = w.id, "ME writer channel closed"); warn!(writer_id = w.id, "ME writer channel closed");
self.remove_writer_and_close_clients(w.id).await; self.remove_writer_and_close_clients(w.id).await;
continue; continue;
@@ -360,15 +387,20 @@ impl MePool {
} }
let Some(blocking_idx) = fallback_blocking_idx else { let Some(blocking_idx) = fallback_blocking_idx else {
self.stats.increment_me_writer_pick_full_total(pick_mode);
continue; continue;
}; };
let w = writers_snapshot[blocking_idx].clone(); let w = writers_snapshot[blocking_idx].clone();
if !self.writer_accepts_new_binding(&w) { if !self.writer_accepts_new_binding(&w) {
self.stats.increment_me_writer_pick_full_total(pick_mode);
continue; continue;
} }
self.stats.increment_me_writer_pick_blocking_fallback_total();
match w.tx.send(WriterCommand::Data(payload.clone())).await { match w.tx.send(WriterCommand::Data(payload.clone())).await {
Ok(()) => { Ok(()) => {
self.stats
.increment_me_writer_pick_success_fallback_total(pick_mode);
self.registry self.registry
.bind_writer(conn_id, w.id, w.tx.clone(), meta.clone()) .bind_writer(conn_id, w.id, w.tx.clone(), meta.clone())
.await; .await;
@@ -378,6 +410,7 @@ impl MePool {
return Ok(()); return Ok(());
} }
Err(_) => { Err(_) => {
self.stats.increment_me_writer_pick_closed_total(pick_mode);
warn!(writer_id = w.id, "ME writer channel closed (blocking)"); warn!(writer_id = w.id, "ME writer channel closed (blocking)");
self.remove_writer_and_close_clients(w.id).await; self.remove_writer_and_close_clients(w.id).await;
} }
@@ -480,31 +513,7 @@ impl MePool {
} }
async fn endpoint_candidates_for_target_dc(&self, routed_dc: i32) -> Vec<SocketAddr> { async fn endpoint_candidates_for_target_dc(&self, routed_dc: i32) -> Vec<SocketAddr> {
let mut preferred = Vec::<SocketAddr>::new(); self.preferred_endpoints_for_dc(routed_dc).await
let mut seen = HashSet::<SocketAddr>::new();
for family in self.family_order() {
let map_guard = match family {
IpFamily::V4 => self.proxy_map_v4.read().await,
IpFamily::V6 => self.proxy_map_v6.read().await,
};
let mut family_selected = Vec::<SocketAddr>::new();
if let Some(addrs) = map_guard.get(&routed_dc) {
for (ip, port) in addrs {
family_selected.push(SocketAddr::new(*ip, *port));
}
}
for addr in family_selected {
if seen.insert(addr) {
preferred.push(addr);
}
}
if !preferred.is_empty() && !self.decision.effective_multipath {
break;
}
}
preferred
} }
async fn maybe_trigger_hybrid_recovery( async fn maybe_trigger_hybrid_recovery(
@@ -591,28 +600,7 @@ impl MePool {
routed_dc: i32, routed_dc: i32,
include_warm: bool, include_warm: bool,
) -> Vec<usize> { ) -> Vec<usize> {
let mut preferred = HashSet::<SocketAddr>::new(); let preferred = self.preferred_endpoints_for_dc(routed_dc).await;
for family in self.family_order() {
let map_guard = match family {
IpFamily::V4 => self.proxy_map_v4.read().await,
IpFamily::V6 => self.proxy_map_v6.read().await,
};
let mut family_selected = Vec::<SocketAddr>::new();
if let Some(v) = map_guard.get(&routed_dc) {
family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port)));
}
for endpoint in family_selected {
preferred.insert(endpoint);
}
drop(map_guard);
if !preferred.is_empty() && !self.decision.effective_multipath {
break;
}
}
if preferred.is_empty() { if preferred.is_empty() {
return Vec::new(); return Vec::new();
} }
@@ -622,7 +610,7 @@ impl MePool {
if !self.writer_eligible_for_selection(w, include_warm) { if !self.writer_eligible_for_selection(w, include_warm) {
continue; continue;
} }
if w.writer_dc == routed_dc && preferred.contains(&w.addr) { if w.writer_dc == routed_dc && preferred.iter().any(|endpoint| *endpoint == w.addr) {
out.push(idx); out.push(idx);
} }
} }
@@ -671,4 +659,87 @@ impl MePool {
0 0
} }
} }
fn writer_pick_score(
&self,
writer: &super::pool::MeWriter,
idle_since_by_writer: &HashMap<u64, u64>,
now_epoch_secs: u64,
) -> u64 {
let contour_penalty = match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) {
WriterContour::Active => 0,
WriterContour::Warm => PICK_PENALTY_WARM,
WriterContour::Draining => PICK_PENALTY_DRAINING,
};
let stale_penalty = if writer.generation < self.current_generation() {
PICK_PENALTY_STALE
} else {
0
};
let degraded_penalty = if writer.degraded.load(Ordering::Relaxed) {
PICK_PENALTY_DEGRADED
} else {
0
};
let idle_penalty =
(self.writer_idle_rank_for_selection(writer, idle_since_by_writer, now_epoch_secs) as u64)
* 100;
let queue_cap = self.writer_cmd_channel_capacity.max(1) as u64;
let queue_remaining = writer.tx.capacity() as u64;
let queue_used = queue_cap.saturating_sub(queue_remaining.min(queue_cap));
let queue_util_pct = queue_used.saturating_mul(100) / queue_cap;
let queue_penalty = queue_util_pct.saturating_mul(4);
let rtt_penalty = ((writer.rtt_ema_ms_x10.load(Ordering::Relaxed) as u64).saturating_add(5) / 10)
.min(400);
contour_penalty
.saturating_add(stale_penalty)
.saturating_add(degraded_penalty)
.saturating_add(idle_penalty)
.saturating_add(queue_penalty)
.saturating_add(rtt_penalty)
}
fn p2c_ordered_candidate_indices(
&self,
candidate_indices: &[usize],
writers_snapshot: &[super::pool::MeWriter],
idle_since_by_writer: &HashMap<u64, u64>,
now_epoch_secs: u64,
start: usize,
sample_size: usize,
) -> Vec<usize> {
let total = candidate_indices.len();
if total == 0 {
return Vec::new();
}
let mut sampled = Vec::<usize>::with_capacity(sample_size.min(total));
let mut seen = HashSet::<usize>::with_capacity(total);
for offset in 0..sample_size.min(total) {
let idx = candidate_indices[(start + offset) % total];
if seen.insert(idx) {
sampled.push(idx);
}
}
sampled.sort_by_key(|idx| {
let writer = &writers_snapshot[*idx];
(
self.writer_pick_score(writer, idle_since_by_writer, now_epoch_secs),
writer.addr,
writer.id,
)
});
let mut ordered = Vec::<usize>::with_capacity(total);
ordered.extend(sampled.iter().copied());
for offset in 0..total {
let idx = candidate_indices[(start + offset) % total];
if seen.insert(idx) {
ordered.push(idx);
}
}
ordered
}
} }

View File

@@ -1,13 +1,16 @@
[Unit] [Unit]
Description=Telemt Description=Telemt
After=network.target After=network-online.target
Wants=network-online.target
[Service] [Service]
Type=simple Type=simple
WorkingDirectory=/bin WorkingDirectory=/etc/telemt
ExecStart=/bin/telemt /etc/telemt.toml ExecStart=/bin/telemt /etc/telemt.toml
Restart=on-failure Restart=on-failure
LimitNOFILE=65536 LimitNOFILE=262144
TasksMax=8192
MemoryAccounting=yes
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target