Compare commits

..

20 Commits
3.4.1 ... flow

Author SHA1 Message Date
Alexey
f1bf95a7de Merge pull request #718 from astronaut808/fix/me-downstream-retry
Improve ME downstream retries for queued fairness backlog
2026-04-18 14:03:37 +03:00
Alexey
959a16af88 Merge pull request #716 from zarv1k/feature/configurable-proxy-confi-urls
feat: make URLS to obtain proxy_secret, getProxyConfig, getProxyConfgV6 files optionally configurable
2026-04-18 11:17:37 +03:00
Alexey
a54f9ba719 Merge branch 'flow' into feature/configurable-proxy-confi-urls 2026-04-18 11:16:38 +03:00
astronaut808
2d5cd9c8e1 Improve ME downstream retries for queued fairness backlog 2026-04-18 02:40:32 +05:00
Alexey
37b6f7b985 Weighted Fairness + 3-Leveled Pressure Model
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-18 00:37:04 +03:00
Alexey
50e9e5cf32 Active Ring and DRR Hardening
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-18 00:34:35 +03:00
Alexey
d72cfd6bc4 Merge branch 'flow' into feature/configurable-proxy-confi-urls 2026-04-17 19:44:46 +03:00
Alexey
1b25bada29 ServerHello fixes + Docker Health-Check + Conntrack Control for Docker: merge pull request #717 from telemt/flow
ServerHello fixes + Docker Health-Check + Conntrack Control for Docker
2026-04-17 19:43:59 +03:00
Dmitry Zarva
fa3566a9cb - fix: fmt issues 2026-04-17 16:20:16 +00:00
Alexey
bde30eaf05 Update emulator.rs 2026-04-17 19:20:06 +03:00
Alexey
b447f60a72 Rustfmt + Bump 2026-04-17 19:08:57 +03:00
Alexey
093faed0c2 Conntrack Control for Docker 2026-04-17 19:06:18 +03:00
Dmitry Zarva
4e59e52454 - fix: ru docs 2026-04-17 14:10:20 +00:00
Alexey
3ca3e8ff0e Docker Health-Check 2026-04-17 16:36:15 +03:00
Dmitry Zarva
7b9b46291d - fix: param name in ru docs 2026-04-17 13:19:29 +00:00
Dmitry Zarva
2a168b2600 feat: make URLS to obtain proxy_secret, getProxyConfig, getProxyConfigV6 files optionally configurable 2026-04-17 13:04:46 +00:00
Alexey
6e3b4a1ce5 ServerHello fixes 2026-04-17 15:11:36 +03:00
Alexey
cd0771eee4 Merge pull request #715 from telemt/flow
Fixes in TLS-F
2026-04-17 13:00:30 +03:00
Alexey
a858dd799e Bump 2026-04-17 12:43:41 +03:00
Alexey
947ef2beb7 Fixes in TLS-F 2026-04-17 12:38:22 +03:00
28 changed files with 1239 additions and 260 deletions

2
Cargo.lock generated
View File

@@ -2780,7 +2780,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "telemt"
version = "3.4.1"
version = "3.4.3"
dependencies = [
"aes",
"anyhow",

View File

@@ -1,6 +1,6 @@
[package]
name = "telemt"
version = "3.4.1"
version = "3.4.3"
edition = "2024"
[features]

View File

@@ -77,6 +77,34 @@ COPY config.toml /app/config.toml
EXPOSE 443 9090 9091
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
ENTRYPOINT ["/app/telemt"]
CMD ["config.toml"]
# ==========================
# Production Netfilter Profile
# ==========================
FROM debian:12-slim AS prod-netfilter
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends \
ca-certificates \
conntrack \
nftables \
iptables; \
rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=minimal /telemt /app/telemt
COPY config.toml /app/config.toml
EXPOSE 443 9090 9091
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
ENTRYPOINT ["/app/telemt"]
CMD ["config.toml"]
@@ -94,5 +122,7 @@ USER nonroot:nonroot
EXPOSE 443 9090 9091
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
ENTRYPOINT ["/app/telemt"]
CMD ["config.toml"]

View File

@@ -0,0 +1,10 @@
services:
telemt:
build:
context: .
target: prod-netfilter
network_mode: host
ports: []
cap_add:
- NET_BIND_SERVICE
- NET_ADMIN

View File

@@ -0,0 +1,8 @@
services:
telemt:
build:
context: .
target: prod-netfilter
cap_add:
- NET_BIND_SERVICE
- NET_ADMIN

View File

@@ -1,7 +1,9 @@
services:
telemt:
image: ghcr.io/telemt/telemt:latest
build: .
build:
context: .
target: prod
container_name: telemt
restart: unless-stopped
ports:
@@ -16,13 +18,18 @@ services:
- /etc/telemt:rw,mode=1777,size=4m
environment:
- RUST_LOG=info
healthcheck:
test: [ "CMD", "/app/telemt", "healthcheck", "/etc/telemt/config.toml", "--mode", "liveness" ]
interval: 30s
timeout: 5s
retries: 3
start_period: 20s
# Uncomment this line if you want to use host network for IPv6, but bridge is default and usually better
# network_mode: host
cap_drop:
- ALL
cap_add:
- NET_BIND_SERVICE
- NET_ADMIN
read_only: true
security_opt:
- no-new-privileges:true

View File

@@ -255,13 +255,22 @@ This document lists all configuration keys accepted by `config.toml`.
```
## proxy_secret_path
- **Constraints / validation**: `String`. When omitted, the default path is `"proxy-secret"`. Empty values are accepted by TOML/serde but will likely fail at runtime (invalid file path).
- **Description**: Path to Telegram infrastructure `proxy-secret` cache file used by ME handshake/RPC auth. Telemt always tries a fresh download from `https://core.telegram.org/getProxySecret` first, caches it to this path on success, and falls back to reading the cached file (any age) on download failure.
- **Description**: Path to Telegram infrastructure `proxy-secret` cache file used by ME handshake/RPC auth. Telemt always tries a fresh download from `https://core.telegram.org/getProxySecret` first (unless `proxy_secret_url` is set) , caches it to this path on success, and falls back to reading the cached file (any age) on download failure.
- **Example**:
```toml
[general]
proxy_secret_path = "proxy-secret"
```
## proxy_secret_url
- **Constraints / validation**: `String`. When omitted, the `"https://core.telegram.org/getProxySecret"` is used.
- **Description**: Optional URL to obtain `proxy-secret` file used by ME handshake/RPC auth. Telemt always tries a fresh download from this URL first (with fallback to `https://core.telegram.org/getProxySecret` if absent).
- **Example**:
```toml
[general]
proxy_secret_url = "https://core.telegram.org/getProxySecret"
```
## proxy_config_v4_cache_path
- **Constraints / validation**: `String`. When set, must not be empty/whitespace-only.
- **Description**: Optional disk cache path for raw `getProxyConfig` (IPv4) snapshot. At startup Telemt tries to fetch a fresh snapshot first; on fetch failure or empty snapshot it falls back to this cache file when present and non-empty.
@@ -271,6 +280,15 @@ This document lists all configuration keys accepted by `config.toml`.
[general]
proxy_config_v4_cache_path = "cache/proxy-config-v4.txt"
```
## proxy_config_v4_url
- **Constraints / validation**: `String`. When omitted, the `"https://core.telegram.org/getProxyConfig"` is used.
- **Description**: Optional URL to obtain raw `getProxyConfig` (IPv4). Telemt always tries a fresh download from this URL first (with fallback to `https://core.telegram.org/getProxyConfig` if absent).
- **Example**:
```toml
[general]
proxy_config_v4_url = "https://core.telegram.org/getProxyConfig"
```
## proxy_config_v6_cache_path
- **Constraints / validation**: `String`. When set, must not be empty/whitespace-only.
- **Description**: Optional disk cache path for raw `getProxyConfigV6` (IPv6) snapshot. At startup Telemt tries to fetch a fresh snapshot first; on fetch failure or empty snapshot it falls back to this cache file when present and non-empty.
@@ -280,6 +298,15 @@ This document lists all configuration keys accepted by `config.toml`.
[general]
proxy_config_v6_cache_path = "cache/proxy-config-v6.txt"
```
## proxy_config_v6_url
- **Constraints / validation**: `String`. When omitted, the `"https://core.telegram.org/getProxyConfigV6"` is used.
- **Description**: Optional URL to obtain raw `getProxyConfigV6` (IPv6). Telemt always tries a fresh download from this URL first (with fallback to `https://core.telegram.org/getProxyConfigV6` if absent).
- **Example**:
```toml
[general]
proxy_config_v6_url = "https://core.telegram.org/getProxyConfigV6"
```
## ad_tag
- **Constraints / validation**: `String` (optional). When set, must be exactly 32 hex characters; invalid values are disabled during config load.
- **Description**: Global fallback sponsored-channel `ad_tag` (used when user has no override in `access.user_ad_tags`). An all-zero tag is accepted but has no effect (and is warned about) until replaced with a real tag from `@MTProxybot`.

View File

@@ -255,13 +255,22 @@
```
## proxy_secret_path
- **Ограничения / валидация**: `String`. Если этот параметр не указан, используется путь по умолчанию — «proxy-secret». Пустые значения принимаются TOML/serde, но во время выполнения произойдет ошибка (invalid file path).
- **Описание**: Путь к файлу кэша `proxy-secret` инфраструктуры Telegram, используемому ME-handshake/аутентификацией RPC. Telemt всегда сначала пытается выполнить новую загрузку с https://core.telegram.org/getProxySecret, в случае успеха кэширует ее по этому пути и возвращается к чтению кэшированного файла в случае сбоя загрузки.
- **Описание**: Путь к файлу кэша `proxy-secret` инфраструктуры Telegram, используемому ME-handshake/аутентификацией RPC. Telemt всегда сначала пытается выполнить новую загрузку с https://core.telegram.org/getProxySecret (если не установлен `proxy_secret_url`), в случае успеха кэширует ее по этому пути и возвращается к чтению кэшированного файла в случае сбоя загрузки.
- **Пример**:
```toml
[general]
proxy_secret_path = "proxy-secret"
```
## proxy_secret_url
- **Ограничения / валидация**: `String`. Если не указан, используется `"https://core.telegram.org/getProxySecret"`.
- **Описание**: Необязательный URL для получения файла `proxy-secret` используемого ME-handshake/аутентификацией RPC. Telemt всегда сначала пытается выполнить новую загрузку с этого URL (если не задан, используется https://core.telegram.org/getProxySecret).
- **Пример**:
```toml
[general]
proxy_secret_url = "https://core.telegram.org/getProxySecret"
```
## proxy_config_v4_cache_path
- **Ограничения / валидация**: `String`. Если используется, значение не должно быть пустым или содержать только пробелы.
- **Описание**: Необязательный путь к кэшу для необработанного (raw) снимка getProxyConfig (IPv4). При запуске Telemt сначала пытается получить свежий снимок; в случае сбоя выборки или пустого снимка он возвращается к этому файлу кэша, если он присутствует и не пуст.
@@ -271,6 +280,15 @@
[general]
proxy_config_v4_cache_path = "cache/proxy-config-v4.txt"
```
## proxy_config_v4_url
- **Ограничения / валидация**: `String`. Если не указан, используется `"https://core.telegram.org/getProxyConfig"`.
- **Описание**: Необязательный URL для получения `getProxyConfig` (IPv4). Telemt при всегда пытается выполнить новую загрузку с этого URL (и если не задан, использует `https://core.telegram.org/getProxyConfig`).
- **Example**:
```toml
[general]
proxy_config_v4_url = "https://core.telegram.org/getProxyConfig"
```
## proxy_config_v6_cache_path
- **Ограничения / валидация**: `String`. Если используется, значение не должно быть пустым или содержать только пробелы.
- **Описание**: Необязательный путь к кэшу для необработанного (raw) снимка getProxyConfigV6 (IPv6). При запуске Telemt сначала пытается получить свежий снимок; в случае сбоя выборки или пустого снимка он возвращается к этому файлу кэша, если он присутствует и не пуст.
@@ -280,6 +298,15 @@
[general]
proxy_config_v6_cache_path = "cache/proxy-config-v6.txt"
```
## proxy_config_v6_url
- **Ограничения / валидация**: `String`. Если не указан, используется `"https://core.telegram.org/getProxyConfigV6"`.
- **Описание**: Необязательный URL для получения `getProxyConfigV6` (IPv6). Telemt при всегда пытается выполнить новую загрузку с этого URL (и если не задан, использует `https://core.telegram.org/getProxyConfigV6`).
- **Example**:
```toml
[general]
proxy_config_v6_url = "https://core.telegram.org/getProxyConfigV6"
```
## ad_tag
- **Ограничения / валидация**: `String` (необязательный параметр). Если используется, значение должно быть ровно 32 символа в шестнадцатеричной системе; недопустимые значения отключаются во время загрузки конфигурации.
- **Описание**: Глобальный резервный спонсируемый канал `ad_tag` (используется, когда у пользователя нет переопределения в `access.user_ad_tags`). Тег со всеми нулями принимается, но не имеет никакого эффекта, пока не будет заменен реальным тегом от `@MTProxybot`.

View File

@@ -41,8 +41,8 @@ use config_store::{current_revision, load_config_from_disk, parse_if_match};
use events::ApiEventStore;
use http_utils::{error_response, read_json, read_optional_json, success_response};
use model::{
ApiFailure, CreateUserRequest, DeleteUserResponse, HealthData, PatchUserRequest,
RotateSecretRequest, SummaryData, UserActiveIps,
ApiFailure, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
PatchUserRequest, RotateSecretRequest, SummaryData, UserActiveIps,
};
use runtime_edge::{
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
@@ -275,6 +275,33 @@ async fn handle(
};
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/health/ready") => {
let revision = current_revision(&shared.config_path).await?;
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
let upstream_health = shared.upstream_manager.api_health_summary().await;
let ready = admission_open && upstream_health.healthy_total > 0;
let reason = if ready {
None
} else if !admission_open {
Some("admission_closed")
} else {
Some("no_healthy_upstreams")
};
let data = HealthReadyData {
ready,
status: if ready { "ready" } else { "not_ready" },
reason,
admission_open,
healthy_upstreams: upstream_health.healthy_total,
total_upstreams: upstream_health.configured_total,
};
let status_code = if ready {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
};
Ok(success_response(status_code, data, revision))
}
("GET", "/v1/system/info") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);

View File

@@ -60,6 +60,17 @@ pub(super) struct HealthData {
pub(super) read_only: bool,
}
#[derive(Serialize)]
pub(super) struct HealthReadyData {
pub(super) ready: bool,
pub(super) status: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) reason: Option<&'static str>,
pub(super) admission_open: bool,
pub(super) healthy_upstreams: usize,
pub(super) total_upstreams: usize,
}
#[derive(Serialize)]
pub(super) struct SummaryData {
pub(super) uptime_seconds: f64,

View File

@@ -6,12 +6,15 @@
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
//! - `status [--pid-file PATH]` - Check daemon status
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
use rand::RngExt;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use crate::healthcheck::{self, HealthcheckMode};
#[cfg(unix)]
use crate::daemon::{self, DEFAULT_PID_FILE, DaemonOptions};
@@ -28,6 +31,8 @@ pub enum Subcommand {
Reload,
/// Check daemon status (`status` subcommand).
Status,
/// Run health probe and exit with status code.
Healthcheck,
/// Fire-and-forget setup (`--init`).
Init,
}
@@ -38,6 +43,8 @@ pub struct ParsedCommand {
pub subcommand: Subcommand,
pub pid_file: PathBuf,
pub config_path: String,
pub healthcheck_mode: HealthcheckMode,
pub healthcheck_mode_invalid: Option<String>,
#[cfg(unix)]
pub daemon_opts: DaemonOptions,
pub init_opts: Option<InitOptions>,
@@ -52,6 +59,8 @@ impl Default for ParsedCommand {
#[cfg(not(unix))]
pid_file: PathBuf::from("/var/run/telemt.pid"),
config_path: "config.toml".to_string(),
healthcheck_mode: HealthcheckMode::Liveness,
healthcheck_mode_invalid: None,
#[cfg(unix)]
daemon_opts: DaemonOptions::default(),
init_opts: None,
@@ -91,6 +100,9 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
"status" => {
cmd.subcommand = Subcommand::Status;
}
"healthcheck" => {
cmd.subcommand = Subcommand::Healthcheck;
}
"run" => {
cmd.subcommand = Subcommand::Run;
#[cfg(unix)]
@@ -113,7 +125,35 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
while i < args.len() {
match args[i].as_str() {
// Skip subcommand names
"start" | "stop" | "reload" | "status" | "run" => {}
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
"--mode" => {
i += 1;
if i < args.len() {
match HealthcheckMode::from_cli_arg(&args[i]) {
Some(mode) => {
cmd.healthcheck_mode = mode;
cmd.healthcheck_mode_invalid = None;
}
None => {
cmd.healthcheck_mode_invalid = Some(args[i].clone());
}
}
} else {
cmd.healthcheck_mode_invalid = Some(String::new());
}
}
s if s.starts_with("--mode=") => {
let raw = s.trim_start_matches("--mode=");
match HealthcheckMode::from_cli_arg(raw) {
Some(mode) => {
cmd.healthcheck_mode = mode;
cmd.healthcheck_mode_invalid = None;
}
None => {
cmd.healthcheck_mode_invalid = Some(raw.to_string());
}
}
}
// PID file option (for stop/reload/status)
"--pid-file" => {
i += 1;
@@ -152,6 +192,20 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
Subcommand::Stop => Some(cmd_stop(&cmd.pid_file)),
Subcommand::Reload => Some(cmd_reload(&cmd.pid_file)),
Subcommand::Status => Some(cmd_status(&cmd.pid_file)),
Subcommand::Healthcheck => {
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
if invalid_mode.is_empty() {
eprintln!("[telemt] Missing value for --mode (supported: liveness, ready)");
} else {
eprintln!(
"[telemt] Invalid --mode value '{invalid_mode}' (supported: liveness, ready)"
);
}
Some(2)
} else {
Some(healthcheck::run(&cmd.config_path, cmd.healthcheck_mode))
}
}
Subcommand::Init => {
if let Some(opts) = cmd.init_opts.clone() {
match run_init(opts) {
@@ -177,6 +231,20 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
eprintln!("[telemt] Subcommand not supported on this platform");
Some(1)
}
Subcommand::Healthcheck => {
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
if invalid_mode.is_empty() {
eprintln!("[telemt] Missing value for --mode (supported: liveness, ready)");
} else {
eprintln!(
"[telemt] Invalid --mode value '{invalid_mode}' (supported: liveness, ready)"
);
}
Some(2)
} else {
Some(healthcheck::run(&cmd.config_path, cmd.healthcheck_mode))
}
}
Subcommand::Init => {
if let Some(opts) = cmd.init_opts.clone() {
match run_init(opts) {

View File

@@ -343,6 +343,10 @@ impl ProxyConfig {
let network_table = parsed_toml
.get("network")
.and_then(|value| value.as_table());
let server_table = parsed_toml.get("server").and_then(|value| value.as_table());
let conntrack_control_table = server_table
.and_then(|table| table.get("conntrack_control"))
.and_then(|value| value.as_table());
let update_every_is_explicit = general_table
.map(|table| table.contains_key("update_every"))
.unwrap_or(false);
@@ -372,10 +376,17 @@ impl ProxyConfig {
let stun_servers_is_explicit = network_table
.map(|table| table.contains_key("stun_servers"))
.unwrap_or(false);
let inline_conntrack_control_is_explicit = conntrack_control_table
.map(|table| table.contains_key("inline_conntrack_control"))
.unwrap_or(false);
let mut config: ProxyConfig = parsed_toml
.try_into()
.map_err(|e| ProxyError::Config(e.to_string()))?;
config
.server
.conntrack_control
.inline_conntrack_control_explicit = inline_conntrack_control_is_explicit;
if !update_every_is_explicit && (legacy_secret_is_explicit || legacy_config_is_explicit) {
config.general.update_every = None;
@@ -1881,6 +1892,43 @@ mod tests {
);
}
#[test]
fn conntrack_inline_explicit_flag_is_false_when_omitted() {
let cfg = load_config_from_temp_toml(
r#"
[general]
[network]
[server]
[server.conntrack_control]
[access]
"#,
);
assert!(
!cfg.server
.conntrack_control
.inline_conntrack_control_explicit
);
}
#[test]
fn conntrack_inline_explicit_flag_is_true_when_present() {
let cfg = load_config_from_temp_toml(
r#"
[general]
[network]
[server]
[server.conntrack_control]
inline_conntrack_control = true
[access]
"#,
);
assert!(
cfg.server
.conntrack_control
.inline_conntrack_control_explicit
);
}
#[test]
fn unknown_sni_action_parses_and_defaults_to_drop() {
let cfg_default: ProxyConfig = toml::from_str(

View File

@@ -392,14 +392,26 @@ pub struct GeneralConfig {
#[serde(default = "default_proxy_secret_path")]
pub proxy_secret_path: Option<String>,
/// Optional custom URL for infrastructure secret (https://core.telegram.org/getProxySecret if absent).
#[serde(default)]
pub proxy_secret_url: Option<String>,
/// Optional path to cache raw getProxyConfig (IPv4) snapshot for startup fallback.
#[serde(default = "default_proxy_config_v4_cache_path")]
pub proxy_config_v4_cache_path: Option<String>,
/// Optional custom URL for getProxyConfig (https://core.telegram.org/getProxyConfig if absent).
#[serde(default)]
pub proxy_config_v4_url: Option<String>,
/// Optional path to cache raw getProxyConfigV6 snapshot for startup fallback.
#[serde(default = "default_proxy_config_v6_cache_path")]
pub proxy_config_v6_cache_path: Option<String>,
/// Optional custom URL for getProxyConfigV6 (https://core.telegram.org/getProxyConfigV6 if absent).
#[serde(default)]
pub proxy_config_v6_url: Option<String>,
/// Global ad_tag (32 hex chars from @MTProxybot). Fallback when user has no per-user tag in access.user_ad_tags.
#[serde(default)]
pub ad_tag: Option<String>,
@@ -960,8 +972,11 @@ impl Default for GeneralConfig {
use_middle_proxy: default_true(),
ad_tag: None,
proxy_secret_path: default_proxy_secret_path(),
proxy_secret_url: None,
proxy_config_v4_cache_path: default_proxy_config_v4_cache_path(),
proxy_config_v4_url: None,
proxy_config_v6_cache_path: default_proxy_config_v6_cache_path(),
proxy_config_v6_url: None,
middle_proxy_nat_ip: None,
middle_proxy_nat_probe: default_true(),
middle_proxy_nat_stun: default_middle_proxy_nat_stun(),
@@ -1329,6 +1344,10 @@ pub struct ConntrackControlConfig {
#[serde(default = "default_conntrack_control_enabled")]
pub inline_conntrack_control: bool,
/// Tracks whether inline_conntrack_control was explicitly set in config.
#[serde(skip)]
pub inline_conntrack_control_explicit: bool,
/// Conntrack mode for listener ingress traffic.
#[serde(default)]
pub mode: ConntrackMode,
@@ -1363,6 +1382,7 @@ impl Default for ConntrackControlConfig {
fn default() -> Self {
Self {
inline_conntrack_control: default_conntrack_control_enabled(),
inline_conntrack_control_explicit: false,
mode: ConntrackMode::default(),
backend: ConntrackBackend::default(),
profile: ConntrackPressureProfile::default(),

View File

@@ -24,6 +24,13 @@ enum NetfilterBackend {
Iptables,
}
#[derive(Clone, Copy)]
struct ConntrackRuntimeSupport {
netfilter_backend: Option<NetfilterBackend>,
has_cap_net_admin: bool,
has_conntrack_binary: bool,
}
#[derive(Clone, Copy)]
struct PressureSample {
conn_pct: Option<u8>,
@@ -56,11 +63,8 @@ pub(crate) fn spawn_conntrack_controller(
shared: Arc<ProxySharedState>,
) {
if !cfg!(target_os = "linux") {
let enabled = config_rx
.borrow()
.server
.conntrack_control
.inline_conntrack_control;
let cfg = config_rx.borrow();
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
stats.set_conntrack_control_enabled(enabled);
stats.set_conntrack_control_available(false);
stats.set_conntrack_pressure_active(false);
@@ -68,9 +72,14 @@ pub(crate) fn spawn_conntrack_controller(
stats.set_conntrack_rule_apply_ok(false);
shared.disable_conntrack_close_sender();
shared.set_conntrack_pressure_active(false);
if enabled {
if enabled
&& cfg
.server
.conntrack_control
.inline_conntrack_control_explicit
{
warn!(
"conntrack control is configured but unsupported on this OS; disabling runtime worker"
"conntrack control explicitly enabled but unsupported on this OS; disabling runtime worker"
);
}
return;
@@ -92,16 +101,17 @@ async fn run_conntrack_controller(
let mut cfg = config_rx.borrow().clone();
let mut pressure_state = PressureState::new(stats.as_ref());
let mut delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
let mut backend = pick_backend(cfg.server.conntrack_control.backend);
let mut runtime_support = probe_runtime_support(cfg.server.conntrack_control.backend);
let mut effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
apply_runtime_state(
stats.as_ref(),
shared.as_ref(),
&cfg,
backend.is_some(),
runtime_support,
false,
);
reconcile_rules(&cfg, backend, stats.as_ref()).await;
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
loop {
tokio::select! {
@@ -110,17 +120,18 @@ async fn run_conntrack_controller(
break;
}
cfg = config_rx.borrow_and_update().clone();
backend = pick_backend(cfg.server.conntrack_control.backend);
runtime_support = probe_runtime_support(cfg.server.conntrack_control.backend);
effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, backend.is_some(), pressure_state.active);
reconcile_rules(&cfg, backend, stats.as_ref()).await;
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, runtime_support, pressure_state.active);
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
}
event = close_rx.recv() => {
let Some(event) = event else {
break;
};
stats.set_conntrack_event_queue_depth(close_rx.len() as u64);
if !cfg.server.conntrack_control.inline_conntrack_control {
if !effective_enabled {
continue;
}
if !pressure_state.active {
@@ -156,6 +167,7 @@ async fn run_conntrack_controller(
stats.as_ref(),
shared.as_ref(),
&cfg,
effective_enabled,
&sample,
&mut pressure_state,
);
@@ -175,20 +187,30 @@ fn apply_runtime_state(
stats: &Stats,
shared: &ProxySharedState,
cfg: &ProxyConfig,
backend_available: bool,
runtime_support: ConntrackRuntimeSupport,
pressure_active: bool,
) {
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
let available = enabled && backend_available && has_cap_net_admin();
if enabled && !available {
let available = effective_conntrack_enabled(cfg, runtime_support);
if enabled
&& !available
&& cfg
.server
.conntrack_control
.inline_conntrack_control_explicit
{
warn!(
"conntrack control enabled but unavailable (missing CAP_NET_ADMIN or backend binaries)"
has_cap_net_admin = runtime_support.has_cap_net_admin,
backend_available = runtime_support.netfilter_backend.is_some(),
conntrack_binary_available = runtime_support.has_conntrack_binary,
configured_backend = ?cfg.server.conntrack_control.backend,
"conntrack control explicitly enabled but unavailable; disabling runtime features"
);
}
stats.set_conntrack_control_enabled(enabled);
stats.set_conntrack_control_available(available);
shared.set_conntrack_pressure_active(enabled && pressure_active);
stats.set_conntrack_pressure_active(enabled && pressure_active);
shared.set_conntrack_pressure_active(available && pressure_active);
stats.set_conntrack_pressure_active(available && pressure_active);
}
fn collect_pressure_sample(
@@ -228,10 +250,11 @@ fn update_pressure_state(
stats: &Stats,
shared: &ProxySharedState,
cfg: &ProxyConfig,
effective_enabled: bool,
sample: &PressureSample,
state: &mut PressureState,
) {
if !cfg.server.conntrack_control.inline_conntrack_control {
if !effective_enabled {
if state.active {
state.active = false;
state.low_streak = 0;
@@ -285,22 +308,26 @@ fn update_pressure_state(
state.low_streak = 0;
}
async fn reconcile_rules(cfg: &ProxyConfig, backend: Option<NetfilterBackend>, stats: &Stats) {
async fn reconcile_rules(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
stats: &Stats,
) {
if !cfg.server.conntrack_control.inline_conntrack_control {
clear_notrack_rules_all_backends().await;
stats.set_conntrack_rule_apply_ok(true);
return;
}
if !has_cap_net_admin() {
if !effective_conntrack_enabled(cfg, runtime_support) {
clear_notrack_rules_all_backends().await;
stats.set_conntrack_rule_apply_ok(false);
return;
}
let Some(backend) = backend else {
stats.set_conntrack_rule_apply_ok(false);
return;
};
let backend = runtime_support
.netfilter_backend
.expect("netfilter backend must be available for effective conntrack control");
let apply_result = match backend {
NetfilterBackend::Nftables => apply_nft_rules(cfg).await,
@@ -315,6 +342,24 @@ async fn reconcile_rules(cfg: &ProxyConfig, backend: Option<NetfilterBackend>, s
}
}
fn probe_runtime_support(configured_backend: ConntrackBackend) -> ConntrackRuntimeSupport {
ConntrackRuntimeSupport {
netfilter_backend: pick_backend(configured_backend),
has_cap_net_admin: has_cap_net_admin(),
has_conntrack_binary: command_exists("conntrack"),
}
}
fn effective_conntrack_enabled(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
) -> bool {
cfg.server.conntrack_control.inline_conntrack_control
&& runtime_support.has_cap_net_admin
&& runtime_support.netfilter_backend.is_some()
&& runtime_support.has_conntrack_binary
}
fn pick_backend(configured: ConntrackBackend) -> Option<NetfilterBackend> {
match configured {
ConntrackBackend::Auto => {
@@ -710,7 +755,7 @@ mod tests {
me_queue_pressure_delta: 0,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &sample, &mut state);
assert!(state.active);
assert!(shared.conntrack_pressure_active());
@@ -731,7 +776,14 @@ mod tests {
accept_timeout_delta: 0,
me_queue_pressure_delta: 0,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &high_sample, &mut state);
update_pressure_state(
&stats,
shared.as_ref(),
&cfg,
true,
&high_sample,
&mut state,
);
assert!(state.active);
let low_sample = PressureSample {
@@ -740,11 +792,11 @@ mod tests {
accept_timeout_delta: 0,
me_queue_pressure_delta: 0,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
assert!(state.active);
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
assert!(state.active);
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
assert!(!state.active);
assert!(!shared.conntrack_pressure_active());
@@ -765,7 +817,7 @@ mod tests {
me_queue_pressure_delta: 10,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, false, &sample, &mut state);
assert!(!state.active);
assert!(!shared.conntrack_pressure_active());

211
src/healthcheck.rs Normal file
View File

@@ -0,0 +1,211 @@
use std::io::{Read, Write};
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream};
use std::time::Duration;
use serde_json::Value;
use crate::config::ProxyConfig;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HealthcheckMode {
Liveness,
Ready,
}
impl HealthcheckMode {
pub(crate) fn from_cli_arg(value: &str) -> Option<Self> {
match value {
"liveness" => Some(Self::Liveness),
"ready" => Some(Self::Ready),
_ => None,
}
}
fn request_path(self) -> &'static str {
match self {
Self::Liveness => "/v1/health",
Self::Ready => "/v1/health/ready",
}
}
}
pub(crate) fn run(config_path: &str, mode: HealthcheckMode) -> i32 {
match run_inner(config_path, mode) {
Ok(()) => 0,
Err(error) => {
eprintln!("[telemt] healthcheck failed: {error}");
1
}
}
}
fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
let config =
ProxyConfig::load(config_path).map_err(|error| format!("config load failed: {error}"))?;
let api_cfg = &config.server.api;
if !api_cfg.enabled {
return Ok(());
}
let listen: SocketAddr = api_cfg
.listen
.parse()
.map_err(|_| format!("invalid API listen address: {}", api_cfg.listen))?;
if listen.port() == 0 {
return Err("API listen port is 0".to_string());
}
let target = probe_target(listen);
let mut stream = TcpStream::connect_timeout(&target, Duration::from_secs(2))
.map_err(|error| format!("connect {target} failed: {error}"))?;
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.map_err(|error| format!("set read timeout failed: {error}"))?;
stream
.set_write_timeout(Some(Duration::from_secs(2)))
.map_err(|error| format!("set write timeout failed: {error}"))?;
let request = build_request(target, mode.request_path(), &api_cfg.auth_header);
stream
.write_all(request.as_bytes())
.map_err(|error| format!("request write failed: {error}"))?;
stream
.flush()
.map_err(|error| format!("request flush failed: {error}"))?;
let mut raw_response = Vec::new();
stream
.read_to_end(&mut raw_response)
.map_err(|error| format!("response read failed: {error}"))?;
let response =
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
let (status_code, body) = split_response(&response)?;
if status_code != 200 {
return Err(format!("HTTP status {status_code}"));
}
validate_payload(mode, body)?;
Ok(())
}
fn probe_target(listen: SocketAddr) -> SocketAddr {
match listen {
SocketAddr::V4(addr) => {
let ip = if addr.ip().is_unspecified() {
Ipv4Addr::LOCALHOST
} else {
*addr.ip()
};
SocketAddr::from((ip, addr.port()))
}
SocketAddr::V6(addr) => {
let ip = if addr.ip().is_unspecified() {
Ipv6Addr::LOCALHOST
} else {
*addr.ip()
};
SocketAddr::from((ip, addr.port()))
}
}
}
fn build_request(target: SocketAddr, path: &str, auth_header: &str) -> String {
let mut request = format!(
"GET {path} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\n",
target
);
if !auth_header.is_empty() {
request.push_str("Authorization: ");
request.push_str(auth_header);
request.push_str("\r\n");
}
request.push_str("\r\n");
request
}
fn split_response(response: &str) -> Result<(u16, &str), String> {
let header_end = response
.find("\r\n\r\n")
.ok_or_else(|| "invalid HTTP response headers".to_string())?;
let header = &response[..header_end];
let body = &response[header_end + 4..];
let status_line = header
.lines()
.next()
.ok_or_else(|| "missing HTTP status line".to_string())?;
let status_code = parse_status_code(status_line)?;
Ok((status_code, body))
}
fn parse_status_code(status_line: &str) -> Result<u16, String> {
let mut parts = status_line.split_whitespace();
let version = parts
.next()
.ok_or_else(|| "missing HTTP version".to_string())?;
if !version.starts_with("HTTP/") {
return Err(format!("invalid HTTP status line: {status_line}"));
}
let code = parts
.next()
.ok_or_else(|| "missing HTTP status code".to_string())?;
code.parse::<u16>()
.map_err(|_| format!("invalid HTTP status code: {code}"))
}
fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
let payload: Value =
serde_json::from_str(body).map_err(|_| "response body is not valid JSON".to_string())?;
if payload.get("ok").and_then(Value::as_bool) != Some(true) {
return Err("response JSON has ok=false".to_string());
}
let data = payload
.get("data")
.ok_or_else(|| "response JSON has no data field".to_string())?;
match mode {
HealthcheckMode::Liveness => {
if data.get("status").and_then(Value::as_str) != Some("ok") {
return Err("liveness status is not ok".to_string());
}
}
HealthcheckMode::Ready => {
if data.get("ready").and_then(Value::as_bool) != Some(true) {
return Err("readiness flag is false".to_string());
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{HealthcheckMode, parse_status_code, split_response, validate_payload};
#[test]
fn parse_status_code_reads_http_200() {
let status = parse_status_code("HTTP/1.1 200 OK").expect("must parse status");
assert_eq!(status, 200);
}
#[test]
fn split_response_extracts_status_and_body() {
let response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"ok\":true}";
let (status, body) = split_response(response).expect("must split response");
assert_eq!(status, 200);
assert_eq!(body, "{\"ok\":true}");
}
#[test]
fn validate_payload_accepts_liveness_contract() {
let body = "{\"ok\":true,\"data\":{\"status\":\"ok\"}}";
validate_payload(HealthcheckMode::Liveness, body).expect("liveness payload must pass");
}
#[test]
fn validate_payload_rejects_not_ready() {
let body = "{\"ok\":true,\"data\":{\"ready\":false}}";
let result = validate_payload(HealthcheckMode::Ready, body);
assert!(result.is_err());
}
}

View File

@@ -66,6 +66,7 @@ pub(crate) async fn initialize_me_pool(
match crate::transport::middle_proxy::fetch_proxy_secret_with_upstream(
proxy_secret_path,
config.general.proxy_secret_len_max,
config.general.proxy_secret_url.as_deref(),
Some(upstream_manager.clone()),
)
.await
@@ -126,7 +127,11 @@ pub(crate) async fn initialize_me_pool(
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4)
.await;
let cfg_v4 = load_startup_proxy_config_snapshot(
"https://core.telegram.org/getProxyConfig",
config
.general
.proxy_config_v4_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfig"),
config.general.proxy_config_v4_cache_path.as_deref(),
me2dc_fallback,
"getProxyConfig",
@@ -158,7 +163,11 @@ pub(crate) async fn initialize_me_pool(
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6)
.await;
let cfg_v6 = load_startup_proxy_config_snapshot(
"https://core.telegram.org/getProxyConfigV6",
config
.general
.proxy_config_v6_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfigV6"),
config.general.proxy_config_v6_cache_path.as_deref(),
me2dc_fallback,
"getProxyConfigV6",

View File

@@ -8,6 +8,7 @@ mod crypto;
#[cfg(unix)]
mod daemon;
mod error;
mod healthcheck;
mod ip_tracker;
#[cfg(test)]
#[path = "tests/ip_tracker_encapsulation_adversarial_tests.rs"]

View File

@@ -66,9 +66,19 @@ fn ensure_payload_capacity(mut sizes: Vec<usize>, payload_len: usize) -> Vec<usi
fn emulated_app_data_sizes(cached: &CachedTlsData) -> Vec<usize> {
match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => {
if !cached.behavior_profile.app_data_record_sizes.is_empty() {
return cached.behavior_profile.app_data_record_sizes.clone();
}
return cached
.app_data_records_sizes
.first()
.copied()
.or_else(|| {
cached
.behavior_profile
.app_data_record_sizes
.first()
.copied()
})
.map(|size| vec![size])
.unwrap_or_else(|| vec![cached.total_app_data_len.max(1024)]);
}
TlsProfileSource::Default | TlsProfileSource::Rustls => {}
}
@@ -80,8 +90,8 @@ fn emulated_app_data_sizes(cached: &CachedTlsData) -> Vec<usize> {
sizes
}
fn emulated_change_cipher_spec_count(cached: &CachedTlsData) -> usize {
usize::from(cached.behavior_profile.change_cipher_spec_count.max(1))
fn emulated_change_cipher_spec_count(_cached: &CachedTlsData) -> usize {
1
}
fn emulated_ticket_record_sizes(
@@ -89,19 +99,20 @@ fn emulated_ticket_record_sizes(
new_session_tickets: u8,
rng: &SecureRandom,
) -> Vec<usize> {
let mut sizes = match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => {
cached.behavior_profile.ticket_record_sizes.clone()
let target_count = usize::from(new_session_tickets.min(MAX_TICKET_RECORDS as u8));
if target_count == 0 {
return Vec::new();
}
TlsProfileSource::Default | TlsProfileSource::Rustls => Vec::new(),
let profiled_sizes = match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => {
cached.behavior_profile.ticket_record_sizes.as_slice()
}
TlsProfileSource::Default | TlsProfileSource::Rustls => &[],
};
let target_count = sizes
.len()
.max(usize::from(
new_session_tickets.min(MAX_TICKET_RECORDS as u8),
))
.min(MAX_TICKET_RECORDS);
let mut sizes = Vec::with_capacity(target_count);
sizes.extend(profiled_sizes.iter().copied().take(target_count));
while sizes.len() < target_count {
sizes.push(rng.range(48) + 48);
@@ -242,7 +253,18 @@ pub fn build_emulated_server_hello(
}
// --- ApplicationData (fake encrypted records) ---
let mut sizes = jitter_and_clamp_sizes(&emulated_app_data_sizes(cached), rng);
let mut sizes = {
let base_sizes = emulated_app_data_sizes(cached);
match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => base_sizes
.into_iter()
.map(|size| size.clamp(MIN_APP_DATA, MAX_APP_DATA))
.collect(),
TlsProfileSource::Default | TlsProfileSource::Rustls => {
jitter_and_clamp_sizes(&base_sizes, rng)
}
}
};
let compact_payload = cached
.cert_info
.as_ref()
@@ -508,7 +530,7 @@ mod tests {
}
#[test]
fn test_build_emulated_server_hello_replays_tail_records_for_profiled_tls() {
fn test_build_emulated_server_hello_ignores_tail_records_for_profiled_tls() {
let mut cached = make_cached(None);
cached.app_data_records_sizes = vec![27, 3905, 537, 69];
cached.total_app_data_len = 4538;
@@ -530,19 +552,11 @@ mod tests {
let hello_len = u16::from_be_bytes([response[3], response[4]]) as usize;
let ccs_start = 5 + hello_len;
let mut pos = ccs_start + 6;
let mut app_lengths = Vec::new();
while pos + 5 <= response.len() {
assert_eq!(response[pos], TLS_RECORD_APPLICATION);
let record_len = u16::from_be_bytes([response[pos + 3], response[pos + 4]]) as usize;
app_lengths.push(record_len);
pos += 5 + record_len;
}
assert_eq!(app_lengths.len(), 4);
assert_eq!(app_lengths[0], 64);
assert_eq!(app_lengths[3], 69);
assert!(app_lengths[1] >= 64);
assert!(app_lengths[2] >= 64);
let app_start = ccs_start + 6;
let app_len =
u16::from_be_bytes([response[app_start + 3], response[app_start + 4]]) as usize;
assert_eq!(response[app_start], TLS_RECORD_APPLICATION);
assert_eq!(app_len, 64);
assert_eq!(app_start + 5 + app_len, response.len());
}
}

View File

@@ -52,7 +52,7 @@ fn record_lengths_by_type(response: &[u8], wanted_type: u8) -> Vec<usize> {
}
#[test]
fn emulated_server_hello_replays_profile_change_cipher_spec_count() {
fn emulated_server_hello_keeps_single_change_cipher_spec_for_client_compatibility() {
let cached = make_cached();
let rng = SecureRandom::new();
@@ -69,12 +69,12 @@ fn emulated_server_hello_replays_profile_change_cipher_spec_count() {
assert_eq!(response[0], TLS_RECORD_HANDSHAKE);
let ccs_records = record_lengths_by_type(&response, TLS_RECORD_CHANGE_CIPHER);
assert_eq!(ccs_records.len(), 2);
assert_eq!(ccs_records.len(), 1);
assert!(ccs_records.iter().all(|len| *len == 1));
}
#[test]
fn emulated_server_hello_replays_profile_ticket_tail_lengths() {
fn emulated_server_hello_does_not_emit_profile_ticket_tail_when_disabled() {
let cached = make_cached();
let rng = SecureRandom::new();
@@ -90,6 +90,25 @@ fn emulated_server_hello_replays_profile_ticket_tail_lengths() {
);
let app_records = record_lengths_by_type(&response, TLS_RECORD_APPLICATION);
assert!(app_records.len() >= 4);
assert_eq!(&app_records[app_records.len() - 2..], &[220, 180]);
assert_eq!(app_records, vec![1200]);
}
#[test]
fn emulated_server_hello_uses_profile_ticket_lengths_when_enabled() {
let cached = make_cached();
let rng = SecureRandom::new();
let response = build_emulated_server_hello(
b"secret",
&[0x91; 32],
&[0x92; 16],
&cached,
false,
&rng,
None,
2,
);
let app_records = record_lengths_by_type(&response, TLS_RECORD_APPLICATION);
assert_eq!(app_records, vec![1200, 220, 180]);
}

View File

@@ -321,7 +321,14 @@ async fn run_update_cycle(
let mut maps_changed = false;
let mut ready_v4: Option<(ProxyConfigData, u64)> = None;
let cfg_v4 = retry_fetch("https://core.telegram.org/getProxyConfig", upstream.clone()).await;
let cfg_v4 = retry_fetch(
cfg.general
.proxy_config_v4_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfig"),
upstream.clone(),
)
.await;
if let Some(cfg_v4) = cfg_v4
&& snapshot_passes_guards(cfg, &cfg_v4, "getProxyConfig")
{
@@ -346,7 +353,10 @@ async fn run_update_cycle(
let mut ready_v6: Option<(ProxyConfigData, u64)> = None;
let cfg_v6 = retry_fetch(
"https://core.telegram.org/getProxyConfigV6",
cfg.general
.proxy_config_v6_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfigV6"),
upstream.clone(),
)
.await;
@@ -430,6 +440,7 @@ async fn run_update_cycle(
match download_proxy_secret_with_max_len_via_upstream(
cfg.general.proxy_secret_len_max,
upstream,
cfg.general.proxy_secret_url.as_deref(),
)
.await
{

View File

@@ -7,7 +7,6 @@ mod model;
mod pressure;
mod scheduler;
#[cfg(test)]
pub(crate) use model::PressureState;
pub(crate) use model::{AdmissionDecision, DispatchAction, DispatchFeedback, SchedulerDecision};
pub(crate) use scheduler::{WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState};

View File

@@ -77,11 +77,12 @@ pub(crate) struct FlowFairnessState {
pub(crate) standing_state: StandingQueueState,
pub(crate) scheduler_state: FlowSchedulerState,
pub(crate) bucket_id: usize,
pub(crate) weight_quanta: u8,
pub(crate) in_active_ring: bool,
}
impl FlowFairnessState {
pub(crate) fn new(flow_id: u64, worker_id: u16, bucket_id: usize) -> Self {
pub(crate) fn new(flow_id: u64, worker_id: u16, bucket_id: usize, weight_quanta: u8) -> Self {
Self {
_flow_id: flow_id,
_worker_id: worker_id,
@@ -97,6 +98,7 @@ impl FlowFairnessState {
standing_state: StandingQueueState::Transient,
scheduler_state: FlowSchedulerState::Idle,
bucket_id,
weight_quanta: weight_quanta.max(1),
in_active_ring: false,
}
}

View File

@@ -146,59 +146,55 @@ impl PressureEvaluator {
((signals.standing_flows.saturating_mul(100)) / signals.active_flows).min(100) as u8
};
let mut pressure_score = 0u8;
let mut pressured = false;
let mut saturated = false;
let queue_saturated_pct = cfg.queue_ratio_shedding_pct.min(cfg.queue_ratio_saturated_pct);
if queue_ratio_pct >= cfg.queue_ratio_pressured_pct {
pressure_score = pressure_score.max(1);
pressured = true;
}
if queue_ratio_pct >= cfg.queue_ratio_shedding_pct {
pressure_score = pressure_score.max(2);
}
if queue_ratio_pct >= cfg.queue_ratio_saturated_pct {
pressure_score = pressure_score.max(3);
if queue_ratio_pct >= queue_saturated_pct {
saturated = true;
}
let standing_saturated_pct = cfg
.standing_ratio_shedding_pct
.min(cfg.standing_ratio_saturated_pct);
if standing_ratio_pct >= cfg.standing_ratio_pressured_pct {
pressure_score = pressure_score.max(1);
pressured = true;
}
if standing_ratio_pct >= cfg.standing_ratio_shedding_pct {
pressure_score = pressure_score.max(2);
}
if standing_ratio_pct >= cfg.standing_ratio_saturated_pct {
pressure_score = pressure_score.max(3);
if standing_ratio_pct >= standing_saturated_pct {
saturated = true;
}
let rejects_saturated = cfg.rejects_shedding.min(cfg.rejects_saturated);
if self.admission_rejects_window >= cfg.rejects_pressured {
pressure_score = pressure_score.max(1);
pressured = true;
}
if self.admission_rejects_window >= cfg.rejects_shedding {
pressure_score = pressure_score.max(2);
}
if self.admission_rejects_window >= cfg.rejects_saturated {
pressure_score = pressure_score.max(3);
if self.admission_rejects_window >= rejects_saturated {
saturated = true;
}
let stalls_saturated = cfg.stalls_shedding.min(cfg.stalls_saturated);
if self.route_stalls_window >= cfg.stalls_pressured {
pressure_score = pressure_score.max(1);
pressured = true;
}
if self.route_stalls_window >= cfg.stalls_shedding {
pressure_score = pressure_score.max(2);
}
if self.route_stalls_window >= cfg.stalls_saturated {
pressure_score = pressure_score.max(3);
if self.route_stalls_window >= stalls_saturated {
saturated = true;
}
if signals.backpressured_flows > signals.active_flows.saturating_div(2)
&& signals.active_flows > 0
{
pressure_score = pressure_score.max(2);
pressured = true;
}
match pressure_score {
0 => PressureState::Normal,
1 => PressureState::Pressured,
2 => PressureState::Shedding,
_ => PressureState::Saturated,
if saturated {
PressureState::Saturated
} else if pressured {
PressureState::Pressured
} else {
PressureState::Normal
}
}

View File

@@ -1,7 +1,8 @@
use std::collections::{HashMap, VecDeque};
use std::collections::{HashMap, HashSet, VecDeque};
use std::time::{Duration, Instant};
use bytes::Bytes;
use crate::protocol::constants::RPC_FLAG_QUICKACK;
use super::model::{
AdmissionDecision, DispatchAction, DispatchCandidate, DispatchFeedback, FlowFairnessState,
@@ -26,6 +27,8 @@ pub(crate) struct WorkerFairnessConfig {
pub(crate) max_consecutive_stalls_before_close: u8,
pub(crate) soft_bucket_count: usize,
pub(crate) soft_bucket_share_pct: u8,
pub(crate) default_flow_weight: u8,
pub(crate) quickack_flow_weight: u8,
pub(crate) pressure: PressureConfig,
}
@@ -46,6 +49,8 @@ impl Default for WorkerFairnessConfig {
max_consecutive_stalls_before_close: 16,
soft_bucket_count: 64,
soft_bucket_share_pct: 25,
default_flow_weight: 1,
quickack_flow_weight: 4,
pressure: PressureConfig::default(),
}
}
@@ -57,9 +62,9 @@ struct FlowEntry {
}
impl FlowEntry {
fn new(flow_id: u64, worker_id: u16, bucket_id: usize) -> Self {
fn new(flow_id: u64, worker_id: u16, bucket_id: usize, weight_quanta: u8) -> Self {
Self {
fairness: FlowFairnessState::new(flow_id, worker_id, bucket_id),
fairness: FlowFairnessState::new(flow_id, worker_id, bucket_id, weight_quanta),
queue: VecDeque::new(),
}
}
@@ -86,6 +91,7 @@ pub(crate) struct WorkerFairnessState {
pressure: PressureEvaluator,
flows: HashMap<u64, FlowEntry>,
active_ring: VecDeque<u64>,
active_ring_members: HashSet<u64>,
total_queued_bytes: u64,
bucket_queued_bytes: Vec<u64>,
bucket_active_flows: Vec<usize>,
@@ -108,6 +114,7 @@ impl WorkerFairnessState {
pressure: PressureEvaluator::new(now),
flows: HashMap::new(),
active_ring: VecDeque::new(),
active_ring_members: HashSet::new(),
total_queued_bytes: 0,
bucket_queued_bytes: vec![0; bucket_count],
bucket_active_flows: vec![0; bucket_count],
@@ -184,6 +191,7 @@ impl WorkerFairnessState {
}
let bucket_id = self.bucket_for(conn_id);
let frame_weight = Self::weight_for_flags(&self.config, flags);
let bucket_cap = self
.config
.max_total_queued_bytes
@@ -205,12 +213,13 @@ impl WorkerFairnessState {
self.bucket_active_flows[bucket_id].saturating_add(1);
self.flows.insert(
conn_id,
FlowEntry::new(conn_id, self.config.worker_id, bucket_id),
FlowEntry::new(conn_id, self.config.worker_id, bucket_id, frame_weight),
);
self.flows
.get_mut(&conn_id)
.expect("flow inserted must be retrievable")
};
entry.fairness.weight_quanta = entry.fairness.weight_quanta.max(frame_weight);
if entry.fairness.pending_bytes.saturating_add(frame_bytes)
> self.config.max_flow_queued_bytes
@@ -242,11 +251,24 @@ impl WorkerFairnessState {
self.bucket_queued_bytes[bucket_id] =
self.bucket_queued_bytes[bucket_id].saturating_add(frame_bytes);
let mut enqueue_active = false;
if !entry.fairness.in_active_ring {
entry.fairness.in_active_ring = true;
self.active_ring.push_back(conn_id);
enqueue_active = true;
}
let pressure_state = self.pressure.state();
let (before_membership, after_membership) = {
let before = Self::flow_membership(&entry.fairness);
Self::classify_flow(&self.config, pressure_state, now, &mut entry.fairness);
let after = Self::flow_membership(&entry.fairness);
(before, after)
};
if enqueue_active {
self.enqueue_active_conn(conn_id);
}
self.apply_flow_membership_delta(before_membership, after_membership);
self.evaluate_pressure(now, true);
AdmissionDecision::Admit
}
@@ -260,32 +282,44 @@ impl WorkerFairnessState {
let Some(conn_id) = self.active_ring.pop_front() else {
break;
};
if !self.active_ring_members.remove(&conn_id) {
continue;
}
let mut candidate = None;
let mut requeue_active = false;
let mut drained_bytes = 0u64;
let mut bucket_id = 0usize;
let mut should_continue = false;
let mut enqueue_active = false;
let mut membership_delta = None;
let pressure_state = self.pressure.state();
if let Some(flow) = self.flows.get_mut(&conn_id) {
bucket_id = flow.fairness.bucket_id;
flow.fairness.in_active_ring = false;
let before_membership = Self::flow_membership(&flow.fairness);
if flow.queue.is_empty() {
flow.fairness.in_active_ring = false;
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
flow.fairness.pending_bytes = 0;
flow.fairness.deficit_bytes = 0;
flow.fairness.queue_started_at = None;
continue;
}
should_continue = true;
} else {
Self::classify_flow(&self.config, pressure_state, now, &mut flow.fairness);
let quantum =
Self::effective_quantum_bytes(&self.config, pressure_state, &flow.fairness);
let quantum = Self::effective_quantum_bytes(
&self.config,
pressure_state,
&flow.fairness,
);
flow.fairness.deficit_bytes = flow
.fairness
.deficit_bytes
.saturating_add(i64::from(quantum));
Self::clamp_deficit_bytes(&self.config, &mut flow.fairness);
self.deficit_grants = self.deficit_grants.saturating_add(1);
let front_len = flow.queue.front().map_or(0, |front| front.queued_bytes());
@@ -294,6 +328,8 @@ impl WorkerFairnessState {
flow.fairness.consecutive_skips.saturating_add(1);
self.deficit_skips = self.deficit_skips.saturating_add(1);
requeue_active = true;
flow.fairness.in_active_ring = true;
enqueue_active = true;
} else if let Some(frame) = flow.queue.pop_front() {
drained_bytes = frame.queued_bytes();
flow.fairness.pending_bytes =
@@ -302,6 +338,7 @@ impl WorkerFairnessState {
.fairness
.deficit_bytes
.saturating_sub(drained_bytes as i64);
Self::clamp_deficit_bytes(&self.config, &mut flow.fairness);
flow.fairness.consecutive_skips = 0;
flow.fairness.queue_started_at =
flow.queue.front().map(|front| front.enqueued_at);
@@ -309,6 +346,10 @@ impl WorkerFairnessState {
if !requeue_active {
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
flow.fairness.in_active_ring = false;
flow.fairness.deficit_bytes = 0;
} else {
flow.fairness.in_active_ring = true;
enqueue_active = true;
}
candidate = Some(DispatchCandidate {
pressure_state,
@@ -318,17 +359,25 @@ impl WorkerFairnessState {
}
}
membership_delta = Some((before_membership, Self::flow_membership(&flow.fairness)));
}
if let Some((before_membership, after_membership)) = membership_delta {
self.apply_flow_membership_delta(before_membership, after_membership);
}
if should_continue {
continue;
}
if drained_bytes > 0 {
self.total_queued_bytes = self.total_queued_bytes.saturating_sub(drained_bytes);
self.bucket_queued_bytes[bucket_id] =
self.bucket_queued_bytes[bucket_id].saturating_sub(drained_bytes);
}
if requeue_active {
if let Some(flow) = self.flows.get_mut(&conn_id) {
flow.fairness.in_active_ring = true;
}
self.active_ring.push_back(conn_id);
if requeue_active && enqueue_active {
self.enqueue_active_conn(conn_id);
}
if let Some(candidate) = candidate {
@@ -348,7 +397,9 @@ impl WorkerFairnessState {
) -> DispatchAction {
match feedback {
DispatchFeedback::Routed => {
let mut membership_delta = None;
if let Some(flow) = self.flows.get_mut(&conn_id) {
let before_membership = Self::flow_membership(&flow.fairness);
flow.fairness.last_drain_at = Some(now);
flow.fairness.recent_drain_bytes = flow
.fairness
@@ -358,6 +409,17 @@ impl WorkerFairnessState {
if flow.fairness.scheduler_state != FlowSchedulerState::Idle {
flow.fairness.scheduler_state = FlowSchedulerState::Active;
}
Self::classify_flow(
&self.config,
self.pressure.state(),
now,
&mut flow.fairness,
);
membership_delta =
Some((before_membership, Self::flow_membership(&flow.fairness)));
}
if let Some((before_membership, after_membership)) = membership_delta {
self.apply_flow_membership_delta(before_membership, after_membership);
}
self.evaluate_pressure(now, false);
DispatchAction::Continue
@@ -365,17 +427,20 @@ impl WorkerFairnessState {
DispatchFeedback::QueueFull => {
self.pressure.note_route_stall(now, &self.config.pressure);
self.downstream_stalls = self.downstream_stalls.saturating_add(1);
let state = self.pressure.state();
let Some(flow) = self.flows.get_mut(&conn_id) else {
self.evaluate_pressure(now, true);
return DispatchAction::Continue;
};
let (before_membership, after_membership, should_close_flow, enqueue_active) = {
let before_membership = Self::flow_membership(&flow.fairness);
let mut enqueue_active = false;
flow.fairness.consecutive_stalls =
flow.fairness.consecutive_stalls.saturating_add(1);
flow.fairness.scheduler_state = FlowSchedulerState::Backpressured;
flow.fairness.pressure_class = FlowPressureClass::Backpressured;
let state = self.pressure.state();
let should_shed_frame = matches!(state, PressureState::Saturated)
|| (matches!(state, PressureState::Shedding)
&& flow.fairness.standing_state == StandingQueueState::Standing
@@ -392,20 +457,35 @@ impl WorkerFairnessState {
flow.fairness.pending_bytes.saturating_add(frame_bytes);
flow.fairness.queue_started_at =
flow.queue.front().map(|front| front.enqueued_at);
self.total_queued_bytes = self.total_queued_bytes.saturating_add(frame_bytes);
self.bucket_queued_bytes[flow.fairness.bucket_id] = self.bucket_queued_bytes
[flow.fairness.bucket_id]
self.total_queued_bytes =
self.total_queued_bytes.saturating_add(frame_bytes);
self.bucket_queued_bytes[flow.fairness.bucket_id] = self
.bucket_queued_bytes[flow.fairness.bucket_id]
.saturating_add(frame_bytes);
if !flow.fairness.in_active_ring {
flow.fairness.in_active_ring = true;
self.active_ring.push_back(conn_id);
enqueue_active = true;
}
}
if flow.fairness.consecutive_stalls
Self::classify_flow(&self.config, state, now, &mut flow.fairness);
let after_membership = Self::flow_membership(&flow.fairness);
let should_close_flow = flow.fairness.consecutive_stalls
>= self.config.max_consecutive_stalls_before_close
&& self.pressure.state() == PressureState::Saturated
{
&& self.pressure.state() == PressureState::Saturated;
(
before_membership,
after_membership,
should_close_flow,
enqueue_active,
)
};
if enqueue_active {
self.enqueue_active_conn(conn_id);
}
self.apply_flow_membership_delta(before_membership, after_membership);
if should_close_flow {
self.remove_flow(conn_id);
self.evaluate_pressure(now, true);
return DispatchAction::CloseFlow;
@@ -426,6 +506,15 @@ impl WorkerFairnessState {
let Some(entry) = self.flows.remove(&conn_id) else {
return;
};
self.active_ring_members.remove(&conn_id);
self.active_ring.retain(|queued_conn_id| *queued_conn_id != conn_id);
let (was_standing, was_backpressured) = Self::flow_membership(&entry.fairness);
if was_standing {
self.standing_flow_count = self.standing_flow_count.saturating_sub(1);
}
if was_backpressured {
self.backpressured_flow_count = self.backpressured_flow_count.saturating_sub(1);
}
self.bucket_active_flows[entry.fairness.bucket_id] =
self.bucket_active_flows[entry.fairness.bucket_id].saturating_sub(1);
@@ -440,27 +529,6 @@ impl WorkerFairnessState {
}
fn evaluate_pressure(&mut self, now: Instant, force: bool) {
let mut standing = 0usize;
let mut backpressured = 0usize;
for flow in self.flows.values_mut() {
Self::classify_flow(&self.config, self.pressure.state(), now, &mut flow.fairness);
if flow.fairness.standing_state == StandingQueueState::Standing {
standing = standing.saturating_add(1);
}
if matches!(
flow.fairness.scheduler_state,
FlowSchedulerState::Backpressured
| FlowSchedulerState::Penalized
| FlowSchedulerState::SheddingCandidate
) {
backpressured = backpressured.saturating_add(1);
}
}
self.standing_flow_count = standing;
self.backpressured_flow_count = backpressured;
let _ = self.pressure.maybe_evaluate(
now,
&self.config.pressure,
@@ -468,8 +536,8 @@ impl WorkerFairnessState {
PressureSignals {
active_flows: self.flows.len(),
total_queued_bytes: self.total_queued_bytes,
standing_flows: standing,
backpressured_flows: backpressured,
standing_flows: self.standing_flow_count,
backpressured_flows: self.backpressured_flow_count,
},
force,
);
@@ -481,12 +549,39 @@ impl WorkerFairnessState {
now: Instant,
fairness: &mut FlowFairnessState,
) {
if fairness.pending_bytes == 0 {
fairness.pressure_class = FlowPressureClass::Healthy;
fairness.standing_state = StandingQueueState::Transient;
fairness.scheduler_state = FlowSchedulerState::Idle;
let (pressure_class, standing_state, scheduler_state, standing) =
Self::derive_flow_classification(config, pressure_state, now, fairness);
fairness.pressure_class = pressure_class;
fairness.standing_state = standing_state;
fairness.scheduler_state = scheduler_state;
if scheduler_state == FlowSchedulerState::Idle {
fairness.deficit_bytes = 0;
}
if standing {
fairness.penalty_score = fairness.penalty_score.saturating_add(1);
} else {
fairness.penalty_score = fairness.penalty_score.saturating_sub(1);
return;
}
}
fn derive_flow_classification(
config: &WorkerFairnessConfig,
pressure_state: PressureState,
now: Instant,
fairness: &FlowFairnessState,
) -> (
FlowPressureClass,
StandingQueueState,
FlowSchedulerState,
bool,
) {
if fairness.pending_bytes == 0 {
return (
FlowPressureClass::Healthy,
StandingQueueState::Transient,
FlowSchedulerState::Idle,
false,
);
}
let queue_age = fairness
@@ -503,29 +598,165 @@ impl WorkerFairnessState {
&& (fairness.consecutive_stalls >= config.standing_stall_threshold || drain_stalled);
if standing {
fairness.standing_state = StandingQueueState::Standing;
fairness.pressure_class = FlowPressureClass::Standing;
fairness.penalty_score = fairness.penalty_score.saturating_add(1);
fairness.scheduler_state = if pressure_state >= PressureState::Shedding {
let scheduler_state = if pressure_state >= PressureState::Shedding {
FlowSchedulerState::SheddingCandidate
} else {
FlowSchedulerState::Penalized
};
return;
return (
FlowPressureClass::Standing,
StandingQueueState::Standing,
scheduler_state,
true,
);
}
fairness.standing_state = StandingQueueState::Transient;
if fairness.consecutive_stalls > 0 {
fairness.pressure_class = FlowPressureClass::Backpressured;
fairness.scheduler_state = FlowSchedulerState::Backpressured;
} else if fairness.pending_bytes >= config.standing_queue_min_backlog_bytes {
fairness.pressure_class = FlowPressureClass::Bursty;
fairness.scheduler_state = FlowSchedulerState::Active;
} else {
fairness.pressure_class = FlowPressureClass::Healthy;
fairness.scheduler_state = FlowSchedulerState::Active;
return (
FlowPressureClass::Backpressured,
StandingQueueState::Transient,
FlowSchedulerState::Backpressured,
false,
);
}
fairness.penalty_score = fairness.penalty_score.saturating_sub(1);
if fairness.pending_bytes >= config.standing_queue_min_backlog_bytes {
return (
FlowPressureClass::Bursty,
StandingQueueState::Transient,
FlowSchedulerState::Active,
false,
);
}
(
FlowPressureClass::Healthy,
StandingQueueState::Transient,
FlowSchedulerState::Active,
false,
)
}
#[inline]
fn flow_membership(fairness: &FlowFairnessState) -> (bool, bool) {
(
fairness.standing_state == StandingQueueState::Standing,
Self::scheduler_state_is_backpressured(fairness.scheduler_state),
)
}
#[inline]
fn scheduler_state_is_backpressured(state: FlowSchedulerState) -> bool {
matches!(
state,
FlowSchedulerState::Backpressured
| FlowSchedulerState::Penalized
| FlowSchedulerState::SheddingCandidate
)
}
fn apply_flow_membership_delta(
&mut self,
before_membership: (bool, bool),
after_membership: (bool, bool),
) {
if before_membership.0 != after_membership.0 {
if after_membership.0 {
self.standing_flow_count = self.standing_flow_count.saturating_add(1);
} else {
self.standing_flow_count = self.standing_flow_count.saturating_sub(1);
}
}
if before_membership.1 != after_membership.1 {
if after_membership.1 {
self.backpressured_flow_count = self.backpressured_flow_count.saturating_add(1);
} else {
self.backpressured_flow_count = self.backpressured_flow_count.saturating_sub(1);
}
}
}
#[inline]
fn clamp_deficit_bytes(config: &WorkerFairnessConfig, fairness: &mut FlowFairnessState) {
let max_deficit = config.max_flow_queued_bytes.min(i64::MAX as u64) as i64;
fairness.deficit_bytes = fairness.deficit_bytes.clamp(0, max_deficit);
}
#[inline]
fn enqueue_active_conn(&mut self, conn_id: u64) {
if self.active_ring_members.insert(conn_id) {
self.active_ring.push_back(conn_id);
}
}
#[inline]
fn weight_for_flags(config: &WorkerFairnessConfig, flags: u32) -> u8 {
if (flags & RPC_FLAG_QUICKACK) != 0 {
return config.quickack_flow_weight.max(1);
}
config.default_flow_weight.max(1)
}
#[cfg(test)]
pub(crate) fn debug_recompute_flow_counters(&self, now: Instant) -> (usize, usize) {
let pressure_state = self.pressure.state();
let mut standing = 0usize;
let mut backpressured = 0usize;
for flow in self.flows.values() {
let (_, standing_state, scheduler_state, _) =
Self::derive_flow_classification(&self.config, pressure_state, now, &flow.fairness);
if standing_state == StandingQueueState::Standing {
standing = standing.saturating_add(1);
}
if Self::scheduler_state_is_backpressured(scheduler_state) {
backpressured = backpressured.saturating_add(1);
}
}
(standing, backpressured)
}
#[cfg(test)]
pub(crate) fn debug_check_active_ring_consistency(&self) -> bool {
if self.active_ring.len() != self.active_ring_members.len() {
return false;
}
let mut seen = HashSet::with_capacity(self.active_ring.len());
for conn_id in self.active_ring.iter().copied() {
if !seen.insert(conn_id) {
return false;
}
if !self.active_ring_members.contains(&conn_id) {
return false;
}
let Some(flow) = self.flows.get(&conn_id) else {
return false;
};
if !flow.fairness.in_active_ring || flow.queue.is_empty() {
return false;
}
}
for (conn_id, flow) in self.flows.iter() {
let in_ring = self.active_ring_members.contains(conn_id);
if flow.fairness.in_active_ring != in_ring {
return false;
}
if in_ring && flow.queue.is_empty() {
return false;
}
}
true
}
#[cfg(test)]
pub(crate) fn debug_max_deficit_bytes(&self) -> i64 {
self.flows
.values()
.map(|entry| entry.fairness.deficit_bytes)
.max()
.unwrap_or(0)
}
fn effective_quantum_bytes(
@@ -542,12 +773,14 @@ impl WorkerFairnessState {
return config.penalized_quantum_bytes.max(1);
}
match pressure_state {
let base_quantum = match pressure_state {
PressureState::Normal => config.base_quantum_bytes.max(1),
PressureState::Pressured => config.pressured_quantum_bytes.max(1),
PressureState::Shedding => config.pressured_quantum_bytes.max(1),
PressureState::Saturated => config.penalized_quantum_bytes.max(1),
}
};
let weighted_quantum = base_quantum.saturating_mul(fairness.weight_quanta.max(1) as u32);
weighted_quantum.max(1)
}
fn bucket_for(&self, conn_id: u64) -> usize {

View File

@@ -4,7 +4,7 @@ use std::collections::HashMap;
use std::io::ErrorKind;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::time::Instant;
use std::time::{Duration, Instant};
use bytes::{Bytes, BytesMut};
use tokio::io::AsyncReadExt;
@@ -21,8 +21,8 @@ use crate::stats::Stats;
use super::codec::{RpcChecksumMode, WriterCommand, rpc_crc};
use super::fairness::{
AdmissionDecision, DispatchAction, DispatchFeedback, SchedulerDecision, WorkerFairnessConfig,
WorkerFairnessSnapshot, WorkerFairnessState,
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState,
};
use super::registry::RouteResult;
use super::{ConnRegistry, MeResponse};
@@ -45,10 +45,22 @@ fn is_data_route_queue_full(result: RouteResult) -> bool {
)
}
fn should_close_on_queue_full_streak(streak: u8) -> bool {
fn should_close_on_queue_full_streak(streak: u8, pressure_state: PressureState) -> bool {
if pressure_state < PressureState::Shedding {
return false;
}
streak >= DATA_ROUTE_QUEUE_FULL_STARVATION_THRESHOLD
}
fn should_schedule_fairness_retry(snapshot: &WorkerFairnessSnapshot) -> bool {
snapshot.total_queued_bytes > 0
}
fn fairness_retry_delay(route_wait_ms: u64) -> Duration {
Duration::from_millis(route_wait_ms.max(1))
}
async fn route_data_with_retry(
reg: &ConnRegistry,
conn_id: u64,
@@ -157,7 +169,7 @@ async fn drain_fairness_scheduler(
break;
};
let cid = candidate.frame.conn_id;
let _pressure_state = candidate.pressure_state;
let pressure_state = candidate.pressure_state;
let _flow_class = candidate.flow_class;
let routed = route_data_with_retry(
reg,
@@ -176,7 +188,7 @@ async fn drain_fairness_scheduler(
if is_data_route_queue_full(routed) {
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
if should_close_on_queue_full_streak(*streak) {
if should_close_on_queue_full_streak(*streak, pressure_state) {
fairness.remove_flow(cid);
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
@@ -231,10 +243,33 @@ pub(crate) async fn reader_loop(
let mut fairness_snapshot = fairness.snapshot();
loop {
let mut tmp = [0u8; 65_536];
let backlog_retry_enabled = should_schedule_fairness_retry(&fairness_snapshot);
let backlog_retry_delay =
fairness_retry_delay(reader_route_data_wait_ms.load(Ordering::Relaxed));
let mut retry_only = false;
let n = tokio::select! {
res = rd.read(&mut tmp) => res.map_err(ProxyError::Io)?,
_ = tokio::time::sleep(backlog_retry_delay), if backlog_retry_enabled => {
retry_only = true;
0usize
},
_ = cancel.cancelled() => return Ok(()),
};
if retry_only {
let route_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed);
drain_fairness_scheduler(
&mut fairness,
reg.as_ref(),
&tx,
&mut data_route_queue_full_streak,
route_wait_ms,
stats.as_ref(),
)
.await;
let current_snapshot = fairness.snapshot();
apply_fairness_metrics_delta(stats.as_ref(), &mut fairness_snapshot, current_snapshot);
continue;
}
if n == 0 {
stats.increment_me_reader_eof_total();
return Err(ProxyError::Io(std::io::Error::new(
@@ -317,12 +352,9 @@ pub(crate) async fn reader_loop(
stats.increment_me_route_drop_queue_full_high();
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
if should_close_on_queue_full_streak(*streak)
|| matches!(
admission,
AdmissionDecision::RejectSaturated
| AdmissionDecision::RejectStandingFlow
)
let pressure_state = fairness.pressure_state();
if should_close_on_queue_full_streak(*streak, pressure_state)
|| matches!(admission, AdmissionDecision::RejectSaturated)
{
fairness.remove_flow(cid);
data_route_queue_full_streak.remove(&cid);
@@ -445,14 +477,18 @@ pub(crate) async fn reader_loop(
#[cfg(test)]
mod tests {
use std::time::Duration;
use bytes::Bytes;
use super::PressureState;
use crate::transport::middle_proxy::ConnRegistry;
use super::{
MeResponse, RouteResult, is_data_route_queue_full, route_data_with_retry,
should_close_on_queue_full_streak, should_close_on_route_result_for_ack,
should_close_on_route_result_for_data,
MeResponse, RouteResult, WorkerFairnessSnapshot, fairness_retry_delay,
is_data_route_queue_full, route_data_with_retry, should_close_on_queue_full_streak,
should_close_on_route_result_for_ack, should_close_on_route_result_for_data,
should_schedule_fairness_retry,
};
#[test]
@@ -475,10 +511,29 @@ mod tests {
assert!(is_data_route_queue_full(RouteResult::QueueFullBase));
assert!(is_data_route_queue_full(RouteResult::QueueFullHigh));
assert!(!is_data_route_queue_full(RouteResult::NoConn));
assert!(!should_close_on_queue_full_streak(1));
assert!(!should_close_on_queue_full_streak(2));
assert!(should_close_on_queue_full_streak(3));
assert!(should_close_on_queue_full_streak(u8::MAX));
assert!(!should_close_on_queue_full_streak(1, PressureState::Normal));
assert!(!should_close_on_queue_full_streak(2, PressureState::Pressured));
assert!(!should_close_on_queue_full_streak(3, PressureState::Pressured));
assert!(should_close_on_queue_full_streak(3, PressureState::Shedding));
assert!(should_close_on_queue_full_streak(
u8::MAX,
PressureState::Saturated
));
}
#[test]
fn fairness_retry_is_scheduled_only_when_queue_has_pending_bytes() {
let mut snapshot = WorkerFairnessSnapshot::default();
assert!(!should_schedule_fairness_retry(&snapshot));
snapshot.total_queued_bytes = 1;
assert!(should_schedule_fairness_retry(&snapshot));
}
#[test]
fn fairness_retry_delay_never_drops_below_one_millisecond() {
assert_eq!(fairness_retry_delay(0), Duration::from_millis(1));
assert_eq!(fairness_retry_delay(2), Duration::from_millis(2));
}
#[test]

View File

@@ -37,20 +37,26 @@ pub(super) fn validate_proxy_secret_len(data_len: usize, max_len: usize) -> Resu
/// Fetch Telegram proxy-secret binary.
#[allow(dead_code)]
pub async fn fetch_proxy_secret(cache_path: Option<&str>, max_len: usize) -> Result<Vec<u8>> {
fetch_proxy_secret_with_upstream(cache_path, max_len, None).await
pub async fn fetch_proxy_secret(
cache_path: Option<&str>,
max_len: usize,
proxy_secret_url: Option<&str>,
) -> Result<Vec<u8>> {
fetch_proxy_secret_with_upstream(cache_path, max_len, proxy_secret_url, None).await
}
/// Fetch Telegram proxy-secret binary, optionally through upstream routing.
pub async fn fetch_proxy_secret_with_upstream(
cache_path: Option<&str>,
max_len: usize,
proxy_secret_url: Option<&str>,
upstream: Option<Arc<UpstreamManager>>,
) -> Result<Vec<u8>> {
let cache = cache_path.unwrap_or("proxy-secret");
// 1) Try fresh download first.
match download_proxy_secret_with_max_len_via_upstream(max_len, upstream).await {
match download_proxy_secret_with_max_len_via_upstream(max_len, upstream, proxy_secret_url).await
{
Ok(data) => {
if let Err(e) = tokio::fs::write(cache, &data).await {
warn!(error = %e, "Failed to cache proxy-secret (non-fatal)");
@@ -91,14 +97,19 @@ pub async fn fetch_proxy_secret_with_upstream(
#[allow(dead_code)]
pub async fn download_proxy_secret_with_max_len(max_len: usize) -> Result<Vec<u8>> {
download_proxy_secret_with_max_len_via_upstream(max_len, None).await
download_proxy_secret_with_max_len_via_upstream(max_len, None, None).await
}
pub async fn download_proxy_secret_with_max_len_via_upstream(
max_len: usize,
upstream: Option<Arc<UpstreamManager>>,
proxy_secret_url: Option<&str>,
) -> Result<Vec<u8>> {
let resp = https_get("https://core.telegram.org/getProxySecret", upstream).await?;
let resp = https_get(
proxy_secret_url.unwrap_or("https://core.telegram.org/getProxySecret"),
upstream,
)
.await?;
if !(200..=299).contains(&resp.status) {
return Err(ProxyError::Proxy(format!(

View File

@@ -2,6 +2,7 @@ use std::time::{Duration, Instant};
use bytes::Bytes;
use crate::protocol::constants::RPC_FLAG_QUICKACK;
use crate::transport::middle_proxy::fairness::{
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
WorkerFairnessConfig, WorkerFairnessState,
@@ -114,6 +115,62 @@ fn fairness_keeps_fast_flow_progress_under_slow_neighbor() {
assert!(snapshot.total_queued_bytes <= 64 * 1024);
}
#[test]
fn fairness_prioritizes_quickack_flow_when_weights_enabled() {
let mut now = Instant::now();
let mut fairness = WorkerFairnessState::new(
WorkerFairnessConfig {
max_total_queued_bytes: 256 * 1024,
max_flow_queued_bytes: 64 * 1024,
base_quantum_bytes: 8 * 1024,
pressured_quantum_bytes: 8 * 1024,
penalized_quantum_bytes: 8 * 1024,
default_flow_weight: 1,
quickack_flow_weight: 4,
..WorkerFairnessConfig::default()
},
now,
);
for _ in 0..8 {
assert_eq!(
fairness.enqueue_data(10, RPC_FLAG_QUICKACK, enqueue_payload(16 * 1024), now),
AdmissionDecision::Admit
);
assert_eq!(
fairness.enqueue_data(20, 0, enqueue_payload(16 * 1024), now),
AdmissionDecision::Admit
);
}
let mut quickack_dispatched = 0u64;
let mut bulk_dispatched = 0u64;
for _ in 0..64 {
now += Duration::from_millis(1);
let SchedulerDecision::Dispatch(candidate) = fairness.next_decision(now) else {
break;
};
if candidate.frame.conn_id == 10 {
quickack_dispatched = quickack_dispatched.saturating_add(1);
} else if candidate.frame.conn_id == 20 {
bulk_dispatched = bulk_dispatched.saturating_add(1);
}
let _ = fairness.apply_dispatch_feedback(
candidate.frame.conn_id,
candidate,
DispatchFeedback::Routed,
now,
);
}
assert!(
quickack_dispatched > bulk_dispatched,
"quickack flow must receive higher dispatch rate with larger weight"
);
}
#[test]
fn fairness_pressure_hysteresis_prevents_instant_flapping() {
let mut now = Instant::now();
@@ -180,6 +237,12 @@ fn fairness_randomized_sequence_preserves_memory_bounds() {
}
let snapshot = fairness.snapshot();
let (standing_recomputed, backpressured_recomputed) =
fairness.debug_recompute_flow_counters(now);
assert!(snapshot.total_queued_bytes <= 32 * 1024);
assert_eq!(snapshot.standing_flows, standing_recomputed);
assert_eq!(snapshot.backpressured_flows, backpressured_recomputed);
assert!(fairness.debug_check_active_ring_consistency());
assert!(fairness.debug_max_deficit_bytes() <= 4 * 1024);
}
}

View File

@@ -279,6 +279,12 @@ pub struct UpstreamApiSummarySnapshot {
pub shadowsocks_total: usize,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct UpstreamApiHealthSummary {
pub configured_total: usize,
pub healthy_total: usize,
}
#[derive(Debug, Clone)]
pub struct UpstreamApiSnapshot {
pub summary: UpstreamApiSummarySnapshot,
@@ -444,6 +450,20 @@ impl UpstreamManager {
Some(UpstreamApiSnapshot { summary, upstreams })
}
pub async fn api_health_summary(&self) -> UpstreamApiHealthSummary {
let guard = self.upstreams.read().await;
let mut summary = UpstreamApiHealthSummary {
configured_total: guard.len(),
healthy_total: 0,
};
for upstream in guard.iter() {
if upstream.healthy {
summary.healthy_total += 1;
}
}
summary
}
fn describe_upstream(upstream_type: &UpstreamType) -> (UpstreamRouteKind, String) {
match upstream_type {
UpstreamType::Direct { .. } => (UpstreamRouteKind::Direct, "direct".to_string()),