From bc432f06e27dd664113cf50db4934a006fb93878 Mon Sep 17 00:00:00 2001 From: sintanial Date: Sun, 1 Mar 2026 13:53:50 +0300 Subject: [PATCH 001/261] Add per-user ad_tag with global fallback and hot-reload - Per-user ad_tag in [access.user_ad_tags], global fallback in general.ad_tag - User tag overrides global; if no user tag, general.ad_tag is used - Both general.ad_tag and user_ad_tags support hot-reload (no restart) --- Cargo.lock | 2 +- README.md | 8 +++--- config.toml | 2 ++ src/config/hot_reload.rs | 42 ++++++++++++++++-------------- src/config/load.rs | 4 +-- src/config/types.rs | 12 ++++++--- src/main.rs | 2 +- src/proxy/middle_relay.rs | 19 +++++++++++++- src/transport/middle_proxy/send.rs | 5 +++- 9 files changed, 65 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 251f0b7..e29b473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2087,7 +2087,7 @@ dependencies = [ [[package]] name = "telemt" -version = "3.0.13" +version = "3.1.3" dependencies = [ "aes", "anyhow", diff --git a/README.md b/README.md index 093f2cd..8ea25e7 100644 --- a/README.md +++ b/README.md @@ -215,10 +215,12 @@ hello = "00000000000000000000000000000000" ``` ### Advanced -#### Adtag -To use channel advertising and usage statistics from Telegram, get Adtag from [@mtproxybot](https://t.me/mtproxybot), add this parameter to section `[General]` +#### Adtag (per-user) +To use channel advertising and usage statistics from Telegram, get an Adtag from [@mtproxybot](https://t.me/mtproxybot). Set it per user in `[access.user_ad_tags]` (32 hex chars): ```toml -ad_tag = "00000000000000000000000000000000" # Replace zeros to your adtag from @mtproxybot +[access.user_ad_tags] +username1 = "11111111111111111111111111111111" # Replace with your tag from @mtproxybot +username2 = "22222222222222222222222222222222" ``` #### Listening and Announce IPs To specify listening address and/or address in links, add to section `[[server.listeners]]` of config.toml: diff --git a/config.toml b/config.toml index b280234..cb33e3d 100644 --- a/config.toml +++ b/config.toml @@ -5,7 +5,9 @@ # === General Settings === [general] use_middle_proxy = false +# Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags] # ad_tag = "00000000000000000000000000000000" +# Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot) # === Log Level === # Log level: debug | verbose | normal | silent diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index eec6b8c..e16cff2 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -4,21 +4,22 @@ //! //! # What can be reloaded without restart //! -//! | Section | Field | Effect | -//! |-----------|-------------------------------|-----------------------------------| -//! | `general` | `log_level` | Filter updated via `log_level_tx` | -//! | `general` | `ad_tag` | Passed on next connection | -//! | `general` | `middle_proxy_pool_size` | Passed on next connection | -//! | `general` | `me_keepalive_*` | Passed on next connection | -//! | `general` | `desync_all_full` | Applied immediately | -//! | `general` | `update_every` | Applied to ME updater immediately | -//! | `general` | `hardswap` | Applied on next ME map update | -//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update | -//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update | -//! | `general` | `me_reinit_drain_timeout_secs`| Applied on next ME map update | -//! | `general` | `telemetry` / `me_*_policy` | Applied immediately | -//! | `network` | `dns_overrides` | Applied immediately | -//! | `access` | All user/quota fields | Effective immediately | +//! | Section | Field | Effect | +//! |-----------|--------------------------------|------------------------------------------------| +//! | `general` | `log_level` | Filter updated via `log_level_tx` | +//! | `access` | `user_ad_tags` | Passed on next connection | +//! | `general` | `ad_tag` | Passed on next connection (fallback per-user) | +//! | `general` | `middle_proxy_pool_size` | Passed on next connection | +//! | `general` | `me_keepalive_*` | Passed on next connection | +//! | `general` | `desync_all_full` | Applied immediately | +//! | `general` | `update_every` | Applied to ME updater immediately | +//! | `general` | `hardswap` | Applied on next ME map update | +//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update | +//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update | +//! | `general` | `me_reinit_drain_timeout_secs` | Applied on next ME map update | +//! | `general` | `telemetry` / `me_*_policy` | Applied immediately | +//! | `network` | `dns_overrides` | Applied immediately | +//! | `access` | All user/quota fields | Effective immediately | //! //! Fields that require re-binding sockets (`server.port`, `censorship.*`, //! `network.*`, `use_middle_proxy`) are **not** applied; a warning is emitted. @@ -207,14 +208,17 @@ fn log_changes( log_tx.send(new_hot.log_level.clone()).ok(); } - if old_hot.ad_tag != new_hot.ad_tag { + if old_hot.access.user_ad_tags != new_hot.access.user_ad_tags { info!( - "config reload: ad_tag: {} → {}", - old_hot.ad_tag.as_deref().unwrap_or("none"), - new_hot.ad_tag.as_deref().unwrap_or("none"), + "config reload: user_ad_tags updated ({} entries)", + new_hot.access.user_ad_tags.len(), ); } + if old_hot.ad_tag != new_hot.ad_tag { + info!("config reload: general.ad_tag updated (applied on next connection)"); + } + if old_hot.dns_overrides != new_hot.dns_overrides { info!( "config reload: network.dns_overrides updated ({} entries)", diff --git a/src/config/load.rs b/src/config/load.rs index 3aafda2..f37791d 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -532,7 +532,7 @@ impl ProxyConfig { ))); } - if let Some(tag) = &self.general.ad_tag { + for (user, tag) in &self.access.user_ad_tags { let zeros = "00000000000000000000000000000000"; if !is_valid_ad_tag(tag) { return Err(ProxyError::Config( @@ -540,7 +540,7 @@ impl ProxyConfig { )); } if tag == zeros { - warn!("ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel"); + warn!(user = %user, "user ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel"); } } diff --git a/src/config/types.rs b/src/config/types.rs index 7a3f6e9..716d78f 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -247,14 +247,15 @@ pub struct GeneralConfig { #[serde(default = "default_true")] pub use_middle_proxy: bool, - #[serde(default)] - pub ad_tag: Option, - /// Path to proxy-secret binary file (auto-downloaded if absent). /// Infrastructure secret from https://core.telegram.org/getProxySecret. #[serde(default = "default_proxy_secret_path")] pub proxy_secret_path: Option, + /// Global ad_tag (32 hex chars from @MTProxybot). Fallback when user has no per-user tag in access.user_ad_tags. + #[serde(default)] + pub ad_tag: Option, + /// Public IP override for middle-proxy NAT environments. /// When set, this IP is used in ME key derivation and RPC_PROXY_REQ "our_addr". #[serde(default)] @@ -807,6 +808,10 @@ pub struct AccessConfig { #[serde(default = "default_access_users")] pub users: HashMap, + /// Per-user ad_tag (32 hex chars from @MTProxybot). + #[serde(default)] + pub user_ad_tags: HashMap, + #[serde(default)] pub user_max_tcp_conns: HashMap, @@ -833,6 +838,7 @@ impl Default for AccessConfig { fn default() -> Self { Self { users: default_access_users(), + user_ad_tags: HashMap::new(), user_max_tcp_conns: HashMap::new(), user_expirations: HashMap::new(), user_data_quota: HashMap::new(), diff --git a/src/main.rs b/src/main.rs index 2675509..b910c64 100644 --- a/src/main.rs +++ b/src/main.rs @@ -448,7 +448,7 @@ async fn main() -> std::result::Result<(), Box> { info!("Middle-proxy STUN probing disabled by network.stun_use=false"); } - // ad_tag (proxy_tag) for advertising + // Global ad_tag (pool default). Used when user has no per-user tag in access.user_ad_tags. let proxy_tag = config .general .ad_tag diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index a4942ba..0690906 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -238,7 +238,22 @@ where stats.increment_user_connects(&user); stats.increment_user_curr_connects(&user); - let proto_flags = proto_flags_for_tag(proto_tag, me_pool.has_proxy_tag()); + // Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable) + let user_tag: Option> = config + .access + .user_ad_tags + .get(&user) + .and_then(|s| hex::decode(s).ok()) + .filter(|v| v.len() == 16); + let global_tag: Option> = config + .general + .ad_tag + .as_ref() + .and_then(|s| hex::decode(s).ok()) + .filter(|v| v.len() == 16); + let effective_tag = user_tag.or(global_tag); + + let proto_flags = proto_flags_for_tag(proto_tag, effective_tag.is_some()); debug!( trace_id = format_args!("0x{:016x}", trace_id), user = %user, @@ -256,6 +271,7 @@ where let (c2me_tx, mut c2me_rx) = mpsc::channel::(C2ME_CHANNEL_CAPACITY); let me_pool_c2me = me_pool.clone(); + let effective_tag = effective_tag; let c2me_sender = tokio::spawn(async move { let mut sent_since_yield = 0usize; while let Some(cmd) = c2me_rx.recv().await { @@ -268,6 +284,7 @@ where translated_local_addr, &payload, flags, + effective_tag.as_deref(), ).await?; sent_since_yield = sent_since_yield.saturating_add(1); if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) { diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index f68b1b9..65bc43a 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -18,6 +18,7 @@ use rand::seq::SliceRandom; use super::registry::ConnMeta; impl MePool { + /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default. pub async fn send_proxy_req( self: &Arc, conn_id: u64, @@ -26,13 +27,15 @@ impl MePool { our_addr: SocketAddr, data: &[u8], proto_flags: u32, + tag_override: Option<&[u8]>, ) -> Result<()> { + let tag = tag_override.or(self.proxy_tag.as_deref()); let payload = build_proxy_req_payload( conn_id, client_addr, our_addr, data, - self.proxy_tag.as_deref(), + tag, proto_flags, ); let meta = ConnMeta { From 1e400d4cc25b5f5a7855e6f7372c638d417b7f61 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Sun, 1 Mar 2026 19:05:53 +0300 Subject: [PATCH 002/261] Update QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index 4bd0ae8..e6aca19 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -114,7 +114,7 @@ WantedBy=multi-user.target **5.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt` -**6.** Для получите ссылки введите `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` +**6.** Для получения ссылки введите `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` --- From 6c1b837d5b9a97f35ab28322d2ac557bb438f1b1 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 1 Mar 2026 23:37:49 +0300 Subject: [PATCH 003/261] Create MIDDLE-END-KDF.en.md --- docs/MIDDLE-END-KDF.en.md | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/MIDDLE-END-KDF.en.md diff --git a/docs/MIDDLE-END-KDF.en.md b/docs/MIDDLE-END-KDF.en.md new file mode 100644 index 0000000..e793a8a --- /dev/null +++ b/docs/MIDDLE-END-KDF.en.md @@ -0,0 +1,41 @@ +# Middle-End Proxy + +## KDF Addressing — Implementation FAQ + +### Does the C-implementation require both external IP address and port for the KDF? + +**Yes!** + +In the C reference implementation, **both IP address and port are included in the KDF input** from both sides of the connection. + +Inside `aes_create_keys()`, the KDF input explicitly contains: + +- `server_ip + client_port` +- `client_ip + server_port` +- followed by shared secret / nonces + +For IPv6: + +- IPv4 fields are zeroed +- IPv6 addresses are inserted + +However, **client_port and server_port remain part of the KDF regardless of IP version**. + +> If externally observed IP or port (e.g. due to NAT, SOCKS, or proxy traversal) differs from what the peer expects, the derived keys will not match and the handshake will fail. + +--- + +### Can port be excluded from KDF (e.g. by using port = 0)? + +**No!** + +The C-implementation provides **no mechanism to ignore the port**: + +- `client_port` and `server_port` are explicitly included in the KDF input +- Real socket ports are always passed: + - `c->our_port` + - `c->remote_port` + +If a port is `0`, it is still incorporated into the KDF as `0`. + +There is **no conditional logic to exclude ports** From 501d802b8ded081f5710083e82888eadd7fd3203 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 1 Mar 2026 23:39:42 +0300 Subject: [PATCH 004/261] Create MIDDLE-END-KDF.de.md --- docs/MIDDLE-END-KDF.de.md | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/MIDDLE-END-KDF.de.md diff --git a/docs/MIDDLE-END-KDF.de.md b/docs/MIDDLE-END-KDF.de.md new file mode 100644 index 0000000..6483f7f --- /dev/null +++ b/docs/MIDDLE-END-KDF.de.md @@ -0,0 +1,40 @@ +# Middle-End Proxy + +## KDF-Adressierung — Implementierungs-FAQ + +### Benötigt die C-Referenzimplementierung sowohl externe IP-Adresse als auch Port für die KDF? + +Ja. + +In der C-Referenzimplementierung werden **sowohl IP-Adresse als auch Port in die KDF einbezogen** — auf beiden Seiten der Verbindung. + +In `aes_create_keys()` enthält der KDF-Input: + +- `server_ip + client_port` +- `client_ip + server_port` +- sowie Secret / Nonces + +Für IPv6: + +- IPv4-Felder werden auf 0 gesetzt +- IPv6-Adressen werden ergänzt + +Die **Ports bleiben weiterhin Bestandteil der KDF**. + +> Wenn sich externe IP oder Port (z. B. durch NAT, SOCKS oder Proxy) von den erwarteten Werten unterscheiden, entstehen unterschiedliche Schlüssel — der Handshake schlägt fehl. + +--- + +### Kann der Port aus der KDF ausgeschlossen werden (z. B. durch Port = 0)? + +**Nein!** + +Die C-Referenzimplementierung enthält **keine Möglichkeit, den Port zu ignorieren**: +- `client_port` und `server_port` sind fester Bestandteil der KDF +- Es werden immer reale Socket-Ports übergeben: + - `c->our_port` + - `c->remote_port` + +Falls ein Port den Wert `0` hat, wird er dennoch als `0` in die KDF übernommen. + +Eine „Port-Ignore“-Logik existiert nicht. From 08609f4b6d3ccbef043cb7212c9fd54282d820d8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 1 Mar 2026 23:40:46 +0300 Subject: [PATCH 005/261] Create MIDDLE-END-KDF.ru.md --- docs/MIDDLE-END-KDF.ru.md | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/MIDDLE-END-KDF.ru.md diff --git a/docs/MIDDLE-END-KDF.ru.md b/docs/MIDDLE-END-KDF.ru.md new file mode 100644 index 0000000..7a71b93 --- /dev/null +++ b/docs/MIDDLE-END-KDF.ru.md @@ -0,0 +1,41 @@ +# Middle-End Proxy + +## KDF Addressing — FAQ по реализации + +### Требует ли C-референсная реализация KDF внешний IP и порт? + +**Да** + +В C-референсе **в KDF участвуют и IP-адрес, и порт** — с обеих сторон соединения. + +В `aes_create_keys()` в строку KDF входят: + +- `server_ip + client_port` +- `client_ip + server_port` +- далее secret / nonces + +Для IPv6: + +- IPv4-поля заполняются нулями +- добавляются IPv6-адреса + +Однако **порты client_port и server_port всё равно участвуют в KDF**. + +> Если внешний IP или порт (например, из-за NAT, SOCKS или прокси) не совпадает с ожидаемым другой стороной — ключи расходятся и handshake ломается. + +--- + +### Можно ли исключить порт из KDF (например, установив порт = 0)? + +**Нет.** + +В C-референсе **нет механики отключения порта**. + +- `client_port` и `server_port` явно включены в KDF +- Передаются реальные порты сокета: + - `c->our_port` + - `c->remote_port` + +Если порт равен `0`, он всё равно попадёт в KDF как `0`. + +Отдельной логики «игнорировать порт» не предусмотрено. From 6f1980dfd7e117a09b64f4d81392acd8dc1fedcb Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 00:17:58 +0300 Subject: [PATCH 006/261] ME Pool improvements Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 36 ++++ src/config/load.rs | 40 +++- src/config/types.rs | 78 ++++++++ src/main.rs | 47 +++-- src/transport/middle_proxy/config_updater.rs | 187 +++++++++++++------ src/transport/middle_proxy/handshake.rs | 14 +- src/transport/middle_proxy/mod.rs | 2 +- src/transport/middle_proxy/pool.rs | 65 ++++++- src/transport/middle_proxy/pool_config.rs | 40 +++- src/transport/middle_proxy/pool_init.rs | 2 +- src/transport/middle_proxy/pool_writer.rs | 27 ++- src/transport/middle_proxy/rotation.rs | 104 ++++++++++- src/transport/middle_proxy/send.rs | 34 +++- 13 files changed, 558 insertions(+), 118 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index ab087fd..0ea6692 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -277,6 +277,18 @@ pub(crate) fn default_me_reinit_every_secs() -> u64 { 15 * 60 } +pub(crate) fn default_me_reinit_singleflight() -> bool { + true +} + +pub(crate) fn default_me_reinit_trigger_channel() -> usize { + 64 +} + +pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 { + 200 +} + pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 { 1000 } @@ -301,6 +313,18 @@ pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 { 300 } +pub(crate) fn default_me_snapshot_require_http_2xx() -> bool { + true +} + +pub(crate) fn default_me_snapshot_reject_empty_map() -> bool { + true +} + +pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 { + 1 +} + pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 { 2 } @@ -309,6 +333,10 @@ pub(crate) fn default_proxy_secret_rotate_runtime() -> bool { true } +pub(crate) fn default_me_secret_atomic_snapshot() -> bool { + true +} + pub(crate) fn default_proxy_secret_len_max() -> usize { 256 } @@ -321,10 +349,18 @@ pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 { 90 } +pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 { + default_me_pool_drain_ttl_secs() +} + pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 { 0.8 } +pub(crate) fn default_me_deterministic_writer_sort() -> bool { + true +} + pub(crate) fn default_hardswap() -> bool { true } diff --git a/src/config/load.rs b/src/config/load.rs index f37791d..17545b9 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -305,12 +305,24 @@ impl ProxyConfig { )); } + if config.general.me_snapshot_min_proxy_for_lines == 0 { + return Err(ProxyError::Config( + "general.me_snapshot_min_proxy_for_lines must be > 0".to_string(), + )); + } + if config.general.proxy_secret_stable_snapshots == 0 { return Err(ProxyError::Config( "general.proxy_secret_stable_snapshots must be > 0".to_string(), )); } + if config.general.me_reinit_trigger_channel == 0 { + return Err(ProxyError::Config( + "general.me_reinit_trigger_channel must be > 0".to_string(), + )); + } + if !(32..=4096).contains(&config.general.proxy_secret_len_max) { return Err(ProxyError::Config( "general.proxy_secret_len_max must be within [32, 4096]".to_string(), @@ -535,9 +547,10 @@ impl ProxyConfig { for (user, tag) in &self.access.user_ad_tags { let zeros = "00000000000000000000000000000000"; if !is_valid_ad_tag(tag) { - return Err(ProxyError::Config( - "general.ad_tag must be exactly 32 hex characters".to_string(), - )); + return Err(ProxyError::Config(format!( + "access.user_ad_tags['{}'] must be exactly 32 hex characters", + user + ))); } if tag == zeros { warn!(user = %user, "user ad_tag is all zeros; register a valid proxy tag via @MTProxybot to enable sponsored channel"); @@ -1100,6 +1113,27 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn invalid_user_ad_tag_reports_access_user_ad_tags_key() { + let toml = r#" + [censorship] + tls_domain = "example.com" + + [access.users] + alice = "00000000000000000000000000000000" + + [access.user_ad_tags] + alice = "not_hex" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_invalid_user_ad_tag_message_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + let err = cfg.validate().unwrap_err().to_string(); + assert!(err.contains("access.user_ad_tags['alice'] must be exactly 32 hex characters")); + let _ = std::fs::remove_file(path); + } + #[test] fn invalid_dns_override_is_rejected() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index 716d78f..d57c890 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -130,6 +130,34 @@ impl MeSocksKdfPolicy { } } +/// Stale ME writer bind policy during drain window. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum MeBindStaleMode { + Never, + #[default] + Ttl, + Always, +} + +impl MeBindStaleMode { + pub fn as_u8(self) -> u8 { + match self { + MeBindStaleMode::Never => 0, + MeBindStaleMode::Ttl => 1, + MeBindStaleMode::Always => 2, + } + } + + pub fn from_u8(raw: u8) -> Self { + match raw { + 0 => MeBindStaleMode::Never, + 2 => MeBindStaleMode::Always, + _ => MeBindStaleMode::Ttl, + } + } +} + /// Telemetry controls for hot-path counters and ME diagnostics. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TelemetryConfig { @@ -454,6 +482,18 @@ pub struct GeneralConfig { #[serde(default = "default_me_config_apply_cooldown_secs")] pub me_config_apply_cooldown_secs: u64, + /// Ensure getProxyConfig snapshots are applied only for 2xx HTTP responses. + #[serde(default = "default_me_snapshot_require_http_2xx")] + pub me_snapshot_require_http_2xx: bool, + + /// Reject empty getProxyConfig snapshots instead of marking them applied. + #[serde(default = "default_me_snapshot_reject_empty_map")] + pub me_snapshot_reject_empty_map: bool, + + /// Minimum parsed `proxy_for` rows required to accept a snapshot. + #[serde(default = "default_me_snapshot_min_proxy_for_lines")] + pub me_snapshot_min_proxy_for_lines: u32, + /// Number of identical getProxySecret snapshots required before runtime secret rotation. #[serde(default = "default_proxy_secret_stable_snapshots")] pub proxy_secret_stable_snapshots: u8, @@ -462,6 +502,10 @@ pub struct GeneralConfig { #[serde(default = "default_proxy_secret_rotate_runtime")] pub proxy_secret_rotate_runtime: bool, + /// Keep key-selector and secret bytes from one snapshot during ME handshake. + #[serde(default = "default_me_secret_atomic_snapshot")] + pub me_secret_atomic_snapshot: bool, + /// Maximum allowed proxy-secret length in bytes for startup and runtime refresh. #[serde(default = "default_proxy_secret_len_max")] pub proxy_secret_len_max: usize, @@ -471,6 +515,14 @@ pub struct GeneralConfig { #[serde(default = "default_me_pool_drain_ttl_secs")] pub me_pool_drain_ttl_secs: u64, + /// Policy for new binds on stale draining writers. + #[serde(default)] + pub me_bind_stale_mode: MeBindStaleMode, + + /// TTL for stale bind allowance when `me_bind_stale_mode = \"ttl\"`. + #[serde(default = "default_me_bind_stale_ttl_secs")] + pub me_bind_stale_ttl_secs: u64, + /// Minimum desired-DC coverage ratio required before draining stale writers. /// Range: 0.0..=1.0. #[serde(default = "default_me_pool_min_fresh_ratio")] @@ -491,6 +543,22 @@ pub struct GeneralConfig { #[serde(default = "default_proxy_config_reload_secs")] pub proxy_config_auto_reload_secs: u64, + /// Serialize ME reinit cycles across all trigger sources. + #[serde(default = "default_me_reinit_singleflight")] + pub me_reinit_singleflight: bool, + + /// Trigger queue capacity for reinit scheduler. + #[serde(default = "default_me_reinit_trigger_channel")] + pub me_reinit_trigger_channel: usize, + + /// Trigger coalescing window before starting a reinit cycle. + #[serde(default = "default_me_reinit_coalesce_window_ms")] + pub me_reinit_coalesce_window_ms: u64, + + /// Deterministic candidate sort for ME writer binding path. + #[serde(default = "default_me_deterministic_writer_sort")] + pub me_deterministic_writer_sort: bool, + /// Enable NTP drift check at startup. #[serde(default = "default_ntp_check")] pub ntp_check: bool, @@ -565,14 +633,24 @@ impl Default for GeneralConfig { me_hardswap_warmup_pass_backoff_base_ms: default_me_hardswap_warmup_pass_backoff_base_ms(), me_config_stable_snapshots: default_me_config_stable_snapshots(), me_config_apply_cooldown_secs: default_me_config_apply_cooldown_secs(), + me_snapshot_require_http_2xx: default_me_snapshot_require_http_2xx(), + me_snapshot_reject_empty_map: default_me_snapshot_reject_empty_map(), + me_snapshot_min_proxy_for_lines: default_me_snapshot_min_proxy_for_lines(), proxy_secret_stable_snapshots: default_proxy_secret_stable_snapshots(), proxy_secret_rotate_runtime: default_proxy_secret_rotate_runtime(), + me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(), proxy_secret_len_max: default_proxy_secret_len_max(), me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(), + me_bind_stale_mode: MeBindStaleMode::default(), + me_bind_stale_ttl_secs: default_me_bind_stale_ttl_secs(), me_pool_min_fresh_ratio: default_me_pool_min_fresh_ratio(), me_reinit_drain_timeout_secs: default_me_reinit_drain_timeout_secs(), proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(), proxy_config_auto_reload_secs: default_proxy_config_reload_secs(), + me_reinit_singleflight: default_me_reinit_singleflight(), + me_reinit_trigger_channel: default_me_reinit_trigger_channel(), + me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(), + me_deterministic_writer_sort: default_me_deterministic_writer_sort(), ntp_check: default_ntp_check(), ntp_servers: default_ntp_servers(), auto_degradation_enabled: default_true(), diff --git a/src/main.rs b/src/main.rs index b910c64..03998cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use std::time::Duration; use rand::Rng; use tokio::net::TcpListener; use tokio::signal; -use tokio::sync::Semaphore; +use tokio::sync::{Semaphore, mpsc}; use tracing::{debug, error, info, warn}; use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; #[cfg(unix)] @@ -40,7 +40,7 @@ use crate::stats::telemetry::TelemetryPolicy; use crate::stats::{ReplayChecker, Stats}; use crate::stream::BufferPool; use crate::transport::middle_proxy::{ - MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, format_sample_line, + MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, MeReinitTrigger, format_sample_line, format_me_route, }; use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes}; @@ -546,6 +546,10 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_hardswap_warmup_delay_max_ms, config.general.me_hardswap_warmup_extra_passes, config.general.me_hardswap_warmup_pass_backoff_base_ms, + config.general.me_bind_stale_mode, + config.general.me_bind_stale_ttl_secs, + config.general.me_secret_atomic_snapshot, + config.general.me_deterministic_writer_sort, config.general.me_socks_kdf_policy, config.general.me_route_backpressure_base_timeout_ms, config.general.me_route_backpressure_high_timeout_ms, @@ -849,26 +853,43 @@ async fn main() -> std::result::Result<(), Box> { }); if let Some(ref pool) = me_pool { - let pool_clone = pool.clone(); - let rng_clone = rng.clone(); - let config_rx_clone = config_rx.clone(); + let reinit_trigger_capacity = config + .general + .me_reinit_trigger_channel + .max(1); + let (reinit_tx, reinit_rx) = mpsc::channel::(reinit_trigger_capacity); + + let pool_clone_sched = pool.clone(); + let rng_clone_sched = rng.clone(); + let config_rx_clone_sched = config_rx.clone(); tokio::spawn(async move { - crate::transport::middle_proxy::me_config_updater( - pool_clone, - rng_clone, - config_rx_clone, + crate::transport::middle_proxy::me_reinit_scheduler( + pool_clone_sched, + rng_clone_sched, + config_rx_clone_sched, + reinit_rx, + ) + .await; + }); + + let pool_clone = pool.clone(); + let config_rx_clone = config_rx.clone(); + let reinit_tx_updater = reinit_tx.clone(); + tokio::spawn(async move { + crate::transport::middle_proxy::me_config_updater( + pool_clone, + config_rx_clone, + reinit_tx_updater, ) .await; }); - let pool_clone_rot = pool.clone(); - let rng_clone_rot = rng.clone(); let config_rx_clone_rot = config_rx.clone(); + let reinit_tx_rotation = reinit_tx.clone(); tokio::spawn(async move { crate::transport::middle_proxy::me_rotation_task( - pool_clone_rot, - rng_clone_rot, config_rx_clone_rot, + reinit_tx_rotation, ) .await; }); diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 4e8e63f..2772b27 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -5,15 +5,15 @@ use std::sync::Arc; use std::time::Duration; use httpdate; -use tokio::sync::watch; +use tokio::sync::{mpsc, watch}; use tracing::{debug, info, warn}; use crate::config::ProxyConfig; use crate::error::Result; use super::MePool; +use super::rotation::{MeReinitTrigger, enqueue_reinit_trigger}; use super::secret::download_proxy_secret_with_max_len; -use crate::crypto::SecureRandom; use std::time::SystemTime; async fn retry_fetch(url: &str) -> Option { @@ -38,6 +38,8 @@ async fn retry_fetch(url: &str) -> Option { pub struct ProxyConfigData { pub map: HashMap>, pub default_dc: Option, + pub http_status: u16, + pub proxy_for_lines: u32, } #[derive(Debug, Default)] @@ -172,6 +174,7 @@ pub async fn fetch_proxy_config(url: &str) -> Result { .await .map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config GET failed: {e}")))? ; + let http_status = resp.status().as_u16(); if let Some(date) = resp.headers().get(reqwest::header::DATE) && let Ok(date_str) = date.to_str() @@ -194,9 +197,11 @@ pub async fn fetch_proxy_config(url: &str) -> Result { .map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config read failed: {e}")))?; let mut map: HashMap> = HashMap::new(); + let mut proxy_for_lines: u32 = 0; for line in text.lines() { if let Some((dc, ip, port)) = parse_proxy_line(line) { map.entry(dc).or_default().push((ip, port)); + proxy_for_lines = proxy_for_lines.saturating_add(1); } } @@ -214,14 +219,49 @@ pub async fn fetch_proxy_config(url: &str) -> Result { None }); - Ok(ProxyConfigData { map, default_dc }) + Ok(ProxyConfigData { + map, + default_dc, + http_status, + proxy_for_lines, + }) +} + +fn snapshot_passes_guards( + cfg: &ProxyConfig, + snapshot: &ProxyConfigData, + snapshot_name: &'static str, +) -> bool { + if cfg.general.me_snapshot_require_http_2xx + && !(200..=299).contains(&snapshot.http_status) + { + warn!( + snapshot = snapshot_name, + http_status = snapshot.http_status, + "ME snapshot rejected by non-2xx HTTP status" + ); + return false; + } + + let min_proxy_for = cfg.general.me_snapshot_min_proxy_for_lines; + if snapshot.proxy_for_lines < min_proxy_for { + warn!( + snapshot = snapshot_name, + parsed_proxy_for_lines = snapshot.proxy_for_lines, + min_proxy_for_lines = min_proxy_for, + "ME snapshot rejected by proxy_for line floor" + ); + return false; + } + + true } async fn run_update_cycle( pool: &Arc, - rng: &Arc, cfg: &ProxyConfig, state: &mut UpdaterState, + reinit_tx: &mpsc::Sender, ) { pool.update_runtime_reinit_policy( cfg.general.hardswap, @@ -232,6 +272,10 @@ async fn run_update_cycle( cfg.general.me_hardswap_warmup_delay_max_ms, cfg.general.me_hardswap_warmup_extra_passes, cfg.general.me_hardswap_warmup_pass_backoff_base_ms, + cfg.general.me_bind_stale_mode, + cfg.general.me_bind_stale_ttl_secs, + cfg.general.me_secret_atomic_snapshot, + cfg.general.me_deterministic_writer_sort, ); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); @@ -242,44 +286,48 @@ async fn run_update_cycle( let mut ready_v4: Option<(ProxyConfigData, u64)> = None; let cfg_v4 = retry_fetch("https://core.telegram.org/getProxyConfig").await; if let Some(cfg_v4) = cfg_v4 { - let cfg_v4_hash = hash_proxy_config(&cfg_v4); - let stable_hits = state.config_v4.observe(cfg_v4_hash); - if stable_hits < required_cfg_snapshots { - debug!( - stable_hits, - required_cfg_snapshots, - snapshot = format_args!("0x{cfg_v4_hash:016x}"), - "ME config v4 candidate observed" - ); - } else if state.config_v4.is_applied(cfg_v4_hash) { - debug!( - snapshot = format_args!("0x{cfg_v4_hash:016x}"), - "ME config v4 stable snapshot already applied" - ); - } else { - ready_v4 = Some((cfg_v4, cfg_v4_hash)); + if snapshot_passes_guards(cfg, &cfg_v4, "getProxyConfig") { + let cfg_v4_hash = hash_proxy_config(&cfg_v4); + let stable_hits = state.config_v4.observe(cfg_v4_hash); + if stable_hits < required_cfg_snapshots { + debug!( + stable_hits, + required_cfg_snapshots, + snapshot = format_args!("0x{cfg_v4_hash:016x}"), + "ME config v4 candidate observed" + ); + } else if state.config_v4.is_applied(cfg_v4_hash) { + debug!( + snapshot = format_args!("0x{cfg_v4_hash:016x}"), + "ME config v4 stable snapshot already applied" + ); + } else { + ready_v4 = Some((cfg_v4, cfg_v4_hash)); + } } } let mut ready_v6: Option<(ProxyConfigData, u64)> = None; let cfg_v6 = retry_fetch("https://core.telegram.org/getProxyConfigV6").await; if let Some(cfg_v6) = cfg_v6 { - let cfg_v6_hash = hash_proxy_config(&cfg_v6); - let stable_hits = state.config_v6.observe(cfg_v6_hash); - if stable_hits < required_cfg_snapshots { - debug!( - stable_hits, - required_cfg_snapshots, - snapshot = format_args!("0x{cfg_v6_hash:016x}"), - "ME config v6 candidate observed" - ); - } else if state.config_v6.is_applied(cfg_v6_hash) { - debug!( - snapshot = format_args!("0x{cfg_v6_hash:016x}"), - "ME config v6 stable snapshot already applied" - ); - } else { - ready_v6 = Some((cfg_v6, cfg_v6_hash)); + if snapshot_passes_guards(cfg, &cfg_v6, "getProxyConfigV6") { + let cfg_v6_hash = hash_proxy_config(&cfg_v6); + let stable_hits = state.config_v6.observe(cfg_v6_hash); + if stable_hits < required_cfg_snapshots { + debug!( + stable_hits, + required_cfg_snapshots, + snapshot = format_args!("0x{cfg_v6_hash:016x}"), + "ME config v6 candidate observed" + ); + } else if state.config_v6.is_applied(cfg_v6_hash) { + debug!( + snapshot = format_args!("0x{cfg_v6_hash:016x}"), + "ME config v6 stable snapshot already applied" + ); + } else { + ready_v6 = Some((cfg_v6, cfg_v6_hash)); + } } } @@ -292,28 +340,40 @@ async fn run_update_cycle( let update_v6 = ready_v6 .as_ref() .map(|(snapshot, _)| snapshot.map.clone()); - - let changed = pool.update_proxy_maps(update_v4, update_v6).await; - - if let Some((snapshot, hash)) = ready_v4 { - if let Some(dc) = snapshot.default_dc { - pool.default_dc - .store(dc, std::sync::atomic::Ordering::Relaxed); - } - state.config_v4.mark_applied(hash); - } - - if let Some((_snapshot, hash)) = ready_v6 { - state.config_v6.mark_applied(hash); - } - - state.last_map_apply_at = Some(tokio::time::Instant::now()); - - if changed { - maps_changed = true; - info!("ME config update applied after stable-gate"); + let update_is_empty = + update_v4.is_empty() && update_v6.as_ref().is_none_or(|v| v.is_empty()); + let apply_outcome = if update_is_empty && !cfg.general.me_snapshot_reject_empty_map { + super::pool_config::SnapshotApplyOutcome::AppliedNoDelta } else { - debug!("ME config stable-gate applied with no map delta"); + pool.update_proxy_maps(update_v4, update_v6).await + }; + + if matches!( + apply_outcome, + super::pool_config::SnapshotApplyOutcome::RejectedEmpty + ) { + warn!("ME config stable snapshot rejected (empty endpoint map)"); + } else { + if let Some((snapshot, hash)) = ready_v4 { + if let Some(dc) = snapshot.default_dc { + pool.default_dc + .store(dc, std::sync::atomic::Ordering::Relaxed); + } + state.config_v4.mark_applied(hash); + } + + if let Some((_snapshot, hash)) = ready_v6 { + state.config_v6.mark_applied(hash); + } + + state.last_map_apply_at = Some(tokio::time::Instant::now()); + + if apply_outcome.changed() { + maps_changed = true; + info!("ME config update applied after stable-gate"); + } else { + debug!("ME config stable-gate applied with no map delta"); + } } } else if let Some(last) = state.last_map_apply_at { let wait_secs = map_apply_cooldown_remaining_secs(last, apply_cooldown); @@ -325,8 +385,7 @@ async fn run_update_cycle( } if maps_changed { - pool.zero_downtime_reinit_after_map_change(rng.as_ref()) - .await; + enqueue_reinit_trigger(reinit_tx, MeReinitTrigger::MapChanged); } pool.reset_stun_state(); @@ -367,8 +426,8 @@ async fn run_update_cycle( pub async fn me_config_updater( pool: Arc, - rng: Arc, mut config_rx: watch::Receiver>, + reinit_tx: mpsc::Sender, ) { let mut state = UpdaterState::default(); let mut update_every_secs = config_rx @@ -387,7 +446,7 @@ pub async fn me_config_updater( tokio::select! { _ = &mut sleep => { let cfg = config_rx.borrow().clone(); - run_update_cycle(&pool, &rng, cfg.as_ref(), &mut state).await; + run_update_cycle(&pool, cfg.as_ref(), &mut state, &reinit_tx).await; let refreshed_secs = cfg.general.effective_update_every_secs().max(1); if refreshed_secs != update_every_secs { info!( @@ -415,6 +474,10 @@ pub async fn me_config_updater( cfg.general.me_hardswap_warmup_delay_max_ms, cfg.general.me_hardswap_warmup_extra_passes, cfg.general.me_hardswap_warmup_pass_backoff_base_ms, + cfg.general.me_bind_stale_mode, + cfg.general.me_bind_stale_ttl_secs, + cfg.general.me_secret_atomic_snapshot, + cfg.general.me_deterministic_writer_sort, ); let new_secs = cfg.general.effective_update_every_secs().max(1); if new_secs == update_every_secs { @@ -429,7 +492,7 @@ pub async fn me_config_updater( ); update_every_secs = new_secs; update_every = Duration::from_secs(update_every_secs); - run_update_cycle(&pool, &rng, cfg.as_ref(), &mut state).await; + run_update_cycle(&pool, cfg.as_ref(), &mut state, &reinit_tx).await; next_tick = tokio::time::Instant::now() + update_every; } else { info!( diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 384ecc9..5daa460 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -1,4 +1,5 @@ use std::net::{IpAddr, SocketAddr}; +use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use socket2::{SockRef, TcpKeepalive}; #[cfg(target_os = "linux")] @@ -267,7 +268,16 @@ impl MePool { .unwrap_or_default() .as_secs() as u32; - let ks = self.key_selector().await; + let secret_atomic_snapshot = self.secret_atomic_snapshot.load(Ordering::Relaxed); + let (ks, secret) = if secret_atomic_snapshot { + let snapshot = self.secret_snapshot().await; + (snapshot.key_selector, snapshot.secret) + } else { + // Backward-compatible mode: key selector and secret may come from different updates. + let key_selector = self.key_selector().await; + let secret = self.secret_snapshot().await.secret; + (key_selector, secret) + }; let nonce_payload = build_nonce_payload(ks, crypto_ts, &my_nonce); let nonce_frame = build_rpc_frame(-2, &nonce_payload, RpcChecksumMode::Crc32); let dump = hex_dump(&nonce_frame[..nonce_frame.len().min(44)]); @@ -357,8 +367,6 @@ impl MePool { let diag_level: u8 = std::env::var("ME_DIAG").ok().and_then(|v| v.parse().ok()).unwrap_or(0); - let secret: Vec = self.proxy_secret.read().await.clone(); - let prekey_client = build_middleproxy_prekey( &srv_nonce, &my_nonce, diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 1072ec8..26c58a6 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -30,7 +30,7 @@ pub use pool_nat::{stun_probe, detect_public_ip}; pub use registry::ConnRegistry; pub use secret::fetch_proxy_secret; pub use config_updater::{fetch_proxy_config, me_config_updater}; -pub use rotation::me_rotation_task; +pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task}; pub use wire::proto_flags_for_tag; #[derive(Debug)] diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index f67b2a8..d87430a 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, Notify, RwLock, mpsc}; use tokio_util::sync::CancellationToken; -use crate::config::MeSocksKdfPolicy; +use crate::config::{MeBindStaleMode, MeSocksKdfPolicy}; use crate::crypto::SecureRandom; use crate::network::IpFamily; use crate::network::probe::NetworkDecision; @@ -29,6 +29,13 @@ pub struct MeWriter { pub allow_drain_fallback: Arc, } +#[derive(Debug, Clone)] +pub struct SecretSnapshot { + pub epoch: u64, + pub key_selector: u32, + pub secret: Vec, +} + #[allow(dead_code)] pub struct MePool { pub(super) registry: Arc, @@ -38,7 +45,7 @@ pub struct MePool { pub(super) upstream: Option>, pub(super) rng: Arc, pub(super) proxy_tag: Option>, - pub(super) proxy_secret: Arc>>, + pub(super) proxy_secret: Arc>, pub(super) nat_ip_cfg: Option, pub(super) nat_ip_detected: Arc>>, pub(super) nat_probe: bool, @@ -83,6 +90,10 @@ pub struct MePool { pub(super) me_hardswap_warmup_delay_max_ms: AtomicU64, pub(super) me_hardswap_warmup_extra_passes: AtomicU32, pub(super) me_hardswap_warmup_pass_backoff_base_ms: AtomicU64, + pub(super) me_bind_stale_mode: AtomicU8, + pub(super) me_bind_stale_ttl_secs: AtomicU64, + pub(super) secret_atomic_snapshot: AtomicBool, + pub(super) me_deterministic_writer_sort: AtomicBool, pub(super) me_socks_kdf_policy: AtomicU8, pool_size: usize, } @@ -147,6 +158,10 @@ impl MePool { me_hardswap_warmup_delay_max_ms: u64, me_hardswap_warmup_extra_passes: u8, me_hardswap_warmup_pass_backoff_base_ms: u64, + me_bind_stale_mode: MeBindStaleMode, + me_bind_stale_ttl_secs: u64, + me_secret_atomic_snapshot: bool, + me_deterministic_writer_sort: bool, me_socks_kdf_policy: MeSocksKdfPolicy, me_route_backpressure_base_timeout_ms: u64, me_route_backpressure_high_timeout_ms: u64, @@ -166,7 +181,20 @@ impl MePool { upstream, rng, proxy_tag, - proxy_secret: Arc::new(RwLock::new(proxy_secret)), + proxy_secret: Arc::new(RwLock::new(SecretSnapshot { + epoch: 1, + key_selector: if proxy_secret.len() >= 4 { + u32::from_le_bytes([ + proxy_secret[0], + proxy_secret[1], + proxy_secret[2], + proxy_secret[3], + ]) + } else { + 0 + }, + secret: proxy_secret, + })), nat_ip_cfg: nat_ip, nat_ip_detected: Arc::new(RwLock::new(None)), nat_probe, @@ -216,6 +244,10 @@ impl MePool { me_hardswap_warmup_pass_backoff_base_ms: AtomicU64::new( me_hardswap_warmup_pass_backoff_base_ms, ), + me_bind_stale_mode: AtomicU8::new(me_bind_stale_mode.as_u8()), + me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs), + secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot), + me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort), me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()), }) } @@ -238,6 +270,10 @@ impl MePool { hardswap_warmup_delay_max_ms: u64, hardswap_warmup_extra_passes: u8, hardswap_warmup_pass_backoff_base_ms: u64, + bind_stale_mode: MeBindStaleMode, + bind_stale_ttl_secs: u64, + secret_atomic_snapshot: bool, + deterministic_writer_sort: bool, ) { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs @@ -254,6 +290,14 @@ impl MePool { .store(hardswap_warmup_extra_passes as u32, Ordering::Relaxed); self.me_hardswap_warmup_pass_backoff_base_ms .store(hardswap_warmup_pass_backoff_base_ms, Ordering::Relaxed); + self.me_bind_stale_mode + .store(bind_stale_mode.as_u8(), Ordering::Relaxed); + self.me_bind_stale_ttl_secs + .store(bind_stale_ttl_secs, Ordering::Relaxed); + self.secret_atomic_snapshot + .store(secret_atomic_snapshot, Ordering::Relaxed); + self.me_deterministic_writer_sort + .store(deterministic_writer_sort, Ordering::Relaxed); } pub fn reset_stun_state(&self) { @@ -307,12 +351,15 @@ impl MePool { } pub(super) async fn key_selector(&self) -> u32 { - let secret = self.proxy_secret.read().await; - if secret.len() >= 4 { - u32::from_le_bytes([secret[0], secret[1], secret[2], secret[3]]) - } else { - 0 - } + self.proxy_secret.read().await.key_selector + } + + pub(super) async fn secret_snapshot(&self) -> SecretSnapshot { + self.proxy_secret.read().await.clone() + } + + pub(super) fn bind_stale_mode(&self) -> MeBindStaleMode { + MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed)) } pub(super) fn family_order(&self) -> Vec { diff --git a/src/transport/middle_proxy/pool_config.rs b/src/transport/middle_proxy/pool_config.rs index fe2aad8..04e3bb5 100644 --- a/src/transport/middle_proxy/pool_config.rs +++ b/src/transport/middle_proxy/pool_config.rs @@ -7,12 +7,29 @@ use tracing::warn; use super::pool::MePool; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SnapshotApplyOutcome { + AppliedChanged, + AppliedNoDelta, + RejectedEmpty, +} + +impl SnapshotApplyOutcome { + pub fn changed(self) -> bool { + matches!(self, SnapshotApplyOutcome::AppliedChanged) + } +} + impl MePool { pub async fn update_proxy_maps( &self, new_v4: HashMap>, new_v6: Option>>, - ) -> bool { + ) -> SnapshotApplyOutcome { + if new_v4.is_empty() && new_v6.as_ref().is_none_or(|v| v.is_empty()) { + return SnapshotApplyOutcome::RejectedEmpty; + } + let mut changed = false; { let mut guard = self.proxy_map_v4.write().await; @@ -51,7 +68,11 @@ impl MePool { } } } - changed + if changed { + SnapshotApplyOutcome::AppliedChanged + } else { + SnapshotApplyOutcome::AppliedNoDelta + } } pub async fn update_secret(self: &Arc, new_secret: Vec) -> bool { @@ -60,8 +81,19 @@ impl MePool { return false; } let mut guard = self.proxy_secret.write().await; - if *guard != new_secret { - *guard = new_secret; + if guard.secret != new_secret { + guard.secret = new_secret; + guard.key_selector = if guard.secret.len() >= 4 { + u32::from_le_bytes([ + guard.secret[0], + guard.secret[1], + guard.secret[2], + guard.secret[3], + ]) + } else { + 0 + }; + guard.epoch = guard.epoch.saturating_add(1); drop(guard); self.reconnect_all().await; return true; diff --git a/src/transport/middle_proxy/pool_init.rs b/src/transport/middle_proxy/pool_init.rs index 623be7f..fef1553 100644 --- a/src/transport/middle_proxy/pool_init.rs +++ b/src/transport/middle_proxy/pool_init.rs @@ -19,7 +19,7 @@ impl MePool { me_servers = self.proxy_map_v4.read().await.len(), pool_size, key_selector = format_args!("0x{ks:08x}"), - secret_len = self.proxy_secret.read().await.len(), + secret_len = self.proxy_secret.read().await.secret.len(), "Initializing ME pool" ); diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 28f5538..a8cc5a5 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -9,6 +9,7 @@ use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; +use crate::config::MeBindStaleMode; use crate::crypto::SecureRandom; use crate::error::{ProxyError, Result}; use crate::protocol::constants::RPC_PING_U32; @@ -42,7 +43,7 @@ impl MePool { } pub(crate) async fn connect_one(self: &Arc, addr: SocketAddr, rng: &SecureRandom) -> Result<()> { - let secret_len = self.proxy_secret.read().await.len(); + let secret_len = self.proxy_secret.read().await.secret.len(); if secret_len < 32 { return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into())); } @@ -351,16 +352,22 @@ impl MePool { return false; } - let ttl_secs = self.me_pool_drain_ttl_secs.load(Ordering::Relaxed); - if ttl_secs == 0 { - return true; - } + match self.bind_stale_mode() { + MeBindStaleMode::Never => false, + MeBindStaleMode::Always => true, + MeBindStaleMode::Ttl => { + let ttl_secs = self.me_bind_stale_ttl_secs.load(Ordering::Relaxed); + if ttl_secs == 0 { + return true; + } - let started = writer.draining_started_at_epoch_secs.load(Ordering::Relaxed); - if started == 0 { - return false; - } + let started = writer.draining_started_at_epoch_secs.load(Ordering::Relaxed); + if started == 0 { + return false; + } - Self::now_epoch_secs().saturating_sub(started) <= ttl_secs + Self::now_epoch_secs().saturating_sub(started) <= ttl_secs + } + } } } diff --git a/src/transport/middle_proxy/rotation.rs b/src/transport/middle_proxy/rotation.rs index cf5f70d..16232c9 100644 --- a/src/transport/middle_proxy/rotation.rs +++ b/src/transport/middle_proxy/rotation.rs @@ -1,19 +1,111 @@ use std::sync::Arc; use std::time::Duration; -use tokio::sync::watch; -use tracing::{info, warn}; +use tokio::sync::{mpsc, watch}; +use tracing::{debug, info, warn}; use crate::config::ProxyConfig; use crate::crypto::SecureRandom; use super::MePool; -/// Periodically reinitialize ME generations and swap them after full warmup. -pub async fn me_rotation_task( +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MeReinitTrigger { + Periodic, + MapChanged, +} + +impl MeReinitTrigger { + fn as_str(self) -> &'static str { + match self { + MeReinitTrigger::Periodic => "periodic", + MeReinitTrigger::MapChanged => "map-change", + } + } +} + +pub fn enqueue_reinit_trigger( + tx: &mpsc::Sender, + trigger: MeReinitTrigger, +) { + match tx.try_send(trigger) { + Ok(()) => {} + Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + debug!(trigger = trigger.as_str(), "ME reinit trigger dropped (queue full)"); + } + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + warn!(trigger = trigger.as_str(), "ME reinit trigger dropped (scheduler closed)"); + } + } +} + +pub async fn me_reinit_scheduler( pool: Arc, rng: Arc, + config_rx: watch::Receiver>, + mut trigger_rx: mpsc::Receiver, +) { + info!("ME reinit scheduler started"); + loop { + let Some(first_trigger) = trigger_rx.recv().await else { + warn!("ME reinit scheduler stopped: trigger channel closed"); + break; + }; + + let mut map_change_seen = matches!(first_trigger, MeReinitTrigger::MapChanged); + let mut periodic_seen = matches!(first_trigger, MeReinitTrigger::Periodic); + let cfg = config_rx.borrow().clone(); + let coalesce_window = Duration::from_millis(cfg.general.me_reinit_coalesce_window_ms); + if !coalesce_window.is_zero() { + let deadline = tokio::time::Instant::now() + coalesce_window; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + break; + } + match tokio::time::timeout(deadline - now, trigger_rx.recv()).await { + Ok(Some(next)) => { + if next == MeReinitTrigger::MapChanged { + map_change_seen = true; + } else { + periodic_seen = true; + } + } + Ok(None) => break, + Err(_) => break, + } + } + } + + let reason = if map_change_seen && periodic_seen { + "map-change+periodic" + } else if map_change_seen { + "map-change" + } else { + "periodic" + }; + + if cfg.general.me_reinit_singleflight { + debug!(reason, "ME reinit scheduled (single-flight)"); + pool.zero_downtime_reinit_periodic(rng.as_ref()).await; + } else { + debug!(reason, "ME reinit scheduled (concurrent mode)"); + let pool_clone = pool.clone(); + let rng_clone = rng.clone(); + tokio::spawn(async move { + pool_clone + .zero_downtime_reinit_periodic(rng_clone.as_ref()) + .await; + }); + } + + } +} + +/// Periodically enqueue reinitialization triggers for ME generations. +pub async fn me_rotation_task( mut config_rx: watch::Receiver>, + reinit_tx: mpsc::Sender, ) { let mut interval_secs = config_rx .borrow() @@ -31,7 +123,7 @@ pub async fn me_rotation_task( tokio::select! { _ = &mut sleep => { - pool.zero_downtime_reinit_periodic(rng.as_ref()).await; + enqueue_reinit_trigger(&reinit_tx, MeReinitTrigger::Periodic); let refreshed_secs = config_rx .borrow() .general @@ -70,7 +162,7 @@ pub async fn me_rotation_task( ); interval_secs = new_secs; interval = Duration::from_secs(interval_secs); - pool.zero_downtime_reinit_periodic(rng.as_ref()).await; + enqueue_reinit_trigger(&reinit_tx, MeReinitTrigger::Periodic); next_tick = tokio::time::Instant::now() + interval; } else { info!( diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 65bc43a..25b8852 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -138,12 +138,34 @@ impl MePool { } } - candidate_indices.sort_by_key(|idx| { - let w = &writers_snapshot[*idx]; - let degraded = w.degraded.load(Ordering::Relaxed); - let stale = (w.generation < self.current_generation()) as usize; - (stale, degraded as usize, Reverse(w.tx.capacity())) - }); + if self.me_deterministic_writer_sort.load(Ordering::Relaxed) { + candidate_indices.sort_by(|lhs, rhs| { + let left = &writers_snapshot[*lhs]; + let right = &writers_snapshot[*rhs]; + let left_key = ( + (left.generation < self.current_generation()) as usize, + left.degraded.load(Ordering::Relaxed) as usize, + Reverse(left.tx.capacity()), + left.addr, + left.id, + ); + let right_key = ( + (right.generation < self.current_generation()) as usize, + right.degraded.load(Ordering::Relaxed) as usize, + Reverse(right.tx.capacity()), + right.addr, + right.id, + ); + left_key.cmp(&right_key) + }); + } else { + candidate_indices.sort_by_key(|idx| { + let w = &writers_snapshot[*idx]; + let degraded = w.degraded.load(Ordering::Relaxed); + let stale = (w.generation < self.current_generation()) as usize; + (stale, degraded as usize, Reverse(w.tx.capacity())) + }); + } let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len(); let mut fallback_blocking_idx: Option = None; From 49f4a7bb2218e0c8fc55fdae2624491c784f37cb Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 00:39:18 +0300 Subject: [PATCH 007/261] ME Hardswap Generation stability Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 13 +- src/transport/middle_proxy/pool.rs | 5 + src/transport/middle_proxy/pool_refill.rs | 137 ++++++++++++++++++---- src/transport/middle_proxy/pool_reinit.rs | 26 +++- src/transport/middle_proxy/pool_writer.rs | 6 + src/transport/middle_proxy/reader.rs | 6 +- 6 files changed, 165 insertions(+), 28 deletions(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 06cca03..afa96c6 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -112,7 +112,18 @@ async fn check_family( let max_concurrent = pool.me_reconnect_max_concurrent_per_dc.max(1) as usize; if *inflight.get(&key).unwrap_or(&0) >= max_concurrent { - return; + continue; + } + if pool.has_refill_inflight_for_endpoints(&endpoints).await { + debug!( + dc = %dc, + ?family, + alive, + required, + endpoint_count = endpoints.len(), + "Skipping health reconnect: immediate refill is already in flight for this DC group" + ); + continue; } *inflight.entry(key).or_insert(0) += 1; diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index d87430a..c3e7533 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -21,6 +21,7 @@ pub struct MeWriter { pub id: u64, pub addr: SocketAddr, pub generation: u64, + pub created_at: Instant, pub tx: mpsc::Sender, pub cancel: CancellationToken, pub degraded: Arc, @@ -82,7 +83,9 @@ pub struct MePool { pub(super) conn_count: AtomicUsize, pub(super) stats: Arc, pub(super) generation: AtomicU64, + pub(super) pending_hardswap_generation: AtomicU64, pub(super) hardswap: AtomicBool, + pub(super) endpoint_quarantine: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, pub(super) me_pool_force_close_secs: AtomicU64, pub(super) me_pool_min_fresh_ratio_permille: AtomicU32, @@ -232,7 +235,9 @@ impl MePool { refill_inflight: Arc::new(Mutex::new(HashSet::new())), conn_count: AtomicUsize::new(0), generation: AtomicU64::new(1), + pending_hardswap_generation: AtomicU64::new(0), hardswap: AtomicBool::new(hardswap), + endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())), me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs), me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs), me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille( diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 6dea6c9..a286e65 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; @@ -9,19 +10,97 @@ use crate::crypto::SecureRandom; use super::pool::MePool; +const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20; +const ME_FLAP_QUARANTINE_SECS: u64 = 25; + impl MePool { + pub(super) async fn maybe_quarantine_flapping_endpoint( + &self, + addr: SocketAddr, + uptime: Duration, + ) { + if uptime > Duration::from_secs(ME_FLAP_UPTIME_THRESHOLD_SECS) { + return; + } + + let until = Instant::now() + Duration::from_secs(ME_FLAP_QUARANTINE_SECS); + let mut guard = self.endpoint_quarantine.lock().await; + guard.retain(|_, expiry| *expiry > Instant::now()); + guard.insert(addr, until); + warn!( + %addr, + uptime_ms = uptime.as_millis(), + quarantine_secs = ME_FLAP_QUARANTINE_SECS, + "ME endpoint temporarily quarantined due to rapid writer flap" + ); + } + + async fn is_endpoint_quarantined(&self, addr: SocketAddr) -> bool { + let mut guard = self.endpoint_quarantine.lock().await; + let now = Instant::now(); + guard.retain(|_, expiry| *expiry > now); + guard.contains_key(&addr) + } + + async fn connectable_endpoints(&self, endpoints: &[SocketAddr]) -> Vec { + if endpoints.is_empty() { + return Vec::new(); + } + + let mut guard = self.endpoint_quarantine.lock().await; + let now = Instant::now(); + guard.retain(|_, expiry| *expiry > now); + + let mut ready = Vec::::with_capacity(endpoints.len()); + let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None; + for addr in endpoints { + if let Some(expiry) = guard.get(addr).copied() { + match earliest_quarantine { + Some((_, current_expiry)) if current_expiry <= expiry => {} + _ => earliest_quarantine = Some((*addr, expiry)), + } + } else { + ready.push(*addr); + } + } + + if !ready.is_empty() { + return ready; + } + + if let Some((addr, expiry)) = earliest_quarantine { + debug!( + %addr, + wait_ms = expiry.saturating_duration_since(now).as_millis(), + "All ME endpoints are quarantined for the DC group; retrying earliest one" + ); + return vec![addr]; + } + + Vec::new() + } + + pub(super) async fn has_refill_inflight_for_endpoints(&self, endpoints: &[SocketAddr]) -> bool { + if endpoints.is_empty() { + return false; + } + let guard = self.refill_inflight.lock().await; + endpoints.iter().any(|addr| guard.contains(addr)) + } + pub(super) async fn connect_endpoints_round_robin( self: &Arc, endpoints: &[SocketAddr], rng: &SecureRandom, ) -> bool { - if endpoints.is_empty() { + let candidates = self.connectable_endpoints(endpoints).await; + if candidates.is_empty() { return false; } - let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % endpoints.len(); - for offset in 0..endpoints.len() { - let idx = (start + offset) % endpoints.len(); - let addr = endpoints[idx]; + let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len(); + for offset in 0..candidates.len() { + let idx = (start + offset) % candidates.len(); + let addr = candidates[idx]; match self.connect_one(addr, rng).await { Ok(()) => return true, Err(e) => debug!(%addr, error = %e, "ME connect failed during round-robin warmup"), @@ -83,29 +162,37 @@ impl MePool { async fn refill_writer_after_loss(self: &Arc, addr: SocketAddr) -> bool { let fast_retries = self.me_reconnect_fast_retry_count.max(1); + let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await; - for attempt in 0..fast_retries { - self.stats.increment_me_reconnect_attempt(); - match self.connect_one(addr, self.rng.as_ref()).await { - Ok(()) => { - self.stats.increment_me_reconnect_success(); - self.stats.increment_me_writer_restored_same_endpoint_total(); - info!( - %addr, - attempt = attempt + 1, - "ME writer restored on the same endpoint" - ); - return true; - } - Err(e) => { - debug!( - %addr, - attempt = attempt + 1, - error = %e, - "ME immediate same-endpoint reconnect failed" - ); + if !same_endpoint_quarantined { + for attempt in 0..fast_retries { + self.stats.increment_me_reconnect_attempt(); + match self.connect_one(addr, self.rng.as_ref()).await { + Ok(()) => { + self.stats.increment_me_reconnect_success(); + self.stats.increment_me_writer_restored_same_endpoint_total(); + info!( + %addr, + attempt = attempt + 1, + "ME writer restored on the same endpoint" + ); + return true; + } + Err(e) => { + debug!( + %addr, + attempt = attempt + 1, + error = %e, + "ME immediate same-endpoint reconnect failed" + ); + } } } + } else { + debug!( + %addr, + "Skipping immediate same-endpoint reconnect because endpoint is quarantined" + ); } let dc_endpoints = self.endpoints_for_same_dc(addr).await; diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 261ac02..5552fb6 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -266,8 +266,26 @@ impl MePool { } let previous_generation = self.current_generation(); - let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; let hardswap = self.hardswap.load(Ordering::Relaxed); + let generation = if hardswap { + let pending_generation = self.pending_hardswap_generation.load(Ordering::Relaxed); + if pending_generation != 0 && pending_generation >= previous_generation { + debug!( + previous_generation, + generation = pending_generation, + "ME hardswap continues with pending generation" + ); + pending_generation + } else { + let next_generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; + self.pending_hardswap_generation + .store(next_generation, Ordering::Relaxed); + next_generation + } + } else { + self.pending_hardswap_generation.store(0, Ordering::Relaxed); + self.generation.fetch_add(1, Ordering::Relaxed) + 1 + }; if hardswap { self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc) @@ -354,6 +372,9 @@ impl MePool { drop(writers); if stale_writer_ids.is_empty() { + if hardswap { + self.pending_hardswap_generation.store(0, Ordering::Relaxed); + } debug!("ME reinit cycle completed with no stale writers"); return; } @@ -375,6 +396,9 @@ impl MePool { self.mark_writer_draining_with_timeout(writer_id, drain_timeout, !hardswap) .await; } + if hardswap { + self.pending_hardswap_generation.store(0, Ordering::Relaxed); + } } pub async fn zero_downtime_reinit_periodic(self: &Arc, rng: &SecureRandom) { diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index a8cc5a5..77ab891 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -89,6 +89,7 @@ impl MePool { id: writer_id, addr, generation, + created_at: Instant::now(), tx: tx.clone(), cancel: cancel.clone(), degraded: degraded.clone(), @@ -249,6 +250,7 @@ impl MePool { async fn remove_writer_only(self: &Arc, writer_id: u64) -> Vec { let mut close_tx: Option> = None; let mut removed_addr: Option = None; + let mut removed_uptime: Option = None; let mut trigger_refill = false; { let mut ws = self.writers.write().await; @@ -261,6 +263,7 @@ impl MePool { self.stats.increment_me_writer_removed_total(); w.cancel.cancel(); removed_addr = Some(w.addr); + removed_uptime = Some(w.created_at.elapsed()); trigger_refill = !was_draining; if trigger_refill { self.stats.increment_me_writer_removed_unexpected_total(); @@ -275,6 +278,9 @@ impl MePool { if trigger_refill && let Some(addr) = removed_addr { + if let Some(uptime) = removed_uptime { + self.maybe_quarantine_flapping_endpoint(addr, uptime).await; + } self.trigger_immediate_refill(addr); } self.rtt_stats.lock().await.remove(&writer_id); diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index ea0dd75..632e34a 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::io::ErrorKind; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -45,7 +46,10 @@ pub(crate) async fn reader_loop( _ = cancel.cancelled() => return Ok(()), }; if n == 0 { - return Ok(()); + return Err(ProxyError::Io(std::io::Error::new( + ErrorKind::UnexpectedEof, + "ME socket closed by peer", + ))); } raw.extend_from_slice(&tmp[..n]); From 74ad9037def08f7d5c591bb4bb1d5da3c8adda28 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 00:54:02 +0300 Subject: [PATCH 008/261] Dead-code deletion: has_proxy_tag --- src/transport/middle_proxy/pool.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index c3e7533..92e83bc 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -257,10 +257,6 @@ impl MePool { }) } - pub fn has_proxy_tag(&self) -> bool { - self.proxy_tag.is_some() - } - pub fn current_generation(&self) -> u64 { self.generation.load(Ordering::Relaxed) } From 265b9a5f11b70577a13098ba4d297d5347ca5360 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 01:23:09 +0300 Subject: [PATCH 009/261] Create XRAY-SINGBOX-ROUTING.ru.md --- docs/XRAY-SINGBOX-ROUTING.ru.md | 321 ++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 docs/XRAY-SINGBOX-ROUTING.ru.md diff --git a/docs/XRAY-SINGBOX-ROUTING.ru.md b/docs/XRAY-SINGBOX-ROUTING.ru.md new file mode 100644 index 0000000..ba269bb --- /dev/null +++ b/docs/XRAY-SINGBOX-ROUTING.ru.md @@ -0,0 +1,321 @@ +# SNI-маршрутизация в xray-core / sing-box + TLS-fronting + +## Термины (в контексте этого кейса) + +- **TLS-fronting домен** — домен, который фигурирует в TLS ClientHello как **SNI** (например, `petrovich.ru`): он используется как "маска" на L7 и как ключ маршрутизации в прокси-роутере. +- **xray-core / sing-box** — локальный или удалённый L7/TLS-роутер (прокси), который: + 1) принимает входящее TCP/TLS-соединение, + 2) читает TLS ClientHello, + 3) извлекает SNI, + 4) по SNI выбирает outbound/апстрим, + 5) устанавливает новое TCP-соединение к целевому хосту уже **от себя**. +- **SNI (Server Name Indication)** — поле в TLS ClientHello, где клиент Telegram сообщает доменное имя для "маскировки" +- **DNS-resolve на стороне L7-роутера** — если выходной адрес задан доменом (или роутер решил "всё равно идти по SNI"), то DNS резолвится **на стороне xray/sing-box**, а не на стороне Telegram-клиента + +--- + +## Ключевая идея: куда на самом деле идёт соединение решает не то, что вы указали клиенту, а то как L7-роутер трактует SNI + +Механика: + +1) Telegram-клиенту вы можете указать **IP/домен telemt**,как "сервер". +2) Между клиентом и telemt стоит xray-core/sing-box, который принимает TCP, читает TLS ClientHello и видит **SNI=petrovich.ru** +3) Дальше роутер говорит: "Вижу SNI - направить на апстрим/маршрут N" +4) И устанавливает исходящее соединение не "по тому IP, который пользователь подразумевал", а **по домену из SNI** (или по сопоставлению SNI→outbound), используя для определния его IP собственный DNS-кеш или резолвер +5) `petrovich.ru` по A-записи указывает **не на IP telemt**, а значит при L7-маршрутизации трафик уйдёт на "оригинальный" сайт за этим доменом, а не в telemt: Telegram-клиент, естественно, не сможет получить ожидаемое поведение, потому что ответить с handshake на той стороне некому + +--- + +## Схема №1 "Как это НЕ работает" + +```text +Telegram Client + | + | (указан IP/домен telemt) + v +telemt instance +```` + +Ожидание: "я указал telemt -> значит трафик попадёт в telemt" - **нет!** + +--- + +## Схема №2. "Как это реально работает с TLS/L7-роутером и SNI" + +```text +Telegram Client + | + | 1) TCP/TLS connection: + | - ClientHello: + | - SNI=petrovich.ru + v +xray-core / sing-box / любой L7 router + | + | 2) читает ClientHello -> вытаскивает SNI + | 3) выбирает маршрут по SNI + | 4) делает DNS для petrovich.ru + | 5) подключается к полученному IP по TLS с этим SNI + v +"Оригинальный" сайт, A-запись которого не на telemt + | + X не telemt -> Telegram-клиент не коннектится как ожидалось +``` + +--- + +## Почему указанный в клиенте IP/домен telemt "не спасает" + +Потому что в таком режиме xray/sing-box выступает как **точка терминации TCP/TLS**, можно сказать - TLS-инспектор на уровне ClientHello, это означает: + +* TCP-сессия от Telegram-клиента заканчивается на xray/sing-box +* Дальше создаётся **новая** TCP-сессия "от имени" xray/sing-box к апстриму +* Выбор апстрима делается правилами роутинга, а в TLS-сценариях самый удобный и распространённый ключ — **SNI** + +То есть, "куда идти дальше" определяется логикой L7-роутера: + +* либо правилами вида `if SNI == petrovich.ru -> outbound X`, +* либо более "автоматическим" поведением: `подключаться к тому хосту, который указан в SNI`, +* плюс кэш DNS и собственные резолверы роутера + +--- + +## Что именно извлекается из TLS ClientHello и почему этого достаточно + +TLS ClientHello отправляется **в начале** TLS-сессии и, в классическом TLS без ECH, содержит SNI в открытом виде. + +Упрощённо: + +```text +ClientHello: + - supported_versions + - cipher_suites + - extensions: + - server_name: petrovich.ru <-- SNI + - alpn: h2/http1.1/... + - ... +``` + +Роутеру не нужно расшифровывать трафик и завершать TLS "как сервер" — часто достаточно просто прочитать первые пакеты и распарсить ClientHello, чтобы получить SNI и принять решение + +--- + +## Типовой алгоритм SNI-роутинга + +1. Принять входящий TCP. +2. Подождать первые байты. +3. Определить протокол: + + * если видим TLS ClientHello → парсим SNI/ALPN +4. Применить route rules: + + * match по `server_name` / `domain` / `tls.sni` +5. Выбрать outbound: + + * direct / proxy / specific upstream / detour +6. Установить исходящее соединение: + + * либо на фиксированный IP:порт, + * либо на домен через DNS-resolve на стороне роутера +7. Начать проксирование данных между входом и выходом + +--- + +## Почему "A-запись фронтинг-домена не на telemt" ломает кейс + +### Ситуация + +* В ClientHello: `SNI = petrovich.ru` +* DNS: `petrovich.ru -> 203.0.113.77` - "оригинальный" сайт +* telemt живёт на: `198.51.100.10` + +### Что делает роутер + +* Видит SNI `petrovich.ru` +* Либо: + + * (а) напрямую коннектится к `petrovich.ru:443`, резолвя A-запись в `203.0.113.77`, + * либо: + * (б) выбирает outbound, который указывает на `petrovich.ru` как destination, + * либо: + * (в) делает sniffing/override destination по SNI + +В итоге исходящий коннект идёт на `203.0.113.77:443`, а не на telemt! +Другой сервер, другой протокол, другая логика, где telemt не участвует + +--- + +## "Где именно происходит подмена destination на SNI" + +Это зависит от конфигурации, но типовые варианты: + +### Вариант A: outbound задан доменом (и он совпадает с SNI) + +Правило по SNI выбирает outbound, у которого destination задан доменом фронтинга, +тогда DNS резолвится на стороне роутера и вы уходите на "оригинальный" хост + +### Вариант B: destination override / sniffing + +Роутер "снифает" SNI и **перезаписывает** destination на домен из SNI (даже если вход изначально был на IP telemt), +это особенно коварно: пользователь видит "я подключаюсь к IP telemt", но роутер после sniffing решает иначе + +### Вариант C: split DNS / кеш / независимый резолвер + +Даже если клиент "где-то" резолвит иначе, это не важно: конечный DNS для исходящего коннекта — на стороне xray/sing-box, +который может иметь: + +* свой DoH/DoT, +* свой кеш, +* свои правила fake-ip / system resolver, +* и, как следствие, своя "карта" **домен/SNI -> IP** + +--- + +## Признаки того, что трафик "утёк на оригинал", а не попал в telemt + +* На стороне telemt отсутствуют входящие соединения/логи +* На стороне роутера видно, что destination — домен фронтинга, а IP соответствует публичному сайту +* TLS-метрики/сертификат на выходе соответствует "оригинальному" сайту в записах трафика +* Telegram-клиент получает неожиданный тип ответов/ошибку handshaking/timeout в debug-режиме + +--- + +## Best-practice решение для этого кейса: свой домен фронтинга + заглушка на telemt + Let's Encrypt + +### Цель + +Сделать так, чтобы: + +* SNI (фронтинг-домен) **резолвился в IP telemt**, +* на IP telemt реально был TLS-сервис с валидным сертификатом под этот домен, +* даже если кто-то "попробует открыть домен как сайт", он увидит нормальную заглушку, а не "пустоту" + +### Что это даёт + +* xray/sing-box, маршрутизируя по SNI, будет неизбежно приходить на telemt, потому что DNS(SNI-домен) → IP telemt +* Внешний вид будет правдоподобным: обычный домен с обычным сертификатом +* Устойчивость: меньше сюрпризов от DNS-кеша/перерезолва/"умных" правил роутера + +--- + +## Рекомендуемая схема (целевое состояние) + +```text +Telegram Client + | + | TLS ClientHello: SNI = hello.example.com + v +xray-core / sing-box + | + | Route by SNI -> outbound -> connect to hello.example.com:443 + | DNS(hello.example.com) = IP telemt + v +telemt instance (IP telemt) + | + | TLS cert for hello.example.com (Let's Encrypt) + | + сайт-заглушка / health endpoint + v +OK +``` + +--- + +## Практический чеклист (минимальный) + +1. Купить/иметь домен: `hello.example.com` +2. В DNS: + + * `A hello.example.com -> ` + * (опционально) AAAA, если используете IPv6 и он стабилен +3. На telemt-хосте: + + * поднять TLS endpoint на 443 с валидным сертификатом LE под `hello.example.com` + * отдать "заглушку" (например, статический сайт), чтобы домен выглядел как обычный веб-сервис +4. В xray/sing-box правилах: + + * маршрутизировать нужный трафик по SNI = `hello.example.com` в "правильный" outbound (к telemt) + * избегать конфигураций, где destination override уводит на чужой домен +5. Важно: + + * если вы используете кеш DNS на роутере — сбросить/обновить его после смены A-записи + +--- + +## Пояснение про сайт-заглушку + +Для эмуляции TLS, telemt имеет подсистему TLS-F в `src/tls_front`: +- её модуль - fetcher, собирает TLS-профили, чтоб максимально поведенчески корректно повторять TLS конкретно указанного сайта + +Когда вы указываете сайт, который не отвечает по TLS: +- fetcher не может собрать TLS-профиль и происходит fallback на `fake_cert_len` - примитивный алгоритм, +- он забивает служебную информацию TLS рандомными байтами, +- простые системы DPI не распознают это +- однако, продвинутые системы, такие как nEdge или Fraud Control в сетях мобильной связи легко заблокируют или замедлят такой трафик + +Создав сайт-заглушку с Let's Encrypt сертификатом, вы даёте TLS-F возможность получить данные сертификата и корректно его "повторять" в дальнейшем + +--- + +## Вариант конфиг-подхода: "SNI строго привязываем к telemt - фиксированный IP" + +Чтобы полностью исключить зависимость от DNS если вам это нужно, можно сделать outbound, который ходит на **фиксированный IP telemt**, но при этом выставляет SNI/Host как `hello.example.com`. + +Идея: + +* destination: `IP:443` +* SNI: `hello.example.com` +* сертификат на telemt именно под `hello.example.com` + +Так вы получаете: + +* TLS выглядит корректно, ведь SNI совпадает с сертификатом, +* а routing никогда не уйдёт на "оригинал", потому что A-запись указывает на telemt и контроллируется вами! + +Но в вашем описании проблема как раз в том, что роутер "сам решает по SNI и резолвит домен", поэтому самый универсальный вариант — сделать так, чтобы DNS всегда приводил в telemt + +--- + +## Пример логики правил на псевдоконфиге L7-роутера + +```text +if inbound is TLS and sni == "hello.example.com": + route -> outbound "telemt" +else: + route -> outbound "default" +``` + +Outbound `telemt`: + +* destination: `hello.example.com:443` +* TLS enabled +* SNI: `hello.example.com` + +--- + +## Отдельно: что может неожиданно сломать даже "правильный" DNS + +* **Кеширование DNS** на xray/sing-box или на системном резолвере, особенно при смене A-записи +* **Split-horizon DNS**: разные ответы внутри/снаружи, попытки подмены/терминирования в других точках +* **IPv6**: если есть AAAA и он указывает не туда, роутер может предпочесть IPv6: помните, что поддержка v6 нестабильна и не рекомендуется в prod +* **DoH/DoT** на роутере: он может резолвить не тем резолвером, которым вы проверяли + +Минимальная гигиена: + +* контролировать A/AAAA, +* держать TTL разумным, +* проверять, каким резолвером пользуется именно роутер, +* при необходимости отключить/ограничить destination override + +--- + +## Итог + +В режиме TLS-fronting с xray-core/sing-box как L7/TLS-роутером **SNI становится приоритетным "source-of-truth" для маршрутизации** + +Если фронтинг-домен по DNS указывает не на IP telemt, роутер честно уводит трафик на "оригинальный" сайт, потому что он строит исходящее соединение "по SNI" + +Надёжное решение для этого кейса: + +* использовать **свой домен** для фронтинга, +* направить его **A/AAAA** на IP telemt, +* поднять на telemt **TLS-сервис с Let’s Encrypt сертификатом** под этот домен, +* (желательно) держать **сайт-заглушку**, чтобы 443 выглядел как обычный HTTPS From e589891706005daf8bbb8d2bb067432ca944ef64 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:41:51 +0300 Subject: [PATCH 010/261] ME Dual-Trio Pool Drafts --- src/metrics.rs | 94 ++++++++++++++++++++ src/stats/mod.rs | 79 +++++++++++++++++ src/transport/middle_proxy/handshake.rs | 49 +++++++++++ src/transport/middle_proxy/pool.rs | 48 +++++++++- src/transport/middle_proxy/pool_refill.rs | 98 ++++++++++++++++++++- src/transport/middle_proxy/pool_reinit.rs | 102 ++++++++++++++++++++-- src/transport/middle_proxy/pool_writer.rs | 25 +++++- src/transport/middle_proxy/reader.rs | 1 + src/transport/middle_proxy/send.rs | 58 ++++++++++-- 9 files changed, 534 insertions(+), 20 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 35f29ca..fcbd03c 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -274,6 +274,43 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!(out, "# HELP telemt_me_handshake_reject_total ME handshake rejects from upstream"); + let _ = writeln!(out, "# TYPE telemt_me_handshake_reject_total counter"); + let _ = writeln!( + out, + "telemt_me_handshake_reject_total {}", + if me_allows_normal { + stats.get_me_handshake_reject_total() + } else { + 0 + } + ); + + let _ = writeln!(out, "# HELP telemt_me_handshake_error_code_total ME handshake reject errors by code"); + let _ = writeln!(out, "# TYPE telemt_me_handshake_error_code_total counter"); + if me_allows_normal { + for (error_code, count) in stats.get_me_handshake_error_code_counts() { + let _ = writeln!( + out, + "telemt_me_handshake_error_code_total{{error_code=\"{}\"}} {}", + error_code, + count + ); + } + } + + let _ = writeln!(out, "# HELP telemt_me_reader_eof_total ME reader EOF terminations"); + let _ = writeln!(out, "# TYPE telemt_me_reader_eof_total counter"); + let _ = writeln!( + out, + "telemt_me_reader_eof_total {}", + if me_allows_normal { + stats.get_me_reader_eof_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_me_crc_mismatch_total ME CRC mismatches"); let _ = writeln!(out, "# TYPE telemt_me_crc_mismatch_total counter"); let _ = writeln!( @@ -385,6 +422,63 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_endpoint_quarantine_total ME endpoint quarantines due to rapid flaps" + ); + let _ = writeln!(out, "# TYPE telemt_me_endpoint_quarantine_total counter"); + let _ = writeln!( + out, + "telemt_me_endpoint_quarantine_total {}", + if me_allows_normal { + stats.get_me_endpoint_quarantine_total() + } else { + 0 + } + ); + + let _ = writeln!(out, "# HELP telemt_me_kdf_drift_total ME KDF input drift detections"); + let _ = writeln!(out, "# TYPE telemt_me_kdf_drift_total counter"); + let _ = writeln!( + out, + "telemt_me_kdf_drift_total {}", + if me_allows_normal { + stats.get_me_kdf_drift_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_hardswap_pending_reuse_total Hardswap cycles that reused an existing pending generation" + ); + let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_reuse_total counter"); + let _ = writeln!( + out, + "telemt_me_hardswap_pending_reuse_total {}", + if me_allows_debug { + stats.get_me_hardswap_pending_reuse_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_hardswap_pending_ttl_expired_total Pending hardswap generations reset by TTL expiration" + ); + let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_ttl_expired_total counter"); + let _ = writeln!( + out, + "telemt_me_hardswap_pending_ttl_expired_total {}", + if me_allows_normal { + stats.get_me_hardswap_pending_ttl_expired_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"); let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter"); let _ = writeln!( diff --git a/src/stats/mod.rs b/src/stats/mod.rs index f5aa2b7..453a73a 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -32,8 +32,15 @@ pub struct Stats { me_keepalive_timeout: AtomicU64, me_reconnect_attempts: AtomicU64, me_reconnect_success: AtomicU64, + me_handshake_reject_total: AtomicU64, + me_reader_eof_total: AtomicU64, me_crc_mismatch: AtomicU64, me_seq_mismatch: AtomicU64, + me_endpoint_quarantine_total: AtomicU64, + me_kdf_drift_total: AtomicU64, + me_hardswap_pending_reuse_total: AtomicU64, + me_hardswap_pending_ttl_expired_total: AtomicU64, + me_handshake_error_codes: DashMap, me_route_drop_no_conn: AtomicU64, me_route_drop_channel_closed: AtomicU64, me_route_drop_queue_full: AtomicU64, @@ -172,6 +179,26 @@ impl Stats { self.me_reconnect_success.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_handshake_reject_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_handshake_reject_total.fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_handshake_error_code(&self, code: i32) { + if !self.telemetry_me_allows_normal() { + return; + } + let entry = self + .me_handshake_error_codes + .entry(code) + .or_insert_with(|| AtomicU64::new(0)); + entry.fetch_add(1, Ordering::Relaxed); + } + pub fn increment_me_reader_eof_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_reader_eof_total.fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_crc_mismatch(&self) { if self.telemetry_me_allows_normal() { self.me_crc_mismatch.fetch_add(1, Ordering::Relaxed); @@ -333,6 +360,29 @@ impl Stats { .fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_endpoint_quarantine_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_endpoint_quarantine_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_kdf_drift_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_kdf_drift_total.fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_hardswap_pending_reuse_total(&self) { + if self.telemetry_me_allows_debug() { + self.me_hardswap_pending_reuse_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_hardswap_pending_ttl_expired_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_hardswap_pending_ttl_expired_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) } pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) } pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) } @@ -341,8 +391,37 @@ impl Stats { pub fn get_me_keepalive_timeout(&self) -> u64 { self.me_keepalive_timeout.load(Ordering::Relaxed) } pub fn get_me_reconnect_attempts(&self) -> u64 { self.me_reconnect_attempts.load(Ordering::Relaxed) } pub fn get_me_reconnect_success(&self) -> u64 { self.me_reconnect_success.load(Ordering::Relaxed) } + pub fn get_me_handshake_reject_total(&self) -> u64 { + self.me_handshake_reject_total.load(Ordering::Relaxed) + } + pub fn get_me_reader_eof_total(&self) -> u64 { + self.me_reader_eof_total.load(Ordering::Relaxed) + } pub fn get_me_crc_mismatch(&self) -> u64 { self.me_crc_mismatch.load(Ordering::Relaxed) } pub fn get_me_seq_mismatch(&self) -> u64 { self.me_seq_mismatch.load(Ordering::Relaxed) } + pub fn get_me_endpoint_quarantine_total(&self) -> u64 { + self.me_endpoint_quarantine_total.load(Ordering::Relaxed) + } + pub fn get_me_kdf_drift_total(&self) -> u64 { + self.me_kdf_drift_total.load(Ordering::Relaxed) + } + pub fn get_me_hardswap_pending_reuse_total(&self) -> u64 { + self.me_hardswap_pending_reuse_total + .load(Ordering::Relaxed) + } + pub fn get_me_hardswap_pending_ttl_expired_total(&self) -> u64 { + self.me_hardswap_pending_ttl_expired_total + .load(Ordering::Relaxed) + } + pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> { + let mut out: Vec<(i32, u64)> = self + .me_handshake_error_codes + .iter() + .map(|entry| (*entry.key(), entry.value().load(Ordering::Relaxed))) + .collect(); + out.sort_by_key(|(code, _)| *code); + out + } pub fn get_me_route_drop_no_conn(&self) -> u64 { self.me_route_drop_no_conn.load(Ordering::Relaxed) } pub fn get_me_route_drop_channel_closed(&self) -> u64 { self.me_route_drop_channel_closed.load(Ordering::Relaxed) diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 5daa460..251c911 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -1,6 +1,8 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use socket2::{SockRef, TcpKeepalive}; #[cfg(target_os = "linux")] use libc; @@ -34,6 +36,8 @@ use super::codec::{ use super::wire::{extract_ip_material, IpMaterial}; use super::MePool; +const ME_KDF_DRIFT_STRICT: bool = false; + /// Result of a successful ME handshake with timings. pub(crate) struct HandshakeOutput { pub rd: ReadHalf, @@ -47,6 +51,22 @@ pub(crate) struct HandshakeOutput { } impl MePool { + fn kdf_material_fingerprint( + local_addr_nat: SocketAddr, + peer_addr_nat: SocketAddr, + client_port_for_kdf: u16, + reflected: Option, + socks_bound_addr: Option, + ) -> u64 { + let mut hasher = DefaultHasher::new(); + local_addr_nat.hash(&mut hasher); + peer_addr_nat.hash(&mut hasher); + client_port_for_kdf.hash(&mut hasher); + reflected.hash(&mut hasher); + socks_bound_addr.hash(&mut hasher); + hasher.finish() + } + async fn resolve_dc_idx_for_endpoint(&self, addr: SocketAddr) -> Option { if addr.is_ipv4() { let map = self.proxy_map_v4.read().await; @@ -343,6 +363,33 @@ impl MePool { .map(|bound| bound.port()) .filter(|port| *port != 0) .unwrap_or(local_addr_nat.port()); + let kdf_fingerprint = Self::kdf_material_fingerprint( + local_addr_nat, + peer_addr_nat, + client_port_for_kdf, + reflected, + socks_bound_addr, + ); + let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.lock().await; + if let Some(prev_fingerprint) = kdf_fingerprint_guard.get(&peer_addr_nat).copied() + && prev_fingerprint != kdf_fingerprint + { + self.stats.increment_me_kdf_drift_total(); + warn!( + %peer_addr_nat, + %local_addr_nat, + client_port_for_kdf, + "ME KDF input drift detected for endpoint" + ); + if ME_KDF_DRIFT_STRICT { + return Err(ProxyError::InvalidHandshake( + "ME KDF input drift detected (strict mode)".to_string(), + )); + } + } + kdf_fingerprint_guard.insert(peer_addr_nat, kdf_fingerprint); + drop(kdf_fingerprint_guard); + let client_port_bytes = client_port_for_kdf.to_le_bytes(); let server_ip = extract_ip_material(peer_addr_nat); @@ -540,6 +587,8 @@ impl MePool { } else { -1 }; + self.stats.increment_me_handshake_reject_total(); + self.stats.increment_me_handshake_error_code(err_code); return Err(ProxyError::InvalidHandshake(format!( "ME rejected handshake (error={err_code})" ))); diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 92e83bc..4a5598a 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -16,11 +16,18 @@ use crate::transport::UpstreamManager; use super::ConnRegistry; use super::codec::WriterCommand; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(super) struct RefillDcKey { + pub dc: i32, + pub family: IpFamily, +} + #[derive(Clone)] pub struct MeWriter { pub id: u64, pub addr: SocketAddr, pub generation: u64, + pub contour: Arc, pub created_at: Instant, pub tx: mpsc::Sender, pub cancel: CancellationToken, @@ -30,6 +37,29 @@ pub struct MeWriter { pub allow_drain_fallback: Arc, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub(super) enum WriterContour { + Warm = 0, + Active = 1, + Draining = 2, +} + +impl WriterContour { + pub(super) fn as_u8(self) -> u8 { + self as u8 + } + + pub(super) fn from_u8(value: u8) -> Self { + match value { + 0 => Self::Warm, + 1 => Self::Active, + 2 => Self::Draining, + _ => Self::Draining, + } + } +} + #[derive(Debug, Clone)] pub struct SecretSnapshot { pub epoch: u64, @@ -80,12 +110,18 @@ pub struct MePool { pub(super) nat_reflection_cache: Arc>, pub(super) writer_available: Arc, pub(super) refill_inflight: Arc>>, + pub(super) refill_inflight_dc: Arc>>, pub(super) conn_count: AtomicUsize, pub(super) stats: Arc, pub(super) generation: AtomicU64, + pub(super) active_generation: AtomicU64, + pub(super) warm_generation: AtomicU64, pub(super) pending_hardswap_generation: AtomicU64, + pub(super) pending_hardswap_started_at_epoch_secs: AtomicU64, + pub(super) pending_hardswap_map_hash: AtomicU64, pub(super) hardswap: AtomicBool, pub(super) endpoint_quarantine: Arc>>, + pub(super) kdf_material_fingerprint: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, pub(super) me_pool_force_close_secs: AtomicU64, pub(super) me_pool_min_fresh_ratio_permille: AtomicU32, @@ -233,11 +269,17 @@ impl MePool { nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())), writer_available: Arc::new(Notify::new()), refill_inflight: Arc::new(Mutex::new(HashSet::new())), + refill_inflight_dc: Arc::new(Mutex::new(HashSet::new())), conn_count: AtomicUsize::new(0), generation: AtomicU64::new(1), + active_generation: AtomicU64::new(1), + warm_generation: AtomicU64::new(0), pending_hardswap_generation: AtomicU64::new(0), + pending_hardswap_started_at_epoch_secs: AtomicU64::new(0), + pending_hardswap_map_hash: AtomicU64::new(0), hardswap: AtomicBool::new(hardswap), endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())), + kdf_material_fingerprint: Arc::new(Mutex::new(HashMap::new())), me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs), me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs), me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille( @@ -258,7 +300,11 @@ impl MePool { } pub fn current_generation(&self) -> u64 { - self.generation.load(Ordering::Relaxed) + self.active_generation.load(Ordering::Relaxed) + } + + pub(super) fn warm_generation(&self) -> u64 { + self.warm_generation.load(Ordering::Relaxed) } pub fn update_runtime_reinit_policy( diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index a286e65..92071bd 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -7,8 +7,9 @@ use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; use crate::crypto::SecureRandom; +use crate::network::IpFamily; -use super::pool::MePool; +use super::pool::{MePool, RefillDcKey, WriterContour}; const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20; const ME_FLAP_QUARANTINE_SECS: u64 = 25; @@ -27,6 +28,7 @@ impl MePool { let mut guard = self.endpoint_quarantine.lock().await; guard.retain(|_, expiry| *expiry > Instant::now()); guard.insert(addr, until); + self.stats.increment_me_endpoint_quarantine_total(); warn!( %addr, uptime_ms = uptime.as_millis(), @@ -84,14 +86,76 @@ impl MePool { if endpoints.is_empty() { return false; } - let guard = self.refill_inflight.lock().await; - endpoints.iter().any(|addr| guard.contains(addr)) + + { + let guard = self.refill_inflight.lock().await; + if endpoints.iter().any(|addr| guard.contains(addr)) { + return true; + } + } + + let dc_keys = self.resolve_refill_dc_keys_for_endpoints(endpoints).await; + if dc_keys.is_empty() { + return false; + } + let guard = self.refill_inflight_dc.lock().await; + dc_keys.iter().any(|key| guard.contains(key)) + } + + async fn resolve_refill_dc_key_for_addr(&self, addr: SocketAddr) -> Option { + let family = if addr.is_ipv4() { + IpFamily::V4 + } else { + IpFamily::V6 + }; + let map = self.proxy_map_for_family(family).await; + for (dc, endpoints) in map { + if endpoints + .into_iter() + .any(|(ip, port)| SocketAddr::new(ip, port) == addr) + { + return Some(RefillDcKey { + dc: dc.abs(), + family, + }); + } + } + None + } + + async fn resolve_refill_dc_keys_for_endpoints( + &self, + endpoints: &[SocketAddr], + ) -> HashSet { + let mut out = HashSet::::new(); + for addr in endpoints { + if let Some(key) = self.resolve_refill_dc_key_for_addr(*addr).await { + out.insert(key); + } + } + out } pub(super) async fn connect_endpoints_round_robin( self: &Arc, endpoints: &[SocketAddr], rng: &SecureRandom, + ) -> bool { + self.connect_endpoints_round_robin_with_generation_contour( + endpoints, + rng, + self.current_generation(), + WriterContour::Active, + ) + .await + } + + pub(super) async fn connect_endpoints_round_robin_with_generation_contour( + self: &Arc, + endpoints: &[SocketAddr], + rng: &SecureRandom, + generation: u64, + contour: WriterContour, ) -> bool { let candidates = self.connectable_endpoints(endpoints).await; if candidates.is_empty() { @@ -101,7 +165,10 @@ impl MePool { for offset in 0..candidates.len() { let idx = (start + offset) % candidates.len(); let addr = candidates[idx]; - match self.connect_one(addr, rng).await { + match self + .connect_one_with_generation_contour(addr, rng, generation, contour) + .await + { Ok(()) => return true, Err(e) => debug!(%addr, error = %e, "ME connect failed during round-robin warmup"), } @@ -225,6 +292,9 @@ impl MePool { pub(crate) fn trigger_immediate_refill(self: &Arc, addr: SocketAddr) { let pool = Arc::clone(self); tokio::spawn(async move { + let dc_endpoints = pool.endpoints_for_same_dc(addr).await; + let dc_keys = pool.resolve_refill_dc_keys_for_endpoints(&dc_endpoints).await; + { let mut guard = pool.refill_inflight.lock().await; if !guard.insert(addr) { @@ -232,6 +302,19 @@ impl MePool { return; } } + + if !dc_keys.is_empty() { + let mut dc_guard = pool.refill_inflight_dc.lock().await; + if dc_keys.iter().any(|key| dc_guard.contains(key)) { + pool.stats.increment_me_refill_skipped_inflight_total(); + drop(dc_guard); + let mut guard = pool.refill_inflight.lock().await; + guard.remove(&addr); + return; + } + dc_guard.extend(dc_keys.iter().copied()); + } + pool.stats.increment_me_refill_triggered_total(); let restored = pool.refill_writer_after_loss(addr).await; @@ -241,6 +324,13 @@ impl MePool { let mut guard = pool.refill_inflight.lock().await; guard.remove(&addr); + drop(guard); + if !dc_keys.is_empty() { + let mut dc_guard = pool.refill_inflight_dc.lock().await; + for key in &dc_keys { + dc_guard.remove(key); + } + } }); } } diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 5552fb6..33b8cc4 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -1,4 +1,5 @@ use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -7,12 +8,58 @@ use std::time::Duration; use rand::Rng; use rand::seq::SliceRandom; use tracing::{debug, info, warn}; +use std::collections::hash_map::DefaultHasher; use crate::crypto::SecureRandom; -use super::pool::MePool; +use super::pool::{MePool, WriterContour}; + +const ME_HARDSWAP_PENDING_TTL_SECS: u64 = 1800; impl MePool { + fn desired_map_hash(desired_by_dc: &HashMap>) -> u64 { + let mut hasher = DefaultHasher::new(); + let mut dcs: Vec = desired_by_dc.keys().copied().collect(); + dcs.sort_unstable(); + for dc in dcs { + dc.hash(&mut hasher); + let mut endpoints: Vec = desired_by_dc + .get(&dc) + .map(|set| set.iter().copied().collect()) + .unwrap_or_default(); + endpoints.sort_unstable(); + for endpoint in endpoints { + endpoint.hash(&mut hasher); + } + } + hasher.finish() + } + + fn clear_pending_hardswap_state(&self) { + self.pending_hardswap_generation.store(0, Ordering::Relaxed); + self.pending_hardswap_started_at_epoch_secs + .store(0, Ordering::Relaxed); + self.pending_hardswap_map_hash.store(0, Ordering::Relaxed); + self.warm_generation.store(0, Ordering::Relaxed); + } + + async fn promote_warm_generation_to_active(&self, generation: u64) { + self.active_generation.store(generation, Ordering::Relaxed); + self.warm_generation.store(0, Ordering::Relaxed); + + let ws = self.writers.read().await; + for writer in ws.iter() { + if writer.draining.load(Ordering::Relaxed) { + continue; + } + if writer.generation == generation { + writer + .contour + .store(WriterContour::Active.as_u8(), Ordering::Relaxed); + } + } + } + fn coverage_ratio( desired_by_dc: &HashMap>, active_writer_addrs: &HashSet, @@ -202,7 +249,14 @@ impl MePool { let delay_ms = self.hardswap_warmup_connect_delay_ms(); tokio::time::sleep(Duration::from_millis(delay_ms)).await; - let connected = self.connect_endpoints_round_robin(&endpoint_list, rng).await; + let connected = self + .connect_endpoints_round_robin_with_generation_contour( + &endpoint_list, + rng, + generation, + WriterContour::Warm, + ) + .await; debug!( dc = *dc, pass = pass_idx + 1, @@ -265,29 +319,61 @@ impl MePool { return; } + let desired_map_hash = Self::desired_map_hash(&desired_by_dc); + let now_epoch_secs = Self::now_epoch_secs(); let previous_generation = self.current_generation(); let hardswap = self.hardswap.load(Ordering::Relaxed); let generation = if hardswap { let pending_generation = self.pending_hardswap_generation.load(Ordering::Relaxed); - if pending_generation != 0 && pending_generation >= previous_generation { + let pending_started_at = self + .pending_hardswap_started_at_epoch_secs + .load(Ordering::Relaxed); + let pending_map_hash = self.pending_hardswap_map_hash.load(Ordering::Relaxed); + let pending_age_secs = now_epoch_secs.saturating_sub(pending_started_at); + let pending_ttl_expired = pending_started_at > 0 && pending_age_secs > ME_HARDSWAP_PENDING_TTL_SECS; + let pending_matches_map = pending_map_hash != 0 && pending_map_hash == desired_map_hash; + + if pending_generation != 0 + && pending_generation >= previous_generation + && pending_matches_map + && !pending_ttl_expired + { + self.stats.increment_me_hardswap_pending_reuse_total(); debug!( previous_generation, generation = pending_generation, + pending_age_secs, "ME hardswap continues with pending generation" ); pending_generation } else { + if pending_generation != 0 && pending_ttl_expired { + self.stats.increment_me_hardswap_pending_ttl_expired_total(); + warn!( + previous_generation, + generation = pending_generation, + pending_age_secs, + pending_ttl_secs = ME_HARDSWAP_PENDING_TTL_SECS, + "ME hardswap pending generation expired by TTL; starting fresh generation" + ); + } let next_generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; self.pending_hardswap_generation .store(next_generation, Ordering::Relaxed); + self.pending_hardswap_started_at_epoch_secs + .store(now_epoch_secs, Ordering::Relaxed); + self.pending_hardswap_map_hash + .store(desired_map_hash, Ordering::Relaxed); + self.warm_generation.store(next_generation, Ordering::Relaxed); next_generation } } else { - self.pending_hardswap_generation.store(0, Ordering::Relaxed); + self.clear_pending_hardswap_state(); self.generation.fetch_add(1, Ordering::Relaxed) + 1 }; if hardswap { + self.warm_generation.store(generation, Ordering::Relaxed); self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc) .await; } else { @@ -352,6 +438,10 @@ impl MePool { return; } + if hardswap { + self.promote_warm_generation_to_active(generation).await; + } + let desired_addrs: HashSet = desired_by_dc .values() .flat_map(|set| set.iter().copied()) @@ -373,7 +463,7 @@ impl MePool { if stale_writer_ids.is_empty() { if hardswap { - self.pending_hardswap_generation.store(0, Ordering::Relaxed); + self.clear_pending_hardswap_state(); } debug!("ME reinit cycle completed with no stale writers"); return; @@ -397,7 +487,7 @@ impl MePool { .await; } if hardswap { - self.pending_hardswap_generation.store(0, Ordering::Relaxed); + self.clear_pending_hardswap_state(); } } diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 77ab891..455757e 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -1,6 +1,6 @@ use std::net::SocketAddr; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use bytes::BytesMut; @@ -15,7 +15,7 @@ use crate::error::{ProxyError, Result}; use crate::protocol::constants::RPC_PING_U32; use super::codec::{RpcWriter, WriterCommand}; -use super::pool::{MePool, MeWriter}; +use super::pool::{MePool, MeWriter, WriterContour}; use super::reader::reader_loop; use super::registry::BoundConn; @@ -43,6 +43,22 @@ impl MePool { } pub(crate) async fn connect_one(self: &Arc, addr: SocketAddr, rng: &SecureRandom) -> Result<()> { + self.connect_one_with_generation_contour( + addr, + rng, + self.current_generation(), + WriterContour::Active, + ) + .await + } + + pub(super) async fn connect_one_with_generation_contour( + self: &Arc, + addr: SocketAddr, + rng: &SecureRandom, + generation: u64, + contour: WriterContour, + ) -> Result<()> { let secret_len = self.proxy_secret.read().await.secret.len(); if secret_len < 32 { return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into())); @@ -52,7 +68,7 @@ impl MePool { let hs = self.handshake_only(stream, addr, upstream_egress, rng).await?; let writer_id = self.next_writer_id.fetch_add(1, Ordering::Relaxed); - let generation = self.current_generation(); + let contour = Arc::new(AtomicU8::new(contour.as_u8())); let cancel = CancellationToken::new(); let degraded = Arc::new(AtomicBool::new(false)); let draining = Arc::new(AtomicBool::new(false)); @@ -89,6 +105,7 @@ impl MePool { id: writer_id, addr, generation, + contour: contour.clone(), created_at: Instant::now(), tx: tx.clone(), cancel: cancel.clone(), @@ -305,6 +322,8 @@ impl MePool { if !already_draining { self.stats.increment_pool_drain_active(); } + w.contour + .store(WriterContour::Draining.as_u8(), Ordering::Relaxed); w.draining.store(true, Ordering::Relaxed); true } else { diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index 632e34a..e907d25 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -46,6 +46,7 @@ pub(crate) async fn reader_loop( _ = cancel.cancelled() => return Ok(()), }; if n == 0 { + stats.increment_me_reader_eof_total(); return Err(ProxyError::Io(std::io::Error::new( ErrorKind::UnexpectedEof, "ME socket closed by peer", diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 25b8852..3b57c4c 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -13,6 +13,7 @@ use crate::protocol::constants::RPC_CLOSE_EXT_U32; use super::MePool; use super::codec::WriterCommand; +use super::pool::WriterContour; use super::wire::build_proxy_req_payload; use rand::seq::SliceRandom; use super::registry::ConnMeta; @@ -101,7 +102,14 @@ impl MePool { ws.clone() }; - let mut candidate_indices = self.candidate_indices_for_dc(&writers_snapshot, target_dc).await; + let mut candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .await; + if candidate_indices.is_empty() { + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .await; + } if candidate_indices.is_empty() { // Emergency connect-on-demand if emergency_attempts >= 3 { @@ -127,7 +135,14 @@ impl MePool { let ws2 = self.writers.read().await; writers_snapshot = ws2.clone(); drop(ws2); - candidate_indices = self.candidate_indices_for_dc(&writers_snapshot, target_dc).await; + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .await; + if candidate_indices.is_empty() { + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .await; + } if !candidate_indices.is_empty() { break; } @@ -143,6 +158,7 @@ impl MePool { let left = &writers_snapshot[*lhs]; let right = &writers_snapshot[*rhs]; let left_key = ( + self.writer_contour_rank_for_selection(left), (left.generation < self.current_generation()) as usize, left.degraded.load(Ordering::Relaxed) as usize, Reverse(left.tx.capacity()), @@ -150,6 +166,7 @@ impl MePool { left.id, ); let right_key = ( + self.writer_contour_rank_for_selection(right), (right.generation < self.current_generation()) as usize, right.degraded.load(Ordering::Relaxed) as usize, Reverse(right.tx.capacity()), @@ -163,7 +180,12 @@ impl MePool { let w = &writers_snapshot[*idx]; let degraded = w.degraded.load(Ordering::Relaxed); let stale = (w.generation < self.current_generation()) as usize; - (stale, degraded as usize, Reverse(w.tx.capacity())) + ( + self.writer_contour_rank_for_selection(w), + stale, + degraded as usize, + Reverse(w.tx.capacity()), + ) }); } @@ -257,6 +279,7 @@ impl MePool { &self, writers: &[super::pool::MeWriter], target_dc: i16, + include_warm: bool, ) -> Vec { let key = target_dc as i32; let mut preferred = Vec::::new(); @@ -300,13 +323,13 @@ impl MePool { if preferred.is_empty() { return (0..writers.len()) - .filter(|i| self.writer_accepts_new_binding(&writers[*i])) + .filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm)) .collect(); } let mut out = Vec::new(); for (idx, w) in writers.iter().enumerate() { - if !self.writer_accepts_new_binding(w) { + if !self.writer_eligible_for_selection(w, include_warm) { continue; } if preferred.contains(&w.addr) { @@ -315,10 +338,33 @@ impl MePool { } if out.is_empty() { return (0..writers.len()) - .filter(|i| self.writer_accepts_new_binding(&writers[*i])) + .filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm)) .collect(); } out } + fn writer_eligible_for_selection( + &self, + writer: &super::pool::MeWriter, + include_warm: bool, + ) -> bool { + if !self.writer_accepts_new_binding(writer) { + return false; + } + + match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) { + WriterContour::Active => true, + WriterContour::Warm => include_warm, + WriterContour::Draining => true, + } + } + + fn writer_contour_rank_for_selection(&self, writer: &super::pool::MeWriter) -> usize { + match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) { + WriterContour::Active => 0, + WriterContour::Warm => 1, + WriterContour::Draining => 2, + } + } } From 9477103f894e13b5062e526594c68922f98a28d9 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:45:43 +0300 Subject: [PATCH 011/261] Update pool.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/pool.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 4a5598a..ed6969a 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -303,10 +303,6 @@ impl MePool { self.active_generation.load(Ordering::Relaxed) } - pub(super) fn warm_generation(&self) -> u64 { - self.warm_generation.load(Ordering::Relaxed) - } - pub fn update_runtime_reinit_policy( &self, hardswap: bool, From a6d22e8a57f87637d321a7c5878718eefefbadc4 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:04:06 +0300 Subject: [PATCH 012/261] ME Pool Shadow Writers Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 25 ++ src/config/load.rs | 117 ++++++++ src/config/types.rs | 31 ++ src/main.rs | 6 + src/metrics.rs | 108 +++++++ src/stats/mod.rs | 66 +++++ src/transport/middle_proxy/config_updater.rs | 12 + src/transport/middle_proxy/health.rs | 297 ++++++++++++++++++- src/transport/middle_proxy/pool.rs | 94 ++++++ src/transport/middle_proxy/pool_reinit.rs | 8 +- 10 files changed, 748 insertions(+), 16 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 0ea6692..5602287 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -8,6 +8,7 @@ const DEFAULT_STUN_TCP_FALLBACK: bool = true; const DEFAULT_MIDDLE_PROXY_WARM_STANDBY: usize = 16; const DEFAULT_ME_RECONNECT_MAX_CONCURRENT_PER_DC: u32 = 8; const DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT: u32 = 16; +const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 3; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 4; const DEFAULT_LISTEN_ADDR_IPV6: &str = "::"; @@ -160,6 +161,30 @@ pub(crate) fn default_me_reconnect_fast_retry_count() -> u32 { DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT } +pub(crate) fn default_me_single_endpoint_shadow_writers() -> u8 { + DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS +} + +pub(crate) fn default_me_single_endpoint_outage_mode_enabled() -> bool { + true +} + +pub(crate) fn default_me_single_endpoint_outage_disable_quarantine() -> bool { + true +} + +pub(crate) fn default_me_single_endpoint_outage_backoff_min_ms() -> u64 { + 250 +} + +pub(crate) fn default_me_single_endpoint_outage_backoff_max_ms() -> u64 { + 3000 +} + +pub(crate) fn default_me_single_endpoint_shadow_rotate_every_secs() -> u64 { + 900 +} + pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS } diff --git a/src/config/load.rs b/src/config/load.rs index 17545b9..a027d58 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -255,6 +255,32 @@ impl ProxyConfig { )); } + if config.general.me_single_endpoint_shadow_writers > 32 { + return Err(ProxyError::Config( + "general.me_single_endpoint_shadow_writers must be within [0, 32]".to_string(), + )); + } + + if config.general.me_single_endpoint_outage_backoff_min_ms == 0 { + return Err(ProxyError::Config( + "general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(), + )); + } + + if config.general.me_single_endpoint_outage_backoff_max_ms == 0 { + return Err(ProxyError::Config( + "general.me_single_endpoint_outage_backoff_max_ms must be > 0".to_string(), + )); + } + + if config.general.me_single_endpoint_outage_backoff_min_ms + > config.general.me_single_endpoint_outage_backoff_max_ms + { + return Err(ProxyError::Config( + "general.me_single_endpoint_outage_backoff_min_ms must be <= general.me_single_endpoint_outage_backoff_max_ms".to_string(), + )); + } + if config.general.beobachten_minutes == 0 { return Err(ProxyError::Config( "general.beobachten_minutes must be > 0".to_string(), @@ -592,6 +618,30 @@ mod tests { cfg.general.me_reconnect_fast_retry_count, default_me_reconnect_fast_retry_count() ); + assert_eq!( + cfg.general.me_single_endpoint_shadow_writers, + default_me_single_endpoint_shadow_writers() + ); + assert_eq!( + cfg.general.me_single_endpoint_outage_mode_enabled, + default_me_single_endpoint_outage_mode_enabled() + ); + assert_eq!( + cfg.general.me_single_endpoint_outage_disable_quarantine, + default_me_single_endpoint_outage_disable_quarantine() + ); + assert_eq!( + cfg.general.me_single_endpoint_outage_backoff_min_ms, + default_me_single_endpoint_outage_backoff_min_ms() + ); + assert_eq!( + cfg.general.me_single_endpoint_outage_backoff_max_ms, + default_me_single_endpoint_outage_backoff_max_ms() + ); + assert_eq!( + cfg.general.me_single_endpoint_shadow_rotate_every_secs, + default_me_single_endpoint_shadow_rotate_every_secs() + ); assert_eq!( cfg.general.upstream_connect_retry_attempts, default_upstream_connect_retry_attempts() @@ -630,6 +680,30 @@ mod tests { general.me_reconnect_fast_retry_count, default_me_reconnect_fast_retry_count() ); + assert_eq!( + general.me_single_endpoint_shadow_writers, + default_me_single_endpoint_shadow_writers() + ); + assert_eq!( + general.me_single_endpoint_outage_mode_enabled, + default_me_single_endpoint_outage_mode_enabled() + ); + assert_eq!( + general.me_single_endpoint_outage_disable_quarantine, + default_me_single_endpoint_outage_disable_quarantine() + ); + assert_eq!( + general.me_single_endpoint_outage_backoff_min_ms, + default_me_single_endpoint_outage_backoff_min_ms() + ); + assert_eq!( + general.me_single_endpoint_outage_backoff_max_ms, + default_me_single_endpoint_outage_backoff_max_ms() + ); + assert_eq!( + general.me_single_endpoint_shadow_rotate_every_secs, + default_me_single_endpoint_shadow_rotate_every_secs() + ); assert_eq!( general.upstream_connect_retry_attempts, default_upstream_connect_retry_attempts() @@ -814,6 +888,49 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn me_single_endpoint_outage_backoff_range_is_validated() { + let toml = r#" + [general] + me_single_endpoint_outage_backoff_min_ms = 4000 + me_single_endpoint_outage_backoff_max_ms = 3000 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_single_endpoint_outage_backoff_range_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains( + "general.me_single_endpoint_outage_backoff_min_ms must be <= general.me_single_endpoint_outage_backoff_max_ms" + )); + let _ = std::fs::remove_file(path); + } + + #[test] + fn me_single_endpoint_shadow_writers_too_large_is_rejected() { + let toml = r#" + [general] + me_single_endpoint_shadow_writers = 33 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_single_endpoint_shadow_writers_limit_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.me_single_endpoint_shadow_writers must be within [0, 32]")); + let _ = std::fs::remove_file(path); + } + #[test] fn upstream_connect_retry_attempts_zero_is_rejected() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index d57c890..324abf2 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -394,6 +394,31 @@ pub struct GeneralConfig { #[serde(default = "default_me_reconnect_fast_retry_count")] pub me_reconnect_fast_retry_count: u32, + /// Number of additional reserve writers for DC groups with exactly one endpoint. + #[serde(default = "default_me_single_endpoint_shadow_writers")] + pub me_single_endpoint_shadow_writers: u8, + + /// Enable aggressive outage recovery mode for single-endpoint DC groups. + #[serde(default = "default_me_single_endpoint_outage_mode_enabled")] + pub me_single_endpoint_outage_mode_enabled: bool, + + /// Ignore endpoint quarantine while in single-endpoint outage mode. + #[serde(default = "default_me_single_endpoint_outage_disable_quarantine")] + pub me_single_endpoint_outage_disable_quarantine: bool, + + /// Minimum reconnect backoff in ms for single-endpoint outage mode. + #[serde(default = "default_me_single_endpoint_outage_backoff_min_ms")] + pub me_single_endpoint_outage_backoff_min_ms: u64, + + /// Maximum reconnect backoff in ms for single-endpoint outage mode. + #[serde(default = "default_me_single_endpoint_outage_backoff_max_ms")] + pub me_single_endpoint_outage_backoff_max_ms: u64, + + /// Periodic shadow writer rotation interval in seconds for single-endpoint DC groups. + /// Set to 0 to disable periodic shadow rotation. + #[serde(default = "default_me_single_endpoint_shadow_rotate_every_secs")] + pub me_single_endpoint_shadow_rotate_every_secs: u64, + /// Connect attempts for the selected upstream before returning error/fallback. #[serde(default = "default_upstream_connect_retry_attempts")] pub upstream_connect_retry_attempts: u32, @@ -603,6 +628,12 @@ impl Default for GeneralConfig { me_reconnect_backoff_base_ms: default_reconnect_backoff_base_ms(), me_reconnect_backoff_cap_ms: default_reconnect_backoff_cap_ms(), me_reconnect_fast_retry_count: default_me_reconnect_fast_retry_count(), + me_single_endpoint_shadow_writers: default_me_single_endpoint_shadow_writers(), + me_single_endpoint_outage_mode_enabled: default_me_single_endpoint_outage_mode_enabled(), + me_single_endpoint_outage_disable_quarantine: default_me_single_endpoint_outage_disable_quarantine(), + me_single_endpoint_outage_backoff_min_ms: default_me_single_endpoint_outage_backoff_min_ms(), + me_single_endpoint_outage_backoff_max_ms: default_me_single_endpoint_outage_backoff_max_ms(), + me_single_endpoint_shadow_rotate_every_secs: default_me_single_endpoint_shadow_rotate_every_secs(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(), diff --git a/src/main.rs b/src/main.rs index 03998cd..4ff79fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -538,6 +538,12 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_reconnect_backoff_base_ms, config.general.me_reconnect_backoff_cap_ms, config.general.me_reconnect_fast_retry_count, + config.general.me_single_endpoint_shadow_writers, + config.general.me_single_endpoint_outage_mode_enabled, + config.general.me_single_endpoint_outage_disable_quarantine, + config.general.me_single_endpoint_outage_backoff_min_ms, + config.general.me_single_endpoint_outage_backoff_max_ms, + config.general.me_single_endpoint_shadow_rotate_every_secs, config.general.hardswap, config.general.me_pool_drain_ttl_secs, config.general.effective_me_pool_force_close_secs(), diff --git a/src/metrics.rs b/src/metrics.rs index fcbd03c..4f4c317 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -479,6 +479,114 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_outage_enter_total Single-endpoint DC outage transitions to active state" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_outage_enter_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_outage_enter_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_outage_enter_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_outage_exit_total Single-endpoint DC outage recovery transitions" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_outage_exit_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_outage_exit_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_outage_exit_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_outage_reconnect_attempt_total Reconnect attempts performed during single-endpoint outages" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_outage_reconnect_attempt_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_outage_reconnect_attempt_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_outage_reconnect_attempt_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_outage_reconnect_success_total Successful reconnect attempts during single-endpoint outages" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_outage_reconnect_success_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_outage_reconnect_success_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_outage_reconnect_success_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_quarantine_bypass_total Outage reconnect attempts that bypassed quarantine" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_quarantine_bypass_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_quarantine_bypass_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_quarantine_bypass_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_shadow_rotate_total Successful periodic shadow rotations for single-endpoint DC groups" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_shadow_rotate_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_shadow_rotate_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_shadow_rotate_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"); let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter"); let _ = writeln!( diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 453a73a..5b2bb94 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -40,6 +40,12 @@ pub struct Stats { me_kdf_drift_total: AtomicU64, me_hardswap_pending_reuse_total: AtomicU64, me_hardswap_pending_ttl_expired_total: AtomicU64, + me_single_endpoint_outage_enter_total: AtomicU64, + me_single_endpoint_outage_exit_total: AtomicU64, + me_single_endpoint_outage_reconnect_attempt_total: AtomicU64, + me_single_endpoint_outage_reconnect_success_total: AtomicU64, + me_single_endpoint_quarantine_bypass_total: AtomicU64, + me_single_endpoint_shadow_rotate_total: AtomicU64, me_handshake_error_codes: DashMap, me_route_drop_no_conn: AtomicU64, me_route_drop_channel_closed: AtomicU64, @@ -383,6 +389,42 @@ impl Stats { .fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_single_endpoint_outage_enter_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_outage_enter_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_single_endpoint_outage_exit_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_outage_exit_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_single_endpoint_outage_reconnect_attempt_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_outage_reconnect_attempt_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_single_endpoint_outage_reconnect_success_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_outage_reconnect_success_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_single_endpoint_quarantine_bypass_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_quarantine_bypass_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_single_endpoint_shadow_rotate_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_shadow_rotate_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) } pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) } pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) } @@ -413,6 +455,30 @@ impl Stats { self.me_hardswap_pending_ttl_expired_total .load(Ordering::Relaxed) } + pub fn get_me_single_endpoint_outage_enter_total(&self) -> u64 { + self.me_single_endpoint_outage_enter_total + .load(Ordering::Relaxed) + } + pub fn get_me_single_endpoint_outage_exit_total(&self) -> u64 { + self.me_single_endpoint_outage_exit_total + .load(Ordering::Relaxed) + } + pub fn get_me_single_endpoint_outage_reconnect_attempt_total(&self) -> u64 { + self.me_single_endpoint_outage_reconnect_attempt_total + .load(Ordering::Relaxed) + } + pub fn get_me_single_endpoint_outage_reconnect_success_total(&self) -> u64 { + self.me_single_endpoint_outage_reconnect_success_total + .load(Ordering::Relaxed) + } + pub fn get_me_single_endpoint_quarantine_bypass_total(&self) -> u64 { + self.me_single_endpoint_quarantine_bypass_total + .load(Ordering::Relaxed) + } + pub fn get_me_single_endpoint_shadow_rotate_total(&self) -> u64 { + self.me_single_endpoint_shadow_rotate_total + .load(Ordering::Relaxed) + } pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> { let mut out: Vec<(i32, u64)> = self .me_handshake_error_codes diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 2772b27..6da14fa 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -276,6 +276,12 @@ async fn run_update_cycle( cfg.general.me_bind_stale_ttl_secs, cfg.general.me_secret_atomic_snapshot, cfg.general.me_deterministic_writer_sort, + cfg.general.me_single_endpoint_shadow_writers, + cfg.general.me_single_endpoint_outage_mode_enabled, + cfg.general.me_single_endpoint_outage_disable_quarantine, + cfg.general.me_single_endpoint_outage_backoff_min_ms, + cfg.general.me_single_endpoint_outage_backoff_max_ms, + cfg.general.me_single_endpoint_shadow_rotate_every_secs, ); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); @@ -478,6 +484,12 @@ pub async fn me_config_updater( cfg.general.me_bind_stale_ttl_secs, cfg.general.me_secret_atomic_snapshot, cfg.general.me_deterministic_writer_sort, + cfg.general.me_single_endpoint_shadow_writers, + cfg.general.me_single_endpoint_outage_mode_enabled, + cfg.general.me_single_endpoint_outage_disable_quarantine, + cfg.general.me_single_endpoint_outage_backoff_min_ms, + cfg.general.me_single_endpoint_outage_backoff_max_ms, + cfg.general.me_single_endpoint_shadow_rotate_every_secs, ); let new_secs = cfg.general.effective_update_every_secs().max(1); if new_secs == update_every_secs { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index afa96c6..c07ec4f 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; +use std::collections::HashSet; use std::net::SocketAddr; use std::sync::Arc; use std::time::{Duration, Instant}; -use tracing::{debug, info, warn}; use rand::Rng; +use tracing::{debug, info, warn}; use crate::crypto::SecureRandom; use crate::network::IpFamily; @@ -15,11 +16,16 @@ const HEALTH_INTERVAL_SECS: u64 = 1; const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff #[allow(dead_code)] const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1; +const SHADOW_ROTATE_RETRY_SECS: u64 = 30; pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_connections: usize) { let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new(); let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut inflight: HashMap<(i32, IpFamily), usize> = HashMap::new(); + let mut outage_backoff: HashMap<(i32, IpFamily), u64> = HashMap::new(); + let mut outage_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new(); + let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new(); loop { tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await; pool.prune_closed_writers().await; @@ -30,6 +36,10 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut backoff, &mut next_attempt, &mut inflight, + &mut outage_backoff, + &mut outage_next_attempt, + &mut single_endpoint_outage, + &mut shadow_rotate_deadline, ) .await; check_family( @@ -39,6 +49,10 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut backoff, &mut next_attempt, &mut inflight, + &mut outage_backoff, + &mut outage_next_attempt, + &mut single_endpoint_outage, + &mut shadow_rotate_deadline, ) .await; } @@ -51,6 +65,10 @@ async fn check_family( backoff: &mut HashMap<(i32, IpFamily), u64>, next_attempt: &mut HashMap<(i32, IpFamily), Instant>, inflight: &mut HashMap<(i32, IpFamily), usize>, + outage_backoff: &mut HashMap<(i32, IpFamily), u64>, + outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, + single_endpoint_outage: &mut HashSet<(i32, IpFamily)>, + shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>, ) { let enabled = match family { IpFamily::V4 => pool.decision.ipv4_me, @@ -78,31 +96,86 @@ async fn check_family( } let mut live_addr_counts = HashMap::::new(); - for writer in pool - .writers - .read() - .await - .iter() - .filter(|w| !w.draining.load(std::sync::atomic::Ordering::Relaxed)) - { + let mut live_writer_ids_by_addr = HashMap::>::new(); + for writer in pool.writers.read().await.iter().filter(|w| { + !w.draining.load(std::sync::atomic::Ordering::Relaxed) + }) { *live_addr_counts.entry(writer.addr).or_insert(0) += 1; + live_writer_ids_by_addr + .entry(writer.addr) + .or_default() + .push(writer.id); } for (dc, endpoints) in dc_endpoints { if endpoints.is_empty() { continue; } - let required = MePool::required_writers_for_dc(endpoints.len()); + let required = pool.required_writers_for_dc(endpoints.len()); let alive = endpoints .iter() .map(|addr| *live_addr_counts.get(addr).unwrap_or(&0)) .sum::(); + let key = (dc, family); + + if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 { + if single_endpoint_outage.insert(key) { + pool.stats.increment_me_single_endpoint_outage_enter_total(); + warn!( + dc = %dc, + ?family, + required, + endpoint_count = endpoints.len(), + "Single-endpoint DC outage detected" + ); + } + + recover_single_endpoint_outage( + pool, + rng, + key, + endpoints[0], + required, + outage_backoff, + outage_next_attempt, + ) + .await; + continue; + } + + if single_endpoint_outage.remove(&key) { + pool.stats.increment_me_single_endpoint_outage_exit_total(); + outage_backoff.remove(&key); + outage_next_attempt.remove(&key); + shadow_rotate_deadline.remove(&key); + info!( + dc = %dc, + ?family, + alive, + required, + endpoint_count = endpoints.len(), + "Single-endpoint DC outage recovered" + ); + } + if alive >= required { + maybe_rotate_single_endpoint_shadow( + pool, + rng, + key, + dc, + family, + &endpoints, + alive, + required, + &live_writer_ids_by_addr, + shadow_rotate_deadline, + ) + .await; continue; } let missing = required - alive; - let key = (dc, family); let now = Instant::now(); if let Some(ts) = next_attempt.get(&key) && now < *ts @@ -188,3 +261,207 @@ async fn check_family( } } } + +async fn recover_single_endpoint_outage( + pool: &Arc, + rng: &Arc, + key: (i32, IpFamily), + endpoint: SocketAddr, + required: usize, + outage_backoff: &mut HashMap<(i32, IpFamily), u64>, + outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, +) { + let now = Instant::now(); + if let Some(ts) = outage_next_attempt.get(&key) + && now < *ts + { + return; + } + + let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms(); + pool.stats + .increment_me_single_endpoint_outage_reconnect_attempt_total(); + + let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine(); + let attempt_ok = if bypass_quarantine { + pool.stats + .increment_me_single_endpoint_quarantine_bypass_total(); + match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + Ok(Ok(())) => true, + Ok(Err(e)) => { + debug!( + dc = %key.0, + family = ?key.1, + %endpoint, + error = %e, + "Single-endpoint outage reconnect failed (quarantine bypass path)" + ); + false + } + Err(_) => { + debug!( + dc = %key.0, + family = ?key.1, + %endpoint, + "Single-endpoint outage reconnect timed out (quarantine bypass path)" + ); + false + } + } + } else { + let one_endpoint = [endpoint]; + match tokio::time::timeout( + pool.me_one_timeout, + pool.connect_endpoints_round_robin(&one_endpoint, rng.as_ref()), + ) + .await + { + Ok(ok) => ok, + Err(_) => { + debug!( + dc = %key.0, + family = ?key.1, + %endpoint, + "Single-endpoint outage reconnect timed out" + ); + false + } + } + }; + + if attempt_ok { + pool.stats + .increment_me_single_endpoint_outage_reconnect_success_total(); + pool.stats.increment_me_reconnect_success(); + outage_backoff.insert(key, min_backoff_ms); + let jitter = min_backoff_ms / JITTER_FRAC_NUM; + let wait = Duration::from_millis(min_backoff_ms) + + Duration::from_millis(rand::rng().random_range(0..=jitter.max(1))); + outage_next_attempt.insert(key, now + wait); + info!( + dc = %key.0, + family = ?key.1, + %endpoint, + required, + backoff_ms = min_backoff_ms, + "Single-endpoint outage reconnect succeeded" + ); + return; + } + + pool.stats.increment_me_reconnect_attempt(); + let current_ms = *outage_backoff.get(&key).unwrap_or(&min_backoff_ms); + let next_ms = current_ms.saturating_mul(2).min(max_backoff_ms); + outage_backoff.insert(key, next_ms); + let jitter = next_ms / JITTER_FRAC_NUM; + let wait = Duration::from_millis(next_ms) + + Duration::from_millis(rand::rng().random_range(0..=jitter.max(1))); + outage_next_attempt.insert(key, now + wait); + warn!( + dc = %key.0, + family = ?key.1, + %endpoint, + required, + backoff_ms = next_ms, + "Single-endpoint outage reconnect scheduled" + ); +} + +async fn maybe_rotate_single_endpoint_shadow( + pool: &Arc, + rng: &Arc, + key: (i32, IpFamily), + dc: i32, + family: IpFamily, + endpoints: &[SocketAddr], + alive: usize, + required: usize, + live_writer_ids_by_addr: &HashMap>, + shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>, +) { + if endpoints.len() != 1 || alive < required { + return; + } + + let Some(interval) = pool.single_endpoint_shadow_rotate_interval() else { + return; + }; + + let now = Instant::now(); + if let Some(deadline) = shadow_rotate_deadline.get(&key) + && now < *deadline + { + return; + } + + let endpoint = endpoints[0]; + let Some(writer_ids) = live_writer_ids_by_addr.get(&endpoint) else { + shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)); + return; + }; + + let mut candidate_writer_id = None; + for writer_id in writer_ids { + if pool.registry.is_writer_empty(*writer_id).await { + candidate_writer_id = Some(*writer_id); + break; + } + } + + let Some(old_writer_id) = candidate_writer_id else { + shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)); + debug!( + dc = %dc, + ?family, + %endpoint, + alive, + required, + "Single-endpoint shadow rotation skipped: no empty writer candidate" + ); + return; + }; + + let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + Ok(Ok(())) => true, + Ok(Err(e)) => { + debug!( + dc = %dc, + ?family, + %endpoint, + error = %e, + "Single-endpoint shadow rotation connect failed" + ); + false + } + Err(_) => { + debug!( + dc = %dc, + ?family, + %endpoint, + "Single-endpoint shadow rotation connect timed out" + ); + false + } + }; + + if !rotate_ok { + shadow_rotate_deadline.insert( + key, + now + interval.min(Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)), + ); + return; + } + + pool.mark_writer_draining_with_timeout(old_writer_id, pool.force_close_timeout(), false) + .await; + pool.stats.increment_me_single_endpoint_shadow_rotate_total(); + shadow_rotate_deadline.insert(key, now + interval); + info!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + rotate_every_secs = interval.as_secs(), + "Single-endpoint shadow writer rotated" + ); +} diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index ed6969a..5a1046b 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -101,6 +101,12 @@ pub struct MePool { pub(super) me_reconnect_backoff_base: Duration, pub(super) me_reconnect_backoff_cap: Duration, pub(super) me_reconnect_fast_retry_count: u32, + pub(super) me_single_endpoint_shadow_writers: AtomicU8, + pub(super) me_single_endpoint_outage_mode_enabled: AtomicBool, + pub(super) me_single_endpoint_outage_disable_quarantine: AtomicBool, + pub(super) me_single_endpoint_outage_backoff_min_ms: AtomicU64, + pub(super) me_single_endpoint_outage_backoff_max_ms: AtomicU64, + pub(super) me_single_endpoint_shadow_rotate_every_secs: AtomicU64, pub(super) proxy_map_v4: Arc>>>, pub(super) proxy_map_v6: Arc>>>, pub(super) default_dc: AtomicI32, @@ -189,6 +195,12 @@ impl MePool { me_reconnect_backoff_base_ms: u64, me_reconnect_backoff_cap_ms: u64, me_reconnect_fast_retry_count: u32, + me_single_endpoint_shadow_writers: u8, + me_single_endpoint_outage_mode_enabled: bool, + me_single_endpoint_outage_disable_quarantine: bool, + me_single_endpoint_outage_backoff_min_ms: u64, + me_single_endpoint_outage_backoff_max_ms: u64, + me_single_endpoint_shadow_rotate_every_secs: u64, hardswap: bool, me_pool_drain_ttl_secs: u64, me_pool_force_close_secs: u64, @@ -259,6 +271,22 @@ impl MePool { me_reconnect_backoff_base: Duration::from_millis(me_reconnect_backoff_base_ms), me_reconnect_backoff_cap: Duration::from_millis(me_reconnect_backoff_cap_ms), me_reconnect_fast_retry_count, + me_single_endpoint_shadow_writers: AtomicU8::new(me_single_endpoint_shadow_writers), + me_single_endpoint_outage_mode_enabled: AtomicBool::new( + me_single_endpoint_outage_mode_enabled, + ), + me_single_endpoint_outage_disable_quarantine: AtomicBool::new( + me_single_endpoint_outage_disable_quarantine, + ), + me_single_endpoint_outage_backoff_min_ms: AtomicU64::new( + me_single_endpoint_outage_backoff_min_ms, + ), + me_single_endpoint_outage_backoff_max_ms: AtomicU64::new( + me_single_endpoint_outage_backoff_max_ms, + ), + me_single_endpoint_shadow_rotate_every_secs: AtomicU64::new( + me_single_endpoint_shadow_rotate_every_secs, + ), pool_size: 2, proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), @@ -317,6 +345,12 @@ impl MePool { bind_stale_ttl_secs: u64, secret_atomic_snapshot: bool, deterministic_writer_sort: bool, + single_endpoint_shadow_writers: u8, + single_endpoint_outage_mode_enabled: bool, + single_endpoint_outage_disable_quarantine: bool, + single_endpoint_outage_backoff_min_ms: u64, + single_endpoint_outage_backoff_max_ms: u64, + single_endpoint_shadow_rotate_every_secs: u64, ) { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs @@ -341,6 +375,18 @@ impl MePool { .store(secret_atomic_snapshot, Ordering::Relaxed); self.me_deterministic_writer_sort .store(deterministic_writer_sort, Ordering::Relaxed); + self.me_single_endpoint_shadow_writers + .store(single_endpoint_shadow_writers, Ordering::Relaxed); + self.me_single_endpoint_outage_mode_enabled + .store(single_endpoint_outage_mode_enabled, Ordering::Relaxed); + self.me_single_endpoint_outage_disable_quarantine + .store(single_endpoint_outage_disable_quarantine, Ordering::Relaxed); + self.me_single_endpoint_outage_backoff_min_ms + .store(single_endpoint_outage_backoff_min_ms, Ordering::Relaxed); + self.me_single_endpoint_outage_backoff_max_ms + .store(single_endpoint_outage_backoff_max_ms, Ordering::Relaxed); + self.me_single_endpoint_shadow_rotate_every_secs + .store(single_endpoint_shadow_rotate_every_secs, Ordering::Relaxed); } pub fn reset_stun_state(&self) { @@ -405,6 +451,54 @@ impl MePool { MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed)) } + pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize { + if endpoint_count == 0 { + return 0; + } + if endpoint_count == 1 { + let shadow = self + .me_single_endpoint_shadow_writers + .load(Ordering::Relaxed) as usize; + return (1 + shadow).max(3); + } + endpoint_count.max(3) + } + + pub(super) fn single_endpoint_outage_mode_enabled(&self) -> bool { + self.me_single_endpoint_outage_mode_enabled + .load(Ordering::Relaxed) + } + + pub(super) fn single_endpoint_outage_disable_quarantine(&self) -> bool { + self.me_single_endpoint_outage_disable_quarantine + .load(Ordering::Relaxed) + } + + pub(super) fn single_endpoint_outage_backoff_bounds_ms(&self) -> (u64, u64) { + let min_ms = self + .me_single_endpoint_outage_backoff_min_ms + .load(Ordering::Relaxed); + let max_ms = self + .me_single_endpoint_outage_backoff_max_ms + .load(Ordering::Relaxed); + if min_ms <= max_ms { + (min_ms, max_ms) + } else { + (max_ms, min_ms) + } + } + + pub(super) fn single_endpoint_shadow_rotate_interval(&self) -> Option { + let secs = self + .me_single_endpoint_shadow_rotate_every_secs + .load(Ordering::Relaxed); + if secs == 0 { + None + } else { + Some(Duration::from_secs(secs)) + } + } + pub(super) fn family_order(&self) -> Vec { let mut order = Vec::new(); if self.decision.prefer_ipv6() { diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 33b8cc4..d5242b7 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -148,10 +148,6 @@ impl MePool { out } - pub(super) fn required_writers_for_dc(endpoint_count: usize) -> usize { - endpoint_count.max(3) - } - fn hardswap_warmup_connect_delay_ms(&self) -> u64 { let min_ms = self.me_hardswap_warmup_delay_min_ms.load(Ordering::Relaxed); let max_ms = self.me_hardswap_warmup_delay_max_ms.load(Ordering::Relaxed); @@ -221,7 +217,7 @@ impl MePool { let mut endpoint_list: Vec = endpoints.iter().copied().collect(); endpoint_list.sort_unstable(); - let required = Self::required_writers_for_dc(endpoint_list.len()); + let required = self.required_writers_for_dc(endpoint_list.len()); let mut completed = false; let mut last_fresh_count = self .fresh_writer_count_for_endpoints(generation, endpoints) @@ -409,7 +405,7 @@ impl MePool { if endpoints.is_empty() { continue; } - let required = Self::required_writers_for_dc(endpoints.len()); + let required = self.required_writers_for_dc(endpoints.len()); let fresh_count = writers .iter() .filter(|w| !w.draining.load(Ordering::Relaxed)) From b7a8e759eb6c18fa9cdca10979e1a33941f843a8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:36:00 +0300 Subject: [PATCH 013/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5fce3d6..46e8be5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.1.3" +version = "3.1.4" edition = "2024" [dependencies] From 762deac511c719a0d77ba89f2d2c31f7899e1fe6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:03:44 +0300 Subject: [PATCH 014/261] ME Healthcheck fixes Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/handshake.rs | 87 +++++++++++++++-------- src/transport/middle_proxy/health.rs | 13 ++++ src/transport/middle_proxy/pool_refill.rs | 2 +- 3 files changed, 72 insertions(+), 30 deletions(-) diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 251c911..1016c6b 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -38,6 +38,22 @@ use super::MePool; const ME_KDF_DRIFT_STRICT: bool = false; +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum KdfClientPortSource { + LocalSocket = 0, + SocksBound = 1, +} + +impl KdfClientPortSource { + fn from_socks_bound_port(socks_bound_port: Option) -> Self { + if socks_bound_port.is_some() { + Self::SocksBound + } else { + Self::LocalSocket + } + } +} + /// Result of a successful ME handshake with timings. pub(crate) struct HandshakeOutput { pub rd: ReadHalf, @@ -52,18 +68,18 @@ pub(crate) struct HandshakeOutput { impl MePool { fn kdf_material_fingerprint( - local_addr_nat: SocketAddr, + local_ip_nat: IpAddr, peer_addr_nat: SocketAddr, - client_port_for_kdf: u16, - reflected: Option, - socks_bound_addr: Option, + reflected_ip: Option, + socks_bound_ip: Option, + client_port_source: KdfClientPortSource, ) -> u64 { let mut hasher = DefaultHasher::new(); - local_addr_nat.hash(&mut hasher); + local_ip_nat.hash(&mut hasher); peer_addr_nat.hash(&mut hasher); - client_port_for_kdf.hash(&mut hasher); - reflected.hash(&mut hasher); - socks_bound_addr.hash(&mut hasher); + reflected_ip.hash(&mut hasher); + socks_bound_ip.hash(&mut hasher); + client_port_source.hash(&mut hasher); hasher.finish() } @@ -359,35 +375,48 @@ impl MePool { let ts_bytes = crypto_ts.to_le_bytes(); let server_port_bytes = peer_addr_nat.port().to_le_bytes(); - let client_port_for_kdf = socks_bound_addr + let socks_bound_port = socks_bound_addr .map(|bound| bound.port()) - .filter(|port| *port != 0) - .unwrap_or(local_addr_nat.port()); + .filter(|port| *port != 0); + let client_port_for_kdf = socks_bound_port.unwrap_or(local_addr_nat.port()); + let client_port_source = KdfClientPortSource::from_socks_bound_port(socks_bound_port); let kdf_fingerprint = Self::kdf_material_fingerprint( - local_addr_nat, + local_addr_nat.ip(), peer_addr_nat, - client_port_for_kdf, - reflected, - socks_bound_addr, + reflected.map(|value| value.ip()), + socks_bound_addr.map(|value| value.ip()), + client_port_source, ); let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.lock().await; - if let Some(prev_fingerprint) = kdf_fingerprint_guard.get(&peer_addr_nat).copied() - && prev_fingerprint != kdf_fingerprint + if let Some((prev_fingerprint, prev_client_port)) = + kdf_fingerprint_guard.get(&peer_addr_nat).copied() { - self.stats.increment_me_kdf_drift_total(); - warn!( - %peer_addr_nat, - %local_addr_nat, - client_port_for_kdf, - "ME KDF input drift detected for endpoint" - ); - if ME_KDF_DRIFT_STRICT { - return Err(ProxyError::InvalidHandshake( - "ME KDF input drift detected (strict mode)".to_string(), - )); + if prev_fingerprint != kdf_fingerprint { + self.stats.increment_me_kdf_drift_total(); + warn!( + %peer_addr_nat, + %local_addr_nat, + client_port_for_kdf, + client_port_source = ?client_port_source, + "ME KDF material drift detected for endpoint" + ); + if ME_KDF_DRIFT_STRICT { + return Err(ProxyError::InvalidHandshake( + "ME KDF material drift detected (strict mode)".to_string(), + )); + } + } else if prev_client_port != client_port_for_kdf { + self.stats.increment_me_kdf_port_only_drift_total(); + debug!( + %peer_addr_nat, + previous_client_port_for_kdf = prev_client_port, + client_port_for_kdf, + client_port_source = ?client_port_source, + "ME KDF client port changed with stable material" + ); } } - kdf_fingerprint_guard.insert(peer_addr_nat, kdf_fingerprint); + kdf_fingerprint_guard.insert(peer_addr_nat, (kdf_fingerprint, client_port_for_kdf)); drop(kdf_fingerprint_guard); let client_port_bytes = client_port_for_kdf.to_le_bytes(); diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index c07ec4f..192bf1b 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -395,6 +395,19 @@ async fn maybe_rotate_single_endpoint_shadow( } let endpoint = endpoints[0]; + if pool.is_endpoint_quarantined(endpoint).await { + pool.stats + .increment_me_single_endpoint_shadow_rotate_skipped_quarantine_total(); + shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)); + debug!( + dc = %dc, + ?family, + %endpoint, + "Single-endpoint shadow rotation skipped: endpoint is quarantined" + ); + return; + } + let Some(writer_ids) = live_writer_ids_by_addr.get(&endpoint) else { shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)); return; diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 92071bd..6e14617 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -37,7 +37,7 @@ impl MePool { ); } - async fn is_endpoint_quarantined(&self, addr: SocketAddr) -> bool { + pub(super) async fn is_endpoint_quarantined(&self, addr: SocketAddr) -> bool { let mut guard = self.endpoint_quarantine.lock().await; let now = Instant::now(); guard.retain(|_, expiry| *expiry > now); From 71261522bdfcf4e8960e3a3f5c14e89f1f4f1109 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:04:07 +0300 Subject: [PATCH 015/261] Update pool.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 5a1046b..14133b4 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -127,7 +127,7 @@ pub struct MePool { pub(super) pending_hardswap_map_hash: AtomicU64, pub(super) hardswap: AtomicBool, pub(super) endpoint_quarantine: Arc>>, - pub(super) kdf_material_fingerprint: Arc>>, + pub(super) kdf_material_fingerprint: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, pub(super) me_pool_force_close_secs: AtomicU64, pub(super) me_pool_min_fresh_ratio_permille: AtomicU32, From 3799fc13c45e77a002163a92833e5fbef2dc1e68 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:04:45 +0300 Subject: [PATCH 016/261] ME Pool in Metrics Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/metrics.rs | 35 ++++++++++++++++++++++++++++++++++- src/stats/mod.rs | 23 ++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 4f4c317..4c03dc2 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -449,6 +449,21 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_kdf_port_only_drift_total ME KDF client-port changes with stable non-port material" + ); + let _ = writeln!(out, "# TYPE telemt_me_kdf_port_only_drift_total counter"); + let _ = writeln!( + out, + "telemt_me_kdf_port_only_drift_total {}", + if me_allows_debug { + stats.get_me_kdf_port_only_drift_total() + } else { + 0 + } + ); + let _ = writeln!( out, "# HELP telemt_me_hardswap_pending_reuse_total Hardswap cycles that reused an existing pending generation" @@ -587,6 +602,24 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total Shadow rotations skipped because endpoint is quarantined" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total counter" + ); + let _ = writeln!( + out, + "telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total {}", + if me_allows_normal { + stats.get_me_single_endpoint_shadow_rotate_skipped_quarantine_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"); let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter"); let _ = writeln!( @@ -679,7 +712,7 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp let _ = writeln!( out, "telemt_pool_swap_total {}", - if me_allows_debug { + if me_allows_normal { stats.get_pool_swap_total() } else { 0 diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 5b2bb94..57b732d 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -38,6 +38,7 @@ pub struct Stats { me_seq_mismatch: AtomicU64, me_endpoint_quarantine_total: AtomicU64, me_kdf_drift_total: AtomicU64, + me_kdf_port_only_drift_total: AtomicU64, me_hardswap_pending_reuse_total: AtomicU64, me_hardswap_pending_ttl_expired_total: AtomicU64, me_single_endpoint_outage_enter_total: AtomicU64, @@ -46,6 +47,7 @@ pub struct Stats { me_single_endpoint_outage_reconnect_success_total: AtomicU64, me_single_endpoint_quarantine_bypass_total: AtomicU64, me_single_endpoint_shadow_rotate_total: AtomicU64, + me_single_endpoint_shadow_rotate_skipped_quarantine_total: AtomicU64, me_handshake_error_codes: DashMap, me_route_drop_no_conn: AtomicU64, me_route_drop_channel_closed: AtomicU64, @@ -290,7 +292,7 @@ impl Stats { } } pub fn increment_pool_swap_total(&self) { - if self.telemetry_me_allows_debug() { + if self.telemetry_me_allows_normal() { self.pool_swap_total.fetch_add(1, Ordering::Relaxed); } } @@ -377,6 +379,12 @@ impl Stats { self.me_kdf_drift_total.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_kdf_port_only_drift_total(&self) { + if self.telemetry_me_allows_debug() { + self.me_kdf_port_only_drift_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_hardswap_pending_reuse_total(&self) { if self.telemetry_me_allows_debug() { self.me_hardswap_pending_reuse_total @@ -425,6 +433,12 @@ impl Stats { .fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_single_endpoint_shadow_rotate_skipped_quarantine_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_single_endpoint_shadow_rotate_skipped_quarantine_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) } pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) } pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) } @@ -447,6 +461,9 @@ impl Stats { pub fn get_me_kdf_drift_total(&self) -> u64 { self.me_kdf_drift_total.load(Ordering::Relaxed) } + pub fn get_me_kdf_port_only_drift_total(&self) -> u64 { + self.me_kdf_port_only_drift_total.load(Ordering::Relaxed) + } pub fn get_me_hardswap_pending_reuse_total(&self) -> u64 { self.me_hardswap_pending_reuse_total .load(Ordering::Relaxed) @@ -479,6 +496,10 @@ impl Stats { self.me_single_endpoint_shadow_rotate_total .load(Ordering::Relaxed) } + pub fn get_me_single_endpoint_shadow_rotate_skipped_quarantine_total(&self) -> u64 { + self.me_single_endpoint_shadow_rotate_skipped_quarantine_total + .load(Ordering::Relaxed) + } pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> { let mut out: Vec<(i32, u64)> = self .me_handshake_error_codes From 235642459ab17c46b52207731db30268c7284ef8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:08:15 +0300 Subject: [PATCH 017/261] ME Keepalive 8/2 Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 5602287..544e328 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -130,11 +130,11 @@ pub(crate) fn default_middle_proxy_warm_standby() -> usize { } pub(crate) fn default_keepalive_interval() -> u64 { - 25 + 8 } pub(crate) fn default_keepalive_jitter() -> u64 { - 5 + 2 } pub(crate) fn default_warmup_step_delay_ms() -> u64 { From 3b8eea762bbaa17e4cc7504c4a1b3772dde857d5 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:28:37 +0300 Subject: [PATCH 018/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 46e8be5..61f4d24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.1.4" +version = "3.1.5" edition = "2024" [dependencies] From 58ff0c7971fee0ded9f63ec5c52e790e6c757594 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:35:47 +0300 Subject: [PATCH 019/261] Update pool.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/pool.rs | 79 +++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 14133b4..5ae922a 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, Notify, RwLock, mpsc}; use tokio_util::sync::CancellationToken; -use crate::config::{MeBindStaleMode, MeSocksKdfPolicy}; +use crate::config::{MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy}; use crate::crypto::SecureRandom; use crate::network::IpFamily; use crate::network::probe::NetworkDecision; @@ -107,6 +107,10 @@ pub struct MePool { pub(super) me_single_endpoint_outage_backoff_min_ms: AtomicU64, pub(super) me_single_endpoint_outage_backoff_max_ms: AtomicU64, pub(super) me_single_endpoint_shadow_rotate_every_secs: AtomicU64, + pub(super) me_floor_mode: AtomicU8, + pub(super) me_adaptive_floor_idle_secs: AtomicU64, + pub(super) me_adaptive_floor_min_writers_single_endpoint: AtomicU8, + pub(super) me_adaptive_floor_recover_grace_secs: AtomicU64, pub(super) proxy_map_v4: Arc>>>, pub(super) proxy_map_v6: Arc>>>, pub(super) default_dc: AtomicI32, @@ -201,6 +205,10 @@ impl MePool { me_single_endpoint_outage_backoff_min_ms: u64, me_single_endpoint_outage_backoff_max_ms: u64, me_single_endpoint_shadow_rotate_every_secs: u64, + me_floor_mode: MeFloorMode, + me_adaptive_floor_idle_secs: u64, + me_adaptive_floor_min_writers_single_endpoint: u8, + me_adaptive_floor_recover_grace_secs: u64, hardswap: bool, me_pool_drain_ttl_secs: u64, me_pool_force_close_secs: u64, @@ -287,6 +295,14 @@ impl MePool { me_single_endpoint_shadow_rotate_every_secs: AtomicU64::new( me_single_endpoint_shadow_rotate_every_secs, ), + me_floor_mode: AtomicU8::new(me_floor_mode.as_u8()), + me_adaptive_floor_idle_secs: AtomicU64::new(me_adaptive_floor_idle_secs), + me_adaptive_floor_min_writers_single_endpoint: AtomicU8::new( + me_adaptive_floor_min_writers_single_endpoint, + ), + me_adaptive_floor_recover_grace_secs: AtomicU64::new( + me_adaptive_floor_recover_grace_secs, + ), pool_size: 2, proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), @@ -351,6 +367,10 @@ impl MePool { single_endpoint_outage_backoff_min_ms: u64, single_endpoint_outage_backoff_max_ms: u64, single_endpoint_shadow_rotate_every_secs: u64, + floor_mode: MeFloorMode, + adaptive_floor_idle_secs: u64, + adaptive_floor_min_writers_single_endpoint: u8, + adaptive_floor_recover_grace_secs: u64, ) { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs @@ -387,6 +407,29 @@ impl MePool { .store(single_endpoint_outage_backoff_max_ms, Ordering::Relaxed); self.me_single_endpoint_shadow_rotate_every_secs .store(single_endpoint_shadow_rotate_every_secs, Ordering::Relaxed); + let previous_floor_mode = self.floor_mode(); + self.me_floor_mode + .store(floor_mode.as_u8(), Ordering::Relaxed); + self.me_adaptive_floor_idle_secs + .store(adaptive_floor_idle_secs, Ordering::Relaxed); + self.me_adaptive_floor_min_writers_single_endpoint + .store(adaptive_floor_min_writers_single_endpoint, Ordering::Relaxed); + self.me_adaptive_floor_recover_grace_secs + .store(adaptive_floor_recover_grace_secs, Ordering::Relaxed); + if previous_floor_mode != floor_mode { + self.stats.increment_me_floor_mode_switch_total(); + match (previous_floor_mode, floor_mode) { + (MeFloorMode::Static, MeFloorMode::Adaptive) => { + self.stats + .increment_me_floor_mode_switch_static_to_adaptive_total(); + } + (MeFloorMode::Adaptive, MeFloorMode::Static) => { + self.stats + .increment_me_floor_mode_switch_adaptive_to_static_total(); + } + _ => {} + } + } } pub fn reset_stun_state(&self) { @@ -464,6 +507,40 @@ impl MePool { endpoint_count.max(3) } + pub(super) fn floor_mode(&self) -> MeFloorMode { + MeFloorMode::from_u8(self.me_floor_mode.load(Ordering::Relaxed)) + } + + pub(super) fn adaptive_floor_idle_duration(&self) -> Duration { + Duration::from_secs(self.me_adaptive_floor_idle_secs.load(Ordering::Relaxed)) + } + + pub(super) fn adaptive_floor_recover_grace_duration(&self) -> Duration { + Duration::from_secs( + self.me_adaptive_floor_recover_grace_secs + .load(Ordering::Relaxed), + ) + } + + pub(super) fn required_writers_for_dc_with_floor_mode( + &self, + endpoint_count: usize, + reduce_for_idle: bool, + ) -> usize { + let base_required = self.required_writers_for_dc(endpoint_count); + if !reduce_for_idle { + return base_required; + } + if endpoint_count != 1 || self.floor_mode() != MeFloorMode::Adaptive { + return base_required; + } + let min_writers = (self + .me_adaptive_floor_min_writers_single_endpoint + .load(Ordering::Relaxed) as usize) + .max(1); + base_required.min(min_writers) + } + pub(super) fn single_endpoint_outage_mode_enabled(&self) -> bool { self.me_single_endpoint_outage_mode_enabled .load(Ordering::Relaxed) From 4a0d88ad43a935eb76d68eda9144274416e47a71 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:35:57 +0300 Subject: [PATCH 020/261] Update health.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 77 +++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 192bf1b..55d8409 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -7,6 +7,7 @@ use std::time::{Duration, Instant}; use rand::Rng; use tracing::{debug, info, warn}; +use crate::config::MeFloorMode; use crate::crypto::SecureRandom; use crate::network::IpFamily; @@ -26,6 +27,8 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c let mut outage_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new(); let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new(); loop { tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await; pool.prune_closed_writers().await; @@ -40,6 +43,8 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut outage_next_attempt, &mut single_endpoint_outage, &mut shadow_rotate_deadline, + &mut adaptive_idle_since, + &mut adaptive_recover_until, ) .await; check_family( @@ -53,6 +58,8 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut outage_next_attempt, &mut single_endpoint_outage, &mut shadow_rotate_deadline, + &mut adaptive_idle_since, + &mut adaptive_recover_until, ) .await; } @@ -69,6 +76,8 @@ async fn check_family( outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, single_endpoint_outage: &mut HashSet<(i32, IpFamily)>, shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>, + adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, + adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, ) { let enabled = match family { IpFamily::V4 => pool.decision.ipv4_me, @@ -95,6 +104,11 @@ async fn check_family( endpoints.dedup(); } + if pool.floor_mode() == MeFloorMode::Static { + adaptive_idle_since.clear(); + adaptive_recover_until.clear(); + } + let mut live_addr_counts = HashMap::::new(); let mut live_writer_ids_by_addr = HashMap::>::new(); for writer in pool.writers.read().await.iter().filter(|w| { @@ -111,12 +125,21 @@ async fn check_family( if endpoints.is_empty() { continue; } - let required = pool.required_writers_for_dc(endpoints.len()); + let key = (dc, family); + let reduce_for_idle = should_reduce_floor_for_idle( + pool, + key, + &endpoints, + &live_writer_ids_by_addr, + adaptive_idle_since, + adaptive_recover_until, + ) + .await; + let required = pool.required_writers_for_dc_with_floor_mode(endpoints.len(), reduce_for_idle); let alive = endpoints .iter() .map(|addr| *live_addr_counts.get(addr).unwrap_or(&0)) .sum::(); - let key = (dc, family); if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 { if single_endpoint_outage.insert(key) { @@ -148,6 +171,8 @@ async fn check_family( outage_backoff.remove(&key); outage_next_attempt.remove(&key); shadow_rotate_deadline.remove(&key); + adaptive_idle_since.remove(&key); + adaptive_recover_until.remove(&key); info!( dc = %dc, ?family, @@ -262,6 +287,54 @@ async fn check_family( } } +async fn should_reduce_floor_for_idle( + pool: &Arc, + key: (i32, IpFamily), + endpoints: &[SocketAddr], + live_writer_ids_by_addr: &HashMap>, + adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, + adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, +) -> bool { + if endpoints.len() != 1 || pool.floor_mode() != MeFloorMode::Adaptive { + adaptive_idle_since.remove(&key); + adaptive_recover_until.remove(&key); + return false; + } + + let now = Instant::now(); + let endpoint = endpoints[0]; + let writer_ids = live_writer_ids_by_addr + .get(&endpoint) + .map(Vec::as_slice) + .unwrap_or(&[]); + let has_bound_clients = has_bound_clients_on_endpoint(pool, writer_ids).await; + if has_bound_clients { + adaptive_idle_since.remove(&key); + adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration()); + return false; + } + + if let Some(recover_until) = adaptive_recover_until.get(&key) + && now < *recover_until + { + adaptive_idle_since.remove(&key); + return false; + } + adaptive_recover_until.remove(&key); + + let idle_since = adaptive_idle_since.entry(key).or_insert(now); + now.saturating_duration_since(*idle_since) >= pool.adaptive_floor_idle_duration() +} + +async fn has_bound_clients_on_endpoint(pool: &Arc, writer_ids: &[u64]) -> bool { + for writer_id in writer_ids { + if !pool.registry.is_writer_empty(*writer_id).await { + return true; + } + } + false +} + async fn recover_single_endpoint_outage( pool: &Arc, rng: &Arc, From 225fc3e4ea7e8fa835dad818c392468deb435735 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:37:00 +0300 Subject: [PATCH 021/261] ME Adaptive Floor Drafts Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 15 +++++++ src/config/load.rs | 35 +++++++++++++++ src/config/types.rs | 45 ++++++++++++++++++++ src/transport/middle_proxy/config_updater.rs | 8 ++++ 4 files changed, 103 insertions(+) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 544e328..d92ae78 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -9,6 +9,9 @@ const DEFAULT_MIDDLE_PROXY_WARM_STANDBY: usize = 16; const DEFAULT_ME_RECONNECT_MAX_CONCURRENT_PER_DC: u32 = 8; const DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT: u32 = 16; const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2; +const DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS: u64 = 90; +const DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT: u8 = 1; +const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 3; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 4; const DEFAULT_LISTEN_ADDR_IPV6: &str = "::"; @@ -185,6 +188,18 @@ pub(crate) fn default_me_single_endpoint_shadow_rotate_every_secs() -> u64 { 900 } +pub(crate) fn default_me_adaptive_floor_idle_secs() -> u64 { + DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS +} + +pub(crate) fn default_me_adaptive_floor_min_writers_single_endpoint() -> u8 { + DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT +} + +pub(crate) fn default_me_adaptive_floor_recover_grace_secs() -> u64 { + DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS +} + pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS } diff --git a/src/config/load.rs b/src/config/load.rs index a027d58..27249c6 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -261,6 +261,15 @@ impl ProxyConfig { )); } + if config.general.me_adaptive_floor_min_writers_single_endpoint == 0 + || config.general.me_adaptive_floor_min_writers_single_endpoint > 32 + { + return Err(ProxyError::Config( + "general.me_adaptive_floor_min_writers_single_endpoint must be within [1, 32]" + .to_string(), + )); + } + if config.general.me_single_endpoint_outage_backoff_min_ms == 0 { return Err(ProxyError::Config( "general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(), @@ -642,6 +651,19 @@ mod tests { cfg.general.me_single_endpoint_shadow_rotate_every_secs, default_me_single_endpoint_shadow_rotate_every_secs() ); + assert_eq!(cfg.general.me_floor_mode, MeFloorMode::default()); + assert_eq!( + cfg.general.me_adaptive_floor_idle_secs, + default_me_adaptive_floor_idle_secs() + ); + assert_eq!( + cfg.general.me_adaptive_floor_min_writers_single_endpoint, + default_me_adaptive_floor_min_writers_single_endpoint() + ); + assert_eq!( + cfg.general.me_adaptive_floor_recover_grace_secs, + default_me_adaptive_floor_recover_grace_secs() + ); assert_eq!( cfg.general.upstream_connect_retry_attempts, default_upstream_connect_retry_attempts() @@ -704,6 +726,19 @@ mod tests { general.me_single_endpoint_shadow_rotate_every_secs, default_me_single_endpoint_shadow_rotate_every_secs() ); + assert_eq!(general.me_floor_mode, MeFloorMode::default()); + assert_eq!( + general.me_adaptive_floor_idle_secs, + default_me_adaptive_floor_idle_secs() + ); + assert_eq!( + general.me_adaptive_floor_min_writers_single_endpoint, + default_me_adaptive_floor_min_writers_single_endpoint() + ); + assert_eq!( + general.me_adaptive_floor_recover_grace_secs, + default_me_adaptive_floor_recover_grace_secs() + ); assert_eq!( general.upstream_connect_retry_attempts, default_upstream_connect_retry_attempts() diff --git a/src/config/types.rs b/src/config/types.rs index 324abf2..0d255ba 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -158,6 +158,31 @@ impl MeBindStaleMode { } } +/// Middle-End writer floor policy mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum MeFloorMode { + #[default] + Static, + Adaptive, +} + +impl MeFloorMode { + pub fn as_u8(self) -> u8 { + match self { + MeFloorMode::Static => 0, + MeFloorMode::Adaptive => 1, + } + } + + pub fn from_u8(raw: u8) -> Self { + match raw { + 1 => MeFloorMode::Adaptive, + _ => MeFloorMode::Static, + } + } +} + /// Telemetry controls for hot-path counters and ME diagnostics. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TelemetryConfig { @@ -419,6 +444,22 @@ pub struct GeneralConfig { #[serde(default = "default_me_single_endpoint_shadow_rotate_every_secs")] pub me_single_endpoint_shadow_rotate_every_secs: u64, + /// Floor policy mode for ME writer targets. + #[serde(default)] + pub me_floor_mode: MeFloorMode, + + /// Idle time in seconds before adaptive floor can reduce single-endpoint writer target. + #[serde(default = "default_me_adaptive_floor_idle_secs")] + pub me_adaptive_floor_idle_secs: u64, + + /// Minimum writer target for single-endpoint DC groups in adaptive floor mode. + #[serde(default = "default_me_adaptive_floor_min_writers_single_endpoint")] + pub me_adaptive_floor_min_writers_single_endpoint: u8, + + /// Grace period in seconds to hold static floor after activity in adaptive mode. + #[serde(default = "default_me_adaptive_floor_recover_grace_secs")] + pub me_adaptive_floor_recover_grace_secs: u64, + /// Connect attempts for the selected upstream before returning error/fallback. #[serde(default = "default_upstream_connect_retry_attempts")] pub upstream_connect_retry_attempts: u32, @@ -634,6 +675,10 @@ impl Default for GeneralConfig { me_single_endpoint_outage_backoff_min_ms: default_me_single_endpoint_outage_backoff_min_ms(), me_single_endpoint_outage_backoff_max_ms: default_me_single_endpoint_outage_backoff_max_ms(), me_single_endpoint_shadow_rotate_every_secs: default_me_single_endpoint_shadow_rotate_every_secs(), + me_floor_mode: MeFloorMode::default(), + me_adaptive_floor_idle_secs: default_me_adaptive_floor_idle_secs(), + me_adaptive_floor_min_writers_single_endpoint: default_me_adaptive_floor_min_writers_single_endpoint(), + me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(), diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 6da14fa..a9c50ab 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -282,6 +282,10 @@ async fn run_update_cycle( cfg.general.me_single_endpoint_outage_backoff_min_ms, cfg.general.me_single_endpoint_outage_backoff_max_ms, cfg.general.me_single_endpoint_shadow_rotate_every_secs, + cfg.general.me_floor_mode, + cfg.general.me_adaptive_floor_idle_secs, + cfg.general.me_adaptive_floor_min_writers_single_endpoint, + cfg.general.me_adaptive_floor_recover_grace_secs, ); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); @@ -490,6 +494,10 @@ pub async fn me_config_updater( cfg.general.me_single_endpoint_outage_backoff_min_ms, cfg.general.me_single_endpoint_outage_backoff_max_ms, cfg.general.me_single_endpoint_shadow_rotate_every_secs, + cfg.general.me_floor_mode, + cfg.general.me_adaptive_floor_idle_secs, + cfg.general.me_adaptive_floor_min_writers_single_endpoint, + cfg.general.me_adaptive_floor_recover_grace_secs, ); let new_secs = cfg.general.effective_update_every_secs().max(1); if new_secs == update_every_secs { From 1fe1acadd4e6a7d1515f16070ec8a49ae26fde77 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:37:24 +0300 Subject: [PATCH 022/261] ME Adaptive Floor in Metrics Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/stats/mod.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 57b732d..8152599 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -48,6 +48,9 @@ pub struct Stats { me_single_endpoint_quarantine_bypass_total: AtomicU64, me_single_endpoint_shadow_rotate_total: AtomicU64, me_single_endpoint_shadow_rotate_skipped_quarantine_total: AtomicU64, + me_floor_mode_switch_total: AtomicU64, + me_floor_mode_switch_static_to_adaptive_total: AtomicU64, + me_floor_mode_switch_adaptive_to_static_total: AtomicU64, me_handshake_error_codes: DashMap, me_route_drop_no_conn: AtomicU64, me_route_drop_channel_closed: AtomicU64, @@ -439,6 +442,24 @@ impl Stats { .fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_floor_mode_switch_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_floor_mode_switch_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_floor_mode_switch_static_to_adaptive_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_floor_mode_switch_static_to_adaptive_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_floor_mode_switch_adaptive_to_static_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_floor_mode_switch_adaptive_to_static_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) } pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) } pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) } @@ -500,6 +521,17 @@ impl Stats { self.me_single_endpoint_shadow_rotate_skipped_quarantine_total .load(Ordering::Relaxed) } + pub fn get_me_floor_mode_switch_total(&self) -> u64 { + self.me_floor_mode_switch_total.load(Ordering::Relaxed) + } + pub fn get_me_floor_mode_switch_static_to_adaptive_total(&self) -> u64 { + self.me_floor_mode_switch_static_to_adaptive_total + .load(Ordering::Relaxed) + } + pub fn get_me_floor_mode_switch_adaptive_to_static_total(&self) -> u64 { + self.me_floor_mode_switch_adaptive_to_static_total + .load(Ordering::Relaxed) + } pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> { let mut out: Vec<(i32, u64)> = self .me_handshake_error_codes From cfec6dbb3c4933f8abffc92f97ab4c47b7b145fa Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:38:06 +0300 Subject: [PATCH 023/261] ME Adaptive Floor pull-up Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/hot_reload.rs | 30 ++++++++++++++++++++- src/main.rs | 4 +++ src/metrics.rs | 58 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index e16cff2..caec078 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -32,7 +32,7 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher}; use tokio::sync::{mpsc, watch}; use tracing::{error, info, warn}; -use crate::config::{LogLevel, MeSocksKdfPolicy, MeTelemetryLevel}; +use crate::config::{LogLevel, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel}; use super::load::ProxyConfig; // ── Hot fields ──────────────────────────────────────────────────────────────── @@ -58,6 +58,10 @@ pub struct HotFields { pub telemetry_user_enabled: bool, pub telemetry_me_level: MeTelemetryLevel, pub me_socks_kdf_policy: MeSocksKdfPolicy, + pub me_floor_mode: MeFloorMode, + pub me_adaptive_floor_idle_secs: u64, + pub me_adaptive_floor_min_writers_single_endpoint: u8, + pub me_adaptive_floor_recover_grace_secs: u64, pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_watermark_pct: u8, @@ -85,6 +89,14 @@ impl HotFields { telemetry_user_enabled: cfg.general.telemetry.user_enabled, telemetry_me_level: cfg.general.telemetry.me_level, me_socks_kdf_policy: cfg.general.me_socks_kdf_policy, + me_floor_mode: cfg.general.me_floor_mode, + me_adaptive_floor_idle_secs: cfg.general.me_adaptive_floor_idle_secs, + me_adaptive_floor_min_writers_single_endpoint: cfg + .general + .me_adaptive_floor_min_writers_single_endpoint, + me_adaptive_floor_recover_grace_secs: cfg + .general + .me_adaptive_floor_recover_grace_secs, me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, @@ -309,6 +321,22 @@ fn log_changes( ); } + if old_hot.me_floor_mode != new_hot.me_floor_mode + || old_hot.me_adaptive_floor_idle_secs != new_hot.me_adaptive_floor_idle_secs + || old_hot.me_adaptive_floor_min_writers_single_endpoint + != new_hot.me_adaptive_floor_min_writers_single_endpoint + || old_hot.me_adaptive_floor_recover_grace_secs + != new_hot.me_adaptive_floor_recover_grace_secs + { + info!( + "config reload: me_floor: mode={:?} idle={}s min_single={} recover_grace={}s", + new_hot.me_floor_mode, + new_hot.me_adaptive_floor_idle_secs, + new_hot.me_adaptive_floor_min_writers_single_endpoint, + new_hot.me_adaptive_floor_recover_grace_secs, + ); + } + if old_hot.me_route_backpressure_base_timeout_ms != new_hot.me_route_backpressure_base_timeout_ms || old_hot.me_route_backpressure_high_timeout_ms diff --git a/src/main.rs b/src/main.rs index 4ff79fc..b890233 100644 --- a/src/main.rs +++ b/src/main.rs @@ -544,6 +544,10 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_single_endpoint_outage_backoff_min_ms, config.general.me_single_endpoint_outage_backoff_max_ms, config.general.me_single_endpoint_shadow_rotate_every_secs, + config.general.me_floor_mode, + config.general.me_adaptive_floor_idle_secs, + config.general.me_adaptive_floor_min_writers_single_endpoint, + config.general.me_adaptive_floor_recover_grace_secs, config.general.hardswap, config.general.me_pool_drain_ttl_secs, config.general.effective_me_pool_force_close_secs(), diff --git a/src/metrics.rs b/src/metrics.rs index 4c03dc2..f8a6716 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -620,6 +620,64 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_floor_mode Runtime ME writer floor policy mode" + ); + let _ = writeln!(out, "# TYPE telemt_me_floor_mode gauge"); + let floor_mode = config.general.me_floor_mode; + let _ = writeln!( + out, + "telemt_me_floor_mode{{mode=\"static\"}} {}", + if matches!(floor_mode, crate::config::MeFloorMode::Static) { + 1 + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_floor_mode{{mode=\"adaptive\"}} {}", + if matches!(floor_mode, crate::config::MeFloorMode::Adaptive) { + 1 + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_floor_mode_switch_all_total Runtime ME floor mode switches" + ); + let _ = writeln!(out, "# TYPE telemt_me_floor_mode_switch_all_total counter"); + let _ = writeln!( + out, + "telemt_me_floor_mode_switch_all_total {}", + if me_allows_normal { + stats.get_me_floor_mode_switch_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_floor_mode_switch_total{{from=\"static\",to=\"adaptive\"}} {}", + if me_allows_normal { + stats.get_me_floor_mode_switch_static_to_adaptive_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_floor_mode_switch_total{{from=\"adaptive\",to=\"static\"}} {}", + if me_allows_normal { + stats.get_me_floor_mode_switch_adaptive_to_static_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"); let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter"); let _ = writeln!( From c01ca40b6d59a732f23ece7d6b26b10bbb842821 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 03:39:28 +0300 Subject: [PATCH 024/261] ME Adaptive Floor in Tests Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/load.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/config/load.rs b/src/config/load.rs index 27249c6..e549b55 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -966,6 +966,50 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn me_adaptive_floor_min_writers_out_of_range_is_rejected() { + let toml = r#" + [general] + me_adaptive_floor_min_writers_single_endpoint = 0 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_adaptive_floor_min_writers_out_of_range_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!( + err.contains( + "general.me_adaptive_floor_min_writers_single_endpoint must be within [1, 32]" + ) + ); + let _ = std::fs::remove_file(path); + } + + #[test] + fn me_floor_mode_adaptive_is_parsed() { + let toml = r#" + [general] + me_floor_mode = "adaptive" + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_floor_mode_adaptive_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert_eq!(cfg.general.me_floor_mode, MeFloorMode::Adaptive); + let _ = std::fs::remove_file(path); + } + #[test] fn upstream_connect_retry_attempts_zero_is_rejected() { let toml = r#" From 1706698a83a0909791d6654a2abc069d2dd76f14 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 04:06:26 +0300 Subject: [PATCH 025/261] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8ea25e7..faa8cd9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Telemt - MTProxy on Rust + Tokio +***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist*** + **Telemt** is a fast, secure, and feature-rich server written in Rust: it fully implements the official Telegram proxy algo and adds many production-ready improvements such as connection pooling, replay protection, detailed statistics, masking from "prying" eyes [**Telemt Chat in Telegram**](https://t.me/telemtrs) From 53d93880adb16fb047284bc0bea83dbdce45509c Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Tue, 3 Mar 2026 15:16:22 +0300 Subject: [PATCH 026/261] Update QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index e6aca19..d9974d3 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -115,6 +115,8 @@ WantedBy=multi-user.target **5.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt` **6.** Для получения ссылки введите `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` +> [!WARNING] +> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо! --- From 430a0ae6b41981193aa66e5a993c56e61eaea3b0 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Tue, 3 Mar 2026 15:20:39 +0300 Subject: [PATCH 027/261] Update FAQ.ru.md --- docs/FAQ.ru.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index b23e839..de8a7b8 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -6,6 +6,8 @@ 4. Открыть конфиг `nano /etc/telemt.toml`. 5. Скопировать и отправить боту секрет пользователя из раздела [access.users]. 6. Скопировать полученный tag у бота. Например 1234567890abcdef1234567890abcdef. +> [!WARNING] +> Ссылка, которую выдает бот, не будет работать. Не копируйте и не используйте её! 7. Раскомментировать параметр ad_tag и вписать tag, полученный у бота. 8. Раскомментировать/добавить параметр use_middle_proxy = true. @@ -61,4 +63,3 @@ metrics_whitelist = ["127.0.0.1/32", "::1/128", "0.0.0.0/0"] 4. Метрики доступны по адресу SERVER_IP:9090/metrics. > [!WARNING] > "0.0.0.0/0" в metrics_whitelist открывает доступ с любого IP. Замените на свой ip. Например "1.2.3.4" - From c29390166960eb8ba2c540a627fb8fd1ab2c9fce Mon Sep 17 00:00:00 2001 From: ivulit Date: Tue, 3 Mar 2026 17:18:19 +0300 Subject: [PATCH 028/261] fix: bind metrics port to localhost only --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 01e06bb..5866c4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: restart: unless-stopped ports: - "443:443" - - "9090:9090" + - "127.0.0.1:9090:9090" # Allow caching 'proxy-secret' in read-only container working_dir: /run/telemt volumes: From bd0dcfff15dec1ddc9a6a0fbcf7a6536f6e90a7a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 20:49:09 +0300 Subject: [PATCH 029/261] Upstream Error classifier Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/upstream.rs | 96 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index fa7b0a6..1e2dd1e 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -19,6 +19,7 @@ use crate::config::{UpstreamConfig, UpstreamType}; use crate::error::{Result, ProxyError}; use crate::network::dns_overrides::{resolve_socket_addr, split_host_port}; use crate::protocol::constants::{TG_DATACENTERS_V4, TG_DATACENTERS_V6, TG_DATACENTER_PORT}; +use crate::stats::Stats; use crate::transport::socket::{create_outgoing_socket_bound, resolve_interface_ip}; use crate::transport::socks::{connect_socks4, connect_socks5}; @@ -188,6 +189,8 @@ pub struct UpstreamManager { connect_retry_attempts: u32, connect_retry_backoff: Duration, unhealthy_fail_threshold: u32, + connect_failfast_hard_errors: bool, + stats: Arc, } impl UpstreamManager { @@ -196,6 +199,8 @@ impl UpstreamManager { connect_retry_attempts: u32, connect_retry_backoff_ms: u64, unhealthy_fail_threshold: u32, + connect_failfast_hard_errors: bool, + stats: Arc, ) -> Self { let states = configs.into_iter() .filter(|c| c.enabled) @@ -207,6 +212,8 @@ impl UpstreamManager { connect_retry_attempts: connect_retry_attempts.max(1), connect_retry_backoff: Duration::from_millis(connect_retry_backoff_ms), unhealthy_fail_threshold: unhealthy_fail_threshold.max(1), + connect_failfast_hard_errors, + stats, } } @@ -349,6 +356,34 @@ impl UpstreamManager { } } + fn retry_backoff_with_jitter(&self) -> Duration { + if self.connect_retry_backoff.is_zero() { + return Duration::ZERO; + } + let base_ms = self.connect_retry_backoff.as_millis() as u64; + if base_ms == 0 { + return self.connect_retry_backoff; + } + let jitter_cap_ms = (base_ms / 2).max(1); + let jitter_ms = rand::rng().gen_range(0..=jitter_cap_ms); + Duration::from_millis(base_ms.saturating_add(jitter_ms)) + } + + fn is_hard_connect_error(error: &ProxyError) -> bool { + match error { + ProxyError::Config(_) | ProxyError::ConnectionRefused { .. } => true, + ProxyError::Io(ioe) => matches!( + ioe.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::AddrInUse + | std::io::ErrorKind::AddrNotAvailable + | std::io::ErrorKind::InvalidInput + | std::io::ErrorKind::Unsupported + ), + _ => false, + } + } + /// Select upstream using latency-weighted random selection. async fn select_upstream(&self, dc_idx: Option, scope: Option<&str>) -> Option { let upstreams = self.upstreams.read().await; @@ -459,8 +494,12 @@ impl UpstreamManager { guard.get(idx).map(|u| u.bind_rr.clone()) }; + let connect_started_at = Instant::now(); let mut last_error: Option = None; + let mut attempts_used = 0u32; for attempt in 1..=self.connect_retry_attempts { + attempts_used = attempt; + self.stats.increment_upstream_connect_attempt_total(); let start = Instant::now(); match self .connect_via_upstream(&upstream, target, bind_rr.clone()) @@ -468,6 +507,13 @@ impl UpstreamManager { { Ok((stream, egress)) => { let rtt_ms = start.elapsed().as_secs_f64() * 1000.0; + self.stats.increment_upstream_connect_success_total(); + self.stats + .observe_upstream_connect_attempts_per_request(attempts_used); + self.stats.observe_upstream_connect_duration_ms( + connect_started_at.elapsed().as_millis() as u64, + true, + ); let mut guard = self.upstreams.write().await; if let Some(u) = guard.get_mut(idx) { if !u.healthy { @@ -491,7 +537,13 @@ impl UpstreamManager { return Ok((stream, egress)); } Err(e) => { - if attempt < self.connect_retry_attempts { + let hard_error = + self.connect_failfast_hard_errors && Self::is_hard_connect_error(&e); + if hard_error { + self.stats + .increment_upstream_connect_failfast_hard_error_total(); + } + if attempt < self.connect_retry_attempts && !hard_error { debug!( attempt, attempts = self.connect_retry_attempts, @@ -499,21 +551,43 @@ impl UpstreamManager { error = %e, "Upstream connect attempt failed, retrying" ); - if !self.connect_retry_backoff.is_zero() { - tokio::time::sleep(self.connect_retry_backoff).await; + let backoff = self.retry_backoff_with_jitter(); + if !backoff.is_zero() { + tokio::time::sleep(backoff).await; } + } else if hard_error { + debug!( + attempt, + attempts = self.connect_retry_attempts, + target = %target, + error = %e, + "Upstream connect failed with hard error, failfast is active" + ); } last_error = Some(e); + if hard_error { + break; + } } } } + self.stats.increment_upstream_connect_fail_total(); + self.stats + .observe_upstream_connect_attempts_per_request(attempts_used); + self.stats.observe_upstream_connect_duration_ms( + connect_started_at.elapsed().as_millis() as u64, + false, + ); + let error = last_error.unwrap_or_else(|| { ProxyError::Config("Upstream connect attempts exhausted".to_string()) }); let mut guard = self.upstreams.write().await; if let Some(u) = guard.get_mut(idx) { + // Intermediate attempts are intentionally ignored here. + // Health state is degraded only when the entire connect cycle fails. u.fails += 1; warn!( fails = u.fails, @@ -1364,4 +1438,20 @@ mod tests { .contains(&"198.51.100.2:443".parse::().unwrap())); assert!(dc9.fallback.is_empty()); } + + #[test] + fn hard_connect_error_classification_detects_connection_refused() { + let error = ProxyError::ConnectionRefused { + addr: "127.0.0.1:443".to_string(), + }; + assert!(UpstreamManager::is_hard_connect_error(&error)); + } + + #[test] + fn hard_connect_error_classification_skips_timeouts() { + let error = ProxyError::ConnectionTimeout { + addr: "127.0.0.1:443".to_string(), + }; + assert!(!UpstreamManager::is_hard_connect_error(&error)); + } } From af5f0b9692ed21d60624949a6996566544ce0609 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 20:49:29 +0300 Subject: [PATCH 030/261] Upstream Connect in Stats Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/stats/mod.rs | 168 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 8152599..6c72b6f 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -26,6 +26,22 @@ pub struct Stats { connects_all: AtomicU64, connects_bad: AtomicU64, handshake_timeouts: AtomicU64, + upstream_connect_attempt_total: AtomicU64, + upstream_connect_success_total: AtomicU64, + upstream_connect_fail_total: AtomicU64, + upstream_connect_failfast_hard_error_total: AtomicU64, + upstream_connect_attempts_bucket_1: AtomicU64, + upstream_connect_attempts_bucket_2: AtomicU64, + upstream_connect_attempts_bucket_3_4: AtomicU64, + upstream_connect_attempts_bucket_gt_4: AtomicU64, + upstream_connect_duration_success_bucket_le_100ms: AtomicU64, + upstream_connect_duration_success_bucket_101_500ms: AtomicU64, + upstream_connect_duration_success_bucket_501_1000ms: AtomicU64, + upstream_connect_duration_success_bucket_gt_1000ms: AtomicU64, + upstream_connect_duration_fail_bucket_le_100ms: AtomicU64, + upstream_connect_duration_fail_bucket_101_500ms: AtomicU64, + upstream_connect_duration_fail_bucket_501_1000ms: AtomicU64, + upstream_connect_duration_fail_bucket_gt_1000ms: AtomicU64, me_keepalive_sent: AtomicU64, me_keepalive_failed: AtomicU64, me_keepalive_pong: AtomicU64, @@ -155,6 +171,99 @@ impl Stats { self.handshake_timeouts.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_upstream_connect_attempt_total(&self) { + if self.telemetry_core_enabled() { + self.upstream_connect_attempt_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_upstream_connect_success_total(&self) { + if self.telemetry_core_enabled() { + self.upstream_connect_success_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_upstream_connect_fail_total(&self) { + if self.telemetry_core_enabled() { + self.upstream_connect_fail_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_upstream_connect_failfast_hard_error_total(&self) { + if self.telemetry_core_enabled() { + self.upstream_connect_failfast_hard_error_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn observe_upstream_connect_attempts_per_request(&self, attempts: u32) { + if !self.telemetry_core_enabled() { + return; + } + match attempts { + 0 => {} + 1 => { + self.upstream_connect_attempts_bucket_1 + .fetch_add(1, Ordering::Relaxed); + } + 2 => { + self.upstream_connect_attempts_bucket_2 + .fetch_add(1, Ordering::Relaxed); + } + 3..=4 => { + self.upstream_connect_attempts_bucket_3_4 + .fetch_add(1, Ordering::Relaxed); + } + _ => { + self.upstream_connect_attempts_bucket_gt_4 + .fetch_add(1, Ordering::Relaxed); + } + } + } + pub fn observe_upstream_connect_duration_ms(&self, duration_ms: u64, success: bool) { + if !self.telemetry_core_enabled() { + return; + } + let bucket = match duration_ms { + 0..=100 => 0u8, + 101..=500 => 1u8, + 501..=1000 => 2u8, + _ => 3u8, + }; + match (success, bucket) { + (true, 0) => { + self.upstream_connect_duration_success_bucket_le_100ms + .fetch_add(1, Ordering::Relaxed); + } + (true, 1) => { + self.upstream_connect_duration_success_bucket_101_500ms + .fetch_add(1, Ordering::Relaxed); + } + (true, 2) => { + self.upstream_connect_duration_success_bucket_501_1000ms + .fetch_add(1, Ordering::Relaxed); + } + (true, _) => { + self.upstream_connect_duration_success_bucket_gt_1000ms + .fetch_add(1, Ordering::Relaxed); + } + (false, 0) => { + self.upstream_connect_duration_fail_bucket_le_100ms + .fetch_add(1, Ordering::Relaxed); + } + (false, 1) => { + self.upstream_connect_duration_fail_bucket_101_500ms + .fetch_add(1, Ordering::Relaxed); + } + (false, 2) => { + self.upstream_connect_duration_fail_bucket_501_1000ms + .fetch_add(1, Ordering::Relaxed); + } + (false, _) => { + self.upstream_connect_duration_fail_bucket_gt_1000ms + .fetch_add(1, Ordering::Relaxed); + } + } + } pub fn increment_me_keepalive_sent(&self) { if self.telemetry_me_allows_debug() { self.me_keepalive_sent.fetch_add(1, Ordering::Relaxed); @@ -703,6 +812,65 @@ impl Stats { } pub fn get_handshake_timeouts(&self) -> u64 { self.handshake_timeouts.load(Ordering::Relaxed) } + pub fn get_upstream_connect_attempt_total(&self) -> u64 { + self.upstream_connect_attempt_total.load(Ordering::Relaxed) + } + pub fn get_upstream_connect_success_total(&self) -> u64 { + self.upstream_connect_success_total.load(Ordering::Relaxed) + } + pub fn get_upstream_connect_fail_total(&self) -> u64 { + self.upstream_connect_fail_total.load(Ordering::Relaxed) + } + pub fn get_upstream_connect_failfast_hard_error_total(&self) -> u64 { + self.upstream_connect_failfast_hard_error_total + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_attempts_bucket_1(&self) -> u64 { + self.upstream_connect_attempts_bucket_1.load(Ordering::Relaxed) + } + pub fn get_upstream_connect_attempts_bucket_2(&self) -> u64 { + self.upstream_connect_attempts_bucket_2.load(Ordering::Relaxed) + } + pub fn get_upstream_connect_attempts_bucket_3_4(&self) -> u64 { + self.upstream_connect_attempts_bucket_3_4 + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_attempts_bucket_gt_4(&self) -> u64 { + self.upstream_connect_attempts_bucket_gt_4 + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_success_bucket_le_100ms(&self) -> u64 { + self.upstream_connect_duration_success_bucket_le_100ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_success_bucket_101_500ms(&self) -> u64 { + self.upstream_connect_duration_success_bucket_101_500ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_success_bucket_501_1000ms(&self) -> u64 { + self.upstream_connect_duration_success_bucket_501_1000ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_success_bucket_gt_1000ms(&self) -> u64 { + self.upstream_connect_duration_success_bucket_gt_1000ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_fail_bucket_le_100ms(&self) -> u64 { + self.upstream_connect_duration_fail_bucket_le_100ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_fail_bucket_101_500ms(&self) -> u64 { + self.upstream_connect_duration_fail_bucket_101_500ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_fail_bucket_501_1000ms(&self) -> u64 { + self.upstream_connect_duration_fail_bucket_501_1000ms + .load(Ordering::Relaxed) + } + pub fn get_upstream_connect_duration_fail_bucket_gt_1000ms(&self) -> u64 { + self.upstream_connect_duration_fail_bucket_gt_1000ms + .load(Ordering::Relaxed) + } pub fn iter_user_stats(&self) -> dashmap::iter::Iter<'_, String, UserStats> { self.user_stats.iter() From 6ef51dbfb090ff64cacbbfe37f6e5fb4851c4d9b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 20:49:53 +0300 Subject: [PATCH 031/261] Upstream Connect pull-up Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/main.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index b890233..54011cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -261,11 +261,16 @@ async fn main() -> std::result::Result<(), Box> { warn!("Using default tls_domain. Consider setting a custom domain."); } + let stats = Arc::new(Stats::new()); + stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry)); + let upstream_manager = Arc::new(UpstreamManager::new( config.upstreams.clone(), config.general.upstream_connect_retry_attempts, config.general.upstream_connect_retry_backoff_ms, config.general.upstream_unhealthy_fail_threshold, + config.general.upstream_connect_failfast_hard_errors, + stats.clone(), )); let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); @@ -411,8 +416,6 @@ async fn main() -> std::result::Result<(), Box> { let prefer_ipv6 = decision.prefer_ipv6(); let mut use_middle_proxy = config.general.use_middle_proxy && (decision.ipv4_me || decision.ipv6_me); - let stats = Arc::new(Stats::new()); - stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry)); let beobachten = Arc::new(BeobachtenStore::new()); let rng = Arc::new(SecureRandom::new()); From cdf829de910b2dbf0fdb4d41c25004f9b38f41b1 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 20:50:08 +0300 Subject: [PATCH 032/261] Upstream Connect in Metrics Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/metrics.rs | 213 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) diff --git a/src/metrics.rs b/src/metrics.rs index f8a6716..2460d8e 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -202,6 +202,195 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_attempt_total Upstream connect attempts across all requests" + ); + let _ = writeln!(out, "# TYPE telemt_upstream_connect_attempt_total counter"); + let _ = writeln!( + out, + "telemt_upstream_connect_attempt_total {}", + if core_enabled { + stats.get_upstream_connect_attempt_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_success_total Successful upstream connect request cycles" + ); + let _ = writeln!(out, "# TYPE telemt_upstream_connect_success_total counter"); + let _ = writeln!( + out, + "telemt_upstream_connect_success_total {}", + if core_enabled { + stats.get_upstream_connect_success_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_fail_total Failed upstream connect request cycles" + ); + let _ = writeln!(out, "# TYPE telemt_upstream_connect_fail_total counter"); + let _ = writeln!( + out, + "telemt_upstream_connect_fail_total {}", + if core_enabled { + stats.get_upstream_connect_fail_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_failfast_hard_error_total Hard errors that triggered upstream connect failfast" + ); + let _ = writeln!( + out, + "# TYPE telemt_upstream_connect_failfast_hard_error_total counter" + ); + let _ = writeln!( + out, + "telemt_upstream_connect_failfast_hard_error_total {}", + if core_enabled { + stats.get_upstream_connect_failfast_hard_error_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_attempts_per_request Histogram-like buckets for attempts per upstream connect request cycle" + ); + let _ = writeln!(out, "# TYPE telemt_upstream_connect_attempts_per_request counter"); + let _ = writeln!( + out, + "telemt_upstream_connect_attempts_per_request{{bucket=\"1\"}} {}", + if core_enabled { + stats.get_upstream_connect_attempts_bucket_1() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_attempts_per_request{{bucket=\"2\"}} {}", + if core_enabled { + stats.get_upstream_connect_attempts_bucket_2() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_attempts_per_request{{bucket=\"3_4\"}} {}", + if core_enabled { + stats.get_upstream_connect_attempts_bucket_3_4() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_attempts_per_request{{bucket=\"gt_4\"}} {}", + if core_enabled { + stats.get_upstream_connect_attempts_bucket_gt_4() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_duration_success_total Histogram-like buckets of successful upstream connect cycle duration" + ); + let _ = writeln!(out, "# TYPE telemt_upstream_connect_duration_success_total counter"); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_success_total{{bucket=\"le_100ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_success_bucket_le_100ms() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_success_total{{bucket=\"101_500ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_success_bucket_101_500ms() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_success_total{{bucket=\"501_1000ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_success_bucket_501_1000ms() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_success_total{{bucket=\"gt_1000ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_success_bucket_gt_1000ms() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_upstream_connect_duration_fail_total Histogram-like buckets of failed upstream connect cycle duration" + ); + let _ = writeln!(out, "# TYPE telemt_upstream_connect_duration_fail_total counter"); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_fail_total{{bucket=\"le_100ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_fail_bucket_le_100ms() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_fail_total{{bucket=\"101_500ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_fail_bucket_101_500ms() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_fail_total{{bucket=\"501_1000ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_fail_bucket_501_1000ms() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_upstream_connect_duration_fail_total{{bucket=\"gt_1000ms\"}} {}", + if core_enabled { + stats.get_upstream_connect_duration_fail_bucket_gt_1000ms() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_me_keepalive_sent_total ME keepalive frames sent"); let _ = writeln!(out, "# TYPE telemt_me_keepalive_sent_total counter"); let _ = writeln!( @@ -1028,6 +1217,14 @@ mod tests { stats.increment_connects_all(); stats.increment_connects_bad(); stats.increment_handshake_timeouts(); + stats.increment_upstream_connect_attempt_total(); + stats.increment_upstream_connect_attempt_total(); + stats.increment_upstream_connect_success_total(); + stats.increment_upstream_connect_fail_total(); + stats.increment_upstream_connect_failfast_hard_error_total(); + stats.observe_upstream_connect_attempts_per_request(2); + stats.observe_upstream_connect_duration_ms(220, true); + stats.observe_upstream_connect_duration_ms(1500, false); stats.increment_user_connects("alice"); stats.increment_user_curr_connects("alice"); stats.add_user_octets_from("alice", 1024); @@ -1045,6 +1242,21 @@ mod tests { assert!(output.contains("telemt_connections_total 2")); assert!(output.contains("telemt_connections_bad_total 1")); assert!(output.contains("telemt_handshake_timeouts_total 1")); + assert!(output.contains("telemt_upstream_connect_attempt_total 2")); + assert!(output.contains("telemt_upstream_connect_success_total 1")); + assert!(output.contains("telemt_upstream_connect_fail_total 1")); + assert!(output.contains("telemt_upstream_connect_failfast_hard_error_total 1")); + assert!( + output.contains("telemt_upstream_connect_attempts_per_request{bucket=\"2\"} 1") + ); + assert!( + output.contains( + "telemt_upstream_connect_duration_success_total{bucket=\"101_500ms\"} 1" + ) + ); + assert!( + output.contains("telemt_upstream_connect_duration_fail_total{bucket=\"gt_1000ms\"} 1") + ); assert!(output.contains("telemt_user_connections_total{user=\"alice\"} 1")); assert!(output.contains("telemt_user_connections_current{user=\"alice\"} 1")); assert!(output.contains("telemt_user_octets_from_client{user=\"alice\"} 1024")); @@ -1078,6 +1290,7 @@ mod tests { assert!(output.contains("# TYPE telemt_connections_total counter")); assert!(output.contains("# TYPE telemt_connections_bad_total counter")); assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter")); + assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter")); assert!(output.contains("# TYPE telemt_me_writer_removed_total counter")); assert!(output.contains( "# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge" From 624870109effe8b6f191704872408150f95c2c3a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 20:50:31 +0300 Subject: [PATCH 033/261] Upstream Connect in defaults Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 10 +++++++--- src/config/hot_reload.rs | 2 ++ src/config/load.rs | 8 ++++++++ src/config/types.rs | 5 +++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index d92ae78..83b263d 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -12,8 +12,8 @@ const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2; const DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS: u64 = 90; const DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT: u8 = 1; const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180; -const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 3; -const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 4; +const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; +const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; const DEFAULT_LISTEN_ADDR_IPV6: &str = "::"; const DEFAULT_ACCESS_USER: &str = "default"; const DEFAULT_ACCESS_SECRET: &str = "00000000000000000000000000000000"; @@ -205,13 +205,17 @@ pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { } pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 { - 250 + 100 } pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 { DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD } +pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool { + false +} + pub(crate) fn default_crypto_pending_buffer() -> usize { 256 * 1024 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index caec078..5ec911c 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -135,6 +135,8 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { != new.general.upstream_connect_retry_backoff_ms || old.general.upstream_unhealthy_fail_threshold != new.general.upstream_unhealthy_fail_threshold + || old.general.upstream_connect_failfast_hard_errors + != new.general.upstream_connect_failfast_hard_errors { warn!("config reload: general.upstream_* changed; restart required"); } diff --git a/src/config/load.rs b/src/config/load.rs index e549b55..9dbd7c3 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -676,6 +676,10 @@ mod tests { cfg.general.upstream_unhealthy_fail_threshold, default_upstream_unhealthy_fail_threshold() ); + assert_eq!( + cfg.general.upstream_connect_failfast_hard_errors, + default_upstream_connect_failfast_hard_errors() + ); assert_eq!(cfg.general.update_every, default_update_every()); assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4()); assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt()); @@ -751,6 +755,10 @@ mod tests { general.upstream_unhealthy_fail_threshold, default_upstream_unhealthy_fail_threshold() ); + assert_eq!( + general.upstream_connect_failfast_hard_errors, + default_upstream_connect_failfast_hard_errors() + ); assert_eq!(general.update_every, default_update_every()); let server = ServerConfig::default(); diff --git a/src/config/types.rs b/src/config/types.rs index 0d255ba..137f585 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -472,6 +472,10 @@ pub struct GeneralConfig { #[serde(default = "default_upstream_unhealthy_fail_threshold")] pub upstream_unhealthy_fail_threshold: u32, + /// Skip additional retries for hard non-transient upstream connect errors. + #[serde(default = "default_upstream_connect_failfast_hard_errors")] + pub upstream_connect_failfast_hard_errors: bool, + /// Ignore STUN/interface IP mismatch (keep using Middle Proxy even if NAT detected). #[serde(default)] pub stun_iface_mismatch_ignore: bool, @@ -682,6 +686,7 @@ impl Default for GeneralConfig { upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(), + upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(), stun_iface_mismatch_ignore: false, unknown_dc_log_path: default_unknown_dc_log_path(), log_level: LogLevel::Normal, From a6132bac380346fddd8dba7daf54eb38725c8006 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 23:16:25 +0300 Subject: [PATCH 034/261] Idle tolerance + Adaptive floor by default + RPC Proxy Req Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 4 + src/config/hot_reload.rs | 1 + src/config/load.rs | 69 +++++++++++ src/config/types.rs | 8 +- src/main.rs | 1 + src/metrics.rs | 116 ++++++++++++++++++ src/stats/mod.rs | 65 ++++++++++ src/transport/middle_proxy/health.rs | 131 ++++++++++++++++++++ src/transport/middle_proxy/pool.rs | 3 + src/transport/middle_proxy/pool_writer.rs | 141 +++++++++++++++++++++- src/transport/middle_proxy/registry.rs | 48 +++++++- src/transport/middle_proxy/send.rs | 40 ++++++ 12 files changed, 621 insertions(+), 6 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 83b263d..41573a4 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -216,6 +216,10 @@ pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool { false } +pub(crate) fn default_rpc_proxy_req_every() -> u64 { + 0 +} + pub(crate) fn default_crypto_pending_buffer() -> usize { 256 * 1024 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 5ec911c..902811c 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -137,6 +137,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { != new.general.upstream_unhealthy_fail_threshold || old.general.upstream_connect_failfast_hard_errors != new.general.upstream_connect_failfast_hard_errors + || old.general.rpc_proxy_req_every != new.general.rpc_proxy_req_every { warn!("config reload: general.upstream_* changed; restart required"); } diff --git a/src/config/load.rs b/src/config/load.rs index 9dbd7c3..c051b8e 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -249,6 +249,14 @@ impl ProxyConfig { )); } + if config.general.rpc_proxy_req_every != 0 + && !(10..=300).contains(&config.general.rpc_proxy_req_every) + { + return Err(ProxyError::Config( + "general.rpc_proxy_req_every must be 0 or within [10, 300]".to_string(), + )); + } + if config.general.me_reinit_every_secs == 0 { return Err(ProxyError::Config( "general.me_reinit_every_secs must be > 0".to_string(), @@ -680,6 +688,10 @@ mod tests { cfg.general.upstream_connect_failfast_hard_errors, default_upstream_connect_failfast_hard_errors() ); + assert_eq!( + cfg.general.rpc_proxy_req_every, + default_rpc_proxy_req_every() + ); assert_eq!(cfg.general.update_every, default_update_every()); assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4()); assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt()); @@ -759,6 +771,7 @@ mod tests { general.upstream_connect_failfast_hard_errors, default_upstream_connect_failfast_hard_errors() ); + assert_eq!(general.rpc_proxy_req_every, default_rpc_proxy_req_every()); assert_eq!(general.update_every, default_update_every()); let server = ServerConfig::default(); @@ -1058,6 +1071,62 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn rpc_proxy_req_every_out_of_range_is_rejected() { + let toml = r#" + [general] + rpc_proxy_req_every = 9 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_rpc_proxy_req_every_out_of_range_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.rpc_proxy_req_every must be 0 or within [10, 300]")); + let _ = std::fs::remove_file(path); + } + + #[test] + fn rpc_proxy_req_every_zero_and_valid_range_are_accepted() { + let toml_zero = r#" + [general] + rpc_proxy_req_every = 0 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path_zero = dir.join("telemt_rpc_proxy_req_every_zero_ok_test.toml"); + std::fs::write(&path_zero, toml_zero).unwrap(); + let cfg_zero = ProxyConfig::load(&path_zero).unwrap(); + assert_eq!(cfg_zero.general.rpc_proxy_req_every, 0); + let _ = std::fs::remove_file(path_zero); + + let toml_valid = r#" + [general] + rpc_proxy_req_every = 40 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let path_valid = dir.join("telemt_rpc_proxy_req_every_valid_ok_test.toml"); + std::fs::write(&path_valid, toml_valid).unwrap(); + let cfg_valid = ProxyConfig::load(&path_valid).unwrap(); + assert_eq!(cfg_valid.general.rpc_proxy_req_every, 40); + let _ = std::fs::remove_file(path_valid); + } + #[test] fn me_hardswap_warmup_defaults_are_set() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index 137f585..64be729 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -162,8 +162,8 @@ impl MeBindStaleMode { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "lowercase")] pub enum MeFloorMode { - #[default] Static, + #[default] Adaptive, } @@ -356,6 +356,11 @@ pub struct GeneralConfig { #[serde(default = "default_true")] pub me_keepalive_payload_random: bool, + /// Interval in seconds for service RPC_PROXY_REQ activity signals to ME. + /// 0 disables service activity signals. + #[serde(default = "default_rpc_proxy_req_every")] + pub rpc_proxy_req_every: u64, + /// Max pending ciphertext buffer per client writer (bytes). /// Controls FakeTLS backpressure vs throughput. #[serde(default = "default_crypto_pending_buffer")] @@ -666,6 +671,7 @@ impl Default for GeneralConfig { me_keepalive_interval_secs: default_keepalive_interval(), me_keepalive_jitter_secs: default_keepalive_jitter(), me_keepalive_payload_random: default_true(), + rpc_proxy_req_every: default_rpc_proxy_req_every(), me_warmup_stagger_enabled: default_true(), me_warmup_step_delay_ms: default_warmup_step_delay_ms(), me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(), diff --git a/src/main.rs b/src/main.rs index 54011cf..f7f9239 100644 --- a/src/main.rs +++ b/src/main.rs @@ -534,6 +534,7 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_keepalive_interval_secs, config.general.me_keepalive_jitter_secs, config.general.me_keepalive_payload_random, + config.general.rpc_proxy_req_every, config.general.me_warmup_stagger_enabled, config.general.me_warmup_step_delay_ms, config.general.me_warmup_step_jitter_ms, diff --git a/src/metrics.rs b/src/metrics.rs index 2460d8e..eae69d1 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -439,6 +439,93 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_rpc_proxy_req_signal_sent_total Service RPC_PROXY_REQ activity signals sent" + ); + let _ = writeln!(out, "# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter"); + let _ = writeln!( + out, + "telemt_me_rpc_proxy_req_signal_sent_total {}", + if me_allows_normal { + stats.get_me_rpc_proxy_req_signal_sent_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_rpc_proxy_req_signal_failed_total Service RPC_PROXY_REQ activity signal failures" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_rpc_proxy_req_signal_failed_total counter" + ); + let _ = writeln!( + out, + "telemt_me_rpc_proxy_req_signal_failed_total {}", + if me_allows_normal { + stats.get_me_rpc_proxy_req_signal_failed_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_rpc_proxy_req_signal_skipped_no_meta_total Service RPC_PROXY_REQ skipped due to missing writer metadata" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_rpc_proxy_req_signal_skipped_no_meta_total counter" + ); + let _ = writeln!( + out, + "telemt_me_rpc_proxy_req_signal_skipped_no_meta_total {}", + if me_allows_normal { + stats.get_me_rpc_proxy_req_signal_skipped_no_meta_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_rpc_proxy_req_signal_response_total Service RPC_PROXY_REQ responses observed" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_rpc_proxy_req_signal_response_total counter" + ); + let _ = writeln!( + out, + "telemt_me_rpc_proxy_req_signal_response_total {}", + if me_allows_normal { + stats.get_me_rpc_proxy_req_signal_response_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_rpc_proxy_req_signal_close_sent_total Service RPC_CLOSE_EXT sent after activity signals" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_rpc_proxy_req_signal_close_sent_total counter" + ); + let _ = writeln!( + out, + "telemt_me_rpc_proxy_req_signal_close_sent_total {}", + if me_allows_normal { + stats.get_me_rpc_proxy_req_signal_close_sent_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_me_reconnect_attempts_total ME reconnect attempts"); let _ = writeln!(out, "# TYPE telemt_me_reconnect_attempts_total counter"); let _ = writeln!( @@ -500,6 +587,21 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_idle_close_by_peer_total ME idle writers closed by peer" + ); + let _ = writeln!(out, "# TYPE telemt_me_idle_close_by_peer_total counter"); + let _ = writeln!( + out, + "telemt_me_idle_close_by_peer_total {}", + if me_allows_normal { + stats.get_me_idle_close_by_peer_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_me_crc_mismatch_total ME CRC mismatches"); let _ = writeln!(out, "# TYPE telemt_me_crc_mismatch_total counter"); let _ = writeln!( @@ -1225,6 +1327,12 @@ mod tests { stats.observe_upstream_connect_attempts_per_request(2); stats.observe_upstream_connect_duration_ms(220, true); stats.observe_upstream_connect_duration_ms(1500, false); + stats.increment_me_rpc_proxy_req_signal_sent_total(); + stats.increment_me_rpc_proxy_req_signal_failed_total(); + stats.increment_me_rpc_proxy_req_signal_skipped_no_meta_total(); + stats.increment_me_rpc_proxy_req_signal_response_total(); + stats.increment_me_rpc_proxy_req_signal_close_sent_total(); + stats.increment_me_idle_close_by_peer_total(); stats.increment_user_connects("alice"); stats.increment_user_curr_connects("alice"); stats.add_user_octets_from("alice", 1024); @@ -1257,6 +1365,12 @@ mod tests { assert!( output.contains("telemt_upstream_connect_duration_fail_total{bucket=\"gt_1000ms\"} 1") ); + assert!(output.contains("telemt_me_rpc_proxy_req_signal_sent_total 1")); + assert!(output.contains("telemt_me_rpc_proxy_req_signal_failed_total 1")); + assert!(output.contains("telemt_me_rpc_proxy_req_signal_skipped_no_meta_total 1")); + assert!(output.contains("telemt_me_rpc_proxy_req_signal_response_total 1")); + assert!(output.contains("telemt_me_rpc_proxy_req_signal_close_sent_total 1")); + assert!(output.contains("telemt_me_idle_close_by_peer_total 1")); assert!(output.contains("telemt_user_connections_total{user=\"alice\"} 1")); assert!(output.contains("telemt_user_connections_current{user=\"alice\"} 1")); assert!(output.contains("telemt_user_octets_from_client{user=\"alice\"} 1024")); @@ -1291,6 +1405,8 @@ mod tests { assert!(output.contains("# TYPE telemt_connections_bad_total counter")); assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter")); assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter")); + assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter")); + assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter")); assert!(output.contains("# TYPE telemt_me_writer_removed_total counter")); assert!(output.contains( "# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge" diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 6c72b6f..29d7f45 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -46,10 +46,16 @@ pub struct Stats { me_keepalive_failed: AtomicU64, me_keepalive_pong: AtomicU64, me_keepalive_timeout: AtomicU64, + me_rpc_proxy_req_signal_sent_total: AtomicU64, + me_rpc_proxy_req_signal_failed_total: AtomicU64, + me_rpc_proxy_req_signal_skipped_no_meta_total: AtomicU64, + me_rpc_proxy_req_signal_response_total: AtomicU64, + me_rpc_proxy_req_signal_close_sent_total: AtomicU64, me_reconnect_attempts: AtomicU64, me_reconnect_success: AtomicU64, me_handshake_reject_total: AtomicU64, me_reader_eof_total: AtomicU64, + me_idle_close_by_peer_total: AtomicU64, me_crc_mismatch: AtomicU64, me_seq_mismatch: AtomicU64, me_endpoint_quarantine_total: AtomicU64, @@ -289,6 +295,36 @@ impl Stats { self.me_keepalive_timeout.fetch_add(value, Ordering::Relaxed); } } + pub fn increment_me_rpc_proxy_req_signal_sent_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_rpc_proxy_req_signal_sent_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_rpc_proxy_req_signal_failed_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_rpc_proxy_req_signal_failed_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_rpc_proxy_req_signal_skipped_no_meta_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_rpc_proxy_req_signal_skipped_no_meta_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_rpc_proxy_req_signal_response_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_rpc_proxy_req_signal_response_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_rpc_proxy_req_signal_close_sent_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_rpc_proxy_req_signal_close_sent_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_reconnect_attempt(&self) { if self.telemetry_me_allows_normal() { self.me_reconnect_attempts.fetch_add(1, Ordering::Relaxed); @@ -319,6 +355,12 @@ impl Stats { self.me_reader_eof_total.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_idle_close_by_peer_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_idle_close_by_peer_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_crc_mismatch(&self) { if self.telemetry_me_allows_normal() { self.me_crc_mismatch.fetch_add(1, Ordering::Relaxed); @@ -575,6 +617,26 @@ impl Stats { pub fn get_me_keepalive_failed(&self) -> u64 { self.me_keepalive_failed.load(Ordering::Relaxed) } pub fn get_me_keepalive_pong(&self) -> u64 { self.me_keepalive_pong.load(Ordering::Relaxed) } pub fn get_me_keepalive_timeout(&self) -> u64 { self.me_keepalive_timeout.load(Ordering::Relaxed) } + pub fn get_me_rpc_proxy_req_signal_sent_total(&self) -> u64 { + self.me_rpc_proxy_req_signal_sent_total + .load(Ordering::Relaxed) + } + pub fn get_me_rpc_proxy_req_signal_failed_total(&self) -> u64 { + self.me_rpc_proxy_req_signal_failed_total + .load(Ordering::Relaxed) + } + pub fn get_me_rpc_proxy_req_signal_skipped_no_meta_total(&self) -> u64 { + self.me_rpc_proxy_req_signal_skipped_no_meta_total + .load(Ordering::Relaxed) + } + pub fn get_me_rpc_proxy_req_signal_response_total(&self) -> u64 { + self.me_rpc_proxy_req_signal_response_total + .load(Ordering::Relaxed) + } + pub fn get_me_rpc_proxy_req_signal_close_sent_total(&self) -> u64 { + self.me_rpc_proxy_req_signal_close_sent_total + .load(Ordering::Relaxed) + } pub fn get_me_reconnect_attempts(&self) -> u64 { self.me_reconnect_attempts.load(Ordering::Relaxed) } pub fn get_me_reconnect_success(&self) -> u64 { self.me_reconnect_success.load(Ordering::Relaxed) } pub fn get_me_handshake_reject_total(&self) -> u64 { @@ -583,6 +645,9 @@ impl Stats { pub fn get_me_reader_eof_total(&self) -> u64 { self.me_reader_eof_total.load(Ordering::Relaxed) } + pub fn get_me_idle_close_by_peer_total(&self) -> u64 { + self.me_idle_close_by_peer_total.load(Ordering::Relaxed) + } pub fn get_me_crc_mismatch(&self) -> u64 { self.me_crc_mismatch.load(Ordering::Relaxed) } pub fn get_me_seq_mismatch(&self) -> u64 { self.me_seq_mismatch.load(Ordering::Relaxed) } pub fn get_me_endpoint_quarantine_total(&self) -> u64 { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 55d8409..c9ad34c 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -18,6 +18,10 @@ const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff #[allow(dead_code)] const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1; const SHADOW_ROTATE_RETRY_SECS: u64 = 30; +const IDLE_REFRESH_TRIGGER_BASE_SECS: u64 = 45; +const IDLE_REFRESH_TRIGGER_JITTER_SECS: u64 = 5; +const IDLE_REFRESH_RETRY_SECS: u64 = 8; +const IDLE_REFRESH_SUCCESS_GUARD_SECS: u64 = 5; pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_connections: usize) { let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new(); @@ -27,6 +31,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c let mut outage_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut single_endpoint_outage: HashSet<(i32, IpFamily)> = HashSet::new(); let mut shadow_rotate_deadline: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new(); loop { @@ -43,6 +48,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut outage_next_attempt, &mut single_endpoint_outage, &mut shadow_rotate_deadline, + &mut idle_refresh_next_attempt, &mut adaptive_idle_since, &mut adaptive_recover_until, ) @@ -58,6 +64,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut outage_next_attempt, &mut single_endpoint_outage, &mut shadow_rotate_deadline, + &mut idle_refresh_next_attempt, &mut adaptive_idle_since, &mut adaptive_recover_until, ) @@ -76,6 +83,7 @@ async fn check_family( outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, single_endpoint_outage: &mut HashSet<(i32, IpFamily)>, shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>, + idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, ) { @@ -120,6 +128,7 @@ async fn check_family( .or_default() .push(writer.id); } + let writer_idle_since = pool.registry.writer_idle_since_snapshot().await; for (dc, endpoints) in dc_endpoints { if endpoints.is_empty() { @@ -171,6 +180,7 @@ async fn check_family( outage_backoff.remove(&key); outage_next_attempt.remove(&key); shadow_rotate_deadline.remove(&key); + idle_refresh_next_attempt.remove(&key); adaptive_idle_since.remove(&key); adaptive_recover_until.remove(&key); info!( @@ -184,6 +194,20 @@ async fn check_family( } if alive >= required { + maybe_refresh_idle_writer_for_dc( + pool, + rng, + key, + dc, + family, + &endpoints, + alive, + required, + &live_writer_ids_by_addr, + &writer_idle_since, + idle_refresh_next_attempt, + ) + .await; maybe_rotate_single_endpoint_shadow( pool, rng, @@ -287,6 +311,113 @@ async fn check_family( } } +async fn maybe_refresh_idle_writer_for_dc( + pool: &Arc, + rng: &Arc, + key: (i32, IpFamily), + dc: i32, + family: IpFamily, + endpoints: &[SocketAddr], + alive: usize, + required: usize, + live_writer_ids_by_addr: &HashMap>, + writer_idle_since: &HashMap, + idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, +) { + if alive < required { + return; + } + + let now = Instant::now(); + if let Some(next) = idle_refresh_next_attempt.get(&key) + && now < *next + { + return; + } + + let now_epoch_secs = MePool::now_epoch_secs(); + let mut candidate: Option<(u64, SocketAddr, u64, u64)> = None; + for endpoint in endpoints { + let Some(writer_ids) = live_writer_ids_by_addr.get(endpoint) else { + continue; + }; + for writer_id in writer_ids { + let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else { + continue; + }; + let idle_age_secs = now_epoch_secs.saturating_sub(idle_since_epoch_secs); + let threshold_secs = IDLE_REFRESH_TRIGGER_BASE_SECS + + (*writer_id % (IDLE_REFRESH_TRIGGER_JITTER_SECS + 1)); + if idle_age_secs < threshold_secs { + continue; + } + if candidate + .as_ref() + .map(|(_, _, age, _)| idle_age_secs > *age) + .unwrap_or(true) + { + candidate = Some((*writer_id, *endpoint, idle_age_secs, threshold_secs)); + } + } + } + + let Some((old_writer_id, endpoint, idle_age_secs, threshold_secs)) = candidate else { + return; + }; + + let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + Ok(Ok(())) => true, + Ok(Err(error)) => { + debug!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + idle_age_secs, + threshold_secs, + %error, + "Idle writer pre-refresh connect failed" + ); + false + } + Err(_) => { + debug!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + idle_age_secs, + threshold_secs, + "Idle writer pre-refresh connect timed out" + ); + false + } + }; + + if !rotate_ok { + idle_refresh_next_attempt.insert(key, now + Duration::from_secs(IDLE_REFRESH_RETRY_SECS)); + return; + } + + pool.mark_writer_draining_with_timeout(old_writer_id, pool.force_close_timeout(), false) + .await; + idle_refresh_next_attempt.insert( + key, + now + Duration::from_secs(IDLE_REFRESH_SUCCESS_GUARD_SECS), + ); + info!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + idle_age_secs, + threshold_secs, + alive, + required, + "Idle writer refreshed before upstream idle timeout" + ); +} + async fn should_reduce_floor_for_idle( pool: &Arc, key: (i32, IpFamily), diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 5ae922a..8c185be 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -94,6 +94,7 @@ pub struct MePool { pub(super) me_keepalive_interval: Duration, pub(super) me_keepalive_jitter: Duration, pub(super) me_keepalive_payload_random: bool, + pub(super) rpc_proxy_req_every_secs: AtomicU64, pub(super) me_warmup_stagger_enabled: bool, pub(super) me_warmup_step_delay: Duration, pub(super) me_warmup_step_jitter: Duration, @@ -192,6 +193,7 @@ impl MePool { me_keepalive_interval_secs: u64, me_keepalive_jitter_secs: u64, me_keepalive_payload_random: bool, + rpc_proxy_req_every_secs: u64, me_warmup_stagger_enabled: bool, me_warmup_step_delay_ms: u64, me_warmup_step_jitter_ms: u64, @@ -272,6 +274,7 @@ impl MePool { me_keepalive_interval: Duration::from_secs(me_keepalive_interval_secs), me_keepalive_jitter: Duration::from_secs(me_keepalive_jitter_secs), me_keepalive_payload_random, + rpc_proxy_req_every_secs: AtomicU64::new(rpc_proxy_req_every_secs), me_warmup_stagger_enabled, me_warmup_step_delay: Duration::from_millis(me_warmup_step_delay_ms), me_warmup_step_jitter: Duration::from_millis(me_warmup_step_jitter_ms), diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 455757e..1e86ea3 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -2,6 +2,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; use std::time::{Duration, Instant}; +use std::io::ErrorKind; use bytes::BytesMut; use rand::Rng; @@ -12,16 +13,22 @@ use tracing::{debug, info, warn}; use crate::config::MeBindStaleMode; use crate::crypto::SecureRandom; use crate::error::{ProxyError, Result}; -use crate::protocol::constants::RPC_PING_U32; +use crate::protocol::constants::{RPC_CLOSE_EXT_U32, RPC_PING_U32}; use super::codec::{RpcWriter, WriterCommand}; use super::pool::{MePool, MeWriter, WriterContour}; use super::reader::reader_loop; use super::registry::BoundConn; +use super::wire::build_proxy_req_payload; const ME_ACTIVE_PING_SECS: u64 = 25; const ME_ACTIVE_PING_JITTER_SECS: i64 = 5; const ME_IDLE_KEEPALIVE_MAX_SECS: u64 = 5; +const ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS: u64 = 700; + +fn is_me_peer_closed_error(error: &ProxyError) -> bool { + matches!(error, ProxyError::Io(ioe) if ioe.kind() == ErrorKind::UnexpectedEof) +} impl MePool { pub(crate) async fn prune_closed_writers(self: &Arc) { @@ -115,6 +122,7 @@ impl MePool { allow_drain_fallback: allow_drain_fallback.clone(), }; self.writers.write().await.push(writer.clone()); + self.registry.mark_writer_idle(writer_id).await; self.conn_count.fetch_add(1, Ordering::Relaxed); self.writer_available.notify_one(); @@ -124,6 +132,7 @@ impl MePool { let ping_tracker_reader = ping_tracker.clone(); let rtt_stats = self.rtt_stats.clone(); let stats_reader = self.stats.clone(); + let stats_reader_close = self.stats.clone(); let stats_ping = self.stats.clone(); let pool = Arc::downgrade(self); let cancel_ping = cancel.clone(); @@ -135,6 +144,13 @@ impl MePool { let keepalive_enabled = self.me_keepalive_enabled; let keepalive_interval = self.me_keepalive_interval; let keepalive_jitter = self.me_keepalive_jitter; + let rpc_proxy_req_every_secs = self.rpc_proxy_req_every_secs.load(Ordering::Relaxed); + let tx_signal = tx.clone(); + let stats_signal = self.stats.clone(); + let cancel_signal = cancel.clone(); + let cleanup_for_signal = cleanup_done.clone(); + let pool_signal = Arc::downgrade(self); + let keepalive_jitter_signal = self.me_keepalive_jitter; let cancel_reader_token = cancel.clone(); let cancel_ping_token = cancel_ping.clone(); @@ -156,6 +172,15 @@ impl MePool { cancel_reader_token.clone(), ) .await; + let idle_close_by_peer = if let Err(e) = res.as_ref() { + is_me_peer_closed_error(e) && reg.is_writer_empty(writer_id).await + } else { + false + }; + if idle_close_by_peer { + stats_reader_close.increment_me_idle_close_by_peer_total(); + info!(writer_id, "ME socket closed by peer on idle writer"); + } if let Some(pool) = pool.upgrade() && cleanup_for_reader .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) @@ -164,7 +189,9 @@ impl MePool { pool.remove_writer_and_close_clients(writer_id).await; } if let Err(e) = res { - warn!(error = %e, "ME reader ended"); + if !idle_close_by_peer { + warn!(error = %e, "ME reader ended"); + } } let mut ws = writers_arc.write().await; ws.retain(|w| w.id != writer_id); @@ -253,6 +280,116 @@ impl MePool { } }); + tokio::spawn(async move { + if rpc_proxy_req_every_secs == 0 { + return; + } + + let interval = Duration::from_secs(rpc_proxy_req_every_secs); + let startup_jitter_ms = { + let jitter_cap_ms = interval.as_millis() / 2; + let effective_jitter_ms = keepalive_jitter_signal + .as_millis() + .min(jitter_cap_ms) + .max(1); + rand::rng().random_range(0..=effective_jitter_ms as u64) + }; + + tokio::select! { + _ = cancel_signal.cancelled() => return, + _ = tokio::time::sleep(Duration::from_millis(startup_jitter_ms)) => {} + } + + loop { + let wait = { + let jitter_cap_ms = interval.as_millis() / 2; + let effective_jitter_ms = keepalive_jitter_signal + .as_millis() + .min(jitter_cap_ms) + .max(1); + interval + Duration::from_millis(rand::rng().random_range(0..=effective_jitter_ms as u64)) + }; + + tokio::select! { + _ = cancel_signal.cancelled() => break, + _ = tokio::time::sleep(wait) => {} + } + + let Some(pool) = pool_signal.upgrade() else { + break; + }; + + let Some(meta) = pool.registry.get_last_writer_meta(writer_id).await else { + stats_signal.increment_me_rpc_proxy_req_signal_skipped_no_meta_total(); + continue; + }; + + let (conn_id, mut service_rx) = pool.registry.register().await; + pool.registry + .bind_writer(conn_id, writer_id, tx_signal.clone(), meta.clone()) + .await; + + let payload = build_proxy_req_payload( + conn_id, + meta.client_addr, + meta.our_addr, + &[], + pool.proxy_tag.as_deref(), + meta.proto_flags, + ); + + if tx_signal.send(WriterCommand::DataAndFlush(payload)).await.is_err() { + stats_signal.increment_me_rpc_proxy_req_signal_failed_total(); + let _ = pool.registry.unregister(conn_id).await; + cancel_signal.cancel(); + if cleanup_for_signal + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + pool.remove_writer_and_close_clients(writer_id).await; + } + break; + } + + stats_signal.increment_me_rpc_proxy_req_signal_sent_total(); + + if matches!( + tokio::time::timeout( + Duration::from_millis(ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS), + service_rx.recv(), + ) + .await, + Ok(Some(_)) + ) { + stats_signal.increment_me_rpc_proxy_req_signal_response_total(); + } + + let mut close_payload = Vec::with_capacity(12); + close_payload.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes()); + close_payload.extend_from_slice(&conn_id.to_le_bytes()); + + if tx_signal + .send(WriterCommand::DataAndFlush(close_payload)) + .await + .is_err() + { + stats_signal.increment_me_rpc_proxy_req_signal_failed_total(); + let _ = pool.registry.unregister(conn_id).await; + cancel_signal.cancel(); + if cleanup_for_signal + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + pool.remove_writer_and_close_clients(writer_id).await; + } + break; + } + + stats_signal.increment_me_rpc_proxy_req_signal_close_sent_total(); + let _ = pool.registry.unregister(conn_id).await; + } + }); + Ok(()) } diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index 223fa71..4a66654 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::{mpsc, RwLock}; use tokio::sync::mpsc::error::TrySendError; @@ -51,6 +51,8 @@ struct RegistryInner { writer_for_conn: HashMap, conns_for_writer: HashMap>, meta: HashMap, + last_meta_for_writer: HashMap, + writer_idle_since_epoch_secs: HashMap, } impl RegistryInner { @@ -61,6 +63,8 @@ impl RegistryInner { writer_for_conn: HashMap::new(), conns_for_writer: HashMap::new(), meta: HashMap::new(), + last_meta_for_writer: HashMap::new(), + writer_idle_since_epoch_secs: HashMap::new(), } } } @@ -74,6 +78,13 @@ pub struct ConnRegistry { } impl ConnRegistry { + fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + } + pub fn new() -> Self { let start = rand::random::() | 1; Self { @@ -121,8 +132,16 @@ impl ConnRegistry { inner.map.remove(&id); inner.meta.remove(&id); if let Some(writer_id) = inner.writer_for_conn.remove(&id) { - if let Some(set) = inner.conns_for_writer.get_mut(&writer_id) { + let became_empty = if let Some(set) = inner.conns_for_writer.get_mut(&writer_id) { set.remove(&id); + set.is_empty() + } else { + false + }; + if became_empty { + inner + .writer_idle_since_epoch_secs + .insert(writer_id, Self::now_epoch_secs()); } return Some(writer_id); } @@ -191,8 +210,10 @@ impl ConnRegistry { meta: ConnMeta, ) { let mut inner = self.inner.write().await; - inner.meta.entry(conn_id).or_insert(meta); + inner.meta.entry(conn_id).or_insert(meta.clone()); inner.writer_for_conn.insert(conn_id, writer_id); + inner.last_meta_for_writer.insert(writer_id, meta); + inner.writer_idle_since_epoch_secs.remove(&writer_id); inner.writers.entry(writer_id).or_insert_with(|| tx.clone()); inner .conns_for_writer @@ -201,6 +222,25 @@ impl ConnRegistry { .insert(conn_id); } + pub async fn mark_writer_idle(&self, writer_id: u64) { + let mut inner = self.inner.write().await; + inner.conns_for_writer.entry(writer_id).or_insert_with(HashSet::new); + inner + .writer_idle_since_epoch_secs + .entry(writer_id) + .or_insert(Self::now_epoch_secs()); + } + + pub async fn get_last_writer_meta(&self, writer_id: u64) -> Option { + let inner = self.inner.read().await; + inner.last_meta_for_writer.get(&writer_id).cloned() + } + + pub async fn writer_idle_since_snapshot(&self) -> HashMap { + let inner = self.inner.read().await; + inner.writer_idle_since_epoch_secs.clone() + } + pub async fn get_writer(&self, conn_id: u64) -> Option { let inner = self.inner.read().await; let writer_id = inner.writer_for_conn.get(&conn_id).cloned()?; @@ -211,6 +251,8 @@ impl ConnRegistry { pub async fn writer_lost(&self, writer_id: u64) -> Vec { let mut inner = self.inner.write().await; inner.writers.remove(&writer_id); + inner.last_meta_for_writer.remove(&writer_id); + inner.writer_idle_since_epoch_secs.remove(&writer_id); let conns = inner .conns_for_writer .remove(&writer_id) diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 3b57c4c..ba4a419 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -1,4 +1,5 @@ use std::cmp::Reverse; +use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -18,6 +19,9 @@ use super::wire::build_proxy_req_payload; use rand::seq::SliceRandom; use super::registry::ConnMeta; +const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45; +const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55; + impl MePool { /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default. pub async fn send_proxy_req( @@ -152,6 +156,8 @@ impl MePool { return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } } + let writer_idle_since = self.registry.writer_idle_since_snapshot().await; + let now_epoch_secs = Self::now_epoch_secs(); if self.me_deterministic_writer_sort.load(Ordering::Relaxed) { candidate_indices.sort_by(|lhs, rhs| { @@ -161,6 +167,11 @@ impl MePool { self.writer_contour_rank_for_selection(left), (left.generation < self.current_generation()) as usize, left.degraded.load(Ordering::Relaxed) as usize, + self.writer_idle_rank_for_selection( + left, + &writer_idle_since, + now_epoch_secs, + ), Reverse(left.tx.capacity()), left.addr, left.id, @@ -169,6 +180,11 @@ impl MePool { self.writer_contour_rank_for_selection(right), (right.generation < self.current_generation()) as usize, right.degraded.load(Ordering::Relaxed) as usize, + self.writer_idle_rank_for_selection( + right, + &writer_idle_since, + now_epoch_secs, + ), Reverse(right.tx.capacity()), right.addr, right.id, @@ -184,6 +200,11 @@ impl MePool { self.writer_contour_rank_for_selection(w), stale, degraded as usize, + self.writer_idle_rank_for_selection( + w, + &writer_idle_since, + now_epoch_secs, + ), Reverse(w.tx.capacity()), ) }); @@ -367,4 +388,23 @@ impl MePool { WriterContour::Draining => 2, } } + + fn writer_idle_rank_for_selection( + &self, + writer: &super::pool::MeWriter, + idle_since_by_writer: &HashMap, + now_epoch_secs: u64, + ) -> usize { + let Some(idle_since) = idle_since_by_writer.get(&writer.id).copied() else { + return 0; + }; + let idle_age_secs = now_epoch_secs.saturating_sub(idle_since); + if idle_age_secs >= IDLE_WRITER_PENALTY_HIGH_SECS { + 2 + } else if idle_age_secs >= IDLE_WRITER_PENALTY_MID_SECS { + 1 + } else { + 0 + } + } } From d31b4cd6c857740e96e65a7ff7d3509e834d5183 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 3 Mar 2026 23:38:15 +0300 Subject: [PATCH 035/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 61f4d24..16586e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.1.5" +version = "3.1.6" edition = "2024" [dependencies] From f7d451e6898e3896bfbf331401053853539fd086 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:08:05 +0300 Subject: [PATCH 036/261] API V1 Drafts Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/config_store.rs | 107 +++++++++++ src/api/mod.rs | 383 ++++++++++++++++++++++++++++++++++++++++ src/api/model.rs | 144 +++++++++++++++ src/api/users.rs | 301 +++++++++++++++++++++++++++++++ 4 files changed, 935 insertions(+) create mode 100644 src/api/config_store.rs create mode 100644 src/api/mod.rs create mode 100644 src/api/model.rs create mode 100644 src/api/users.rs diff --git a/src/api/config_store.rs b/src/api/config_store.rs new file mode 100644 index 0000000..e7fbbca --- /dev/null +++ b/src/api/config_store.rs @@ -0,0 +1,107 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; + +use hyper::header::IF_MATCH; +use sha2::{Digest, Sha256}; + +use crate::config::ProxyConfig; + +use super::model::ApiFailure; + +pub(super) fn parse_if_match(headers: &hyper::HeaderMap) -> Option { + headers + .get(IF_MATCH) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| value.trim_matches('"').to_string()) +} + +pub(super) async fn ensure_expected_revision( + config_path: &Path, + expected_revision: Option<&str>, +) -> Result<(), ApiFailure> { + let Some(expected) = expected_revision else { + return Ok(()); + }; + let current = current_revision(config_path).await?; + if current != expected { + return Err(ApiFailure::new( + hyper::StatusCode::CONFLICT, + "revision_conflict", + "Config revision mismatch", + )); + } + Ok(()) +} + +pub(super) async fn current_revision(config_path: &Path) -> Result { + let content = tokio::fs::read_to_string(config_path) + .await + .map_err(|e| ApiFailure::internal(format!("failed to read config: {}", e)))?; + Ok(compute_revision(&content)) +} + +pub(super) fn compute_revision(content: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(content.as_bytes()); + hex::encode(hasher.finalize()) +} + +pub(super) async fn load_config_from_disk(config_path: &Path) -> Result { + let config_path = config_path.to_path_buf(); + tokio::task::spawn_blocking(move || ProxyConfig::load(config_path)) + .await + .map_err(|e| ApiFailure::internal(format!("failed to join config loader: {}", e)))? + .map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e))) +} + +pub(super) async fn save_config_to_disk( + config_path: &Path, + cfg: &ProxyConfig, +) -> Result { + let serialized = toml::to_string_pretty(cfg) + .map_err(|e| ApiFailure::internal(format!("failed to serialize config: {}", e)))?; + write_atomic(config_path.to_path_buf(), serialized.clone()).await?; + Ok(compute_revision(&serialized)) +} + +async fn write_atomic(path: PathBuf, contents: String) -> Result<(), ApiFailure> { + tokio::task::spawn_blocking(move || write_atomic_sync(&path, &contents)) + .await + .map_err(|e| ApiFailure::internal(format!("failed to join writer: {}", e)))? + .map_err(|e| ApiFailure::internal(format!("failed to write config: {}", e))) +} + +fn write_atomic_sync(path: &Path, contents: &str) -> std::io::Result<()> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + std::fs::create_dir_all(parent)?; + + let tmp_name = format!( + ".{}.tmp-{}", + path.file_name() + .and_then(|s| s.to_str()) + .unwrap_or("config.toml"), + rand::random::() + ); + let tmp_path = parent.join(tmp_name); + + let write_result = (|| { + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&tmp_path)?; + file.write_all(contents.as_bytes())?; + file.sync_all()?; + std::fs::rename(&tmp_path, path)?; + if let Ok(dir) = std::fs::File::open(parent) { + let _ = dir.sync_all(); + } + Ok(()) + })(); + + if write_result.is_err() { + let _ = std::fs::remove_file(&tmp_path); + } + write_result +} diff --git a/src/api/mod.rs b/src/api/mod.rs new file mode 100644 index 0000000..c13828e --- /dev/null +++ b/src/api/mod.rs @@ -0,0 +1,383 @@ +use std::convert::Infallible; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use http_body_util::{BodyExt, Full}; +use hyper::body::{Bytes, Incoming}; +use hyper::header::AUTHORIZATION; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use tokio::net::TcpListener; +use tokio::sync::{Mutex, watch}; +use tracing::{debug, info, warn}; + +use crate::config::ProxyConfig; +use crate::ip_tracker::UserIpTracker; +use crate::stats::Stats; + +mod config_store; +mod model; +mod users; + +use config_store::{current_revision, parse_if_match}; +use model::{ + ApiFailure, CreateUserRequest, ErrorBody, ErrorResponse, HealthData, PatchUserRequest, + RotateSecretRequest, SuccessResponse, SummaryData, +}; +use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config}; + +#[derive(Clone)] +pub(super) struct ApiShared { + pub(super) stats: Arc, + pub(super) ip_tracker: Arc, + pub(super) config_path: PathBuf, + pub(super) mutation_lock: Arc>, + pub(super) request_id: Arc, +} + +impl ApiShared { + fn next_request_id(&self) -> u64 { + self.request_id.fetch_add(1, Ordering::Relaxed) + } +} + +pub async fn serve( + listen: SocketAddr, + stats: Arc, + ip_tracker: Arc, + config_rx: watch::Receiver>, + config_path: PathBuf, +) { + let listener = match TcpListener::bind(listen).await { + Ok(listener) => listener, + Err(error) => { + warn!( + error = %error, + listen = %listen, + "Failed to bind API listener" + ); + return; + } + }; + + info!("API endpoint: http://{}/v1/*", listen); + + let shared = Arc::new(ApiShared { + stats, + ip_tracker, + config_path, + mutation_lock: Arc::new(Mutex::new(())), + request_id: Arc::new(AtomicU64::new(1)), + }); + + loop { + let (stream, peer) = match listener.accept().await { + Ok(v) => v, + Err(error) => { + warn!(error = %error, "API accept error"); + continue; + } + }; + + let shared_conn = shared.clone(); + let config_rx_conn = config_rx.clone(); + tokio::spawn(async move { + let svc = service_fn(move |req: Request| { + let shared_req = shared_conn.clone(); + let config_rx_req = config_rx_conn.clone(); + async move { handle(req, peer, shared_req, config_rx_req).await } + }); + if let Err(error) = http1::Builder::new() + .serve_connection(hyper_util::rt::TokioIo::new(stream), svc) + .await + { + debug!(error = %error, "API connection error"); + } + }); + } +} + +async fn handle( + req: Request, + peer: SocketAddr, + shared: Arc, + config_rx: watch::Receiver>, +) -> Result>, Infallible> { + let request_id = shared.next_request_id(); + let cfg = config_rx.borrow().clone(); + let api_cfg = &cfg.server.api; + + if !api_cfg.enabled { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::SERVICE_UNAVAILABLE, + "api_disabled", + "API is disabled", + ), + )); + } + + if !api_cfg.whitelist.is_empty() + && !api_cfg + .whitelist + .iter() + .any(|net| net.contains(peer.ip())) + { + return Ok(error_response( + request_id, + ApiFailure::new(StatusCode::FORBIDDEN, "forbidden", "Source IP is not allowed"), + )); + } + + if !api_cfg.auth_header.is_empty() { + let auth_ok = req + .headers() + .get(AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .map(|v| v == api_cfg.auth_header) + .unwrap_or(false); + if !auth_ok { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::UNAUTHORIZED, + "unauthorized", + "Missing or invalid Authorization header", + ), + )); + } + } + + let method = req.method().clone(); + let path = req.uri().path().to_string(); + let body_limit = api_cfg.request_body_limit_bytes; + + let result: Result>, ApiFailure> = async { + match (method.as_str(), path.as_str()) { + ("GET", "/v1/health") => { + let revision = current_revision(&shared.config_path).await?; + let data = HealthData { + status: "ok", + read_only: api_cfg.read_only, + }; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/stats/summary") => { + let revision = current_revision(&shared.config_path).await?; + let data = SummaryData { + uptime_seconds: shared.stats.uptime_secs(), + connections_total: shared.stats.get_connects_all(), + connections_bad_total: shared.stats.get_connects_bad(), + handshake_timeouts_total: shared.stats.get_handshake_timeouts(), + configured_users: cfg.access.users.len(), + }; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/stats/users") | ("GET", "/v1/users") => { + let revision = current_revision(&shared.config_path).await?; + let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + Ok(success_response(StatusCode::OK, users, revision)) + } + ("POST", "/v1/users") => { + if api_cfg.read_only { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::FORBIDDEN, + "read_only", + "API runs in read-only mode", + ), + )); + } + let expected_revision = parse_if_match(req.headers()); + let body = read_json::(req.into_body(), body_limit).await?; + let (data, revision) = create_user(body, expected_revision, &shared).await?; + Ok(success_response(StatusCode::CREATED, data, revision)) + } + _ => { + if let Some(user) = path.strip_prefix("/v1/users/") + && !user.is_empty() + && !user.contains('/') + { + if method == Method::GET { + let revision = current_revision(&shared.config_path).await?; + let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) + { + return Ok(success_response(StatusCode::OK, user_info, revision)); + } + return Ok(error_response( + request_id, + ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"), + )); + } + if method == Method::PATCH { + if api_cfg.read_only { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::FORBIDDEN, + "read_only", + "API runs in read-only mode", + ), + )); + } + let expected_revision = parse_if_match(req.headers()); + let body = read_json::(req.into_body(), body_limit).await?; + let (data, revision) = + patch_user(user, body, expected_revision, &shared).await?; + return Ok(success_response(StatusCode::OK, data, revision)); + } + if method == Method::DELETE { + if api_cfg.read_only { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::FORBIDDEN, + "read_only", + "API runs in read-only mode", + ), + )); + } + let expected_revision = parse_if_match(req.headers()); + let (deleted_user, revision) = + delete_user(user, expected_revision, &shared).await?; + return Ok(success_response(StatusCode::OK, deleted_user, revision)); + } + if method == Method::POST + && let Some(base_user) = user.strip_suffix("/rotate-secret") + && !base_user.is_empty() + && !base_user.contains('/') + { + if api_cfg.read_only { + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::FORBIDDEN, + "read_only", + "API runs in read-only mode", + ), + )); + } + let expected_revision = parse_if_match(req.headers()); + let body = + read_optional_json::(req.into_body(), body_limit) + .await?; + let (data, revision) = + rotate_secret(base_user, body.unwrap_or_default(), expected_revision, &shared) + .await?; + return Ok(success_response(StatusCode::OK, data, revision)); + } + if method == Method::POST { + return Ok(error_response( + request_id, + ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"), + )); + } + return Ok(error_response( + request_id, + ApiFailure::new( + StatusCode::METHOD_NOT_ALLOWED, + "method_not_allowed", + "Unsupported HTTP method for this route", + ), + )); + } + Ok(error_response( + request_id, + ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"), + )) + } + } + } + .await; + + match result { + Ok(resp) => Ok(resp), + Err(error) => Ok(error_response(request_id, error)), + } +} + +fn success_response( + status: StatusCode, + data: T, + revision: String, +) -> Response> { + let payload = SuccessResponse { + ok: true, + data, + revision, + }; + let body = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{\"ok\":false}".to_vec()); + Response::builder() + .status(status) + .header("content-type", "application/json; charset=utf-8") + .body(Full::new(Bytes::from(body))) + .unwrap() +} + +fn error_response(request_id: u64, failure: ApiFailure) -> Response> { + let payload = ErrorResponse { + ok: false, + error: ErrorBody { + code: failure.code, + message: failure.message, + }, + request_id, + }; + let body = serde_json::to_vec(&payload).unwrap_or_else(|_| { + format!( + "{{\"ok\":false,\"error\":{{\"code\":\"internal_error\",\"message\":\"serialization failed\"}},\"request_id\":{}}}", + request_id + ) + .into_bytes() + }); + Response::builder() + .status(failure.status) + .header("content-type", "application/json; charset=utf-8") + .body(Full::new(Bytes::from(body))) + .unwrap() +} + +async fn read_json(body: Incoming, limit: usize) -> Result { + let bytes = read_body_with_limit(body, limit).await?; + serde_json::from_slice(&bytes).map_err(|_| ApiFailure::bad_request("Invalid JSON body")) +} + +async fn read_optional_json( + body: Incoming, + limit: usize, +) -> Result, ApiFailure> { + let bytes = read_body_with_limit(body, limit).await?; + if bytes.is_empty() { + return Ok(None); + } + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|_| ApiFailure::bad_request("Invalid JSON body")) +} + +async fn read_body_with_limit(body: Incoming, limit: usize) -> Result, ApiFailure> { + let mut collected = Vec::new(); + let mut body = body; + while let Some(frame_result) = body.frame().await { + let frame = frame_result.map_err(|_| ApiFailure::bad_request("Invalid request body"))?; + if let Some(chunk) = frame.data_ref() { + if collected.len().saturating_add(chunk.len()) > limit { + return Err(ApiFailure::new( + StatusCode::PAYLOAD_TOO_LARGE, + "payload_too_large", + format!("Body exceeds {} bytes", limit), + )); + } + collected.extend_from_slice(chunk); + } + } + Ok(collected) +} diff --git a/src/api/model.rs b/src/api/model.rs new file mode 100644 index 0000000..bea2301 --- /dev/null +++ b/src/api/model.rs @@ -0,0 +1,144 @@ +use chrono::{DateTime, Utc}; +use hyper::StatusCode; +use rand::Rng; +use serde::{Deserialize, Serialize}; + +const MAX_USERNAME_LEN: usize = 64; + +#[derive(Debug)] +pub(super) struct ApiFailure { + pub(super) status: StatusCode, + pub(super) code: &'static str, + pub(super) message: String, +} + +impl ApiFailure { + pub(super) fn new(status: StatusCode, code: &'static str, message: impl Into) -> Self { + Self { + status, + code, + message: message.into(), + } + } + + pub(super) fn internal(message: impl Into) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal_error", message) + } + + pub(super) fn bad_request(message: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, "bad_request", message) + } +} + +#[derive(Serialize)] +pub(super) struct ErrorBody { + pub(super) code: &'static str, + pub(super) message: String, +} + +#[derive(Serialize)] +pub(super) struct ErrorResponse { + pub(super) ok: bool, + pub(super) error: ErrorBody, + pub(super) request_id: u64, +} + +#[derive(Serialize)] +pub(super) struct SuccessResponse { + pub(super) ok: bool, + pub(super) data: T, + pub(super) revision: String, +} + +#[derive(Serialize)] +pub(super) struct HealthData { + pub(super) status: &'static str, + pub(super) read_only: bool, +} + +#[derive(Serialize)] +pub(super) struct SummaryData { + pub(super) uptime_seconds: f64, + pub(super) connections_total: u64, + pub(super) connections_bad_total: u64, + pub(super) handshake_timeouts_total: u64, + pub(super) configured_users: usize, +} + +#[derive(Serialize)] +pub(super) struct UserInfo { + pub(super) username: String, + pub(super) user_ad_tag: Option, + pub(super) max_tcp_conns: Option, + pub(super) expiration_rfc3339: Option, + pub(super) data_quota_bytes: Option, + pub(super) max_unique_ips: Option, + pub(super) current_connections: u64, + pub(super) active_unique_ips: usize, + pub(super) total_octets: u64, +} + +#[derive(Serialize)] +pub(super) struct CreateUserResponse { + pub(super) user: UserInfo, + pub(super) secret: String, +} + +#[derive(Deserialize)] +pub(super) struct CreateUserRequest { + pub(super) username: String, + pub(super) secret: Option, + pub(super) user_ad_tag: Option, + pub(super) max_tcp_conns: Option, + pub(super) expiration_rfc3339: Option, + pub(super) data_quota_bytes: Option, + pub(super) max_unique_ips: Option, +} + +#[derive(Deserialize)] +pub(super) struct PatchUserRequest { + pub(super) secret: Option, + pub(super) user_ad_tag: Option, + pub(super) max_tcp_conns: Option, + pub(super) expiration_rfc3339: Option, + pub(super) data_quota_bytes: Option, + pub(super) max_unique_ips: Option, +} + +#[derive(Default, Deserialize)] +pub(super) struct RotateSecretRequest { + pub(super) secret: Option, +} + +pub(super) fn parse_optional_expiration( + value: Option<&str>, +) -> Result>, ApiFailure> { + let Some(raw) = value else { + return Ok(None); + }; + let parsed = DateTime::parse_from_rfc3339(raw) + .map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?; + Ok(Some(parsed.with_timezone(&Utc))) +} + +pub(super) fn is_valid_user_secret(secret: &str) -> bool { + secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit()) +} + +pub(super) fn is_valid_ad_tag(tag: &str) -> bool { + tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit()) +} + +pub(super) fn is_valid_username(user: &str) -> bool { + !user.is_empty() + && user.len() <= MAX_USERNAME_LEN + && user + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) +} + +pub(super) fn random_user_secret() -> String { + let mut bytes = [0u8; 16]; + rand::rng().fill(&mut bytes); + hex::encode(bytes) +} diff --git a/src/api/users.rs b/src/api/users.rs new file mode 100644 index 0000000..75d659f --- /dev/null +++ b/src/api/users.rs @@ -0,0 +1,301 @@ +use std::collections::HashMap; + +use hyper::StatusCode; + +use crate::config::ProxyConfig; +use crate::ip_tracker::UserIpTracker; +use crate::stats::Stats; + +use super::ApiShared; +use super::config_store::{ + ensure_expected_revision, load_config_from_disk, save_config_to_disk, +}; +use super::model::{ + ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest, + UserInfo, is_valid_ad_tag, is_valid_user_secret, is_valid_username, parse_optional_expiration, + random_user_secret, +}; + +pub(super) async fn create_user( + body: CreateUserRequest, + expected_revision: Option, + shared: &ApiShared, +) -> Result<(CreateUserResponse, String), ApiFailure> { + if !is_valid_username(&body.username) { + return Err(ApiFailure::bad_request( + "username must match [A-Za-z0-9_.-] and be 1..64 chars", + )); + } + + let secret = match body.secret { + Some(secret) => { + if !is_valid_user_secret(&secret) { + return Err(ApiFailure::bad_request( + "secret must be exactly 32 hex characters", + )); + } + secret + } + None => random_user_secret(), + }; + + if let Some(ad_tag) = body.user_ad_tag.as_ref() && !is_valid_ad_tag(ad_tag) { + return Err(ApiFailure::bad_request( + "user_ad_tag must be exactly 32 hex characters", + )); + } + + let expiration = parse_optional_expiration(body.expiration_rfc3339.as_deref())?; + let _guard = shared.mutation_lock.lock().await; + let mut cfg = load_config_from_disk(&shared.config_path).await?; + ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?; + + if cfg.access.users.contains_key(&body.username) { + return Err(ApiFailure::new( + StatusCode::CONFLICT, + "user_exists", + "User already exists", + )); + } + + cfg.access.users.insert(body.username.clone(), secret.clone()); + if let Some(ad_tag) = body.user_ad_tag { + cfg.access.user_ad_tags.insert(body.username.clone(), ad_tag); + } + if let Some(limit) = body.max_tcp_conns { + cfg.access.user_max_tcp_conns.insert(body.username.clone(), limit); + } + if let Some(expiration) = expiration { + cfg.access + .user_expirations + .insert(body.username.clone(), expiration); + } + if let Some(quota) = body.data_quota_bytes { + cfg.access.user_data_quota.insert(body.username.clone(), quota); + } + + let updated_limit = body.max_unique_ips; + if let Some(limit) = updated_limit { + cfg.access + .user_max_unique_ips + .insert(body.username.clone(), limit); + } + + cfg.validate() + .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; + + let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + drop(_guard); + + if let Some(limit) = updated_limit { + shared.ip_tracker.set_user_limit(&body.username, limit).await; + } + + let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let user = users + .into_iter() + .find(|entry| entry.username == body.username) + .unwrap_or(UserInfo { + username: body.username.clone(), + user_ad_tag: None, + max_tcp_conns: None, + expiration_rfc3339: None, + data_quota_bytes: None, + max_unique_ips: updated_limit, + current_connections: 0, + active_unique_ips: 0, + total_octets: 0, + }); + + Ok((CreateUserResponse { user, secret }, revision)) +} + +pub(super) async fn patch_user( + user: &str, + body: PatchUserRequest, + expected_revision: Option, + shared: &ApiShared, +) -> Result<(UserInfo, String), ApiFailure> { + if let Some(secret) = body.secret.as_ref() && !is_valid_user_secret(secret) { + return Err(ApiFailure::bad_request( + "secret must be exactly 32 hex characters", + )); + } + if let Some(ad_tag) = body.user_ad_tag.as_ref() && !is_valid_ad_tag(ad_tag) { + return Err(ApiFailure::bad_request( + "user_ad_tag must be exactly 32 hex characters", + )); + } + let expiration = parse_optional_expiration(body.expiration_rfc3339.as_deref())?; + let _guard = shared.mutation_lock.lock().await; + let mut cfg = load_config_from_disk(&shared.config_path).await?; + ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?; + + if !cfg.access.users.contains_key(user) { + return Err(ApiFailure::new( + StatusCode::NOT_FOUND, + "not_found", + "User not found", + )); + } + + if let Some(secret) = body.secret { + cfg.access.users.insert(user.to_string(), secret); + } + if let Some(ad_tag) = body.user_ad_tag { + cfg.access.user_ad_tags.insert(user.to_string(), ad_tag); + } + if let Some(limit) = body.max_tcp_conns { + cfg.access.user_max_tcp_conns.insert(user.to_string(), limit); + } + if let Some(expiration) = expiration { + cfg.access.user_expirations.insert(user.to_string(), expiration); + } + if let Some(quota) = body.data_quota_bytes { + cfg.access.user_data_quota.insert(user.to_string(), quota); + } + + let mut updated_limit = None; + if let Some(limit) = body.max_unique_ips { + cfg.access.user_max_unique_ips.insert(user.to_string(), limit); + updated_limit = Some(limit); + } + + cfg.validate() + .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; + + let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + drop(_guard); + if let Some(limit) = updated_limit { + shared.ip_tracker.set_user_limit(user, limit).await; + } + let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let user_info = users + .into_iter() + .find(|entry| entry.username == user) + .ok_or_else(|| ApiFailure::internal("failed to build updated user view"))?; + + Ok((user_info, revision)) +} + +pub(super) async fn rotate_secret( + user: &str, + body: RotateSecretRequest, + expected_revision: Option, + shared: &ApiShared, +) -> Result<(CreateUserResponse, String), ApiFailure> { + let secret = body.secret.unwrap_or_else(random_user_secret); + if !is_valid_user_secret(&secret) { + return Err(ApiFailure::bad_request( + "secret must be exactly 32 hex characters", + )); + } + + let _guard = shared.mutation_lock.lock().await; + let mut cfg = load_config_from_disk(&shared.config_path).await?; + ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?; + + if !cfg.access.users.contains_key(user) { + return Err(ApiFailure::new( + StatusCode::NOT_FOUND, + "not_found", + "User not found", + )); + } + + cfg.access.users.insert(user.to_string(), secret.clone()); + cfg.validate() + .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; + let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + drop(_guard); + + let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let user_info = users + .into_iter() + .find(|entry| entry.username == user) + .ok_or_else(|| ApiFailure::internal("failed to build updated user view"))?; + + Ok(( + CreateUserResponse { + user: user_info, + secret, + }, + revision, + )) +} + +pub(super) async fn delete_user( + user: &str, + expected_revision: Option, + shared: &ApiShared, +) -> Result<(String, String), ApiFailure> { + let _guard = shared.mutation_lock.lock().await; + let mut cfg = load_config_from_disk(&shared.config_path).await?; + ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?; + + if !cfg.access.users.contains_key(user) { + return Err(ApiFailure::new( + StatusCode::NOT_FOUND, + "not_found", + "User not found", + )); + } + if cfg.access.users.len() <= 1 { + return Err(ApiFailure::new( + StatusCode::CONFLICT, + "last_user_forbidden", + "Cannot delete the last configured user", + )); + } + + cfg.access.users.remove(user); + cfg.access.user_ad_tags.remove(user); + cfg.access.user_max_tcp_conns.remove(user); + cfg.access.user_expirations.remove(user); + cfg.access.user_data_quota.remove(user); + cfg.access.user_max_unique_ips.remove(user); + + cfg.validate() + .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; + let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + drop(_guard); + shared.ip_tracker.clear_user_ips(user).await; + + Ok((user.to_string(), revision)) +} + +pub(super) async fn users_from_config( + cfg: &ProxyConfig, + stats: &Stats, + ip_tracker: &UserIpTracker, +) -> Vec { + let ip_counts = ip_tracker + .get_stats() + .await + .into_iter() + .map(|(user, count, _)| (user, count)) + .collect::>(); + + let mut names = cfg.access.users.keys().cloned().collect::>(); + names.sort(); + + let mut users = Vec::with_capacity(names.len()); + for username in names { + users.push(UserInfo { + user_ad_tag: cfg.access.user_ad_tags.get(&username).cloned(), + max_tcp_conns: cfg.access.user_max_tcp_conns.get(&username).copied(), + expiration_rfc3339: cfg + .access + .user_expirations + .get(&username) + .map(chrono::DateTime::::to_rfc3339), + data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(), + max_unique_ips: cfg.access.user_max_unique_ips.get(&username).copied(), + current_connections: stats.get_user_curr_connects(&username), + active_unique_ips: ip_counts.get(&username).copied().unwrap_or(0), + total_octets: stats.get_user_total_octets(&username), + username, + }); + } + users +} From 1236505502559689ec878681bf7143fcc5e34c53 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:08:19 +0300 Subject: [PATCH 037/261] API Docs V1 Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- docs/API.md | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 docs/API.md diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..7b737dd --- /dev/null +++ b/docs/API.md @@ -0,0 +1,149 @@ +# Telemt Control API + +## Purpose +This document specifies the control-plane HTTP API used for: +- runtime statistics access, +- user management, +- safe configuration mutations. + +The data-plane (MTProto proxy traffic) is out of scope. + +## Design Principles +1. Keep data-plane isolated. +The API must not affect MTProto hot paths. + +2. Keep configuration authoritative. +`config.toml` is the single source of truth for managed entities. + +3. Make writes safe. +All config mutations are validated and persisted atomically. + +4. Be explicit about concurrency. +Mutating endpoints support optimistic concurrency through revision matching. + +5. Prefer fail-fast contract errors. +Input validation errors are returned with machine-readable error codes. + +## Runtime and Configuration +Control API runtime is configured under `[server.api]`. + +Parameters: +- `enabled: bool` +- `listen: "IP:PORT"` +- `whitelist: [CIDR, ...]` +- `auth_header: string` (exact match against `Authorization` header; empty disables header auth) +- `request_body_limit_bytes: usize` +- `read_only: bool` + +Backward compatibility: +- `server.admin_api` is accepted as an alias while `server.api` is canonical. + +Operational note: +- Changes in `server.api` require process restart to take effect. + +## Protocol Contract +- Transport: HTTP/1.1 +- Payload format: JSON (`application/json; charset=utf-8`) +- API prefix: `/v1` + +### Success Envelope +```json +{ + "ok": true, + "data": {}, + "revision": "sha256-of-config" +} +``` + +### Error Envelope +```json +{ + "ok": false, + "error": { + "code": "machine_code", + "message": "human-readable text" + }, + "request_id": 1 +} +``` + +### Revision / Concurrency Contract +- Mutating operations MAY include `If-Match: `. +- If provided and stale, API returns `409 revision_conflict`. +- Revision is a SHA-256 hash of current config file content. + +## Endpoints + +### Read endpoints +- `GET /v1/health` +- `GET /v1/stats/summary` +- `GET /v1/stats/users` +- `GET /v1/users` +- `GET /v1/users/{username}` + +### Mutating endpoints +- `POST /v1/users` +- `PATCH /v1/users/{username}` +- `POST /v1/users/{username}/rotate-secret` +- `DELETE /v1/users/{username}` + +## Entity Contract: User +Managed user fields: +- `username` +- `secret` (32 hex chars) +- `user_ad_tag` (32 hex chars, optional) +- `max_tcp_conns` (optional) +- `expiration_rfc3339` (optional) +- `data_quota_bytes` (optional) +- `max_unique_ips` (optional) + +Derived runtime fields (read-only in API responses): +- `current_connections` +- `active_unique_ips` +- `total_octets` + +## Validation Rules +- `username` must match `[A-Za-z0-9_.-]`, length `1..64`. +- `secret` must be exactly 32 hexadecimal characters. +- `user_ad_tag` must be exactly 32 hexadecimal characters. +- Request body size must not exceed `request_body_limit_bytes`. + +## Security Model +1. Network perimeter. +Access is limited by CIDR whitelist. + +2. Optional application header auth. +If `auth_header` is configured, `Authorization` must match exactly. + +3. Read-only mode. +If `read_only = true`, mutating endpoints are rejected with `403`. + +## Mutation Approach +1. Acquire mutation lock. +2. Load config from disk. +3. Validate optional `If-Match` revision. +4. Apply in-memory mutation. +5. Run config validation. +6. Persist via atomic write (`tmp + fsync + rename`). +7. Return updated revision. + +Runtime apply path: +- Existing config watcher picks up persisted changes and applies them through the standard hot-reload path. + +## Known Limitations +1. Built-in TLS/mTLS is not provided by this API server. +Use loopback bind plus reverse proxy for external exposure. + +2. No pagination/filtering for user list in current version. + +3. `PATCH` updates present fields only. +Field deletion semantics are not implemented as explicit nullable operations. + +4. Config comments and manual formatting are not preserved after mutation. +Config is serialized from structured state. + +5. API configuration itself (`server.api`) is not hot-applied. +Restart is required. + +6. Atomic file replacement can conflict with external editors/tools writing the same config concurrently. +Use revision checks to reduce race impact. From 5e93ce258f2ca8e4b5d1d4b2b2c0ecc61c6a4e67 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:08:42 +0300 Subject: [PATCH 038/261] API pull-up Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/main.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main.rs b/src/main.rs index f7f9239..7a775d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; use tokio::net::UnixListener; mod cli; +mod api; mod config; mod crypto; mod error; @@ -1152,6 +1153,36 @@ async fn main() -> std::result::Result<(), Box> { }); } + if config.server.api.enabled { + let listen = match config.server.api.listen.parse::() { + Ok(listen) => listen, + Err(error) => { + warn!( + error = %error, + listen = %config.server.api.listen, + "Invalid server.api.listen; API is disabled" + ); + SocketAddr::from(([127, 0, 0, 1], 0)) + } + }; + if listen.port() != 0 { + let stats = stats.clone(); + let ip_tracker_api = ip_tracker.clone(); + let config_rx_api = config_rx.clone(); + let config_path_api = std::path::PathBuf::from(&config_path); + tokio::spawn(async move { + api::serve( + listen, + stats, + ip_tracker_api, + config_rx_api, + config_path_api, + ) + .await; + }); + } + } + for (listener, listener_proxy_protocol) in listeners { let mut config_rx: tokio::sync::watch::Receiver> = config_rx.clone(); let stats = stats.clone(); From 533708f885c3a0aaae3785d833204efee5cc8c71 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:08:59 +0300 Subject: [PATCH 039/261] API in defaults Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/types.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/config/types.rs b/src/config/types.rs index 64be729..dfb6b89 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -793,6 +793,48 @@ impl Default for LinksConfig { } } +/// API settings for control-plane endpoints. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ApiConfig { + /// Enable or disable REST API. + #[serde(default)] + pub enabled: bool, + + /// Listen address for API in `IP:PORT` format. + #[serde(default = "default_api_listen")] + pub listen: String, + + /// CIDR whitelist allowed to access API. + #[serde(default = "default_api_whitelist")] + pub whitelist: Vec, + + /// Optional static value for `Authorization` header validation. + /// Empty string disables header auth. + #[serde(default)] + pub auth_header: String, + + /// Maximum accepted HTTP request body size in bytes. + #[serde(default = "default_api_request_body_limit_bytes")] + pub request_body_limit_bytes: usize, + + /// Read-only mode: mutating endpoints are rejected. + #[serde(default)] + pub read_only: bool, +} + +impl Default for ApiConfig { + fn default() -> Self { + Self { + enabled: false, + listen: default_api_listen(), + whitelist: default_api_whitelist(), + auth_header: String::new(), + request_body_limit_bytes: default_api_request_body_limit_bytes(), + read_only: false, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ServerConfig { #[serde(default = "default_port")] @@ -828,6 +870,9 @@ pub struct ServerConfig { #[serde(default = "default_metrics_whitelist")] pub metrics_whitelist: Vec, + #[serde(default, alias = "admin_api")] + pub api: ApiConfig, + #[serde(default)] pub listeners: Vec, } @@ -844,6 +889,7 @@ impl Default for ServerConfig { proxy_protocol: false, metrics_port: None, metrics_whitelist: default_metrics_whitelist(), + api: ApiConfig::default(), listeners: Vec::new(), } } From 7269dfbdc5ca890d00a438c8d4d88e36b78b503a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:09:32 +0300 Subject: [PATCH 040/261] API in defaults+load+reload Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 12 ++++++++++++ src/config/hot_reload.rs | 9 +++++++++ src/config/load.rs | 26 +++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 41573a4..d5eed59 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -92,6 +92,18 @@ pub(crate) fn default_metrics_whitelist() -> Vec { ] } +pub(crate) fn default_api_listen() -> String { + "127.0.0.1:9091".to_string() +} + +pub(crate) fn default_api_whitelist() -> Vec { + default_metrics_whitelist() +} + +pub(crate) fn default_api_request_body_limit_bytes() -> usize { + 64 * 1024 +} + pub(crate) fn default_prefer_4() -> u8 { 4 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 902811c..29a6d70 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -115,6 +115,15 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { old.server.port, new.server.port ); } + if old.server.api.enabled != new.server.api.enabled + || old.server.api.listen != new.server.api.listen + || old.server.api.whitelist != new.server.api.whitelist + || old.server.api.auth_header != new.server.api.auth_header + || old.server.api.request_body_limit_bytes != new.server.api.request_body_limit_bytes + || old.server.api.read_only != new.server.api.read_only + { + warn!("config reload: server.api changed; restart required"); + } if old.censorship.tls_domain != new.censorship.tls_domain { warn!( "config reload: censorship.tls_domain changed ('{}' → '{}'); restart required", diff --git a/src/config/load.rs b/src/config/load.rs index c051b8e..825824d 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -1,7 +1,7 @@ #![allow(deprecated)] use std::collections::HashMap; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::Path; use rand::Rng; @@ -398,6 +398,18 @@ impl ProxyConfig { )); } + if config.server.api.request_body_limit_bytes == 0 { + return Err(ProxyError::Config( + "server.api.request_body_limit_bytes must be > 0".to_string(), + )); + } + + if config.server.api.listen.parse::().is_err() { + return Err(ProxyError::Config( + "server.api.listen must be in IP:PORT format".to_string(), + )); + } + if config.general.effective_me_pool_force_close_secs() > 0 && config.general.effective_me_pool_force_close_secs() < config.general.me_pool_drain_ttl_secs @@ -695,6 +707,12 @@ mod tests { assert_eq!(cfg.general.update_every, default_update_every()); assert_eq!(cfg.server.listen_addr_ipv4, default_listen_addr_ipv4()); assert_eq!(cfg.server.listen_addr_ipv6, default_listen_addr_ipv6_opt()); + assert_eq!(cfg.server.api.listen, default_api_listen()); + assert_eq!(cfg.server.api.whitelist, default_api_whitelist()); + assert_eq!( + cfg.server.api.request_body_limit_bytes, + default_api_request_body_limit_bytes() + ); assert_eq!(cfg.access.users, default_access_users()); } @@ -776,6 +794,12 @@ mod tests { let server = ServerConfig::default(); assert_eq!(server.listen_addr_ipv6, Some(default_listen_addr_ipv6())); + assert_eq!(server.api.listen, default_api_listen()); + assert_eq!(server.api.whitelist, default_api_whitelist()); + assert_eq!( + server.api.request_body_limit_bytes, + default_api_request_body_limit_bytes() + ); let access = AccessConfig::default(); assert_eq!(access.users, default_access_users()); From 068cf825b976ab854a6812fbfac7e047f0725c1b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:40:58 +0300 Subject: [PATCH 041/261] API Pool Status Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/mod.rs | 1 + src/transport/middle_proxy/pool_status.rs | 237 ++++++++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 src/transport/middle_proxy/pool_status.rs diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 26c58a6..e7c7957 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -18,6 +18,7 @@ mod rotation; mod send; mod secret; mod wire; +mod pool_status; use bytes::Bytes; diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs new file mode 100644 index 0000000..8b46a30 --- /dev/null +++ b/src/transport/middle_proxy/pool_status.rs @@ -0,0 +1,237 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::net::SocketAddr; +use std::sync::atomic::Ordering; + +use super::pool::{MePool, WriterContour}; + +#[derive(Clone, Debug)] +pub(crate) struct MeApiWriterStatusSnapshot { + pub writer_id: u64, + pub dc: Option, + pub endpoint: SocketAddr, + pub generation: u64, + pub state: &'static str, + pub draining: bool, + pub degraded: bool, + pub bound_clients: usize, + pub idle_for_secs: Option, + pub rtt_ema_ms: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiDcStatusSnapshot { + pub dc: i16, + pub endpoints: Vec, + pub available_endpoints: usize, + pub available_pct: f64, + pub required_writers: usize, + pub alive_writers: usize, + pub coverage_pct: f64, + pub rtt_ms: Option, + pub load: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiStatusSnapshot { + pub generated_at_epoch_secs: u64, + pub configured_dc_groups: usize, + pub configured_endpoints: usize, + pub available_endpoints: usize, + pub available_pct: f64, + pub required_writers: usize, + pub alive_writers: usize, + pub coverage_pct: f64, + pub writers: Vec, + pub dcs: Vec, +} + +impl MePool { + pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot { + let now_epoch_secs = Self::now_epoch_secs(); + + let mut endpoints_by_dc = BTreeMap::>::new(); + if self.decision.ipv4_me { + let map = self.proxy_map_v4.read().await.clone(); + for (dc, addrs) in map { + let abs_dc = dc.abs(); + if abs_dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(abs_dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + if self.decision.ipv6_me { + let map = self.proxy_map_v6.read().await.clone(); + for (dc, addrs) in map { + let abs_dc = dc.abs(); + if abs_dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(abs_dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + + let mut endpoint_to_dc = HashMap::::new(); + for (dc, endpoints) in &endpoints_by_dc { + for endpoint in endpoints { + endpoint_to_dc.entry(*endpoint).or_insert(*dc); + } + } + + let configured_dc_groups = endpoints_by_dc.len(); + let configured_endpoints = endpoints_by_dc.values().map(BTreeSet::len).sum(); + + let required_writers = endpoints_by_dc + .values() + .map(|endpoints| self.required_writers_for_dc_with_floor_mode(endpoints.len(), false)) + .sum(); + + let idle_since = self.registry.writer_idle_since_snapshot().await; + let activity = self.registry.writer_activity_snapshot().await; + let rtt = self.rtt_stats.lock().await.clone(); + let writers = self.writers.read().await.clone(); + + let mut live_writers_by_endpoint = HashMap::::new(); + let mut live_writers_by_dc = HashMap::::new(); + let mut dc_rtt_agg = HashMap::::new(); + let mut writer_rows = Vec::::with_capacity(writers.len()); + + for writer in writers { + let endpoint = writer.addr; + let dc = endpoint_to_dc.get(&endpoint).copied(); + let draining = writer.draining.load(Ordering::Relaxed); + let degraded = writer.degraded.load(Ordering::Relaxed); + let bound_clients = activity + .bound_clients_by_writer + .get(&writer.id) + .copied() + .unwrap_or(0); + let idle_for_secs = idle_since + .get(&writer.id) + .map(|idle_ts| now_epoch_secs.saturating_sub(*idle_ts)); + let rtt_ema_ms = rtt.get(&writer.id).map(|(_, ema)| *ema); + let state = match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) { + WriterContour::Warm => "warm", + WriterContour::Active => "active", + WriterContour::Draining => "draining", + }; + + if !draining { + *live_writers_by_endpoint.entry(endpoint).or_insert(0) += 1; + if let Some(dc_idx) = dc { + *live_writers_by_dc.entry(dc_idx).or_insert(0) += 1; + if let Some(ema_ms) = rtt_ema_ms { + let entry = dc_rtt_agg.entry(dc_idx).or_insert((0.0, 0)); + entry.0 += ema_ms; + entry.1 += 1; + } + } + } + + writer_rows.push(MeApiWriterStatusSnapshot { + writer_id: writer.id, + dc, + endpoint, + generation: writer.generation, + state, + draining, + degraded, + bound_clients, + idle_for_secs, + rtt_ema_ms, + }); + } + + writer_rows.sort_by_key(|row| (row.dc.unwrap_or(i16::MAX), row.endpoint, row.writer_id)); + + let mut dcs = Vec::::with_capacity(endpoints_by_dc.len()); + let mut available_endpoints = 0usize; + let mut alive_writers = 0usize; + for (dc, endpoints) in endpoints_by_dc { + let endpoint_count = endpoints.len(); + let dc_available_endpoints = endpoints + .iter() + .filter(|endpoint| live_writers_by_endpoint.contains_key(endpoint)) + .count(); + let dc_required_writers = + self.required_writers_for_dc_with_floor_mode(endpoint_count, false); + let dc_alive_writers = live_writers_by_dc.get(&dc).copied().unwrap_or(0); + let dc_load = activity + .active_sessions_by_target_dc + .get(&dc) + .copied() + .unwrap_or(0); + let dc_rtt_ms = dc_rtt_agg + .get(&dc) + .and_then(|(sum, count)| (*count > 0).then_some(*sum / (*count as f64))); + + available_endpoints += dc_available_endpoints; + alive_writers += dc_alive_writers; + + dcs.push(MeApiDcStatusSnapshot { + dc, + endpoints: endpoints.into_iter().collect(), + available_endpoints: dc_available_endpoints, + available_pct: ratio_pct(dc_available_endpoints, endpoint_count), + required_writers: dc_required_writers, + alive_writers: dc_alive_writers, + coverage_pct: ratio_pct(dc_alive_writers, dc_required_writers), + rtt_ms: dc_rtt_ms, + load: dc_load, + }); + } + + MeApiStatusSnapshot { + generated_at_epoch_secs: now_epoch_secs, + configured_dc_groups, + configured_endpoints, + available_endpoints, + available_pct: ratio_pct(available_endpoints, configured_endpoints), + required_writers, + alive_writers, + coverage_pct: ratio_pct(alive_writers, required_writers), + writers: writer_rows, + dcs, + } + } +} + +fn ratio_pct(part: usize, total: usize) -> f64 { + if total == 0 { + return 0.0; + } + let pct = ((part as f64) / (total as f64)) * 100.0; + pct.clamp(0.0, 100.0) +} + +#[cfg(test)] +mod tests { + use super::ratio_pct; + + #[test] + fn ratio_pct_is_zero_when_denominator_is_zero() { + assert_eq!(ratio_pct(1, 0), 0.0); + } + + #[test] + fn ratio_pct_is_capped_at_100() { + assert_eq!(ratio_pct(7, 3), 100.0); + } + + #[test] + fn ratio_pct_reports_expected_value() { + assert_eq!(ratio_pct(1, 4), 25.0); + } +} From ead08981e7206b9a103d0b17626eb31709f53b00 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:41:11 +0300 Subject: [PATCH 042/261] API Pool Status pull-up Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index 7a775d1..c4f0e68 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1168,6 +1168,7 @@ async fn main() -> std::result::Result<(), Box> { if listen.port() != 0 { let stats = stats.clone(); let ip_tracker_api = ip_tracker.clone(); + let me_pool_api = me_pool.clone(); let config_rx_api = config_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); tokio::spawn(async move { @@ -1175,6 +1176,7 @@ async fn main() -> std::result::Result<(), Box> { listen, stats, ip_tracker_api, + me_pool_api, config_rx_api, config_path_api, ) From 349bbbb8fa9977d1cc404670b7d18fad9d9df6ec Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:41:33 +0300 Subject: [PATCH 043/261] API Pool Status Model Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/model.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/api/model.rs b/src/api/model.rs index bea2301..43d4173 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -65,6 +65,59 @@ pub(super) struct SummaryData { pub(super) configured_users: usize, } +#[derive(Serialize)] +pub(super) struct MeWritersSummary { + pub(super) configured_dc_groups: usize, + pub(super) configured_endpoints: usize, + pub(super) available_endpoints: usize, + pub(super) available_pct: f64, + pub(super) required_writers: usize, + pub(super) alive_writers: usize, + pub(super) coverage_pct: f64, +} + +#[derive(Serialize)] +pub(super) struct MeWriterStatus { + pub(super) writer_id: u64, + pub(super) dc: Option, + pub(super) endpoint: String, + pub(super) generation: u64, + pub(super) state: &'static str, + pub(super) draining: bool, + pub(super) degraded: bool, + pub(super) bound_clients: usize, + pub(super) idle_for_secs: Option, + pub(super) rtt_ema_ms: Option, +} + +#[derive(Serialize)] +pub(super) struct MeWritersData { + pub(super) middle_proxy_enabled: bool, + pub(super) generated_at_epoch_secs: u64, + pub(super) summary: MeWritersSummary, + pub(super) writers: Vec, +} + +#[derive(Serialize)] +pub(super) struct DcStatus { + pub(super) dc: i16, + pub(super) endpoints: Vec, + pub(super) available_endpoints: usize, + pub(super) available_pct: f64, + pub(super) required_writers: usize, + pub(super) alive_writers: usize, + pub(super) coverage_pct: f64, + pub(super) rtt_ms: Option, + pub(super) load: usize, +} + +#[derive(Serialize)] +pub(super) struct DcStatusData { + pub(super) middle_proxy_enabled: bool, + pub(super) generated_at_epoch_secs: u64, + pub(super) dcs: Vec, +} + #[derive(Serialize)] pub(super) struct UserInfo { pub(super) username: String, From 3492566842853d9a7e1912900e870ac3586cec31 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:41:43 +0300 Subject: [PATCH 044/261] Update mod.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/mod.rs | 104 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index c13828e..56fb588 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -3,6 +3,7 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; use http_body_util::{BodyExt, Full}; use hyper::body::{Bytes, Incoming}; @@ -19,6 +20,7 @@ use tracing::{debug, info, warn}; use crate::config::ProxyConfig; use crate::ip_tracker::UserIpTracker; use crate::stats::Stats; +use crate::transport::middle_proxy::MePool; mod config_store; mod model; @@ -26,8 +28,9 @@ mod users; use config_store::{current_revision, parse_if_match}; use model::{ - ApiFailure, CreateUserRequest, ErrorBody, ErrorResponse, HealthData, PatchUserRequest, - RotateSecretRequest, SuccessResponse, SummaryData, + ApiFailure, CreateUserRequest, DcStatus, DcStatusData, ErrorBody, ErrorResponse, HealthData, + MeWriterStatus, MeWritersData, MeWritersSummary, PatchUserRequest, RotateSecretRequest, + SuccessResponse, SummaryData, }; use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config}; @@ -35,6 +38,7 @@ use users::{create_user, delete_user, patch_user, rotate_secret, users_from_conf pub(super) struct ApiShared { pub(super) stats: Arc, pub(super) ip_tracker: Arc, + pub(super) me_pool: Option>, pub(super) config_path: PathBuf, pub(super) mutation_lock: Arc>, pub(super) request_id: Arc, @@ -50,6 +54,7 @@ pub async fn serve( listen: SocketAddr, stats: Arc, ip_tracker: Arc, + me_pool: Option>, config_rx: watch::Receiver>, config_path: PathBuf, ) { @@ -70,6 +75,7 @@ pub async fn serve( let shared = Arc::new(ApiShared { stats, ip_tracker, + me_pool, config_path, mutation_lock: Arc::new(Mutex::new(())), request_id: Arc::new(AtomicU64::new(1)), @@ -179,6 +185,93 @@ async fn handle( }; Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/stats/me-writers") => { + let revision = current_revision(&shared.config_path).await?; + let data = match &shared.me_pool { + Some(pool) => { + let snapshot = pool.api_status_snapshot().await; + let writers = snapshot + .writers + .into_iter() + .map(|entry| MeWriterStatus { + writer_id: entry.writer_id, + dc: entry.dc, + endpoint: entry.endpoint.to_string(), + generation: entry.generation, + state: entry.state, + draining: entry.draining, + degraded: entry.degraded, + bound_clients: entry.bound_clients, + idle_for_secs: entry.idle_for_secs, + rtt_ema_ms: entry.rtt_ema_ms, + }) + .collect(); + MeWritersData { + middle_proxy_enabled: true, + generated_at_epoch_secs: snapshot.generated_at_epoch_secs, + summary: MeWritersSummary { + configured_dc_groups: snapshot.configured_dc_groups, + configured_endpoints: snapshot.configured_endpoints, + available_endpoints: snapshot.available_endpoints, + available_pct: snapshot.available_pct, + required_writers: snapshot.required_writers, + alive_writers: snapshot.alive_writers, + coverage_pct: snapshot.coverage_pct, + }, + writers, + } + } + None => MeWritersData { + middle_proxy_enabled: false, + generated_at_epoch_secs: now_epoch_secs(), + summary: MeWritersSummary { + configured_dc_groups: 0, + configured_endpoints: 0, + available_endpoints: 0, + available_pct: 0.0, + required_writers: 0, + alive_writers: 0, + coverage_pct: 0.0, + }, + writers: Vec::new(), + }, + }; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/stats/dcs") => { + let revision = current_revision(&shared.config_path).await?; + let data = match &shared.me_pool { + Some(pool) => { + let snapshot = pool.api_status_snapshot().await; + let dcs = snapshot + .dcs + .into_iter() + .map(|entry| DcStatus { + dc: entry.dc, + endpoints: entry.endpoints.into_iter().map(|value| value.to_string()).collect(), + available_endpoints: entry.available_endpoints, + available_pct: entry.available_pct, + required_writers: entry.required_writers, + alive_writers: entry.alive_writers, + coverage_pct: entry.coverage_pct, + rtt_ms: entry.rtt_ms, + load: entry.load, + }) + .collect(); + DcStatusData { + middle_proxy_enabled: true, + generated_at_epoch_secs: snapshot.generated_at_epoch_secs, + dcs, + } + } + None => DcStatusData { + middle_proxy_enabled: false, + generated_at_epoch_secs: now_epoch_secs(), + dcs: Vec::new(), + }, + }; + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/users") | ("GET", "/v1/users") => { let revision = current_revision(&shared.config_path).await?; let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; @@ -304,6 +397,13 @@ async fn handle( } } +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + fn success_response( status: StatusCode, data: T, From c86f35f05923436cdd328164a8429e1b4ada95af Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:41:57 +0300 Subject: [PATCH 045/261] Pool Status in Docs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- docs/API.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/API.md b/docs/API.md index 7b737dd..27013e3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -77,6 +77,8 @@ Operational note: ### Read endpoints - `GET /v1/health` - `GET /v1/stats/summary` +- `GET /v1/stats/me-writers` +- `GET /v1/stats/dcs` - `GET /v1/stats/users` - `GET /v1/users` - `GET /v1/users/{username}` @@ -102,6 +104,62 @@ Derived runtime fields (read-only in API responses): - `active_unique_ips` - `total_octets` +## Transport Status Endpoints +### `GET /v1/stats/me-writers` +Returns current Middle-End writer status and aggregated coverage/availability summary. + +Top-level fields: +- `middle_proxy_enabled` +- `generated_at_epoch_secs` +- `summary` +- `writers` + +Summary fields: +- `configured_dc_groups` +- `configured_endpoints` +- `available_endpoints` +- `available_pct` +- `required_writers` +- `alive_writers` +- `coverage_pct` + +Writer fields: +- `writer_id` +- `dc` +- `endpoint` (`ip:port`) +- `generation` +- `state` (`warm|active|draining`) +- `draining` +- `degraded` +- `bound_clients` +- `idle_for_secs` +- `rtt_ema_ms` + +### `GET /v1/stats/dcs` +Returns per-DC status aggregated from current ME pool. + +Top-level fields: +- `middle_proxy_enabled` +- `generated_at_epoch_secs` +- `dcs` + +DC row fields: +- `dc` +- `endpoints` (`ip:port[]`) +- `available_endpoints` +- `available_pct` +- `required_writers` +- `alive_writers` +- `coverage_pct` +- `rtt_ms` +- `load` + +Metrics formulas: +- `available_pct = available_endpoints / configured_endpoints * 100` +- `coverage_pct = alive_writers / required_writers * 100` +- `required_writers` uses the runtime writer floor policy for each DC group. +- `load` is the number of active client sessions currently bound to that DC. + ## Validation Rules - `username` must match `[A-Za-z0-9_.-]`, length `1..64`. - `secret` must be exactly 32 hexadecimal characters. From 4c2bc2f41fe52b161cb21062de722f44fab26f40 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 01:42:24 +0300 Subject: [PATCH 046/261] Pool Status hooks in ME Registry Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/registry.rs | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index 4a66654..869030a 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -45,6 +45,12 @@ pub struct ConnWriter { pub tx: mpsc::Sender, } +#[derive(Clone, Debug, Default)] +pub(super) struct WriterActivitySnapshot { + pub bound_clients_by_writer: HashMap, + pub active_sessions_by_target_dc: HashMap, +} + struct RegistryInner { map: HashMap>, writers: HashMap>, @@ -241,6 +247,30 @@ impl ConnRegistry { inner.writer_idle_since_epoch_secs.clone() } + pub(super) async fn writer_activity_snapshot(&self) -> WriterActivitySnapshot { + let inner = self.inner.read().await; + let mut bound_clients_by_writer = HashMap::::new(); + let mut active_sessions_by_target_dc = HashMap::::new(); + + for (writer_id, conn_ids) in &inner.conns_for_writer { + bound_clients_by_writer.insert(*writer_id, conn_ids.len()); + } + for conn_meta in inner.meta.values() { + let dc_u16 = conn_meta.target_dc.unsigned_abs(); + if dc_u16 == 0 { + continue; + } + if let Ok(dc) = i16::try_from(dc_u16) { + *active_sessions_by_target_dc.entry(dc).or_insert(0) += 1; + } + } + + WriterActivitySnapshot { + bound_clients_by_writer, + active_sessions_by_target_dc, + } + } + pub async fn get_writer(&self, conn_id: u64) -> Option { let inner = self.inner.read().await; let writer_id = inner.writer_for_conn.get(&conn_id).cloned()?; @@ -288,3 +318,69 @@ impl ConnRegistry { .unwrap_or(true) } } + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use super::ConnMeta; + use super::ConnRegistry; + + #[tokio::test] + async fn writer_activity_snapshot_tracks_writer_and_dc_load() { + let registry = ConnRegistry::new(); + + let (conn_a, _rx_a) = registry.register().await; + let (conn_b, _rx_b) = registry.register().await; + let (conn_c, _rx_c) = registry.register().await; + let (writer_tx_a, _writer_rx_a) = tokio::sync::mpsc::channel(8); + let (writer_tx_b, _writer_rx_b) = tokio::sync::mpsc::channel(8); + + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + registry + .bind_writer( + conn_a, + 10, + writer_tx_a.clone(), + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await; + registry + .bind_writer( + conn_b, + 10, + writer_tx_a, + ConnMeta { + target_dc: -2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await; + registry + .bind_writer( + conn_c, + 20, + writer_tx_b, + ConnMeta { + target_dc: 4, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await; + + let snapshot = registry.writer_activity_snapshot().await; + assert_eq!(snapshot.bound_clients_by_writer.get(&10), Some(&2)); + assert_eq!(snapshot.bound_clients_by_writer.get(&20), Some(&1)); + assert_eq!(snapshot.active_sessions_by_target_dc.get(&2), Some(&2)); + assert_eq!(snapshot.active_sessions_by_target_dc.get(&4), Some(&1)); + } +} From 7ea63872785f6e79e4b43e0dfa90d82583b03b74 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:45:32 +0300 Subject: [PATCH 047/261] API ME Pool Status Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/pool_status.rs | 187 ++++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 8b46a30..c01f74b 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -1,8 +1,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::net::SocketAddr; use std::sync::atomic::Ordering; +use std::time::Instant; use super::pool::{MePool, WriterContour}; +use crate::config::{MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy}; +use crate::transport::upstream::IpPreference; #[derive(Clone, Debug)] pub(crate) struct MeApiWriterStatusSnapshot { @@ -45,6 +48,57 @@ pub(crate) struct MeApiStatusSnapshot { pub dcs: Vec, } +#[derive(Clone, Debug)] +pub(crate) struct MeApiQuarantinedEndpointSnapshot { + pub endpoint: SocketAddr, + pub remaining_ms: u64, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiDcPathSnapshot { + pub dc: i16, + pub ip_preference: Option<&'static str>, + pub selected_addr_v4: Option, + pub selected_addr_v6: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiRuntimeSnapshot { + pub active_generation: u64, + pub warm_generation: u64, + pub pending_hardswap_generation: u64, + pub pending_hardswap_age_secs: Option, + pub hardswap_enabled: bool, + pub floor_mode: &'static str, + pub adaptive_floor_idle_secs: u64, + pub adaptive_floor_min_writers_single_endpoint: u8, + pub adaptive_floor_recover_grace_secs: u64, + pub me_keepalive_enabled: bool, + pub me_keepalive_interval_secs: u64, + pub me_keepalive_jitter_secs: u64, + pub me_keepalive_payload_random: bool, + pub rpc_proxy_req_every_secs: u64, + pub me_reconnect_max_concurrent_per_dc: u32, + pub me_reconnect_backoff_base_ms: u64, + pub me_reconnect_backoff_cap_ms: u64, + pub me_reconnect_fast_retry_count: u32, + pub me_pool_drain_ttl_secs: u64, + pub me_pool_force_close_secs: u64, + pub me_pool_min_fresh_ratio: f32, + pub me_bind_stale_mode: &'static str, + pub me_bind_stale_ttl_secs: u64, + pub me_single_endpoint_shadow_writers: u8, + pub me_single_endpoint_outage_mode_enabled: bool, + pub me_single_endpoint_outage_disable_quarantine: bool, + pub me_single_endpoint_outage_backoff_min_ms: u64, + pub me_single_endpoint_outage_backoff_max_ms: u64, + pub me_single_endpoint_shadow_rotate_every_secs: u64, + pub me_deterministic_writer_sort: bool, + pub me_socks_kdf_policy: &'static str, + pub quarantined_endpoints: Vec, + pub network_path: Vec, +} + impl MePool { pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot { let now_epoch_secs = Self::now_epoch_secs(); @@ -206,6 +260,107 @@ impl MePool { dcs, } } + + pub(crate) async fn api_runtime_snapshot(&self) -> MeApiRuntimeSnapshot { + let now = Instant::now(); + let now_epoch_secs = Self::now_epoch_secs(); + let pending_started_at = self + .pending_hardswap_started_at_epoch_secs + .load(Ordering::Relaxed); + let pending_hardswap_age_secs = (pending_started_at > 0) + .then_some(now_epoch_secs.saturating_sub(pending_started_at)); + + let mut quarantined_endpoints = Vec::::new(); + { + let guard = self.endpoint_quarantine.lock().await; + for (endpoint, expires_at) in guard.iter() { + if *expires_at <= now { + continue; + } + let remaining_ms = expires_at.duration_since(now).as_millis() as u64; + quarantined_endpoints.push(MeApiQuarantinedEndpointSnapshot { + endpoint: *endpoint, + remaining_ms, + }); + } + } + quarantined_endpoints.sort_by_key(|entry| entry.endpoint); + + let mut network_path = Vec::::new(); + if let Some(upstream) = &self.upstream { + for dc in 1..=5 { + let dc_idx = dc as i16; + let ip_preference = upstream + .get_dc_ip_preference(dc_idx) + .await + .map(ip_preference_label); + let selected_addr_v4 = upstream.get_dc_addr(dc_idx, false).await; + let selected_addr_v6 = upstream.get_dc_addr(dc_idx, true).await; + network_path.push(MeApiDcPathSnapshot { + dc: dc_idx, + ip_preference, + selected_addr_v4, + selected_addr_v6, + }); + } + } + + MeApiRuntimeSnapshot { + active_generation: self.active_generation.load(Ordering::Relaxed), + warm_generation: self.warm_generation.load(Ordering::Relaxed), + pending_hardswap_generation: self.pending_hardswap_generation.load(Ordering::Relaxed), + pending_hardswap_age_secs, + hardswap_enabled: self.hardswap.load(Ordering::Relaxed), + floor_mode: floor_mode_label(self.floor_mode()), + adaptive_floor_idle_secs: self.me_adaptive_floor_idle_secs.load(Ordering::Relaxed), + adaptive_floor_min_writers_single_endpoint: self + .me_adaptive_floor_min_writers_single_endpoint + .load(Ordering::Relaxed), + adaptive_floor_recover_grace_secs: self + .me_adaptive_floor_recover_grace_secs + .load(Ordering::Relaxed), + me_keepalive_enabled: self.me_keepalive_enabled, + me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(), + me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(), + me_keepalive_payload_random: self.me_keepalive_payload_random, + rpc_proxy_req_every_secs: self.rpc_proxy_req_every_secs.load(Ordering::Relaxed), + me_reconnect_max_concurrent_per_dc: self.me_reconnect_max_concurrent_per_dc, + me_reconnect_backoff_base_ms: self.me_reconnect_backoff_base.as_millis() as u64, + me_reconnect_backoff_cap_ms: self.me_reconnect_backoff_cap.as_millis() as u64, + me_reconnect_fast_retry_count: self.me_reconnect_fast_retry_count, + me_pool_drain_ttl_secs: self.me_pool_drain_ttl_secs.load(Ordering::Relaxed), + me_pool_force_close_secs: self.me_pool_force_close_secs.load(Ordering::Relaxed), + me_pool_min_fresh_ratio: Self::permille_to_ratio( + self.me_pool_min_fresh_ratio_permille.load(Ordering::Relaxed), + ), + me_bind_stale_mode: bind_stale_mode_label(self.bind_stale_mode()), + me_bind_stale_ttl_secs: self.me_bind_stale_ttl_secs.load(Ordering::Relaxed), + me_single_endpoint_shadow_writers: self + .me_single_endpoint_shadow_writers + .load(Ordering::Relaxed), + me_single_endpoint_outage_mode_enabled: self + .me_single_endpoint_outage_mode_enabled + .load(Ordering::Relaxed), + me_single_endpoint_outage_disable_quarantine: self + .me_single_endpoint_outage_disable_quarantine + .load(Ordering::Relaxed), + me_single_endpoint_outage_backoff_min_ms: self + .me_single_endpoint_outage_backoff_min_ms + .load(Ordering::Relaxed), + me_single_endpoint_outage_backoff_max_ms: self + .me_single_endpoint_outage_backoff_max_ms + .load(Ordering::Relaxed), + me_single_endpoint_shadow_rotate_every_secs: self + .me_single_endpoint_shadow_rotate_every_secs + .load(Ordering::Relaxed), + me_deterministic_writer_sort: self + .me_deterministic_writer_sort + .load(Ordering::Relaxed), + me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()), + quarantined_endpoints, + network_path, + } + } } fn ratio_pct(part: usize, total: usize) -> f64 { @@ -216,6 +371,38 @@ fn ratio_pct(part: usize, total: usize) -> f64 { pct.clamp(0.0, 100.0) } +fn floor_mode_label(mode: MeFloorMode) -> &'static str { + match mode { + MeFloorMode::Static => "static", + MeFloorMode::Adaptive => "adaptive", + } +} + +fn bind_stale_mode_label(mode: MeBindStaleMode) -> &'static str { + match mode { + MeBindStaleMode::Never => "never", + MeBindStaleMode::Ttl => "ttl", + MeBindStaleMode::Always => "always", + } +} + +fn socks_kdf_policy_label(policy: MeSocksKdfPolicy) -> &'static str { + match policy { + MeSocksKdfPolicy::Strict => "strict", + MeSocksKdfPolicy::Compat => "compat", + } +} + +fn ip_preference_label(preference: IpPreference) -> &'static str { + match preference { + IpPreference::Unknown => "unknown", + IpPreference::PreferV6 => "prefer_v6", + IpPreference::PreferV4 => "prefer_v4", + IpPreference::BothWork => "both", + IpPreference::Unavailable => "unavailable", + } +} + #[cfg(test)] mod tests { use super::ratio_pct; From 6b9c7f7862da68da45446faa7e848e465e87900b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:46:12 +0300 Subject: [PATCH 048/261] Runtime API in defaults Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 8 ++++++++ src/config/hot_reload.rs | 3 +++ src/config/load.rs | 44 ++++++++++++++++++++++++++++++++++++++++ src/config/types.rs | 10 +++++++++ 4 files changed, 65 insertions(+) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index d5eed59..86f569b 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -104,6 +104,14 @@ pub(crate) fn default_api_request_body_limit_bytes() -> usize { 64 * 1024 } +pub(crate) fn default_api_minimal_runtime_enabled() -> bool { + false +} + +pub(crate) fn default_api_minimal_runtime_cache_ttl_ms() -> u64 { + 1000 +} + pub(crate) fn default_prefer_4() -> u8 { 4 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 29a6d70..d752d45 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -120,6 +120,9 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { || old.server.api.whitelist != new.server.api.whitelist || old.server.api.auth_header != new.server.api.auth_header || old.server.api.request_body_limit_bytes != new.server.api.request_body_limit_bytes + || old.server.api.minimal_runtime_enabled != new.server.api.minimal_runtime_enabled + || old.server.api.minimal_runtime_cache_ttl_ms + != new.server.api.minimal_runtime_cache_ttl_ms || old.server.api.read_only != new.server.api.read_only { warn!("config reload: server.api changed; restart required"); diff --git a/src/config/load.rs b/src/config/load.rs index 825824d..b469299 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -404,6 +404,12 @@ impl ProxyConfig { )); } + if config.server.api.minimal_runtime_cache_ttl_ms > 60_000 { + return Err(ProxyError::Config( + "server.api.minimal_runtime_cache_ttl_ms must be within [0, 60000]".to_string(), + )); + } + if config.server.api.listen.parse::().is_err() { return Err(ProxyError::Config( "server.api.listen must be in IP:PORT format".to_string(), @@ -713,6 +719,14 @@ mod tests { cfg.server.api.request_body_limit_bytes, default_api_request_body_limit_bytes() ); + assert_eq!( + cfg.server.api.minimal_runtime_enabled, + default_api_minimal_runtime_enabled() + ); + assert_eq!( + cfg.server.api.minimal_runtime_cache_ttl_ms, + default_api_minimal_runtime_cache_ttl_ms() + ); assert_eq!(cfg.access.users, default_access_users()); } @@ -800,6 +814,14 @@ mod tests { server.api.request_body_limit_bytes, default_api_request_body_limit_bytes() ); + assert_eq!( + server.api.minimal_runtime_enabled, + default_api_minimal_runtime_enabled() + ); + assert_eq!( + server.api.minimal_runtime_cache_ttl_ms, + default_api_minimal_runtime_cache_ttl_ms() + ); let access = AccessConfig::default(); assert_eq!(access.users, default_access_users()); @@ -1346,6 +1368,28 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn api_minimal_runtime_cache_ttl_out_of_range_is_rejected() { + let toml = r#" + [server.api] + enabled = true + listen = "127.0.0.1:9091" + minimal_runtime_cache_ttl_ms = 70000 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_api_minimal_runtime_cache_ttl_invalid_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("server.api.minimal_runtime_cache_ttl_ms must be within [0, 60000]")); + let _ = std::fs::remove_file(path); + } + #[test] fn force_close_bumped_when_below_drain_ttl() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index dfb6b89..ee17108 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -817,6 +817,14 @@ pub struct ApiConfig { #[serde(default = "default_api_request_body_limit_bytes")] pub request_body_limit_bytes: usize, + /// Enable runtime snapshots that require read-lock aggregation on API request path. + #[serde(default = "default_api_minimal_runtime_enabled")] + pub minimal_runtime_enabled: bool, + + /// Cache TTL for minimal runtime snapshots in milliseconds (0 disables caching). + #[serde(default = "default_api_minimal_runtime_cache_ttl_ms")] + pub minimal_runtime_cache_ttl_ms: u64, + /// Read-only mode: mutating endpoints are rejected. #[serde(default)] pub read_only: bool, @@ -830,6 +838,8 @@ impl Default for ApiConfig { whitelist: default_api_whitelist(), auth_header: String::new(), request_body_limit_bytes: default_api_request_body_limit_bytes(), + minimal_runtime_enabled: default_api_minimal_runtime_enabled(), + minimal_runtime_cache_ttl_ms: default_api_minimal_runtime_cache_ttl_ms(), read_only: false, } } From 5876623bb0512ea98a4be75266cdfc9a74f93218 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:46:26 +0300 Subject: [PATCH 049/261] Runtime API Stats Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/runtime_stats.rs | 392 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 src/api/runtime_stats.rs diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs new file mode 100644 index 0000000..53fdeff --- /dev/null +++ b/src/api/runtime_stats.rs @@ -0,0 +1,392 @@ +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use crate::config::ApiConfig; +use crate::stats::Stats; + +use super::ApiShared; +use super::model::{ + DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary, MinimalAllData, + MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData, MinimalQuarantineData, + ZeroAllData, ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData, + ZeroUpstreamData, +}; + +const FEATURE_DISABLED_REASON: &str = "feature_disabled"; +const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable"; + +#[derive(Clone)] +pub(crate) struct MinimalCacheEntry { + pub(super) expires_at: Instant, + pub(super) payload: MinimalAllPayload, + pub(super) generated_at_epoch_secs: u64, +} + +pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> ZeroAllData { + let telemetry = stats.telemetry_policy(); + let handshake_error_codes = stats + .get_me_handshake_error_code_counts() + .into_iter() + .map(|(code, total)| ZeroCodeCount { code, total }) + .collect(); + + ZeroAllData { + generated_at_epoch_secs: now_epoch_secs(), + core: ZeroCoreData { + uptime_seconds: stats.uptime_secs(), + connections_total: stats.get_connects_all(), + connections_bad_total: stats.get_connects_bad(), + handshake_timeouts_total: stats.get_handshake_timeouts(), + configured_users, + telemetry_core_enabled: telemetry.core_enabled, + telemetry_user_enabled: telemetry.user_enabled, + telemetry_me_level: telemetry.me_level.to_string(), + }, + upstream: ZeroUpstreamData { + connect_attempt_total: stats.get_upstream_connect_attempt_total(), + connect_success_total: stats.get_upstream_connect_success_total(), + connect_fail_total: stats.get_upstream_connect_fail_total(), + connect_failfast_hard_error_total: stats.get_upstream_connect_failfast_hard_error_total(), + connect_attempts_bucket_1: stats.get_upstream_connect_attempts_bucket_1(), + connect_attempts_bucket_2: stats.get_upstream_connect_attempts_bucket_2(), + connect_attempts_bucket_3_4: stats.get_upstream_connect_attempts_bucket_3_4(), + connect_attempts_bucket_gt_4: stats.get_upstream_connect_attempts_bucket_gt_4(), + connect_duration_success_bucket_le_100ms: stats + .get_upstream_connect_duration_success_bucket_le_100ms(), + connect_duration_success_bucket_101_500ms: stats + .get_upstream_connect_duration_success_bucket_101_500ms(), + connect_duration_success_bucket_501_1000ms: stats + .get_upstream_connect_duration_success_bucket_501_1000ms(), + connect_duration_success_bucket_gt_1000ms: stats + .get_upstream_connect_duration_success_bucket_gt_1000ms(), + connect_duration_fail_bucket_le_100ms: stats + .get_upstream_connect_duration_fail_bucket_le_100ms(), + connect_duration_fail_bucket_101_500ms: stats + .get_upstream_connect_duration_fail_bucket_101_500ms(), + connect_duration_fail_bucket_501_1000ms: stats + .get_upstream_connect_duration_fail_bucket_501_1000ms(), + connect_duration_fail_bucket_gt_1000ms: stats + .get_upstream_connect_duration_fail_bucket_gt_1000ms(), + }, + middle_proxy: ZeroMiddleProxyData { + keepalive_sent_total: stats.get_me_keepalive_sent(), + keepalive_failed_total: stats.get_me_keepalive_failed(), + keepalive_pong_total: stats.get_me_keepalive_pong(), + keepalive_timeout_total: stats.get_me_keepalive_timeout(), + rpc_proxy_req_signal_sent_total: stats.get_me_rpc_proxy_req_signal_sent_total(), + rpc_proxy_req_signal_failed_total: stats.get_me_rpc_proxy_req_signal_failed_total(), + rpc_proxy_req_signal_skipped_no_meta_total: stats + .get_me_rpc_proxy_req_signal_skipped_no_meta_total(), + rpc_proxy_req_signal_response_total: stats.get_me_rpc_proxy_req_signal_response_total(), + rpc_proxy_req_signal_close_sent_total: stats + .get_me_rpc_proxy_req_signal_close_sent_total(), + reconnect_attempt_total: stats.get_me_reconnect_attempts(), + reconnect_success_total: stats.get_me_reconnect_success(), + handshake_reject_total: stats.get_me_handshake_reject_total(), + handshake_error_codes, + reader_eof_total: stats.get_me_reader_eof_total(), + idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(), + route_drop_no_conn_total: stats.get_me_route_drop_no_conn(), + route_drop_channel_closed_total: stats.get_me_route_drop_channel_closed(), + route_drop_queue_full_total: stats.get_me_route_drop_queue_full(), + route_drop_queue_full_base_total: stats.get_me_route_drop_queue_full_base(), + route_drop_queue_full_high_total: stats.get_me_route_drop_queue_full_high(), + socks_kdf_strict_reject_total: stats.get_me_socks_kdf_strict_reject(), + socks_kdf_compat_fallback_total: stats.get_me_socks_kdf_compat_fallback(), + endpoint_quarantine_total: stats.get_me_endpoint_quarantine_total(), + kdf_drift_total: stats.get_me_kdf_drift_total(), + kdf_port_only_drift_total: stats.get_me_kdf_port_only_drift_total(), + hardswap_pending_reuse_total: stats.get_me_hardswap_pending_reuse_total(), + hardswap_pending_ttl_expired_total: stats.get_me_hardswap_pending_ttl_expired_total(), + single_endpoint_outage_enter_total: stats.get_me_single_endpoint_outage_enter_total(), + single_endpoint_outage_exit_total: stats.get_me_single_endpoint_outage_exit_total(), + single_endpoint_outage_reconnect_attempt_total: stats + .get_me_single_endpoint_outage_reconnect_attempt_total(), + single_endpoint_outage_reconnect_success_total: stats + .get_me_single_endpoint_outage_reconnect_success_total(), + single_endpoint_quarantine_bypass_total: stats + .get_me_single_endpoint_quarantine_bypass_total(), + single_endpoint_shadow_rotate_total: stats.get_me_single_endpoint_shadow_rotate_total(), + single_endpoint_shadow_rotate_skipped_quarantine_total: stats + .get_me_single_endpoint_shadow_rotate_skipped_quarantine_total(), + floor_mode_switch_total: stats.get_me_floor_mode_switch_total(), + floor_mode_switch_static_to_adaptive_total: stats + .get_me_floor_mode_switch_static_to_adaptive_total(), + floor_mode_switch_adaptive_to_static_total: stats + .get_me_floor_mode_switch_adaptive_to_static_total(), + }, + pool: ZeroPoolData { + pool_swap_total: stats.get_pool_swap_total(), + pool_drain_active: stats.get_pool_drain_active(), + pool_force_close_total: stats.get_pool_force_close_total(), + pool_stale_pick_total: stats.get_pool_stale_pick_total(), + writer_removed_total: stats.get_me_writer_removed_total(), + writer_removed_unexpected_total: stats.get_me_writer_removed_unexpected_total(), + refill_triggered_total: stats.get_me_refill_triggered_total(), + refill_skipped_inflight_total: stats.get_me_refill_skipped_inflight_total(), + refill_failed_total: stats.get_me_refill_failed_total(), + writer_restored_same_endpoint_total: stats.get_me_writer_restored_same_endpoint_total(), + writer_restored_fallback_total: stats.get_me_writer_restored_fallback_total(), + }, + desync: ZeroDesyncData { + secure_padding_invalid_total: stats.get_secure_padding_invalid(), + desync_total: stats.get_desync_total(), + desync_full_logged_total: stats.get_desync_full_logged(), + desync_suppressed_total: stats.get_desync_suppressed(), + desync_frames_bucket_0: stats.get_desync_frames_bucket_0(), + desync_frames_bucket_1_2: stats.get_desync_frames_bucket_1_2(), + desync_frames_bucket_3_10: stats.get_desync_frames_bucket_3_10(), + desync_frames_bucket_gt_10: stats.get_desync_frames_bucket_gt_10(), + }, + } +} + +pub(super) async fn build_minimal_all_data( + shared: &ApiShared, + api_cfg: &ApiConfig, +) -> MinimalAllData { + let now = now_epoch_secs(); + if !api_cfg.minimal_runtime_enabled { + return MinimalAllData { + enabled: false, + reason: Some(FEATURE_DISABLED_REASON), + generated_at_epoch_secs: now, + data: None, + }; + } + + let Some((generated_at_epoch_secs, payload)) = + get_minimal_payload_cached(shared, api_cfg.minimal_runtime_cache_ttl_ms).await + else { + return MinimalAllData { + enabled: true, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs: now, + data: Some(MinimalAllPayload { + me_writers: disabled_me_writers(now, SOURCE_UNAVAILABLE_REASON), + dcs: disabled_dcs(now, SOURCE_UNAVAILABLE_REASON), + me_runtime: None, + network_path: Vec::new(), + }), + }; + }; + + MinimalAllData { + enabled: true, + reason: None, + generated_at_epoch_secs, + data: Some(payload), + } +} + +pub(super) async fn build_me_writers_data( + shared: &ApiShared, + api_cfg: &ApiConfig, +) -> MeWritersData { + let now = now_epoch_secs(); + if !api_cfg.minimal_runtime_enabled { + return disabled_me_writers(now, FEATURE_DISABLED_REASON); + } + + let Some((_, payload)) = + get_minimal_payload_cached(shared, api_cfg.minimal_runtime_cache_ttl_ms).await + else { + return disabled_me_writers(now, SOURCE_UNAVAILABLE_REASON); + }; + payload.me_writers +} + +pub(super) async fn build_dcs_data(shared: &ApiShared, api_cfg: &ApiConfig) -> DcStatusData { + let now = now_epoch_secs(); + if !api_cfg.minimal_runtime_enabled { + return disabled_dcs(now, FEATURE_DISABLED_REASON); + } + + let Some((_, payload)) = + get_minimal_payload_cached(shared, api_cfg.minimal_runtime_cache_ttl_ms).await + else { + return disabled_dcs(now, SOURCE_UNAVAILABLE_REASON); + }; + payload.dcs +} + +async fn get_minimal_payload_cached( + shared: &ApiShared, + cache_ttl_ms: u64, +) -> Option<(u64, MinimalAllPayload)> { + if cache_ttl_ms > 0 { + let now = Instant::now(); + let cached = shared.minimal_cache.lock().await.clone(); + if let Some(entry) = cached + && now < entry.expires_at + { + return Some((entry.generated_at_epoch_secs, entry.payload)); + } + } + + let pool = shared.me_pool.as_ref()?; + let status = pool.api_status_snapshot().await; + let runtime = pool.api_runtime_snapshot().await; + let generated_at_epoch_secs = status.generated_at_epoch_secs; + + let me_writers = MeWritersData { + middle_proxy_enabled: true, + reason: None, + generated_at_epoch_secs, + summary: MeWritersSummary { + configured_dc_groups: status.configured_dc_groups, + configured_endpoints: status.configured_endpoints, + available_endpoints: status.available_endpoints, + available_pct: status.available_pct, + required_writers: status.required_writers, + alive_writers: status.alive_writers, + coverage_pct: status.coverage_pct, + }, + writers: status + .writers + .into_iter() + .map(|entry| MeWriterStatus { + writer_id: entry.writer_id, + dc: entry.dc, + endpoint: entry.endpoint.to_string(), + generation: entry.generation, + state: entry.state, + draining: entry.draining, + degraded: entry.degraded, + bound_clients: entry.bound_clients, + idle_for_secs: entry.idle_for_secs, + rtt_ema_ms: entry.rtt_ema_ms, + }) + .collect(), + }; + let dcs = DcStatusData { + middle_proxy_enabled: true, + reason: None, + generated_at_epoch_secs, + dcs: status + .dcs + .into_iter() + .map(|entry| DcStatus { + dc: entry.dc, + endpoints: entry + .endpoints + .into_iter() + .map(|value| value.to_string()) + .collect(), + available_endpoints: entry.available_endpoints, + available_pct: entry.available_pct, + required_writers: entry.required_writers, + alive_writers: entry.alive_writers, + coverage_pct: entry.coverage_pct, + rtt_ms: entry.rtt_ms, + load: entry.load, + }) + .collect(), + }; + let me_runtime = MinimalMeRuntimeData { + active_generation: runtime.active_generation, + warm_generation: runtime.warm_generation, + pending_hardswap_generation: runtime.pending_hardswap_generation, + pending_hardswap_age_secs: runtime.pending_hardswap_age_secs, + hardswap_enabled: runtime.hardswap_enabled, + floor_mode: runtime.floor_mode, + adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs, + adaptive_floor_min_writers_single_endpoint: runtime + .adaptive_floor_min_writers_single_endpoint, + adaptive_floor_recover_grace_secs: runtime.adaptive_floor_recover_grace_secs, + me_keepalive_enabled: runtime.me_keepalive_enabled, + me_keepalive_interval_secs: runtime.me_keepalive_interval_secs, + me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs, + me_keepalive_payload_random: runtime.me_keepalive_payload_random, + rpc_proxy_req_every_secs: runtime.rpc_proxy_req_every_secs, + me_reconnect_max_concurrent_per_dc: runtime.me_reconnect_max_concurrent_per_dc, + me_reconnect_backoff_base_ms: runtime.me_reconnect_backoff_base_ms, + me_reconnect_backoff_cap_ms: runtime.me_reconnect_backoff_cap_ms, + me_reconnect_fast_retry_count: runtime.me_reconnect_fast_retry_count, + me_pool_drain_ttl_secs: runtime.me_pool_drain_ttl_secs, + me_pool_force_close_secs: runtime.me_pool_force_close_secs, + me_pool_min_fresh_ratio: runtime.me_pool_min_fresh_ratio, + me_bind_stale_mode: runtime.me_bind_stale_mode, + me_bind_stale_ttl_secs: runtime.me_bind_stale_ttl_secs, + me_single_endpoint_shadow_writers: runtime.me_single_endpoint_shadow_writers, + me_single_endpoint_outage_mode_enabled: runtime.me_single_endpoint_outage_mode_enabled, + me_single_endpoint_outage_disable_quarantine: runtime + .me_single_endpoint_outage_disable_quarantine, + me_single_endpoint_outage_backoff_min_ms: runtime.me_single_endpoint_outage_backoff_min_ms, + me_single_endpoint_outage_backoff_max_ms: runtime.me_single_endpoint_outage_backoff_max_ms, + me_single_endpoint_shadow_rotate_every_secs: runtime + .me_single_endpoint_shadow_rotate_every_secs, + me_deterministic_writer_sort: runtime.me_deterministic_writer_sort, + me_socks_kdf_policy: runtime.me_socks_kdf_policy, + quarantined_endpoints_total: runtime.quarantined_endpoints.len(), + quarantined_endpoints: runtime + .quarantined_endpoints + .into_iter() + .map(|entry| MinimalQuarantineData { + endpoint: entry.endpoint.to_string(), + remaining_ms: entry.remaining_ms, + }) + .collect(), + }; + let network_path = runtime + .network_path + .into_iter() + .map(|entry| MinimalDcPathData { + dc: entry.dc, + ip_preference: entry.ip_preference, + selected_addr_v4: entry.selected_addr_v4.map(|value| value.to_string()), + selected_addr_v6: entry.selected_addr_v6.map(|value| value.to_string()), + }) + .collect(); + + let payload = MinimalAllPayload { + me_writers, + dcs, + me_runtime: Some(me_runtime), + network_path, + }; + + if cache_ttl_ms > 0 { + let entry = MinimalCacheEntry { + expires_at: Instant::now() + Duration::from_millis(cache_ttl_ms), + payload: payload.clone(), + generated_at_epoch_secs, + }; + *shared.minimal_cache.lock().await = Some(entry); + } + + Some((generated_at_epoch_secs, payload)) +} + +fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData { + MeWritersData { + middle_proxy_enabled: false, + reason: Some(reason), + generated_at_epoch_secs: now_epoch_secs, + summary: MeWritersSummary { + configured_dc_groups: 0, + configured_endpoints: 0, + available_endpoints: 0, + available_pct: 0.0, + required_writers: 0, + alive_writers: 0, + coverage_pct: 0.0, + }, + writers: Vec::new(), + } +} + +fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData { + DcStatusData { + middle_proxy_enabled: false, + reason: Some(reason), + generated_at_epoch_secs: now_epoch_secs, + dcs: Vec::new(), + } +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} From 716b4adef25ec5661821dad45dae27ffad8fea6d Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:46:47 +0300 Subject: [PATCH 050/261] Runtime Stats in API Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/mod.rs | 111 +++++--------------------- src/api/model.rs | 200 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 216 insertions(+), 95 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 56fb588..299d5a1 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -3,7 +3,6 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; use http_body_util::{BodyExt, Full}; use hyper::body::{Bytes, Incoming}; @@ -24,13 +23,17 @@ use crate::transport::middle_proxy::MePool; mod config_store; mod model; +mod runtime_stats; mod users; use config_store::{current_revision, parse_if_match}; use model::{ - ApiFailure, CreateUserRequest, DcStatus, DcStatusData, ErrorBody, ErrorResponse, HealthData, - MeWriterStatus, MeWritersData, MeWritersSummary, PatchUserRequest, RotateSecretRequest, - SuccessResponse, SummaryData, + ApiFailure, CreateUserRequest, ErrorBody, ErrorResponse, HealthData, PatchUserRequest, + RotateSecretRequest, SuccessResponse, SummaryData, +}; +use runtime_stats::{ + MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data, + build_zero_all_data, }; use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config}; @@ -41,6 +44,7 @@ pub(super) struct ApiShared { pub(super) me_pool: Option>, pub(super) config_path: PathBuf, pub(super) mutation_lock: Arc>, + pub(super) minimal_cache: Arc>>, pub(super) request_id: Arc, } @@ -78,6 +82,7 @@ pub async fn serve( me_pool, config_path, mutation_lock: Arc::new(Mutex::new(())), + minimal_cache: Arc::new(Mutex::new(None)), request_id: Arc::new(AtomicU64::new(1)), }); @@ -185,91 +190,24 @@ async fn handle( }; Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/stats/zero/all") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_zero_all_data(&shared.stats, cfg.access.users.len()); + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/stats/minimal/all") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_minimal_all_data(shared.as_ref(), api_cfg).await; + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/me-writers") => { let revision = current_revision(&shared.config_path).await?; - let data = match &shared.me_pool { - Some(pool) => { - let snapshot = pool.api_status_snapshot().await; - let writers = snapshot - .writers - .into_iter() - .map(|entry| MeWriterStatus { - writer_id: entry.writer_id, - dc: entry.dc, - endpoint: entry.endpoint.to_string(), - generation: entry.generation, - state: entry.state, - draining: entry.draining, - degraded: entry.degraded, - bound_clients: entry.bound_clients, - idle_for_secs: entry.idle_for_secs, - rtt_ema_ms: entry.rtt_ema_ms, - }) - .collect(); - MeWritersData { - middle_proxy_enabled: true, - generated_at_epoch_secs: snapshot.generated_at_epoch_secs, - summary: MeWritersSummary { - configured_dc_groups: snapshot.configured_dc_groups, - configured_endpoints: snapshot.configured_endpoints, - available_endpoints: snapshot.available_endpoints, - available_pct: snapshot.available_pct, - required_writers: snapshot.required_writers, - alive_writers: snapshot.alive_writers, - coverage_pct: snapshot.coverage_pct, - }, - writers, - } - } - None => MeWritersData { - middle_proxy_enabled: false, - generated_at_epoch_secs: now_epoch_secs(), - summary: MeWritersSummary { - configured_dc_groups: 0, - configured_endpoints: 0, - available_endpoints: 0, - available_pct: 0.0, - required_writers: 0, - alive_writers: 0, - coverage_pct: 0.0, - }, - writers: Vec::new(), - }, - }; + let data = build_me_writers_data(shared.as_ref(), api_cfg).await; Ok(success_response(StatusCode::OK, data, revision)) } ("GET", "/v1/stats/dcs") => { let revision = current_revision(&shared.config_path).await?; - let data = match &shared.me_pool { - Some(pool) => { - let snapshot = pool.api_status_snapshot().await; - let dcs = snapshot - .dcs - .into_iter() - .map(|entry| DcStatus { - dc: entry.dc, - endpoints: entry.endpoints.into_iter().map(|value| value.to_string()).collect(), - available_endpoints: entry.available_endpoints, - available_pct: entry.available_pct, - required_writers: entry.required_writers, - alive_writers: entry.alive_writers, - coverage_pct: entry.coverage_pct, - rtt_ms: entry.rtt_ms, - load: entry.load, - }) - .collect(); - DcStatusData { - middle_proxy_enabled: true, - generated_at_epoch_secs: snapshot.generated_at_epoch_secs, - dcs, - } - } - None => DcStatusData { - middle_proxy_enabled: false, - generated_at_epoch_secs: now_epoch_secs(), - dcs: Vec::new(), - }, - }; + let data = build_dcs_data(shared.as_ref(), api_cfg).await; Ok(success_response(StatusCode::OK, data, revision)) } ("GET", "/v1/stats/users") | ("GET", "/v1/users") => { @@ -397,13 +335,6 @@ async fn handle( } } -fn now_epoch_secs() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - fn success_response( status: StatusCode, data: T, diff --git a/src/api/model.rs b/src/api/model.rs index 43d4173..8b2d279 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -65,7 +65,123 @@ pub(super) struct SummaryData { pub(super) configured_users: usize, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] +pub(super) struct ZeroCodeCount { + pub(super) code: i32, + pub(super) total: u64, +} + +#[derive(Serialize, Clone)] +pub(super) struct ZeroCoreData { + pub(super) uptime_seconds: f64, + pub(super) connections_total: u64, + pub(super) connections_bad_total: u64, + pub(super) handshake_timeouts_total: u64, + pub(super) configured_users: usize, + pub(super) telemetry_core_enabled: bool, + pub(super) telemetry_user_enabled: bool, + pub(super) telemetry_me_level: String, +} + +#[derive(Serialize, Clone)] +pub(super) struct ZeroUpstreamData { + pub(super) connect_attempt_total: u64, + pub(super) connect_success_total: u64, + pub(super) connect_fail_total: u64, + pub(super) connect_failfast_hard_error_total: u64, + pub(super) connect_attempts_bucket_1: u64, + pub(super) connect_attempts_bucket_2: u64, + pub(super) connect_attempts_bucket_3_4: u64, + pub(super) connect_attempts_bucket_gt_4: u64, + pub(super) connect_duration_success_bucket_le_100ms: u64, + pub(super) connect_duration_success_bucket_101_500ms: u64, + pub(super) connect_duration_success_bucket_501_1000ms: u64, + pub(super) connect_duration_success_bucket_gt_1000ms: u64, + pub(super) connect_duration_fail_bucket_le_100ms: u64, + pub(super) connect_duration_fail_bucket_101_500ms: u64, + pub(super) connect_duration_fail_bucket_501_1000ms: u64, + pub(super) connect_duration_fail_bucket_gt_1000ms: u64, +} + +#[derive(Serialize, Clone)] +pub(super) struct ZeroMiddleProxyData { + pub(super) keepalive_sent_total: u64, + pub(super) keepalive_failed_total: u64, + pub(super) keepalive_pong_total: u64, + pub(super) keepalive_timeout_total: u64, + pub(super) rpc_proxy_req_signal_sent_total: u64, + pub(super) rpc_proxy_req_signal_failed_total: u64, + pub(super) rpc_proxy_req_signal_skipped_no_meta_total: u64, + pub(super) rpc_proxy_req_signal_response_total: u64, + pub(super) rpc_proxy_req_signal_close_sent_total: u64, + pub(super) reconnect_attempt_total: u64, + pub(super) reconnect_success_total: u64, + pub(super) handshake_reject_total: u64, + pub(super) handshake_error_codes: Vec, + pub(super) reader_eof_total: u64, + pub(super) idle_close_by_peer_total: u64, + pub(super) route_drop_no_conn_total: u64, + pub(super) route_drop_channel_closed_total: u64, + pub(super) route_drop_queue_full_total: u64, + pub(super) route_drop_queue_full_base_total: u64, + pub(super) route_drop_queue_full_high_total: u64, + pub(super) socks_kdf_strict_reject_total: u64, + pub(super) socks_kdf_compat_fallback_total: u64, + pub(super) endpoint_quarantine_total: u64, + pub(super) kdf_drift_total: u64, + pub(super) kdf_port_only_drift_total: u64, + pub(super) hardswap_pending_reuse_total: u64, + pub(super) hardswap_pending_ttl_expired_total: u64, + pub(super) single_endpoint_outage_enter_total: u64, + pub(super) single_endpoint_outage_exit_total: u64, + pub(super) single_endpoint_outage_reconnect_attempt_total: u64, + pub(super) single_endpoint_outage_reconnect_success_total: u64, + pub(super) single_endpoint_quarantine_bypass_total: u64, + pub(super) single_endpoint_shadow_rotate_total: u64, + pub(super) single_endpoint_shadow_rotate_skipped_quarantine_total: u64, + pub(super) floor_mode_switch_total: u64, + pub(super) floor_mode_switch_static_to_adaptive_total: u64, + pub(super) floor_mode_switch_adaptive_to_static_total: u64, +} + +#[derive(Serialize, Clone)] +pub(super) struct ZeroPoolData { + pub(super) pool_swap_total: u64, + pub(super) pool_drain_active: u64, + pub(super) pool_force_close_total: u64, + pub(super) pool_stale_pick_total: u64, + pub(super) writer_removed_total: u64, + pub(super) writer_removed_unexpected_total: u64, + pub(super) refill_triggered_total: u64, + pub(super) refill_skipped_inflight_total: u64, + pub(super) refill_failed_total: u64, + pub(super) writer_restored_same_endpoint_total: u64, + pub(super) writer_restored_fallback_total: u64, +} + +#[derive(Serialize, Clone)] +pub(super) struct ZeroDesyncData { + pub(super) secure_padding_invalid_total: u64, + pub(super) desync_total: u64, + pub(super) desync_full_logged_total: u64, + pub(super) desync_suppressed_total: u64, + pub(super) desync_frames_bucket_0: u64, + pub(super) desync_frames_bucket_1_2: u64, + pub(super) desync_frames_bucket_3_10: u64, + pub(super) desync_frames_bucket_gt_10: u64, +} + +#[derive(Serialize, Clone)] +pub(super) struct ZeroAllData { + pub(super) generated_at_epoch_secs: u64, + pub(super) core: ZeroCoreData, + pub(super) upstream: ZeroUpstreamData, + pub(super) middle_proxy: ZeroMiddleProxyData, + pub(super) pool: ZeroPoolData, + pub(super) desync: ZeroDesyncData, +} + +#[derive(Serialize, Clone)] pub(super) struct MeWritersSummary { pub(super) configured_dc_groups: usize, pub(super) configured_endpoints: usize, @@ -76,7 +192,7 @@ pub(super) struct MeWritersSummary { pub(super) coverage_pct: f64, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub(super) struct MeWriterStatus { pub(super) writer_id: u64, pub(super) dc: Option, @@ -90,15 +206,17 @@ pub(super) struct MeWriterStatus { pub(super) rtt_ema_ms: Option, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub(super) struct MeWritersData { pub(super) middle_proxy_enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, pub(super) generated_at_epoch_secs: u64, pub(super) summary: MeWritersSummary, pub(super) writers: Vec, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub(super) struct DcStatus { pub(super) dc: i16, pub(super) endpoints: Vec, @@ -111,13 +229,85 @@ pub(super) struct DcStatus { pub(super) load: usize, } -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub(super) struct DcStatusData { pub(super) middle_proxy_enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, pub(super) generated_at_epoch_secs: u64, pub(super) dcs: Vec, } +#[derive(Serialize, Clone)] +pub(super) struct MinimalQuarantineData { + pub(super) endpoint: String, + pub(super) remaining_ms: u64, +} + +#[derive(Serialize, Clone)] +pub(super) struct MinimalDcPathData { + pub(super) dc: i16, + pub(super) ip_preference: Option<&'static str>, + pub(super) selected_addr_v4: Option, + pub(super) selected_addr_v6: Option, +} + +#[derive(Serialize, Clone)] +pub(super) struct MinimalMeRuntimeData { + pub(super) active_generation: u64, + pub(super) warm_generation: u64, + pub(super) pending_hardswap_generation: u64, + pub(super) pending_hardswap_age_secs: Option, + pub(super) hardswap_enabled: bool, + pub(super) floor_mode: &'static str, + pub(super) adaptive_floor_idle_secs: u64, + pub(super) adaptive_floor_min_writers_single_endpoint: u8, + pub(super) adaptive_floor_recover_grace_secs: u64, + pub(super) me_keepalive_enabled: bool, + pub(super) me_keepalive_interval_secs: u64, + pub(super) me_keepalive_jitter_secs: u64, + pub(super) me_keepalive_payload_random: bool, + pub(super) rpc_proxy_req_every_secs: u64, + pub(super) me_reconnect_max_concurrent_per_dc: u32, + pub(super) me_reconnect_backoff_base_ms: u64, + pub(super) me_reconnect_backoff_cap_ms: u64, + pub(super) me_reconnect_fast_retry_count: u32, + pub(super) me_pool_drain_ttl_secs: u64, + pub(super) me_pool_force_close_secs: u64, + pub(super) me_pool_min_fresh_ratio: f32, + pub(super) me_bind_stale_mode: &'static str, + pub(super) me_bind_stale_ttl_secs: u64, + pub(super) me_single_endpoint_shadow_writers: u8, + pub(super) me_single_endpoint_outage_mode_enabled: bool, + pub(super) me_single_endpoint_outage_disable_quarantine: bool, + pub(super) me_single_endpoint_outage_backoff_min_ms: u64, + pub(super) me_single_endpoint_outage_backoff_max_ms: u64, + pub(super) me_single_endpoint_shadow_rotate_every_secs: u64, + pub(super) me_deterministic_writer_sort: bool, + pub(super) me_socks_kdf_policy: &'static str, + pub(super) quarantined_endpoints_total: usize, + pub(super) quarantined_endpoints: Vec, +} + +#[derive(Serialize, Clone)] +pub(super) struct MinimalAllPayload { + pub(super) me_writers: MeWritersData, + pub(super) dcs: DcStatusData, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) me_runtime: Option, + pub(super) network_path: Vec, +} + +#[derive(Serialize, Clone)] +pub(super) struct MinimalAllData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + #[derive(Serialize)] pub(super) struct UserInfo { pub(super) username: String, From f1efaf4491176591bcf1aff2372464f44b5fcca7 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:48:43 +0300 Subject: [PATCH 051/261] User-links in API Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/model.rs | 8 +++ src/api/users.rs | 138 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index 8b2d279..be76c4e 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -308,6 +308,13 @@ pub(super) struct MinimalAllData { pub(super) data: Option, } +#[derive(Serialize)] +pub(super) struct UserLinks { + pub(super) classic: Vec, + pub(super) secure: Vec, + pub(super) tls: Vec, +} + #[derive(Serialize)] pub(super) struct UserInfo { pub(super) username: String, @@ -319,6 +326,7 @@ pub(super) struct UserInfo { pub(super) current_connections: u64, pub(super) active_unique_ips: usize, pub(super) total_octets: u64, + pub(super) links: UserLinks, } #[derive(Serialize)] diff --git a/src/api/users.rs b/src/api/users.rs index 75d659f..9fc03e9 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::net::IpAddr; use hyper::StatusCode; @@ -12,8 +13,8 @@ use super::config_store::{ }; use super::model::{ ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest, - UserInfo, is_valid_ad_tag, is_valid_user_secret, is_valid_username, parse_optional_expiration, - random_user_secret, + UserInfo, UserLinks, is_valid_ad_tag, is_valid_user_secret, is_valid_username, + parse_optional_expiration, random_user_secret, }; pub(super) async fn create_user( @@ -105,6 +106,7 @@ pub(super) async fn create_user( current_connections: 0, active_unique_ips: 0, total_octets: 0, + links: build_user_links(&cfg, &secret), }); Ok((CreateUserResponse { user, secret }, revision)) @@ -281,6 +283,16 @@ pub(super) async fn users_from_config( let mut users = Vec::with_capacity(names.len()); for username in names { + let links = cfg + .access + .users + .get(&username) + .map(|secret| build_user_links(cfg, secret)) + .unwrap_or(UserLinks { + classic: Vec::new(), + secure: Vec::new(), + tls: Vec::new(), + }); users.push(UserInfo { user_ad_tag: cfg.access.user_ad_tags.get(&username).cloned(), max_tcp_conns: cfg.access.user_max_tcp_conns.get(&username).copied(), @@ -294,8 +306,130 @@ pub(super) async fn users_from_config( current_connections: stats.get_user_curr_connects(&username), active_unique_ips: ip_counts.get(&username).copied().unwrap_or(0), total_octets: stats.get_user_total_octets(&username), + links, username, }); } users } + +fn build_user_links(cfg: &ProxyConfig, secret: &str) -> UserLinks { + let hosts = resolve_link_hosts(cfg); + let port = cfg.general.links.public_port.unwrap_or(cfg.server.port); + let tls_domains = resolve_tls_domains(cfg); + + let mut classic = Vec::new(); + let mut secure = Vec::new(); + let mut tls = Vec::new(); + + for host in &hosts { + if cfg.general.modes.classic { + classic.push(format!( + "tg://proxy?server={}&port={}&secret={}", + host, port, secret + )); + } + if cfg.general.modes.secure { + secure.push(format!( + "tg://proxy?server={}&port={}&secret=dd{}", + host, port, secret + )); + } + if cfg.general.modes.tls { + for domain in &tls_domains { + let domain_hex = hex::encode(domain); + tls.push(format!( + "tg://proxy?server={}&port={}&secret=ee{}{}", + host, port, secret, domain_hex + )); + } + } + } + + UserLinks { + classic, + secure, + tls, + } +} + +fn resolve_link_hosts(cfg: &ProxyConfig) -> Vec { + if let Some(host) = cfg + .general + .links + .public_host + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return vec![host.to_string()]; + } + + let mut hosts = Vec::new(); + for listener in &cfg.server.listeners { + if let Some(host) = listener + .announce + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + push_unique_host(&mut hosts, host); + continue; + } + if let Some(ip) = listener.announce_ip { + if !ip.is_unspecified() { + push_unique_host(&mut hosts, &ip.to_string()); + } + continue; + } + if !listener.ip.is_unspecified() { + push_unique_host(&mut hosts, &listener.ip.to_string()); + } + } + + if hosts.is_empty() { + if let Some(host) = cfg.server.listen_addr_ipv4.as_deref() { + push_host_from_legacy_listen(&mut hosts, host); + } + if let Some(host) = cfg.server.listen_addr_ipv6.as_deref() { + push_host_from_legacy_listen(&mut hosts, host); + } + } + + hosts +} + +fn push_host_from_legacy_listen(hosts: &mut Vec, raw: &str) { + let candidate = raw.trim(); + if candidate.is_empty() { + return; + } + + match candidate.parse::() { + Ok(ip) if ip.is_unspecified() => {} + Ok(ip) => push_unique_host(hosts, &ip.to_string()), + Err(_) => push_unique_host(hosts, candidate), + } +} + +fn push_unique_host(hosts: &mut Vec, candidate: &str) { + if !hosts.iter().any(|existing| existing == candidate) { + hosts.push(candidate.to_string()); + } +} + +fn resolve_tls_domains(cfg: &ProxyConfig) -> Vec<&str> { + let mut domains = Vec::with_capacity(1 + cfg.censorship.tls_domains.len()); + let primary = cfg.censorship.tls_domain.as_str(); + if !primary.is_empty() { + domains.push(primary); + } + for domain in &cfg.censorship.tls_domains { + let value = domain.as_str(); + if value.is_empty() || domains.contains(&value) { + continue; + } + domains.push(value); + } + domains +} From c86a511638dc617b2fb8ea6d4a4c70f63caee052 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:53:17 +0300 Subject: [PATCH 052/261] Update API.md Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- docs/API.md | 528 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 374 insertions(+), 154 deletions(-) diff --git a/docs/API.md b/docs/API.md index 27013e3..bd8f892 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,57 +1,41 @@ # Telemt Control API ## Purpose -This document specifies the control-plane HTTP API used for: -- runtime statistics access, -- user management, -- safe configuration mutations. +Control-plane HTTP API for runtime visibility and user/config management. +Data-plane MTProto traffic is out of scope. -The data-plane (MTProto proxy traffic) is out of scope. +## Runtime Configuration +API runtime is configured in `[server.api]`. -## Design Principles -1. Keep data-plane isolated. -The API must not affect MTProto hot paths. +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `enabled` | `bool` | `false` | Enables REST API listener. | +| `listen` | `string` (`IP:PORT`) | `127.0.0.1:9091` | API bind address. | +| `whitelist` | `CIDR[]` | `127.0.0.1/32, ::1/128` | Source IP allowlist. Empty list means allow all. | +| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. | +| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. | +| `minimal_runtime_enabled` | `bool` | `false` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. | +| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache. | +| `read_only` | `bool` | `false` | Disables mutating endpoints. | -2. Keep configuration authoritative. -`config.toml` is the single source of truth for managed entities. - -3. Make writes safe. -All config mutations are validated and persisted atomically. - -4. Be explicit about concurrency. -Mutating endpoints support optimistic concurrency through revision matching. - -5. Prefer fail-fast contract errors. -Input validation errors are returned with machine-readable error codes. - -## Runtime and Configuration -Control API runtime is configured under `[server.api]`. - -Parameters: -- `enabled: bool` -- `listen: "IP:PORT"` -- `whitelist: [CIDR, ...]` -- `auth_header: string` (exact match against `Authorization` header; empty disables header auth) -- `request_body_limit_bytes: usize` -- `read_only: bool` - -Backward compatibility: -- `server.admin_api` is accepted as an alias while `server.api` is canonical. - -Operational note: -- Changes in `server.api` require process restart to take effect. +`server.admin_api` is accepted as an alias for backward compatibility. ## Protocol Contract -- Transport: HTTP/1.1 -- Payload format: JSON (`application/json; charset=utf-8`) -- API prefix: `/v1` + +| Item | Value | +| --- | --- | +| Transport | HTTP/1.1 | +| Content type | `application/json; charset=utf-8` | +| Prefix | `/v1` | +| Optimistic concurrency | `If-Match: ` on mutating requests (optional) | +| Revision format | SHA-256 hex of current `config.toml` content | ### Success Envelope ```json { "ok": true, "data": {}, - "revision": "sha256-of-config" + "revision": "sha256-hex" } ``` @@ -61,147 +45,383 @@ Operational note: "ok": false, "error": { "code": "machine_code", - "message": "human-readable text" + "message": "human-readable" }, "request_id": 1 } ``` -### Revision / Concurrency Contract -- Mutating operations MAY include `If-Match: `. -- If provided and stale, API returns `409 revision_conflict`. -- Revision is a SHA-256 hash of current config file content. +## Endpoint Matrix -## Endpoints +| Method | Path | Body | Success | `data` contract | +| --- | --- | --- | --- | --- | +| `GET` | `/v1/health` | none | `200` | `HealthData` | +| `GET` | `/v1/stats/summary` | none | `200` | `SummaryData` | +| `GET` | `/v1/stats/zero/all` | none | `200` | `ZeroAllData` | +| `GET` | `/v1/stats/minimal/all` | none | `200` | `MinimalAllData` | +| `GET` | `/v1/stats/me-writers` | none | `200` | `MeWritersData` | +| `GET` | `/v1/stats/dcs` | none | `200` | `DcStatusData` | +| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` | +| `GET` | `/v1/users` | none | `200` | `UserInfo[]` | +| `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` | +| `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` | +| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` | `UserInfo` | +| `DELETE` | `/v1/users/{username}` | none | `200` | `string` (deleted username) | +| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` | `CreateUserResponse` | -### Read endpoints -- `GET /v1/health` -- `GET /v1/stats/summary` -- `GET /v1/stats/me-writers` -- `GET /v1/stats/dcs` -- `GET /v1/stats/users` -- `GET /v1/users` -- `GET /v1/users/{username}` +## Common Error Codes -### Mutating endpoints -- `POST /v1/users` -- `PATCH /v1/users/{username}` -- `POST /v1/users/{username}/rotate-secret` -- `DELETE /v1/users/{username}` +| HTTP | `error.code` | Trigger | +| --- | --- | --- | +| `400` | `bad_request` | Invalid JSON, validation failures, malformed request body. | +| `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. | +| `403` | `forbidden` | Source IP is not allowed by whitelist. | +| `403` | `read_only` | Mutating endpoint called while `read_only=true`. | +| `404` | `not_found` | Unknown route or unknown user. | +| `405` | `method_not_allowed` | Unsupported method for an existing user route. | +| `409` | `revision_conflict` | `If-Match` revision mismatch. | +| `409` | `user_exists` | User already exists on create. | +| `409` | `last_user_forbidden` | Attempt to delete last configured user. | +| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. | +| `500` | `internal_error` | Internal error (I/O, serialization, config load/save). | +| `503` | `api_disabled` | API disabled in config. | -## Entity Contract: User -Managed user fields: -- `username` -- `secret` (32 hex chars) -- `user_ad_tag` (32 hex chars, optional) -- `max_tcp_conns` (optional) -- `expiration_rfc3339` (optional) -- `data_quota_bytes` (optional) -- `max_unique_ips` (optional) +## Request Contracts -Derived runtime fields (read-only in API responses): -- `current_connections` -- `active_unique_ips` -- `total_octets` +### `CreateUserRequest` +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `username` | `string` | yes | `[A-Za-z0-9_.-]`, length `1..64`. | +| `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. | +| `user_ad_tag` | `string` | no | Exactly 32 hex chars. | +| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. | +| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. | +| `data_quota_bytes` | `u64` | no | Per-user traffic quota. | +| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. | -## Transport Status Endpoints -### `GET /v1/stats/me-writers` -Returns current Middle-End writer status and aggregated coverage/availability summary. +### `PatchUserRequest` +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `secret` | `string` | no | Exactly 32 hex chars. | +| `user_ad_tag` | `string` | no | Exactly 32 hex chars. | +| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. | +| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. | +| `data_quota_bytes` | `u64` | no | Per-user traffic quota. | +| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. | -Top-level fields: -- `middle_proxy_enabled` -- `generated_at_epoch_secs` -- `summary` -- `writers` +### `RotateSecretRequest` +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. | -Summary fields: -- `configured_dc_groups` -- `configured_endpoints` -- `available_endpoints` -- `available_pct` -- `required_writers` -- `alive_writers` -- `coverage_pct` +## Response Data Contracts -Writer fields: -- `writer_id` -- `dc` -- `endpoint` (`ip:port`) -- `generation` -- `state` (`warm|active|draining`) -- `draining` -- `degraded` -- `bound_clients` -- `idle_for_secs` -- `rtt_ema_ms` +### `HealthData` +| Field | Type | Description | +| --- | --- | --- | +| `status` | `string` | Always `"ok"`. | +| `read_only` | `bool` | Mirrors current API `read_only` mode. | -### `GET /v1/stats/dcs` -Returns per-DC status aggregated from current ME pool. +### `SummaryData` +| Field | Type | Description | +| --- | --- | --- | +| `uptime_seconds` | `f64` | Process uptime in seconds. | +| `connections_total` | `u64` | Total accepted client connections. | +| `connections_bad_total` | `u64` | Failed/invalid client connections. | +| `handshake_timeouts_total` | `u64` | Handshake timeout count. | +| `configured_users` | `usize` | Number of configured users in config. | -Top-level fields: -- `middle_proxy_enabled` -- `generated_at_epoch_secs` -- `dcs` +### `ZeroAllData` +| Field | Type | Description | +| --- | --- | --- | +| `generated_at_epoch_secs` | `u64` | Snapshot time (Unix epoch seconds). | +| `core` | `ZeroCoreData` | Core counters and telemetry policy snapshot. | +| `upstream` | `ZeroUpstreamData` | Upstream connect counters/histogram buckets. | +| `middle_proxy` | `ZeroMiddleProxyData` | ME protocol/health counters. | +| `pool` | `ZeroPoolData` | ME pool lifecycle counters. | +| `desync` | `ZeroDesyncData` | Frame desync counters. | -DC row fields: -- `dc` -- `endpoints` (`ip:port[]`) -- `available_endpoints` -- `available_pct` -- `required_writers` -- `alive_writers` -- `coverage_pct` -- `rtt_ms` -- `load` +#### `ZeroCoreData` +| Field | Type | Description | +| --- | --- | --- | +| `uptime_seconds` | `f64` | Process uptime. | +| `connections_total` | `u64` | Total accepted connections. | +| `connections_bad_total` | `u64` | Failed/invalid connections. | +| `handshake_timeouts_total` | `u64` | Handshake timeouts. | +| `configured_users` | `usize` | Configured user count. | +| `telemetry_core_enabled` | `bool` | Core telemetry toggle. | +| `telemetry_user_enabled` | `bool` | User telemetry toggle. | +| `telemetry_me_level` | `string` | ME telemetry level (`off|normal|verbose`). | -Metrics formulas: -- `available_pct = available_endpoints / configured_endpoints * 100` -- `coverage_pct = alive_writers / required_writers * 100` -- `required_writers` uses the runtime writer floor policy for each DC group. -- `load` is the number of active client sessions currently bound to that DC. +#### `ZeroUpstreamData` +| Field | Type | Description | +| --- | --- | --- | +| `connect_attempt_total` | `u64` | Total upstream connect attempts. | +| `connect_success_total` | `u64` | Successful upstream connects. | +| `connect_fail_total` | `u64` | Failed upstream connects. | +| `connect_failfast_hard_error_total` | `u64` | Fail-fast hard errors. | +| `connect_attempts_bucket_1` | `u64` | Connect attempts resolved in 1 try. | +| `connect_attempts_bucket_2` | `u64` | Connect attempts resolved in 2 tries. | +| `connect_attempts_bucket_3_4` | `u64` | Connect attempts resolved in 3-4 tries. | +| `connect_attempts_bucket_gt_4` | `u64` | Connect attempts requiring more than 4 tries. | +| `connect_duration_success_bucket_le_100ms` | `u64` | Successful connects <=100 ms. | +| `connect_duration_success_bucket_101_500ms` | `u64` | Successful connects 101-500 ms. | +| `connect_duration_success_bucket_501_1000ms` | `u64` | Successful connects 501-1000 ms. | +| `connect_duration_success_bucket_gt_1000ms` | `u64` | Successful connects >1000 ms. | +| `connect_duration_fail_bucket_le_100ms` | `u64` | Failed connects <=100 ms. | +| `connect_duration_fail_bucket_101_500ms` | `u64` | Failed connects 101-500 ms. | +| `connect_duration_fail_bucket_501_1000ms` | `u64` | Failed connects 501-1000 ms. | +| `connect_duration_fail_bucket_gt_1000ms` | `u64` | Failed connects >1000 ms. | -## Validation Rules -- `username` must match `[A-Za-z0-9_.-]`, length `1..64`. -- `secret` must be exactly 32 hexadecimal characters. -- `user_ad_tag` must be exactly 32 hexadecimal characters. -- Request body size must not exceed `request_body_limit_bytes`. +#### `ZeroMiddleProxyData` +| Field | Type | Description | +| --- | --- | --- | +| `keepalive_sent_total` | `u64` | ME keepalive packets sent. | +| `keepalive_failed_total` | `u64` | ME keepalive send failures. | +| `keepalive_pong_total` | `u64` | Keepalive pong responses received. | +| `keepalive_timeout_total` | `u64` | Keepalive timeout events. | +| `rpc_proxy_req_signal_sent_total` | `u64` | RPC proxy activity signals sent. | +| `rpc_proxy_req_signal_failed_total` | `u64` | RPC proxy activity signal failures. | +| `rpc_proxy_req_signal_skipped_no_meta_total` | `u64` | Signals skipped due to missing metadata. | +| `rpc_proxy_req_signal_response_total` | `u64` | RPC proxy signal responses received. | +| `rpc_proxy_req_signal_close_sent_total` | `u64` | RPC proxy close signals sent. | +| `reconnect_attempt_total` | `u64` | ME reconnect attempts. | +| `reconnect_success_total` | `u64` | Successful reconnects. | +| `handshake_reject_total` | `u64` | ME handshake rejects. | +| `handshake_error_codes` | `ZeroCodeCount[]` | Handshake rejects grouped by code. | +| `reader_eof_total` | `u64` | ME reader EOF events. | +| `idle_close_by_peer_total` | `u64` | Idle closes initiated by peer. | +| `route_drop_no_conn_total` | `u64` | Route drops due to missing bound connection. | +| `route_drop_channel_closed_total` | `u64` | Route drops due to closed channel. | +| `route_drop_queue_full_total` | `u64` | Route drops due to full queue (total). | +| `route_drop_queue_full_base_total` | `u64` | Route drops in base queue mode. | +| `route_drop_queue_full_high_total` | `u64` | Route drops in high queue mode. | +| `socks_kdf_strict_reject_total` | `u64` | SOCKS KDF strict rejects. | +| `socks_kdf_compat_fallback_total` | `u64` | SOCKS KDF compat fallbacks. | +| `endpoint_quarantine_total` | `u64` | Endpoint quarantine activations. | +| `kdf_drift_total` | `u64` | KDF drift detections. | +| `kdf_port_only_drift_total` | `u64` | KDF port-only drift detections. | +| `hardswap_pending_reuse_total` | `u64` | Pending hardswap reused events. | +| `hardswap_pending_ttl_expired_total` | `u64` | Pending hardswap TTL expiry events. | +| `single_endpoint_outage_enter_total` | `u64` | Entered single-endpoint outage mode. | +| `single_endpoint_outage_exit_total` | `u64` | Exited single-endpoint outage mode. | +| `single_endpoint_outage_reconnect_attempt_total` | `u64` | Reconnect attempts in outage mode. | +| `single_endpoint_outage_reconnect_success_total` | `u64` | Reconnect successes in outage mode. | +| `single_endpoint_quarantine_bypass_total` | `u64` | Quarantine bypasses in outage mode. | +| `single_endpoint_shadow_rotate_total` | `u64` | Shadow writer rotations. | +| `single_endpoint_shadow_rotate_skipped_quarantine_total` | `u64` | Shadow rotations skipped because of quarantine. | +| `floor_mode_switch_total` | `u64` | Total floor mode switches. | +| `floor_mode_switch_static_to_adaptive_total` | `u64` | Static -> adaptive switches. | +| `floor_mode_switch_adaptive_to_static_total` | `u64` | Adaptive -> static switches. | -## Security Model -1. Network perimeter. -Access is limited by CIDR whitelist. +#### `ZeroCodeCount` +| Field | Type | Description | +| --- | --- | --- | +| `code` | `i32` | Handshake error code. | +| `total` | `u64` | Events with this code. | -2. Optional application header auth. -If `auth_header` is configured, `Authorization` must match exactly. +#### `ZeroPoolData` +| Field | Type | Description | +| --- | --- | --- | +| `pool_swap_total` | `u64` | Pool swap count. | +| `pool_drain_active` | `u64` | Current active draining pools. | +| `pool_force_close_total` | `u64` | Forced pool closes by timeout. | +| `pool_stale_pick_total` | `u64` | Stale writer picks for binding. | +| `writer_removed_total` | `u64` | Writer removals total. | +| `writer_removed_unexpected_total` | `u64` | Unexpected writer removals. | +| `refill_triggered_total` | `u64` | Refill triggers. | +| `refill_skipped_inflight_total` | `u64` | Refill skipped because refill already in-flight. | +| `refill_failed_total` | `u64` | Refill failures. | +| `writer_restored_same_endpoint_total` | `u64` | Restores on same endpoint. | +| `writer_restored_fallback_total` | `u64` | Restores on fallback endpoint. | -3. Read-only mode. -If `read_only = true`, mutating endpoints are rejected with `403`. +#### `ZeroDesyncData` +| Field | Type | Description | +| --- | --- | --- | +| `secure_padding_invalid_total` | `u64` | Invalid secure padding events. | +| `desync_total` | `u64` | Desync events total. | +| `desync_full_logged_total` | `u64` | Fully logged desync events. | +| `desync_suppressed_total` | `u64` | Suppressed desync logs. | +| `desync_frames_bucket_0` | `u64` | Desync frames bucket 0. | +| `desync_frames_bucket_1_2` | `u64` | Desync frames bucket 1-2. | +| `desync_frames_bucket_3_10` | `u64` | Desync frames bucket 3-10. | +| `desync_frames_bucket_gt_10` | `u64` | Desync frames bucket >10. | -## Mutation Approach -1. Acquire mutation lock. -2. Load config from disk. -3. Validate optional `If-Match` revision. -4. Apply in-memory mutation. -5. Run config validation. -6. Persist via atomic write (`tmp + fsync + rename`). -7. Return updated revision. +### `MinimalAllData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Whether minimal runtime snapshots are enabled by config. | +| `reason` | `string?` | `feature_disabled` or `source_unavailable` when applicable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation time. | +| `data` | `MinimalAllPayload?` | Null when disabled; fallback payload when source unavailable. | -Runtime apply path: -- Existing config watcher picks up persisted changes and applies them through the standard hot-reload path. +#### `MinimalAllPayload` +| Field | Type | Description | +| --- | --- | --- | +| `me_writers` | `MeWritersData` | ME writer status block. | +| `dcs` | `DcStatusData` | DC aggregate status block. | +| `me_runtime` | `MinimalMeRuntimeData?` | Runtime ME control snapshot. | +| `network_path` | `MinimalDcPathData[]` | Active IP path selection per DC. | -## Known Limitations -1. Built-in TLS/mTLS is not provided by this API server. -Use loopback bind plus reverse proxy for external exposure. +#### `MinimalMeRuntimeData` +| Field | Type | Description | +| --- | --- | --- | +| `active_generation` | `u64` | Active pool generation. | +| `warm_generation` | `u64` | Warm pool generation. | +| `pending_hardswap_generation` | `u64` | Pending hardswap generation. | +| `pending_hardswap_age_secs` | `u64?` | Pending hardswap age in seconds. | +| `hardswap_enabled` | `bool` | Hardswap mode toggle. | +| `floor_mode` | `string` | Writer floor mode. | +| `adaptive_floor_idle_secs` | `u64` | Idle threshold for adaptive floor. | +| `adaptive_floor_min_writers_single_endpoint` | `u8` | Minimum writers for single-endpoint DC in adaptive mode. | +| `adaptive_floor_recover_grace_secs` | `u64` | Grace period for floor recovery. | +| `me_keepalive_enabled` | `bool` | ME keepalive toggle. | +| `me_keepalive_interval_secs` | `u64` | Keepalive period. | +| `me_keepalive_jitter_secs` | `u64` | Keepalive jitter. | +| `me_keepalive_payload_random` | `bool` | Randomized keepalive payload toggle. | +| `rpc_proxy_req_every_secs` | `u64` | Period for RPC proxy request signal. | +| `me_reconnect_max_concurrent_per_dc` | `u32` | Reconnect concurrency per DC. | +| `me_reconnect_backoff_base_ms` | `u64` | Base reconnect backoff. | +| `me_reconnect_backoff_cap_ms` | `u64` | Max reconnect backoff. | +| `me_reconnect_fast_retry_count` | `u32` | Fast retry attempts before normal backoff. | +| `me_pool_drain_ttl_secs` | `u64` | Pool drain TTL. | +| `me_pool_force_close_secs` | `u64` | Hard close timeout for draining writers. | +| `me_pool_min_fresh_ratio` | `f32` | Minimum fresh ratio before swap. | +| `me_bind_stale_mode` | `string` | Stale writer bind policy. | +| `me_bind_stale_ttl_secs` | `u64` | Stale writer TTL. | +| `me_single_endpoint_shadow_writers` | `u8` | Shadow writers for single-endpoint DCs. | +| `me_single_endpoint_outage_mode_enabled` | `bool` | Outage mode toggle for single-endpoint DCs. | +| `me_single_endpoint_outage_disable_quarantine` | `bool` | Quarantine behavior in outage mode. | +| `me_single_endpoint_outage_backoff_min_ms` | `u64` | Outage mode min reconnect backoff. | +| `me_single_endpoint_outage_backoff_max_ms` | `u64` | Outage mode max reconnect backoff. | +| `me_single_endpoint_shadow_rotate_every_secs` | `u64` | Shadow rotation interval. | +| `me_deterministic_writer_sort` | `bool` | Deterministic writer ordering toggle. | +| `me_socks_kdf_policy` | `string` | Current SOCKS KDF policy mode. | +| `quarantined_endpoints_total` | `usize` | Total quarantined endpoints. | +| `quarantined_endpoints` | `MinimalQuarantineData[]` | Quarantine details. | -2. No pagination/filtering for user list in current version. +#### `MinimalQuarantineData` +| Field | Type | Description | +| --- | --- | --- | +| `endpoint` | `string` | Endpoint (`ip:port`). | +| `remaining_ms` | `u64` | Remaining quarantine duration. | -3. `PATCH` updates present fields only. -Field deletion semantics are not implemented as explicit nullable operations. +#### `MinimalDcPathData` +| Field | Type | Description | +| --- | --- | --- | +| `dc` | `i16` | Telegram DC identifier. | +| `ip_preference` | `string?` | Runtime IP family preference. | +| `selected_addr_v4` | `string?` | Selected IPv4 endpoint for this DC. | +| `selected_addr_v6` | `string?` | Selected IPv6 endpoint for this DC. | -4. Config comments and manual formatting are not preserved after mutation. -Config is serialized from structured state. +### `MeWritersData` +| Field | Type | Description | +| --- | --- | --- | +| `middle_proxy_enabled` | `bool` | `false` when minimal runtime is disabled or source unavailable. | +| `reason` | `string?` | `feature_disabled` or `source_unavailable` when not fully available. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation time. | +| `summary` | `MeWritersSummary` | Coverage/availability summary. | +| `writers` | `MeWriterStatus[]` | Per-writer statuses. | -5. API configuration itself (`server.api`) is not hot-applied. -Restart is required. +#### `MeWritersSummary` +| Field | Type | Description | +| --- | --- | --- | +| `configured_dc_groups` | `usize` | Number of configured DC groups. | +| `configured_endpoints` | `usize` | Total configured ME endpoints. | +| `available_endpoints` | `usize` | Endpoints currently available. | +| `available_pct` | `f64` | `available_endpoints / configured_endpoints * 100`. | +| `required_writers` | `usize` | Required writers based on current floor policy. | +| `alive_writers` | `usize` | Writers currently alive. | +| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. | -6. Atomic file replacement can conflict with external editors/tools writing the same config concurrently. -Use revision checks to reduce race impact. +#### `MeWriterStatus` +| Field | Type | Description | +| --- | --- | --- | +| `writer_id` | `u64` | Runtime writer identifier. | +| `dc` | `i16?` | DC id if mapped. | +| `endpoint` | `string` | Endpoint (`ip:port`). | +| `generation` | `u64` | Pool generation owning this writer. | +| `state` | `string` | Writer state (`warm`, `active`, `draining`). | +| `draining` | `bool` | Draining flag. | +| `degraded` | `bool` | Degraded flag. | +| `bound_clients` | `usize` | Number of currently bound clients. | +| `idle_for_secs` | `u64?` | Idle age in seconds if idle. | +| `rtt_ema_ms` | `f64?` | RTT exponential moving average. | + +### `DcStatusData` +| Field | Type | Description | +| --- | --- | --- | +| `middle_proxy_enabled` | `bool` | `false` when minimal runtime is disabled or source unavailable. | +| `reason` | `string?` | `feature_disabled` or `source_unavailable` when not fully available. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation time. | +| `dcs` | `DcStatus[]` | Per-DC status rows. | + +#### `DcStatus` +| Field | Type | Description | +| --- | --- | --- | +| `dc` | `i16` | Telegram DC id. | +| `endpoints` | `string[]` | Endpoints in this DC (`ip:port`). | +| `available_endpoints` | `usize` | Endpoints currently available in this DC. | +| `available_pct` | `f64` | `available_endpoints / endpoints_total * 100`. | +| `required_writers` | `usize` | Required writer count for this DC. | +| `alive_writers` | `usize` | Alive writers in this DC. | +| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. | +| `rtt_ms` | `f64?` | Aggregated RTT for DC. | +| `load` | `usize` | Active client sessions bound to this DC. | + +### `UserInfo` +| Field | Type | Description | +| --- | --- | --- | +| `username` | `string` | Username. | +| `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). | +| `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. | +| `expiration_rfc3339` | `string?` | Optional expiration timestamp. | +| `data_quota_bytes` | `u64?` | Optional data quota. | +| `max_unique_ips` | `usize?` | Optional unique IP limit. | +| `current_connections` | `u64` | Current live connections. | +| `active_unique_ips` | `usize` | Current active unique source IPs. | +| `total_octets` | `u64` | Total traffic octets for this user. | +| `links` | `UserLinks` | Active connection links derived from current config. | + +#### `UserLinks` +| Field | Type | Description | +| --- | --- | --- | +| `classic` | `string[]` | Active `tg://proxy` links for classic mode. | +| `secure` | `string[]` | Active `tg://proxy` links for secure/DD mode. | +| `tls` | `string[]` | Active `tg://proxy` links for EE-TLS mode (for each host+TLS domain). | + +Link generation uses active config and enabled modes: +- `[general.links].public_host/public_port` have priority. +- Fallback host sources: listener `announce`, `announce_ip`, explicit listener `ip`. +- Legacy fallback: `listen_addr_ipv4` and `listen_addr_ipv6` when routable. + +### `CreateUserResponse` +| Field | Type | Description | +| --- | --- | --- | +| `user` | `UserInfo` | Created or updated user view. | +| `secret` | `string` | Effective user secret. | + +## Mutation Semantics + +| Endpoint | Notes | +| --- | --- | +| `POST /v1/users` | Creates user and validates resulting config before atomic save. | +| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. | +| `POST /v1/users/{username}/rotate-secret` | Replaces secret. Empty body is allowed and auto-generates secret. | +| `DELETE /v1/users/{username}` | Deletes user and related optional settings. Last user deletion is blocked. | + +All mutating endpoints: +- Respect `read_only` mode. +- Accept optional `If-Match` for optimistic concurrency. +- Return new `revision` after successful write. + +## Operational Notes + +| Topic | Details | +| --- | --- | +| API startup | API binds only when `[server.api].enabled=true`. | +| Restart requirements | Changes in `server.api` settings require process restart. | +| Runtime apply path | Successful writes are picked up by existing config watcher/hot-reload path. | +| Exposure | Built-in TLS/mTLS is not provided. Use loopback bind + reverse proxy if needed. | +| Pagination | User list currently has no pagination/filtering. | +| Serialization side effect | Config comments/manual formatting are not preserved on write. | From 314f30a434bee00dce00836e99c9ac8d813deef3 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 02:53:47 +0300 Subject: [PATCH 053/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 16586e0..324af49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.1.6" +version = "3.2.0" edition = "2024" [dependencies] From 5df2fe9f97f5ccc3c48616973e5442f6c44e8c4b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:04:54 +0300 Subject: [PATCH 054/261] Autodetect IP in API User-links Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/mod.rs | 26 ++++++++++++++++-- src/api/users.rs | 71 ++++++++++++++++++++++++++++++++++++++++++------ src/main.rs | 4 +++ 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 299d5a1..55d790f 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,5 +1,5 @@ use std::convert::Infallible; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -43,6 +43,8 @@ pub(super) struct ApiShared { pub(super) ip_tracker: Arc, pub(super) me_pool: Option>, pub(super) config_path: PathBuf, + pub(super) startup_detected_ip_v4: Option, + pub(super) startup_detected_ip_v6: Option, pub(super) mutation_lock: Arc>, pub(super) minimal_cache: Arc>>, pub(super) request_id: Arc, @@ -61,6 +63,8 @@ pub async fn serve( me_pool: Option>, config_rx: watch::Receiver>, config_path: PathBuf, + startup_detected_ip_v4: Option, + startup_detected_ip_v6: Option, ) { let listener = match TcpListener::bind(listen).await { Ok(listener) => listener, @@ -81,6 +85,8 @@ pub async fn serve( ip_tracker, me_pool, config_path, + startup_detected_ip_v4, + startup_detected_ip_v6, mutation_lock: Arc::new(Mutex::new(())), minimal_cache: Arc::new(Mutex::new(None)), request_id: Arc::new(AtomicU64::new(1)), @@ -212,7 +218,14 @@ async fn handle( } ("GET", "/v1/stats/users") | ("GET", "/v1/users") => { let revision = current_revision(&shared.config_path).await?; - let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let users = users_from_config( + &cfg, + &shared.stats, + &shared.ip_tracker, + shared.startup_detected_ip_v4, + shared.startup_detected_ip_v6, + ) + .await; Ok(success_response(StatusCode::OK, users, revision)) } ("POST", "/v1/users") => { @@ -238,7 +251,14 @@ async fn handle( { if method == Method::GET { let revision = current_revision(&shared.config_path).await?; - let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let users = users_from_config( + &cfg, + &shared.stats, + &shared.ip_tracker, + shared.startup_detected_ip_v4, + shared.startup_detected_ip_v6, + ) + .await; if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) { return Ok(success_response(StatusCode::OK, user_info, revision)); diff --git a/src/api/users.rs b/src/api/users.rs index 9fc03e9..c907070 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -92,7 +92,14 @@ pub(super) async fn create_user( shared.ip_tracker.set_user_limit(&body.username, limit).await; } - let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let users = users_from_config( + &cfg, + &shared.stats, + &shared.ip_tracker, + shared.startup_detected_ip_v4, + shared.startup_detected_ip_v6, + ) + .await; let user = users .into_iter() .find(|entry| entry.username == body.username) @@ -106,7 +113,12 @@ pub(super) async fn create_user( current_connections: 0, active_unique_ips: 0, total_octets: 0, - links: build_user_links(&cfg, &secret), + links: build_user_links( + &cfg, + &secret, + shared.startup_detected_ip_v4, + shared.startup_detected_ip_v6, + ), }); Ok((CreateUserResponse { user, secret }, revision)) @@ -171,7 +183,14 @@ pub(super) async fn patch_user( if let Some(limit) = updated_limit { shared.ip_tracker.set_user_limit(user, limit).await; } - let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let users = users_from_config( + &cfg, + &shared.stats, + &shared.ip_tracker, + shared.startup_detected_ip_v4, + shared.startup_detected_ip_v6, + ) + .await; let user_info = users .into_iter() .find(|entry| entry.username == user) @@ -211,7 +230,14 @@ pub(super) async fn rotate_secret( let revision = save_config_to_disk(&shared.config_path, &cfg).await?; drop(_guard); - let users = users_from_config(&cfg, &shared.stats, &shared.ip_tracker).await; + let users = users_from_config( + &cfg, + &shared.stats, + &shared.ip_tracker, + shared.startup_detected_ip_v4, + shared.startup_detected_ip_v6, + ) + .await; let user_info = users .into_iter() .find(|entry| entry.username == user) @@ -270,6 +296,8 @@ pub(super) async fn users_from_config( cfg: &ProxyConfig, stats: &Stats, ip_tracker: &UserIpTracker, + startup_detected_ip_v4: Option, + startup_detected_ip_v6: Option, ) -> Vec { let ip_counts = ip_tracker .get_stats() @@ -287,7 +315,14 @@ pub(super) async fn users_from_config( .access .users .get(&username) - .map(|secret| build_user_links(cfg, secret)) + .map(|secret| { + build_user_links( + cfg, + secret, + startup_detected_ip_v4, + startup_detected_ip_v6, + ) + }) .unwrap_or(UserLinks { classic: Vec::new(), secure: Vec::new(), @@ -313,8 +348,13 @@ pub(super) async fn users_from_config( users } -fn build_user_links(cfg: &ProxyConfig, secret: &str) -> UserLinks { - let hosts = resolve_link_hosts(cfg); +fn build_user_links( + cfg: &ProxyConfig, + secret: &str, + startup_detected_ip_v4: Option, + startup_detected_ip_v6: Option, +) -> UserLinks { + let hosts = resolve_link_hosts(cfg, startup_detected_ip_v4, startup_detected_ip_v6); let port = cfg.general.links.public_port.unwrap_or(cfg.server.port); let tls_domains = resolve_tls_domains(cfg); @@ -353,7 +393,11 @@ fn build_user_links(cfg: &ProxyConfig, secret: &str) -> UserLinks { } } -fn resolve_link_hosts(cfg: &ProxyConfig) -> Vec { +fn resolve_link_hosts( + cfg: &ProxyConfig, + startup_detected_ip_v4: Option, + startup_detected_ip_v6: Option, +) -> Vec { if let Some(host) = cfg .general .links @@ -365,6 +409,17 @@ fn resolve_link_hosts(cfg: &ProxyConfig) -> Vec { return vec![host.to_string()]; } + let mut startup_hosts = Vec::new(); + if let Some(ip) = startup_detected_ip_v4 { + push_unique_host(&mut startup_hosts, &ip.to_string()); + } + if let Some(ip) = startup_detected_ip_v6 { + push_unique_host(&mut startup_hosts, &ip.to_string()); + } + if !startup_hosts.is_empty() { + return startup_hosts; + } + let mut hosts = Vec::new(); for listener in &cfg.server.listeners { if let Some(host) = listener diff --git a/src/main.rs b/src/main.rs index c4f0e68..1845fdb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1171,6 +1171,8 @@ async fn main() -> std::result::Result<(), Box> { let me_pool_api = me_pool.clone(); let config_rx_api = config_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); + let startup_detected_ip_v4 = detected_ip_v4; + let startup_detected_ip_v6 = detected_ip_v6; tokio::spawn(async move { api::serve( listen, @@ -1179,6 +1181,8 @@ async fn main() -> std::result::Result<(), Box> { me_pool_api, config_rx_api, config_path_api, + startup_detected_ip_v4, + startup_detected_ip_v6, ) .await; }); From de2047adf269e52b82c3519d64319cf0af88a1b0 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:41:41 +0300 Subject: [PATCH 055/261] API UpstreamManager Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/mod.rs | 11 ++- src/api/model.rs | 44 ++++++++++++ src/api/runtime_stats.rs | 146 +++++++++++++++++++++++++++++++------- src/main.rs | 2 + src/transport/upstream.rs | 95 +++++++++++++++++++++++++ 5 files changed, 270 insertions(+), 28 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 55d790f..c01566a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -20,6 +20,7 @@ use crate::config::ProxyConfig; use crate::ip_tracker::UserIpTracker; use crate::stats::Stats; use crate::transport::middle_proxy::MePool; +use crate::transport::UpstreamManager; mod config_store; mod model; @@ -33,7 +34,7 @@ use model::{ }; use runtime_stats::{ MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data, - build_zero_all_data, + build_upstreams_data, build_zero_all_data, }; use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config}; @@ -42,6 +43,7 @@ pub(super) struct ApiShared { pub(super) stats: Arc, pub(super) ip_tracker: Arc, pub(super) me_pool: Option>, + pub(super) upstream_manager: Arc, pub(super) config_path: PathBuf, pub(super) startup_detected_ip_v4: Option, pub(super) startup_detected_ip_v6: Option, @@ -61,6 +63,7 @@ pub async fn serve( stats: Arc, ip_tracker: Arc, me_pool: Option>, + upstream_manager: Arc, config_rx: watch::Receiver>, config_path: PathBuf, startup_detected_ip_v4: Option, @@ -84,6 +87,7 @@ pub async fn serve( stats, ip_tracker, me_pool, + upstream_manager, config_path, startup_detected_ip_v4, startup_detected_ip_v6, @@ -201,6 +205,11 @@ async fn handle( let data = build_zero_all_data(&shared.stats, cfg.access.users.len()); Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/stats/upstreams") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_upstreams_data(shared.as_ref(), api_cfg); + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/minimal/all") => { let revision = current_revision(&shared.config_path).await?; let data = build_minimal_all_data(shared.as_ref(), api_cfg).await; diff --git a/src/api/model.rs b/src/api/model.rs index be76c4e..efe8ebb 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -103,6 +103,50 @@ pub(super) struct ZeroUpstreamData { pub(super) connect_duration_fail_bucket_gt_1000ms: u64, } +#[derive(Serialize, Clone)] +pub(super) struct UpstreamDcStatus { + pub(super) dc: i16, + pub(super) latency_ema_ms: Option, + pub(super) ip_preference: &'static str, +} + +#[derive(Serialize, Clone)] +pub(super) struct UpstreamStatus { + pub(super) upstream_id: usize, + pub(super) route_kind: &'static str, + pub(super) address: String, + pub(super) weight: u16, + pub(super) scopes: String, + pub(super) healthy: bool, + pub(super) fails: u32, + pub(super) last_check_age_secs: u64, + pub(super) effective_latency_ms: Option, + pub(super) dc: Vec, +} + +#[derive(Serialize, Clone)] +pub(super) struct UpstreamSummaryData { + pub(super) configured_total: usize, + pub(super) healthy_total: usize, + pub(super) unhealthy_total: usize, + pub(super) direct_total: usize, + pub(super) socks4_total: usize, + pub(super) socks5_total: usize, +} + +#[derive(Serialize, Clone)] +pub(super) struct UpstreamsData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + pub(super) zero: ZeroUpstreamData, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) upstreams: Option>, +} + #[derive(Serialize, Clone)] pub(super) struct ZeroMiddleProxyData { pub(super) keepalive_sent_total: u64, diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 53fdeff..3019636 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -2,12 +2,15 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::config::ApiConfig; use crate::stats::Stats; +use crate::transport::upstream::IpPreference; +use crate::transport::UpstreamRouteKind; use super::ApiShared; use super::model::{ DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary, MinimalAllData, MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData, MinimalQuarantineData, - ZeroAllData, ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData, + UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData, ZeroAllData, + ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData, ZeroUpstreamData, }; @@ -41,32 +44,7 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer telemetry_user_enabled: telemetry.user_enabled, telemetry_me_level: telemetry.me_level.to_string(), }, - upstream: ZeroUpstreamData { - connect_attempt_total: stats.get_upstream_connect_attempt_total(), - connect_success_total: stats.get_upstream_connect_success_total(), - connect_fail_total: stats.get_upstream_connect_fail_total(), - connect_failfast_hard_error_total: stats.get_upstream_connect_failfast_hard_error_total(), - connect_attempts_bucket_1: stats.get_upstream_connect_attempts_bucket_1(), - connect_attempts_bucket_2: stats.get_upstream_connect_attempts_bucket_2(), - connect_attempts_bucket_3_4: stats.get_upstream_connect_attempts_bucket_3_4(), - connect_attempts_bucket_gt_4: stats.get_upstream_connect_attempts_bucket_gt_4(), - connect_duration_success_bucket_le_100ms: stats - .get_upstream_connect_duration_success_bucket_le_100ms(), - connect_duration_success_bucket_101_500ms: stats - .get_upstream_connect_duration_success_bucket_101_500ms(), - connect_duration_success_bucket_501_1000ms: stats - .get_upstream_connect_duration_success_bucket_501_1000ms(), - connect_duration_success_bucket_gt_1000ms: stats - .get_upstream_connect_duration_success_bucket_gt_1000ms(), - connect_duration_fail_bucket_le_100ms: stats - .get_upstream_connect_duration_fail_bucket_le_100ms(), - connect_duration_fail_bucket_101_500ms: stats - .get_upstream_connect_duration_fail_bucket_101_500ms(), - connect_duration_fail_bucket_501_1000ms: stats - .get_upstream_connect_duration_fail_bucket_501_1000ms(), - connect_duration_fail_bucket_gt_1000ms: stats - .get_upstream_connect_duration_fail_bucket_gt_1000ms(), - }, + upstream: build_zero_upstream_data(stats), middle_proxy: ZeroMiddleProxyData { keepalive_sent_total: stats.get_me_keepalive_sent(), keepalive_failed_total: stats.get_me_keepalive_failed(), @@ -140,6 +118,102 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer } } +fn build_zero_upstream_data(stats: &Stats) -> ZeroUpstreamData { + ZeroUpstreamData { + connect_attempt_total: stats.get_upstream_connect_attempt_total(), + connect_success_total: stats.get_upstream_connect_success_total(), + connect_fail_total: stats.get_upstream_connect_fail_total(), + connect_failfast_hard_error_total: stats.get_upstream_connect_failfast_hard_error_total(), + connect_attempts_bucket_1: stats.get_upstream_connect_attempts_bucket_1(), + connect_attempts_bucket_2: stats.get_upstream_connect_attempts_bucket_2(), + connect_attempts_bucket_3_4: stats.get_upstream_connect_attempts_bucket_3_4(), + connect_attempts_bucket_gt_4: stats.get_upstream_connect_attempts_bucket_gt_4(), + connect_duration_success_bucket_le_100ms: stats + .get_upstream_connect_duration_success_bucket_le_100ms(), + connect_duration_success_bucket_101_500ms: stats + .get_upstream_connect_duration_success_bucket_101_500ms(), + connect_duration_success_bucket_501_1000ms: stats + .get_upstream_connect_duration_success_bucket_501_1000ms(), + connect_duration_success_bucket_gt_1000ms: stats + .get_upstream_connect_duration_success_bucket_gt_1000ms(), + connect_duration_fail_bucket_le_100ms: stats.get_upstream_connect_duration_fail_bucket_le_100ms(), + connect_duration_fail_bucket_101_500ms: stats + .get_upstream_connect_duration_fail_bucket_101_500ms(), + connect_duration_fail_bucket_501_1000ms: stats + .get_upstream_connect_duration_fail_bucket_501_1000ms(), + connect_duration_fail_bucket_gt_1000ms: stats + .get_upstream_connect_duration_fail_bucket_gt_1000ms(), + } +} + +pub(super) fn build_upstreams_data(shared: &ApiShared, api_cfg: &ApiConfig) -> UpstreamsData { + let generated_at_epoch_secs = now_epoch_secs(); + let zero = build_zero_upstream_data(&shared.stats); + if !api_cfg.minimal_runtime_enabled { + return UpstreamsData { + enabled: false, + reason: Some(FEATURE_DISABLED_REASON), + generated_at_epoch_secs, + zero, + summary: None, + upstreams: None, + }; + } + + let Some(snapshot) = shared.upstream_manager.try_api_snapshot() else { + return UpstreamsData { + enabled: true, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs, + zero, + summary: None, + upstreams: None, + }; + }; + + let summary = UpstreamSummaryData { + configured_total: snapshot.summary.configured_total, + healthy_total: snapshot.summary.healthy_total, + unhealthy_total: snapshot.summary.unhealthy_total, + direct_total: snapshot.summary.direct_total, + socks4_total: snapshot.summary.socks4_total, + socks5_total: snapshot.summary.socks5_total, + }; + let upstreams = snapshot + .upstreams + .into_iter() + .map(|upstream| UpstreamStatus { + upstream_id: upstream.upstream_id, + route_kind: map_route_kind(upstream.route_kind), + address: upstream.address, + weight: upstream.weight, + scopes: upstream.scopes, + healthy: upstream.healthy, + fails: upstream.fails, + last_check_age_secs: upstream.last_check_age_secs, + effective_latency_ms: upstream.effective_latency_ms, + dc: upstream + .dc + .into_iter() + .map(|dc| UpstreamDcStatus { + dc: dc.dc, + latency_ema_ms: dc.latency_ema_ms, + ip_preference: map_ip_preference(dc.ip_preference), + }) + .collect(), + }) + .collect(); + + UpstreamsData { + enabled: true, + reason: None, + generated_at_epoch_secs, + zero, + summary: Some(summary), + upstreams: Some(upstreams), + } +} + pub(super) async fn build_minimal_all_data( shared: &ApiShared, api_cfg: &ApiConfig, @@ -384,6 +458,24 @@ fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData { } } +fn map_route_kind(value: UpstreamRouteKind) -> &'static str { + match value { + UpstreamRouteKind::Direct => "direct", + UpstreamRouteKind::Socks4 => "socks4", + UpstreamRouteKind::Socks5 => "socks5", + } +} + +fn map_ip_preference(value: IpPreference) -> &'static str { + match value { + IpPreference::Unknown => "unknown", + IpPreference::PreferV6 => "prefer_v6", + IpPreference::PreferV4 => "prefer_v4", + IpPreference::BothWork => "both_work", + IpPreference::Unavailable => "unavailable", + } +} + fn now_epoch_secs() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/src/main.rs b/src/main.rs index 1845fdb..0aec195 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1169,6 +1169,7 @@ async fn main() -> std::result::Result<(), Box> { let stats = stats.clone(); let ip_tracker_api = ip_tracker.clone(); let me_pool_api = me_pool.clone(); + let upstream_manager_api = upstream_manager.clone(); let config_rx_api = config_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); let startup_detected_ip_v4 = detected_ip_v4; @@ -1179,6 +1180,7 @@ async fn main() -> std::result::Result<(), Box> { stats, ip_tracker_api, me_pool_api, + upstream_manager_api, config_rx_api, config_path_api, startup_detected_ip_v4, diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index 1e2dd1e..d9f0ede 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -165,6 +165,43 @@ pub enum UpstreamRouteKind { Socks5, } +#[derive(Debug, Clone)] +pub struct UpstreamApiDcSnapshot { + pub dc: i16, + pub latency_ema_ms: Option, + pub ip_preference: IpPreference, +} + +#[derive(Debug, Clone)] +pub struct UpstreamApiItemSnapshot { + pub upstream_id: usize, + pub route_kind: UpstreamRouteKind, + pub address: String, + pub weight: u16, + pub scopes: String, + pub healthy: bool, + pub fails: u32, + pub last_check_age_secs: u64, + pub effective_latency_ms: Option, + pub dc: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct UpstreamApiSummarySnapshot { + pub configured_total: usize, + pub healthy_total: usize, + pub unhealthy_total: usize, + pub direct_total: usize, + pub socks4_total: usize, + pub socks5_total: usize, +} + +#[derive(Debug, Clone)] +pub struct UpstreamApiSnapshot { + pub summary: UpstreamApiSummarySnapshot, + pub upstreams: Vec, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct UpstreamEgressInfo { pub route_kind: UpstreamRouteKind, @@ -217,6 +254,64 @@ impl UpstreamManager { } } + pub fn try_api_snapshot(&self) -> Option { + let guard = self.upstreams.try_read().ok()?; + let now = std::time::Instant::now(); + + let mut summary = UpstreamApiSummarySnapshot { + configured_total: guard.len(), + ..UpstreamApiSummarySnapshot::default() + }; + let mut upstreams = Vec::with_capacity(guard.len()); + + for (idx, upstream) in guard.iter().enumerate() { + if upstream.healthy { + summary.healthy_total += 1; + } else { + summary.unhealthy_total += 1; + } + + let (route_kind, address) = match &upstream.config.upstream_type { + UpstreamType::Direct { .. } => { + summary.direct_total += 1; + (UpstreamRouteKind::Direct, "direct".to_string()) + } + UpstreamType::Socks4 { address, .. } => { + summary.socks4_total += 1; + (UpstreamRouteKind::Socks4, address.clone()) + } + UpstreamType::Socks5 { address, .. } => { + summary.socks5_total += 1; + (UpstreamRouteKind::Socks5, address.clone()) + } + }; + + let mut dc = Vec::with_capacity(NUM_DCS); + for dc_idx in 0..NUM_DCS { + dc.push(UpstreamApiDcSnapshot { + dc: (dc_idx + 1) as i16, + latency_ema_ms: upstream.dc_latency[dc_idx].get(), + ip_preference: upstream.dc_ip_pref[dc_idx], + }); + } + + upstreams.push(UpstreamApiItemSnapshot { + upstream_id: idx, + route_kind, + address, + weight: upstream.config.weight, + scopes: upstream.config.scopes.clone(), + healthy: upstream.healthy, + fails: upstream.fails, + last_check_age_secs: now.saturating_duration_since(upstream.last_check).as_secs(), + effective_latency_ms: upstream.effective_latency(None), + dc, + }); + } + + Some(UpstreamApiSnapshot { summary, upstreams }) + } + #[cfg(unix)] fn resolve_interface_addrs(name: &str, want_ipv6: bool) -> Vec { use nix::ifaddrs::getifaddrs; From 173624c838863030eb0e61446bd9d5b867fc6dd3 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:44:50 +0300 Subject: [PATCH 056/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 324af49..d38431d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.2.0" +version = "3.2.1" edition = "2024" [dependencies] From dbadbf0221ebed1bea3aa851bb528cb2a69147f5 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:45:32 +0300 Subject: [PATCH 057/261] Update config.toml --- config.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config.toml b/config.toml index cb33e3d..ab35789 100644 --- a/config.toml +++ b/config.toml @@ -34,6 +34,13 @@ port = 443 # metrics_port = 9090 # metrics_whitelist = ["127.0.0.1", "::1", "0.0.0.0/0"] +[server.api] +enabled = true +listen = "0.0.0.0:9091" +whitelist = ["127.0.0.0/8"] +minimal_runtime_enabled = false +minimal_runtime_cache_ttl_ms = 1000 + # Listen on multiple interfaces/IPs - IPv4 [[server.listeners]] ip = "0.0.0.0" From 0d320c20e0bcea179906764a5af4ed5367bd528e Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Wed, 4 Mar 2026 14:10:12 +0300 Subject: [PATCH 058/261] Update QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index d9974d3..0169953 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -60,6 +60,7 @@ nano /etc/telemt.toml # === General Settings === [general] # ad_tag = "00000000000000000000000000000000" +use_middle_proxy = false [general.modes] classic = false From 0d9470a8405717d5e02d58d9f14edf1a5457ccc7 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Wed, 4 Mar 2026 14:10:46 +0300 Subject: [PATCH 059/261] Update QUICK_START_GUIDE.en.md --- docs/QUICK_START_GUIDE.en.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/QUICK_START_GUIDE.en.md b/docs/QUICK_START_GUIDE.en.md index 6d1f920..6d94242 100644 --- a/docs/QUICK_START_GUIDE.en.md +++ b/docs/QUICK_START_GUIDE.en.md @@ -60,6 +60,7 @@ paste your config # === General Settings === [general] # ad_tag = "00000000000000000000000000000000" +use_middle_proxy = false [general.modes] classic = false From 306b653318ef8ffaf7c050e867c558b5fdc4e5f4 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Wed, 4 Mar 2026 14:23:48 +0300 Subject: [PATCH 060/261] Update README.md --- README.md | 103 ++---------------------------------------------------- 1 file changed, 2 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index faa8cd9..75feb38 100644 --- a/README.md +++ b/README.md @@ -112,110 +112,11 @@ We welcome ideas, architectural feedback, and pull requests. - Extensive logging via `trace` and `debug` with `RUST_LOG` method ## Quick Start Guide -**This software is designed for Debian-based OS: in addition to Debian, these are Ubuntu, Mint, Kali, MX and many other Linux** -1. Download release -```bash -wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz -``` -2. Move to Bin Folder -```bash -mv telemt /bin -``` -4. Make Executable -```bash -chmod +x /bin/telemt -``` -5. Go to [How to use?](#how-to-use) section for for further steps -## How to use? -### Telemt via Systemd -**This instruction "assume" that you:** -- logged in as root or executed `su -` / `sudo su` -- you already have an assembled and executable `telemt` in /bin folder as a result of the [Quick Start Guide](#quick-start-guide) or [Build](#build) +### [Quick Start Guid RU](docs/QUICK_START_GUIDE.ru.md) +### [Quick Start Guid EN](docs/QUICK_START_GUIDE.en.md) -**0. Check port and generate secrets** -The port you have selected for use should be MISSING from the list, when: -```bash -netstat -lnp -``` - -Generate 16 bytes/32 characters HEX with OpenSSL or another way: -```bash -openssl rand -hex 16 -``` -OR -```bash -xxd -l 16 -p /dev/urandom -``` -OR -```bash -python3 -c 'import os; print(os.urandom(16).hex())' -``` - -**1. Place your config to /etc/telemt.toml** - -Open nano -```bash -nano /etc/telemt.toml -``` -paste your config from [Configuration](#configuration) section - -then Ctrl+X -> Y -> Enter to save - -**2. Create service on /etc/systemd/system/telemt.service** - -Open nano -```bash -nano /etc/systemd/system/telemt.service -``` -paste this Systemd Module -```bash -[Unit] -Description=Telemt -After=network.target - -[Service] -Type=simple -WorkingDirectory=/bin -ExecStart=/bin/telemt /etc/telemt.toml -Restart=on-failure -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target -``` -then Ctrl+X -> Y -> Enter to save - -**3.** In Shell type `systemctl start telemt` - it must start with zero exit-code - -**4.** In Shell type `systemctl status telemt` - there you can reach info about current MTProxy status - -**5.** In Shell type `systemctl enable telemt` - then telemt will start with system startup, after the network is up - -**6.** In Shell type `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` - get the connection links - -## Configuration -### Minimal Configuration for First Start -```toml -# === General Settings === -[general] -# ad_tag = "00000000000000000000000000000000" - -[general.modes] -classic = false -secure = false -tls = true - -# === Anti-Censorship & Masking === -[censorship] -tls_domain = "petrovich.ru" - -[access.users] -# format: "username" = "32_hex_chars_secret" -hello = "00000000000000000000000000000000" - -``` ### Advanced #### Adtag (per-user) To use channel advertising and usage statistics from Telegram, get an Adtag from [@mtproxybot](https://t.me/mtproxybot). Set it per user in `[access.user_ad_tags]` (32 hex chars): From 12251e730f3beffcf5c101af7c44028443069418 Mon Sep 17 00:00:00 2001 From: badcdd <114914117+badcdd@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:24:00 +0300 Subject: [PATCH 061/261] Add new prometheus metrics to zabbix template --- tools/zbx_telemt_template.yaml | 903 +++++++++++++++++++++++++++++++++ 1 file changed, 903 insertions(+) diff --git a/tools/zbx_telemt_template.yaml b/tools/zbx_telemt_template.yaml index 493f3f2..27995b9 100644 --- a/tools/zbx_telemt_template.yaml +++ b/tools/zbx_telemt_template.yaml @@ -47,6 +47,54 @@ zabbix_export: tags: - tag: Application value: 'Server connections' + - uuid: 2af8ff0f27e4408db3f9798dc3141457 + name: 'Full forensic desync logs emitted' + type: DEPENDENT + key: telemt.desync_full_logged_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_desync_full_logged_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: f4439948a49f4b1d85c3eeee963259bc + name: 'Suppressed desync forensic events' + type: DEPENDENT + key: telemt.desync_suppressed_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_desync_suppressed_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 721627b8c10a414a82be1e08873604c1 + name: 'Total crypto-desync detections' + type: DEPENDENT + key: telemt.desync_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_desync_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' - uuid: 1618272cf68e44509425f5fab029db7b name: 'Handshake timeouts total' type: DEPENDENT @@ -64,6 +112,152 @@ zabbix_export: tags: - tag: Application value: 'Server connections' + - uuid: 4e5c0d10a4494c959445b4cd7a2e696e + name: 'ME CRC mismatches' + type: DEPENDENT + key: telemt.me_crc_mismatch_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_crc_mismatch_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Middle-End connections' + - uuid: 21a4a48b6e98457d87c56c3ae7b56c55 + name: 'ME endpoint quarantines due to rapid flaps' + type: DEPENDENT + key: telemt.me_endpoint_quarantine_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_endpoint_quarantine_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: c8ffc30dc3d94a6d9085ac79413fbdd6 + name: 'Runtime ME writer floor policy mode' + type: DEPENDENT + key: telemt.me_floor_mode + delay: '0' + value_type: TEXT + trends: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'telemt_me_floor_mode == 1' + - label + - mode + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 4814b52d5d184f63b64654e7635bdf6a + name: 'ME handshake rejects from upstream' + type: DEPENDENT + key: telemt.me_handshake_reject_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_handshake_reject_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 72d11caecefb4472b6c3e07f1ee90053 + name: 'Hardswap cycles that reused an existing pending generation' + type: DEPENDENT + key: telemt.me_hardswap_pending_reuse_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_hardswap_pending_reuse_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 447030854e8840a393874f54e25861d5 + name: 'Pending hardswap generations reset by TTL expiration' + type: DEPENDENT + key: telemt.me_hardswap_pending_ttl_expired_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_hardswap_pending_ttl_expired_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 47f55dd7d9394405b1c0eba6e6eb3e5c + name: 'ME idle writers closed by peer' + type: DEPENDENT + key: telemt.me_idle_close_by_peer_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_idle_close_by_peer_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 9e4598efbfe246fab9360270002b0cfa + name: 'ME KDF input drift detections' + type: DEPENDENT + key: telemt.me_kdf_drift_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_kdf_drift_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 565cc9780c5541bfb7acbb1f4973b5fc + name: 'ME KDF client-port changes with stable non-port material' + type: DEPENDENT + key: telemt.me_kdf_port_only_drift_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_kdf_port_only_drift_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' - uuid: fb95391c7f894e3eb6984b92885813d2 name: 'ME keepalive send failures' type: DEPENDENT @@ -81,6 +275,22 @@ zabbix_export: tags: - tag: Application value: 'Middle-End connections' + - uuid: 7b5995401195430e9f9e02e5dd8c3313 + name: 'ME keepalive pong replies' + type: DEPENDENT + key: telemt.me_keepalive_pong_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_keepalive_pong_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Middle-End connections' - uuid: fb95391c7f894e3eb6984b92885813c2 name: 'ME keepalive frames sent' type: DEPENDENT @@ -98,6 +308,38 @@ zabbix_export: tags: - tag: Application value: 'Middle-End connections' + - uuid: da5af5fd691d4f40bc6cad78b4758eac + name: 'ME keepalive ping timeouts' + type: DEPENDENT + key: telemt.me_keepalive_timeout_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_keepalive_timeout_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Middle-End connections' + - uuid: 50b45e494d584a7b86fca8b80c727411 + name: 'ME reader EOF terminations' + type: DEPENDENT + key: telemt.me_reader_eof_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_reader_eof_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' - uuid: fb95391c7f894e3eb6984b92885811a2 name: 'ME reconnect attempts' type: DEPENDENT @@ -132,6 +374,470 @@ zabbix_export: tags: - tag: Application value: 'Middle-End connections' + - uuid: 6288b537b7964aadb8a483abd716855a + name: 'Immediate ME refill failures' + type: DEPENDENT + key: telemt.me_refill_failed_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_refill_failed_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 8450bdb48f9b4505beb8fdfc665b37c5 + name: 'Immediate ME refill skips due to inflight dedup' + type: DEPENDENT + key: telemt.me_refill_skipped_inflight_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_refill_skipped_inflight_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: cb192264c03a40578140863970333515 + name: 'Immediate ME refill runs started' + type: DEPENDENT + key: telemt.me_refill_triggered_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_refill_triggered_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 8f46b374332848fba0daba72e17eaad0 + name: 'ME route drops: channel closed' + type: DEPENDENT + key: telemt.me_route_drop_channel_closed_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_route_drop_channel_closed_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Middle-End connections' + - uuid: de5fa7a316554d099bcf5e000b33bfed + name: 'ME route drops: no conn' + type: DEPENDENT + key: telemt.me_route_drop_no_conn_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_route_drop_no_conn_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Middle-End connections' + - uuid: d9e1630ce38946f7a8d179187793f12c + name: 'ME route drops: queue full by adaptive profile' + type: DEPENDENT + key: telemt.me_route_drop_queue_full_profile_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'telemt_me_route_drop_queue_full_profile_total == 1' + - label + - profile + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: d5caefb8978e4f3eac4dcdecd4655c46 + name: 'ME route drops: queue full' + type: DEPENDENT + key: telemt.me_route_drop_queue_full_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_route_drop_queue_full_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: f682298c2dfc46dda45771a58faa9ffa + name: 'Service RPC_CLOSE_EXT sent after activity signals' + type: DEPENDENT + key: telemt.me_rpc_proxy_req_signal_close_sent_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_rpc_proxy_req_signal_close_sent_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 5db4bdc93959473eade9281c221e34b6 + name: 'Service RPC_PROXY_REQ activity signal failures' + type: DEPENDENT + key: telemt.me_rpc_proxy_req_signal_failed_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_rpc_proxy_req_signal_failed_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 4e75611bc3854415b63a1863e9bf176f + name: 'Service RPC_PROXY_REQ responses observed' + type: DEPENDENT + key: telemt.me_rpc_proxy_req_signal_response_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_rpc_proxy_req_signal_response_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: ecbffb29f2784839bea0ce2a38393438 + name: 'Service RPC_PROXY_REQ activity signals sent' + type: DEPENDENT + key: telemt.me_rpc_proxy_req_signal_sent_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_rpc_proxy_req_signal_sent_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 078eff3deeec435597f0c531457bb906 + name: 'Service RPC_PROXY_REQ skipped due to missing writer metadata' + type: DEPENDENT + key: telemt.me_rpc_proxy_req_signal_skipped_no_meta_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_rpc_proxy_req_signal_skipped_no_meta_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 7429ffbd94a340d7a600bc1690eb57e7 + name: 'ME sequence mismatches' + type: DEPENDENT + key: telemt.me_seq_mismatch_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_seq_mismatch_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 0f1f77ae34df4a48b36ad263359b5ad3 + name: 'Single-endpoint DC outage transitions to active state' + type: DEPENDENT + key: telemt.me_single_endpoint_outage_enter_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_outage_enter_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 63d44ef672ff4df288914eb98f6fa72c + name: 'Single-endpoint DC outage recovery transitions' + type: DEPENDENT + key: telemt.me_single_endpoint_outage_exit_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_outage_exit_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 1b72ff95f1ba4fb2924aa3a129b22f4d + name: 'Reconnect attempts performed during single-endpoint outages' + type: DEPENDENT + key: telemt.me_single_endpoint_outage_reconnect_attempt_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_outage_reconnect_attempt_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 466bb352d55946a0bb78efc63e1ed71e + name: 'Successful reconnect attempts during single-endpoint outages' + type: DEPENDENT + key: telemt.me_single_endpoint_outage_reconnect_success_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_outage_reconnect_success_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 295b4a519a4d46f7b1ddbdf5b5268751 + name: 'Outage reconnect attempts that bypassed quarantine' + type: DEPENDENT + key: telemt.me_single_endpoint_quarantine_bypass_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_quarantine_bypass_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: bffa4861f83f4445bb0b2259e100e04c + name: 'Shadow rotations skipped because endpoint is quarantined' + type: DEPENDENT + key: telemt.me_single_endpoint_shadow_rotate_skipped_quarantine_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_shadow_rotate_skipped_quarantine_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: f80ce02b50824f8ea0ddabac9ff97757 + name: 'Successful periodic shadow rotations for single-endpoint DC groups' + type: DEPENDENT + key: telemt.me_single_endpoint_shadow_rotate_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_single_endpoint_shadow_rotate_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: bf2a0ff89c314f78904aa43351601111 + name: 'Total ME writer removals' + type: DEPENDENT + key: telemt.me_writer_removed_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_writer_removed_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 0d12ea02187745eba55498dfb16daa5c + name: 'Unexpected writer removals not yet compensated by restore' + type: DEPENDENT + key: telemt.me_writer_removed_unexpected_minus_restored_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_writer_removed_unexpected_minus_restored_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 644278e7f87947e1a49483ba4487e32b + name: 'Unexpected ME writer removals that triggered refill' + type: DEPENDENT + key: telemt.me_writer_removed_unexpected_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_writer_removed_unexpected_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: a6c24dfc85d643dab1c81fc1e63fe3cc + name: 'Refilled ME writer restored via fallback endpoint' + type: DEPENDENT + key: telemt.me_writer_restored_fallback_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_writer_restored_fallback_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: d7d0a78ca6da4bb9b4a0991fd83149cf + name: 'Refilled ME writer restored on the same endpoint' + type: DEPENDENT + key: telemt.me_writer_restored_same_endpoint_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_me_writer_restored_same_endpoint_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: beb906ab89564cf9adfbb7b1d4553c44 + name: 'Active draining ME writers' + type: DEPENDENT + key: telemt.pool_drain_active + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_pool_drain_active + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 2f0926e00d7a4e5aa1783cb33b1192ea + name: 'Forced close events for draining writers' + type: DEPENDENT + key: telemt.pool_force_close_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_pool_force_close_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 70d0b4da6079435ebe978e99bda8f1d3 + name: 'Stale writer fallback picks for new binds' + type: DEPENDENT + key: telemt.pool_stale_pick_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_pool_stale_pick_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 8a1d240b9b554905a8add9bf730bf1f4 + name: 'Successful ME pool swaps' + type: DEPENDENT + key: telemt.pool_swap_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_pool_swap_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' - uuid: 991b1858e3f94b3098ff0f84859efc41 name: 'Prometheus metrics' type: HTTP_AGENT @@ -139,11 +845,158 @@ zabbix_export: value_type: TEXT trends: '0' url: '{$TELEMT_URL}' + - uuid: cef2547bb9464d10b11b6c19beac089d + name: 'Invalid secure frame lengths' + type: DEPENDENT + key: telemt.secure_padding_invalid_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_secure_padding_invalid_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: c164d7b59bdc4429a23b908558de8cf4 + name: 'Runtime core telemetry switch' + type: DEPENDENT + key: telemt.telemetry_core_enabled + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_telemetry_core_enabled + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: ff16438417d842178d26033d13520833 + name: 'Runtime ME telemetry level flag' + type: DEPENDENT + key: telemt.telemetry_me_level + delay: '0' + value_type: TEXT + trends: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'telemt_telemetry_me_level == 1' + - label + - level + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 9fec0bb7c3c84ada96668b74d5849556 + name: 'Runtime per-user telemetry switch' + type: DEPENDENT + key: telemt.telemetry_user_enabled + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_telemetry_user_enabled + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 378b765aa7bc4a4ea87d3bc876c50d12 + name: 'User-labeled metric series suppression flag' + type: DEPENDENT + key: telemt.telemetry_user_series_suppressed + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_telemetry_user_series_suppressed + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 17972d992fa84fc1b53fdefed123ccd8 + name: 'Upstream connect attempts across all requests' + type: DEPENDENT + key: telemt.upstream_connect_attempt_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_upstream_connect_attempt_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 38627dd1cb7145e180d111bdee1d2c23 + name: 'Hard errors that triggered upstream connect failfast' + type: DEPENDENT + key: telemt.upstream_connect_failfast_hard_error_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_upstream_connect_failfast_hard_error_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 0ffd4c35b6734c83bd77c59f30bf3246 + name: 'Failed upstream connect request cycles' + type: DEPENDENT + key: telemt.upstream_connect_fail_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_upstream_connect_fail_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' + - uuid: 7da255f4f38c4095921bc876d16d3586 + name: 'Successful upstream connect request cycles' + type: DEPENDENT + key: telemt.upstream_connect_success_total + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - telemt_upstream_connect_success_total + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Telemt other' - uuid: fb95391c7f894e3eb6984b92885813b2 name: 'Telemt Uptime' type: DEPENDENT key: telemt.uptime delay: '0' + value_type: FLOAT trends: '0' units: s preprocessing: @@ -180,6 +1033,56 @@ zabbix_export: tags: - tag: Application value: 'Users connections' + - uuid: f7ad02d1635542b584bba5941375ae41 + name: 'Current number of unique active IPs by {#TELEMT_USER}' + type: DEPENDENT + key: 'telemt.ips_current_[{#TELEMT_USER}]' + delay: '0' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'telemt_user_unique_ips_current{user="{#TELEMT_USER}"}' + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Users IPs' + - uuid: 100b09bf1cff420495c5c105bdb0af6c + name: 'Configured unique IP limit to {#TELEMT_USER}' + type: DEPENDENT + key: 'telemt.ips_limit_[{#TELEMT_USER}]' + delay: '0' + description: '0 means unlimited' + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'telemt_user_unique_ips_limit{user="{#TELEMT_USER}"}' + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Users IPs' + - uuid: ef3ac8f5c5d746bbaa4b0b698ba0d9f6 + name: 'Unique IP usage ratio by {#TELEMT_USER}' + type: DEPENDENT + key: 'telemt.ips_utilization_[{#TELEMT_USER}]' + delay: '0' + value_type: FLOAT + preprocessing: + - type: PROMETHEUS_PATTERN + parameters: + - 'telemt_user_unique_ips_utilization{user="{#TELEMT_USER}"}' + - value + - '' + master_item: + key: telemt.prom_metrics + tags: + - tag: Application + value: 'Users IPs' - uuid: 3ccce91ab5d54b4d972280c7b7bda910 name: 'Messages received from {#TELEMT_USER}' type: DEPENDENT From b55fa5ec8feb71e3b497ac151f9e594ca657014e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 4 Mar 2026 17:52:02 +0300 Subject: [PATCH 062/261] Update CONTRIBUTING.md --- CONTRIBUTING.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 571a400..2fcce51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,8 @@ +# Issues - Rules +## What it is not +- NOT Question and Answer +- NOT Helpdesk + # Pull Requests - Rules ## General - ONLY signed and verified commits From 4ae4ca8ca85611925da91a1d4e787b8cebc8d83c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 02:28:19 +0300 Subject: [PATCH 063/261] New IP Limit Method Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/users.rs | 1 + src/config/defaults.rs | 5 + src/config/hot_reload.rs | 10 ++ src/config/load.rs | 14 ++ src/config/types.rs | 21 +++ src/ip_tracker.rs | 357 +++++++++++++++++++++------------------ src/main.rs | 51 ++++++ 7 files changed, 293 insertions(+), 166 deletions(-) diff --git a/src/api/users.rs b/src/api/users.rs index c907070..8e90c7f 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -287,6 +287,7 @@ pub(super) async fn delete_user( .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; let revision = save_config_to_disk(&shared.config_path, &cfg).await?; drop(_guard); + shared.ip_tracker.remove_user_limit(user).await; shared.ip_tracker.clear_user_ips(user).await; Ok((user.to_string(), revision)) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 86f569b..9ac6c53 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -12,6 +12,7 @@ const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2; const DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS: u64 = 90; const DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT: u8 = 1; const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180; +const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; const DEFAULT_LISTEN_ADDR_IPV6: &str = "::"; @@ -464,6 +465,10 @@ pub(crate) fn default_access_users() -> HashMap { )]) } +pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 { + DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS +} + // Custom deserializer helpers #[derive(Deserialize)] diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index d752d45..e2f4246 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -438,6 +438,16 @@ fn log_changes( new_hot.access.user_max_unique_ips.len() ); } + if old_hot.access.user_max_unique_ips_mode != new_hot.access.user_max_unique_ips_mode + || old_hot.access.user_max_unique_ips_window_secs + != new_hot.access.user_max_unique_ips_window_secs + { + info!( + "config reload: user_max_unique_ips policy mode={:?} window={}s", + new_hot.access.user_max_unique_ips_mode, + new_hot.access.user_max_unique_ips_window_secs + ); + } } /// Load config, validate, diff against current, and broadcast if changed. diff --git a/src/config/load.rs b/src/config/load.rs index b469299..666c938 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -257,6 +257,12 @@ impl ProxyConfig { )); } + if config.access.user_max_unique_ips_window_secs == 0 { + return Err(ProxyError::Config( + "access.user_max_unique_ips_window_secs must be > 0".to_string(), + )); + } + if config.general.me_reinit_every_secs == 0 { return Err(ProxyError::Config( "general.me_reinit_every_secs must be > 0".to_string(), @@ -728,6 +734,14 @@ mod tests { default_api_minimal_runtime_cache_ttl_ms() ); assert_eq!(cfg.access.users, default_access_users()); + assert_eq!( + cfg.access.user_max_unique_ips_mode, + UserMaxUniqueIpsMode::default() + ); + assert_eq!( + cfg.access.user_max_unique_ips_window_secs, + default_user_max_unique_ips_window_secs() + ); } #[test] diff --git a/src/config/types.rs b/src/config/types.rs index ee17108..00260a8 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -183,6 +183,19 @@ impl MeFloorMode { } } +/// Per-user unique source IP limit mode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum UserMaxUniqueIpsMode { + /// Count only currently active source IPs. + #[default] + ActiveWindow, + /// Count source IPs seen within the recent time window. + TimeWindow, + /// Enforce both active and recent-window limits at the same time. + Combined, +} + /// Telemetry controls for hot-path counters and ME diagnostics. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TelemetryConfig { @@ -1045,6 +1058,12 @@ pub struct AccessConfig { #[serde(default)] pub user_max_unique_ips: HashMap, + #[serde(default)] + pub user_max_unique_ips_mode: UserMaxUniqueIpsMode, + + #[serde(default = "default_user_max_unique_ips_window_secs")] + pub user_max_unique_ips_window_secs: u64, + #[serde(default = "default_replay_check_len")] pub replay_check_len: usize, @@ -1064,6 +1083,8 @@ impl Default for AccessConfig { user_expirations: HashMap::new(), user_data_quota: HashMap::new(), user_max_unique_ips: HashMap::new(), + user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(), + user_max_unique_ips_window_secs: default_user_max_unique_ips_window_secs(), replay_check_len: default_replay_check_len(), replay_window_secs: default_replay_window_secs(), ignore_time_skew: false, diff --git a/src/ip_tracker.rs b/src/ip_tracker.rs index 32fcbe3..626d591 100644 --- a/src/ip_tracker.rs +++ b/src/ip_tracker.rs @@ -1,153 +1,151 @@ -// src/ip_tracker.rs -// IP address tracking and limiting for users +// IP address tracking and per-user unique IP limiting. #![allow(dead_code)] use std::collections::{HashMap, HashSet}; use std::net::IpAddr; use std::sync::Arc; +use std::time::{Duration, Instant}; + use tokio::sync::RwLock; -/// Трекер уникальных IP-адресов для каждого пользователя MTProxy -/// -/// Предоставляет thread-safe механизм для: -/// - Отслеживания активных IP-адресов каждого пользователя -/// - Ограничения количества уникальных IP на пользователя -/// - Автоматической очистки при отключении клиентов +use crate::config::UserMaxUniqueIpsMode; + #[derive(Debug, Clone)] pub struct UserIpTracker { - /// Маппинг: Имя пользователя -> Множество активных IP-адресов active_ips: Arc>>>, - - /// Маппинг: Имя пользователя -> Максимально разрешенное количество уникальных IP + recent_ips: Arc>>>, max_ips: Arc>>, + limit_mode: Arc>, + limit_window: Arc>, } impl UserIpTracker { - /// Создать новый пустой трекер pub fn new() -> Self { Self { active_ips: Arc::new(RwLock::new(HashMap::new())), + recent_ips: Arc::new(RwLock::new(HashMap::new())), max_ips: Arc::new(RwLock::new(HashMap::new())), + limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)), + limit_window: Arc::new(RwLock::new(Duration::from_secs(30))), } } - /// Установить лимит уникальных IP для конкретного пользователя - /// - /// # Arguments - /// * `username` - Имя пользователя - /// * `max_ips` - Максимальное количество одновременно активных IP-адресов + pub async fn set_limit_policy(&self, mode: UserMaxUniqueIpsMode, window_secs: u64) { + { + let mut current_mode = self.limit_mode.write().await; + *current_mode = mode; + } + let mut current_window = self.limit_window.write().await; + *current_window = Duration::from_secs(window_secs.max(1)); + } + pub async fn set_user_limit(&self, username: &str, max_ips: usize) { let mut limits = self.max_ips.write().await; limits.insert(username.to_string(), max_ips); } - /// Загрузить лимиты из конфигурации - /// - /// # Arguments - /// * `limits` - HashMap с лимитами из config.toml - pub async fn load_limits(&self, limits: &HashMap) { - let mut max_ips = self.max_ips.write().await; - for (user, limit) in limits { - max_ips.insert(user.clone(), *limit); - } + pub async fn remove_user_limit(&self, username: &str) { + let mut limits = self.max_ips.write().await; + limits.remove(username); } - /// Проверить, может ли пользователь подключиться с данного IP-адреса - /// и добавить IP в список активных, если проверка успешна - /// - /// # Arguments - /// * `username` - Имя пользователя - /// * `ip` - IP-адрес клиента - /// - /// # Returns - /// * `Ok(())` - Подключение разрешено, IP добавлен в активные - /// * `Err(String)` - Подключение отклонено с описанием причины - pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> { - // Получаем лимит для пользователя - let max_ips = self.max_ips.read().await; - let limit = match max_ips.get(username) { - Some(limit) => *limit, - None => { - // Если лимит не задан - разрешаем безлимитный доступ - drop(max_ips); - let mut active_ips = self.active_ips.write().await; - let user_ips = active_ips - .entry(username.to_string()) - .or_insert_with(HashSet::new); - user_ips.insert(ip); - return Ok(()); - } - }; - drop(max_ips); + pub async fn load_limits(&self, limits: &HashMap) { + let mut max_ips = self.max_ips.write().await; + max_ips.clone_from(limits); + } + + fn prune_recent(user_recent: &mut HashMap, now: Instant, window: Duration) { + if user_recent.is_empty() { + return; + } + user_recent.retain(|_, seen_at| now.duration_since(*seen_at) <= window); + } + + pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> { + let limit = { + let max_ips = self.max_ips.read().await; + max_ips.get(username).copied() + }; - // Проверяем и обновляем активные IP let mut active_ips = self.active_ips.write().await; - let user_ips = active_ips + let user_active = active_ips .entry(username.to_string()) .or_insert_with(HashSet::new); - // Если IP уже есть в списке - это повторное подключение, разрешаем - if user_ips.contains(&ip) { + if limit.is_none() { + user_active.insert(ip); return Ok(()); } - // Проверяем, не превышен ли лимит - if user_ips.len() >= limit { + let limit = limit.unwrap_or_default(); + let mode = *self.limit_mode.read().await; + let window = *self.limit_window.read().await; + let now = Instant::now(); + + let mut recent_ips = self.recent_ips.write().await; + let user_recent = recent_ips + .entry(username.to_string()) + .or_insert_with(HashMap::new); + Self::prune_recent(user_recent, now, window); + + if user_active.contains(&ip) { + user_recent.insert(ip, now); + return Ok(()); + } + + let active_limit_reached = user_active.len() >= limit; + let recent_limit_reached = user_recent.len() >= limit; + let deny = match mode { + UserMaxUniqueIpsMode::ActiveWindow => active_limit_reached, + UserMaxUniqueIpsMode::TimeWindow => recent_limit_reached, + UserMaxUniqueIpsMode::Combined => active_limit_reached || recent_limit_reached, + }; + + if deny { return Err(format!( - "IP limit reached for user '{}': {}/{} unique IPs already connected", + "IP limit reached for user '{}': active={}/{} recent={}/{} mode={:?}", username, - user_ips.len(), - limit + user_active.len(), + limit, + user_recent.len(), + limit, + mode )); } - // Лимит не превышен - добавляем новый IP - user_ips.insert(ip); + user_active.insert(ip); + user_recent.insert(ip, now); Ok(()) } - /// Удалить IP-адрес из списка активных при отключении клиента - /// - /// # Arguments - /// * `username` - Имя пользователя - /// * `ip` - IP-адрес отключившегося клиента pub async fn remove_ip(&self, username: &str, ip: IpAddr) { let mut active_ips = self.active_ips.write().await; - if let Some(user_ips) = active_ips.get_mut(username) { user_ips.remove(&ip); - - // Если у пользователя не осталось активных IP - удаляем запись - // для экономии памяти if user_ips.is_empty() { active_ips.remove(username); } } + drop(active_ips); + + let mode = *self.limit_mode.read().await; + if matches!(mode, UserMaxUniqueIpsMode::ActiveWindow) { + let mut recent_ips = self.recent_ips.write().await; + if let Some(user_recent) = recent_ips.get_mut(username) { + user_recent.remove(&ip); + if user_recent.is_empty() { + recent_ips.remove(username); + } + } + } } - /// Получить текущее количество активных IP-адресов для пользователя - /// - /// # Arguments - /// * `username` - Имя пользователя - /// - /// # Returns - /// Количество уникальных активных IP-адресов pub async fn get_active_ip_count(&self, username: &str) -> usize { let active_ips = self.active_ips.read().await; - active_ips - .get(username) - .map(|ips| ips.len()) - .unwrap_or(0) + active_ips.get(username).map(|ips| ips.len()).unwrap_or(0) } - /// Получить список всех активных IP-адресов для пользователя - /// - /// # Arguments - /// * `username` - Имя пользователя - /// - /// # Returns - /// Вектор с активными IP-адресами pub async fn get_active_ips(&self, username: &str) -> Vec { let active_ips = self.active_ips.read().await; active_ips @@ -156,49 +154,38 @@ impl UserIpTracker { .unwrap_or_else(Vec::new) } - /// Получить статистику по всем пользователям - /// - /// # Returns - /// Вектор кортежей: (имя_пользователя, количество_активных_IP, лимит) pub async fn get_stats(&self) -> Vec<(String, usize, usize)> { let active_ips = self.active_ips.read().await; let max_ips = self.max_ips.read().await; let mut stats = Vec::new(); - - // Собираем статистику по пользователям с активными подключениями for (username, user_ips) in active_ips.iter() { let limit = max_ips.get(username).copied().unwrap_or(0); stats.push((username.clone(), user_ips.len(), limit)); } - - stats.sort_by(|a, b| a.0.cmp(&b.0)); // Сортируем по имени пользователя + + stats.sort_by(|a, b| a.0.cmp(&b.0)); stats } - /// Очистить все активные IP для пользователя (при необходимости) - /// - /// # Arguments - /// * `username` - Имя пользователя pub async fn clear_user_ips(&self, username: &str) { let mut active_ips = self.active_ips.write().await; active_ips.remove(username); + drop(active_ips); + + let mut recent_ips = self.recent_ips.write().await; + recent_ips.remove(username); } - /// Очистить всю статистику (использовать с осторожностью!) pub async fn clear_all(&self) { let mut active_ips = self.active_ips.write().await; active_ips.clear(); + drop(active_ips); + + let mut recent_ips = self.recent_ips.write().await; + recent_ips.clear(); } - /// Проверить, подключен ли пользователь с данного IP - /// - /// # Arguments - /// * `username` - Имя пользователя - /// * `ip` - IP-адрес для проверки - /// - /// # Returns - /// `true` если IP активен, `false` если нет pub async fn is_ip_active(&self, username: &str, ip: IpAddr) -> bool { let active_ips = self.active_ips.read().await; active_ips @@ -207,46 +194,39 @@ impl UserIpTracker { .unwrap_or(false) } - /// Получить лимит для пользователя - /// - /// # Arguments - /// * `username` - Имя пользователя - /// - /// # Returns - /// Лимит IP-адресов или None, если лимит не установлен pub async fn get_user_limit(&self, username: &str) -> Option { let max_ips = self.max_ips.read().await; max_ips.get(username).copied() } - /// Форматировать статистику в читаемый текст - /// - /// # Returns - /// Строка со статистикой для логов или мониторинга pub async fn format_stats(&self) -> String { let stats = self.get_stats().await; - + if stats.is_empty() { return String::from("No active users"); } - + let mut output = String::from("User IP Statistics:\n"); output.push_str("==================\n"); - + for (username, active_count, limit) in stats { output.push_str(&format!( "User: {:<20} Active IPs: {}/{}\n", username, active_count, - if limit > 0 { limit.to_string() } else { "unlimited".to_string() } + if limit > 0 { + limit.to_string() + } else { + "unlimited".to_string() + } )); - + let ips = self.get_active_ips(&username).await; for ip in ips { - output.push_str(&format!(" └─ {}\n", ip)); + output.push_str(&format!(" - {}\n", ip)); } } - + output } } @@ -257,10 +237,6 @@ impl Default for UserIpTracker { } } -// ============================================================================ -// ТЕСТЫ -// ============================================================================ - #[cfg(test)] mod tests { use super::*; @@ -283,14 +259,10 @@ mod tests { let ip2 = test_ipv4(192, 168, 1, 2); let ip3 = test_ipv4(192, 168, 1, 3); - // Первые два IP должны быть приняты assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); assert!(tracker.check_and_add("test_user", ip2).await.is_ok()); - - // Третий IP должен быть отклонен assert!(tracker.check_and_add("test_user", ip3).await.is_err()); - // Проверяем счетчик assert_eq!(tracker.get_active_ip_count("test_user").await, 2); } @@ -301,13 +273,8 @@ mod tests { let ip1 = test_ipv4(192, 168, 1, 1); - // Первое подключение assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); - - // Повторное подключение с того же IP должно пройти assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); - - // Счетчик не должен увеличиться assert_eq!(tracker.get_active_ip_count("test_user").await, 1); } @@ -320,36 +287,28 @@ mod tests { let ip2 = test_ipv4(192, 168, 1, 2); let ip3 = test_ipv4(192, 168, 1, 3); - // Добавляем два IP assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); assert!(tracker.check_and_add("test_user", ip2).await.is_ok()); - - // Третий не должен пройти assert!(tracker.check_and_add("test_user", ip3).await.is_err()); - // Удаляем первый IP tracker.remove_ip("test_user", ip1).await; - - // Теперь третий должен пройти + assert!(tracker.check_and_add("test_user", ip3).await.is_ok()); - assert_eq!(tracker.get_active_ip_count("test_user").await, 2); } #[tokio::test] async fn test_no_limit() { let tracker = UserIpTracker::new(); - // Не устанавливаем лимит для test_user let ip1 = test_ipv4(192, 168, 1, 1); let ip2 = test_ipv4(192, 168, 1, 2); let ip3 = test_ipv4(192, 168, 1, 3); - // Без лимита все IP должны проходить assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); assert!(tracker.check_and_add("test_user", ip2).await.is_ok()); assert!(tracker.check_and_add("test_user", ip3).await.is_ok()); - + assert_eq!(tracker.get_active_ip_count("test_user").await, 3); } @@ -362,11 +321,9 @@ mod tests { let ip1 = test_ipv4(192, 168, 1, 1); let ip2 = test_ipv4(192, 168, 1, 2); - // user1 может использовать 2 IP assert!(tracker.check_and_add("user1", ip1).await.is_ok()); assert!(tracker.check_and_add("user1", ip2).await.is_ok()); - // user2 может использовать только 1 IP assert!(tracker.check_and_add("user2", ip1).await.is_ok()); assert!(tracker.check_and_add("user2", ip2).await.is_err()); } @@ -379,10 +336,9 @@ mod tests { let ipv4 = test_ipv4(192, 168, 1, 1); let ipv6 = test_ipv6(); - // Должны работать оба типа адресов assert!(tracker.check_and_add("test_user", ipv4).await.is_ok()); assert!(tracker.check_and_add("test_user", ipv6).await.is_ok()); - + assert_eq!(tracker.get_active_ip_count("test_user").await, 2); } @@ -417,8 +373,7 @@ mod tests { let stats = tracker.get_stats().await; assert_eq!(stats.len(), 2); - - // Проверяем наличие обоих пользователей в статистике + assert!(stats.iter().any(|(name, _, _)| name == "user1")); assert!(stats.iter().any(|(name, _, _)| name == "user2")); } @@ -427,10 +382,10 @@ mod tests { async fn test_clear_user_ips() { let tracker = UserIpTracker::new(); let ip1 = test_ipv4(192, 168, 1, 1); - + tracker.check_and_add("test_user", ip1).await.unwrap(); assert_eq!(tracker.get_active_ip_count("test_user").await, 1); - + tracker.clear_user_ips("test_user").await; assert_eq!(tracker.get_active_ip_count("test_user").await, 0); } @@ -440,9 +395,9 @@ mod tests { let tracker = UserIpTracker::new(); let ip1 = test_ipv4(192, 168, 1, 1); let ip2 = test_ipv4(192, 168, 1, 2); - + tracker.check_and_add("test_user", ip1).await.unwrap(); - + assert!(tracker.is_ip_active("test_user", ip1).await); assert!(!tracker.is_ip_active("test_user", ip2).await); } @@ -450,15 +405,85 @@ mod tests { #[tokio::test] async fn test_load_limits_from_config() { let tracker = UserIpTracker::new(); - + let mut config_limits = HashMap::new(); config_limits.insert("user1".to_string(), 5); config_limits.insert("user2".to_string(), 3); - + tracker.load_limits(&config_limits).await; - + assert_eq!(tracker.get_user_limit("user1").await, Some(5)); assert_eq!(tracker.get_user_limit("user2").await, Some(3)); assert_eq!(tracker.get_user_limit("user3").await, None); } + + #[tokio::test] + async fn test_load_limits_replaces_previous_map() { + let tracker = UserIpTracker::new(); + + let mut first = HashMap::new(); + first.insert("user1".to_string(), 2); + first.insert("user2".to_string(), 3); + tracker.load_limits(&first).await; + + let mut second = HashMap::new(); + second.insert("user2".to_string(), 5); + tracker.load_limits(&second).await; + + assert_eq!(tracker.get_user_limit("user1").await, None); + assert_eq!(tracker.get_user_limit("user2").await, Some(5)); + } + + #[tokio::test] + async fn test_time_window_mode_blocks_recent_ip_churn() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("test_user", 1).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::TimeWindow, 30) + .await; + + let ip1 = test_ipv4(10, 0, 0, 1); + let ip2 = test_ipv4(10, 0, 0, 2); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + tracker.remove_ip("test_user", ip1).await; + assert!(tracker.check_and_add("test_user", ip2).await.is_err()); + } + + #[tokio::test] + async fn test_combined_mode_enforces_active_and_recent_limits() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("test_user", 1).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::Combined, 30) + .await; + + let ip1 = test_ipv4(10, 0, 1, 1); + let ip2 = test_ipv4(10, 0, 1, 2); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + assert!(tracker.check_and_add("test_user", ip2).await.is_err()); + + tracker.remove_ip("test_user", ip1).await; + assert!(tracker.check_and_add("test_user", ip2).await.is_err()); + } + + #[tokio::test] + async fn test_time_window_expires() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("test_user", 1).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::TimeWindow, 1) + .await; + + let ip1 = test_ipv4(10, 1, 0, 1); + let ip2 = test_ipv4(10, 1, 0, 2); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + tracker.remove_ip("test_user", ip1).await; + assert!(tracker.check_and_add("test_user", ip2).await.is_err()); + + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!(tracker.check_and_add("test_user", ip2).await.is_ok()); + } } diff --git a/src/main.rs b/src/main.rs index 0aec195..6bff525 100644 --- a/src/main.rs +++ b/src/main.rs @@ -423,6 +423,12 @@ async fn main() -> std::result::Result<(), Box> { // IP Tracker initialization let ip_tracker = Arc::new(UserIpTracker::new()); ip_tracker.load_limits(&config.access.user_max_unique_ips).await; + ip_tracker + .set_limit_policy( + config.access.user_max_unique_ips_mode, + config.access.user_max_unique_ips_window_secs, + ) + .await; if !config.access.user_max_unique_ips.is_empty() { info!("IP limits configured for {} users", config.access.user_max_unique_ips.len()); @@ -847,6 +853,51 @@ async fn main() -> std::result::Result<(), Box> { } }); + let ip_tracker_policy = ip_tracker.clone(); + let mut config_rx_ip_limits = config_rx.clone(); + tokio::spawn(async move { + let mut prev_limits = config_rx_ip_limits + .borrow() + .access + .user_max_unique_ips + .clone(); + let mut prev_mode = config_rx_ip_limits + .borrow() + .access + .user_max_unique_ips_mode; + let mut prev_window = config_rx_ip_limits + .borrow() + .access + .user_max_unique_ips_window_secs; + + loop { + if config_rx_ip_limits.changed().await.is_err() { + break; + } + let cfg = config_rx_ip_limits.borrow_and_update().clone(); + + if prev_limits != cfg.access.user_max_unique_ips { + ip_tracker_policy + .load_limits(&cfg.access.user_max_unique_ips) + .await; + prev_limits = cfg.access.user_max_unique_ips.clone(); + } + + if prev_mode != cfg.access.user_max_unique_ips_mode + || prev_window != cfg.access.user_max_unique_ips_window_secs + { + ip_tracker_policy + .set_limit_policy( + cfg.access.user_max_unique_ips_mode, + cfg.access.user_max_unique_ips_window_secs, + ) + .await; + prev_mode = cfg.access.user_max_unique_ips_mode; + prev_window = cfg.access.user_max_unique_ips_window_secs; + } + } + }); + let beobachten_writer = beobachten.clone(); let config_rx_beobachten = config_rx.clone(); tokio::spawn(async move { From a9209fd3c7151aa70c5953f3b7e8a840cc07eaea Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:18:09 +0300 Subject: [PATCH 064/261] Hot-Reload fixes Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/hot_reload.rs | 547 ++++++++++++++++++++++++++++++++++----- 1 file changed, 479 insertions(+), 68 deletions(-) diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index e2f4246..6d88800 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -9,20 +9,17 @@ //! | `general` | `log_level` | Filter updated via `log_level_tx` | //! | `access` | `user_ad_tags` | Passed on next connection | //! | `general` | `ad_tag` | Passed on next connection (fallback per-user) | -//! | `general` | `middle_proxy_pool_size` | Passed on next connection | -//! | `general` | `me_keepalive_*` | Passed on next connection | //! | `general` | `desync_all_full` | Applied immediately | //! | `general` | `update_every` | Applied to ME updater immediately | -//! | `general` | `hardswap` | Applied on next ME map update | -//! | `general` | `me_pool_drain_ttl_secs` | Applied on next ME map update | -//! | `general` | `me_pool_min_fresh_ratio` | Applied on next ME map update | -//! | `general` | `me_reinit_drain_timeout_secs` | Applied on next ME map update | +//! | `general` | `me_reinit_*` | Applied to ME reinit scheduler immediately | +//! | `general` | `hardswap` / `me_*_reinit` | Applied on next ME map update | //! | `general` | `telemetry` / `me_*_policy` | Applied immediately | //! | `network` | `dns_overrides` | Applied immediately | //! | `access` | All user/quota fields | Effective immediately | //! //! Fields that require re-binding sockets (`server.port`, `censorship.*`, //! `network.*`, `use_middle_proxy`) are **not** applied; a warning is emitted. +//! Non-hot changes are never mixed into the runtime config snapshot. use std::net::IpAddr; use std::path::PathBuf; @@ -32,7 +29,7 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher}; use tokio::sync::{mpsc, watch}; use tracing::{error, info, warn}; -use crate::config::{LogLevel, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel}; +use crate::config::{LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel}; use super::load::ProxyConfig; // ── Hot fields ──────────────────────────────────────────────────────────────── @@ -43,17 +40,37 @@ pub struct HotFields { pub log_level: LogLevel, pub ad_tag: Option, pub dns_overrides: Vec, - pub middle_proxy_pool_size: usize, pub desync_all_full: bool, pub update_every_secs: u64, + pub me_reinit_every_secs: u64, + pub me_reinit_singleflight: bool, + pub me_reinit_coalesce_window_ms: u64, pub hardswap: bool, pub me_pool_drain_ttl_secs: u64, pub me_pool_min_fresh_ratio: f32, pub me_reinit_drain_timeout_secs: u64, - pub me_keepalive_enabled: bool, - pub me_keepalive_interval_secs: u64, - pub me_keepalive_jitter_secs: u64, - pub me_keepalive_payload_random: bool, + pub me_hardswap_warmup_delay_min_ms: u64, + pub me_hardswap_warmup_delay_max_ms: u64, + pub me_hardswap_warmup_extra_passes: u8, + pub me_hardswap_warmup_pass_backoff_base_ms: u64, + pub me_bind_stale_mode: MeBindStaleMode, + pub me_bind_stale_ttl_secs: u64, + pub me_secret_atomic_snapshot: bool, + pub me_deterministic_writer_sort: bool, + pub me_single_endpoint_shadow_writers: u8, + pub me_single_endpoint_outage_mode_enabled: bool, + pub me_single_endpoint_outage_disable_quarantine: bool, + pub me_single_endpoint_outage_backoff_min_ms: u64, + pub me_single_endpoint_outage_backoff_max_ms: u64, + pub me_single_endpoint_shadow_rotate_every_secs: u64, + pub me_config_stable_snapshots: u8, + pub me_config_apply_cooldown_secs: u64, + pub me_snapshot_require_http_2xx: bool, + pub me_snapshot_reject_empty_map: bool, + pub me_snapshot_min_proxy_for_lines: u32, + pub proxy_secret_stable_snapshots: u8, + pub proxy_secret_rotate_runtime: bool, + pub proxy_secret_len_max: usize, pub telemetry_core_enabled: bool, pub telemetry_user_enabled: bool, pub telemetry_me_level: MeTelemetryLevel, @@ -65,7 +82,14 @@ pub struct HotFields { pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_watermark_pct: u8, - pub access: crate::config::AccessConfig, + pub users: std::collections::HashMap, + pub user_ad_tags: std::collections::HashMap, + pub user_max_tcp_conns: std::collections::HashMap, + pub user_expirations: std::collections::HashMap>, + pub user_data_quota: std::collections::HashMap, + pub user_max_unique_ips: std::collections::HashMap, + pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode, + pub user_max_unique_ips_window_secs: u64, } impl HotFields { @@ -74,17 +98,49 @@ impl HotFields { log_level: cfg.general.log_level.clone(), ad_tag: cfg.general.ad_tag.clone(), dns_overrides: cfg.network.dns_overrides.clone(), - middle_proxy_pool_size: cfg.general.middle_proxy_pool_size, desync_all_full: cfg.general.desync_all_full, update_every_secs: cfg.general.effective_update_every_secs(), + me_reinit_every_secs: cfg.general.me_reinit_every_secs, + me_reinit_singleflight: cfg.general.me_reinit_singleflight, + me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms, hardswap: cfg.general.hardswap, me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs, me_pool_min_fresh_ratio: cfg.general.me_pool_min_fresh_ratio, me_reinit_drain_timeout_secs: cfg.general.me_reinit_drain_timeout_secs, - me_keepalive_enabled: cfg.general.me_keepalive_enabled, - me_keepalive_interval_secs: cfg.general.me_keepalive_interval_secs, - me_keepalive_jitter_secs: cfg.general.me_keepalive_jitter_secs, - me_keepalive_payload_random: cfg.general.me_keepalive_payload_random, + me_hardswap_warmup_delay_min_ms: cfg.general.me_hardswap_warmup_delay_min_ms, + me_hardswap_warmup_delay_max_ms: cfg.general.me_hardswap_warmup_delay_max_ms, + me_hardswap_warmup_extra_passes: cfg.general.me_hardswap_warmup_extra_passes, + me_hardswap_warmup_pass_backoff_base_ms: cfg + .general + .me_hardswap_warmup_pass_backoff_base_ms, + me_bind_stale_mode: cfg.general.me_bind_stale_mode, + me_bind_stale_ttl_secs: cfg.general.me_bind_stale_ttl_secs, + me_secret_atomic_snapshot: cfg.general.me_secret_atomic_snapshot, + me_deterministic_writer_sort: cfg.general.me_deterministic_writer_sort, + me_single_endpoint_shadow_writers: cfg.general.me_single_endpoint_shadow_writers, + me_single_endpoint_outage_mode_enabled: cfg + .general + .me_single_endpoint_outage_mode_enabled, + me_single_endpoint_outage_disable_quarantine: cfg + .general + .me_single_endpoint_outage_disable_quarantine, + me_single_endpoint_outage_backoff_min_ms: cfg + .general + .me_single_endpoint_outage_backoff_min_ms, + me_single_endpoint_outage_backoff_max_ms: cfg + .general + .me_single_endpoint_outage_backoff_max_ms, + me_single_endpoint_shadow_rotate_every_secs: cfg + .general + .me_single_endpoint_shadow_rotate_every_secs, + me_config_stable_snapshots: cfg.general.me_config_stable_snapshots, + me_config_apply_cooldown_secs: cfg.general.me_config_apply_cooldown_secs, + me_snapshot_require_http_2xx: cfg.general.me_snapshot_require_http_2xx, + me_snapshot_reject_empty_map: cfg.general.me_snapshot_reject_empty_map, + me_snapshot_min_proxy_for_lines: cfg.general.me_snapshot_min_proxy_for_lines, + proxy_secret_stable_snapshots: cfg.general.proxy_secret_stable_snapshots, + proxy_secret_rotate_runtime: cfg.general.proxy_secret_rotate_runtime, + proxy_secret_len_max: cfg.general.proxy_secret_len_max, telemetry_core_enabled: cfg.general.telemetry.core_enabled, telemetry_user_enabled: cfg.general.telemetry.user_enabled, telemetry_me_level: cfg.general.telemetry.me_level, @@ -100,16 +156,149 @@ impl HotFields { me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, - access: cfg.access.clone(), + users: cfg.access.users.clone(), + user_ad_tags: cfg.access.user_ad_tags.clone(), + user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(), + user_expirations: cfg.access.user_expirations.clone(), + user_data_quota: cfg.access.user_data_quota.clone(), + user_max_unique_ips: cfg.access.user_max_unique_ips.clone(), + user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode, + user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs, } } } // ── Helpers ─────────────────────────────────────────────────────────────────── +fn canonicalize_json(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(map) => { + let mut pairs: Vec<(String, serde_json::Value)> = + std::mem::take(map).into_iter().collect(); + pairs.sort_by(|a, b| a.0.cmp(&b.0)); + for (_, item) in pairs.iter_mut() { + canonicalize_json(item); + } + for (key, item) in pairs { + map.insert(key, item); + } + } + serde_json::Value::Array(items) => { + for item in items { + canonicalize_json(item); + } + } + _ => {} + } +} + +fn config_equal(lhs: &ProxyConfig, rhs: &ProxyConfig) -> bool { + let mut left = match serde_json::to_value(lhs) { + Ok(value) => value, + Err(_) => return false, + }; + let mut right = match serde_json::to_value(rhs) { + Ok(value) => value, + Err(_) => return false, + }; + canonicalize_json(&mut left); + canonicalize_json(&mut right); + left == right +} + +fn listeners_equal( + lhs: &[crate::config::ListenerConfig], + rhs: &[crate::config::ListenerConfig], +) -> bool { + if lhs.len() != rhs.len() { + return false; + } + lhs.iter().zip(rhs.iter()).all(|(a, b)| { + a.ip == b.ip + && a.announce == b.announce + && a.announce_ip == b.announce_ip + && a.proxy_protocol == b.proxy_protocol + && a.reuse_allow == b.reuse_allow + }) +} + +fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { + let mut cfg = old.clone(); + + cfg.general.log_level = new.general.log_level.clone(); + cfg.general.ad_tag = new.general.ad_tag.clone(); + cfg.network.dns_overrides = new.network.dns_overrides.clone(); + cfg.general.desync_all_full = new.general.desync_all_full; + cfg.general.update_every = new.general.update_every; + cfg.general.proxy_secret_auto_reload_secs = new.general.proxy_secret_auto_reload_secs; + cfg.general.proxy_config_auto_reload_secs = new.general.proxy_config_auto_reload_secs; + cfg.general.me_reinit_every_secs = new.general.me_reinit_every_secs; + cfg.general.me_reinit_singleflight = new.general.me_reinit_singleflight; + cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms; + cfg.general.hardswap = new.general.hardswap; + cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs; + cfg.general.me_pool_min_fresh_ratio = new.general.me_pool_min_fresh_ratio; + cfg.general.me_reinit_drain_timeout_secs = new.general.me_reinit_drain_timeout_secs; + cfg.general.me_hardswap_warmup_delay_min_ms = new.general.me_hardswap_warmup_delay_min_ms; + cfg.general.me_hardswap_warmup_delay_max_ms = new.general.me_hardswap_warmup_delay_max_ms; + cfg.general.me_hardswap_warmup_extra_passes = new.general.me_hardswap_warmup_extra_passes; + cfg.general.me_hardswap_warmup_pass_backoff_base_ms = + new.general.me_hardswap_warmup_pass_backoff_base_ms; + cfg.general.me_bind_stale_mode = new.general.me_bind_stale_mode; + cfg.general.me_bind_stale_ttl_secs = new.general.me_bind_stale_ttl_secs; + cfg.general.me_secret_atomic_snapshot = new.general.me_secret_atomic_snapshot; + cfg.general.me_deterministic_writer_sort = new.general.me_deterministic_writer_sort; + cfg.general.me_single_endpoint_shadow_writers = new.general.me_single_endpoint_shadow_writers; + cfg.general.me_single_endpoint_outage_mode_enabled = + new.general.me_single_endpoint_outage_mode_enabled; + cfg.general.me_single_endpoint_outage_disable_quarantine = + new.general.me_single_endpoint_outage_disable_quarantine; + cfg.general.me_single_endpoint_outage_backoff_min_ms = + new.general.me_single_endpoint_outage_backoff_min_ms; + cfg.general.me_single_endpoint_outage_backoff_max_ms = + new.general.me_single_endpoint_outage_backoff_max_ms; + cfg.general.me_single_endpoint_shadow_rotate_every_secs = + new.general.me_single_endpoint_shadow_rotate_every_secs; + cfg.general.me_config_stable_snapshots = new.general.me_config_stable_snapshots; + cfg.general.me_config_apply_cooldown_secs = new.general.me_config_apply_cooldown_secs; + cfg.general.me_snapshot_require_http_2xx = new.general.me_snapshot_require_http_2xx; + cfg.general.me_snapshot_reject_empty_map = new.general.me_snapshot_reject_empty_map; + cfg.general.me_snapshot_min_proxy_for_lines = new.general.me_snapshot_min_proxy_for_lines; + cfg.general.proxy_secret_stable_snapshots = new.general.proxy_secret_stable_snapshots; + cfg.general.proxy_secret_rotate_runtime = new.general.proxy_secret_rotate_runtime; + cfg.general.proxy_secret_len_max = new.general.proxy_secret_len_max; + cfg.general.telemetry = new.general.telemetry.clone(); + cfg.general.me_socks_kdf_policy = new.general.me_socks_kdf_policy; + cfg.general.me_floor_mode = new.general.me_floor_mode; + cfg.general.me_adaptive_floor_idle_secs = new.general.me_adaptive_floor_idle_secs; + cfg.general.me_adaptive_floor_min_writers_single_endpoint = + new.general.me_adaptive_floor_min_writers_single_endpoint; + cfg.general.me_adaptive_floor_recover_grace_secs = + new.general.me_adaptive_floor_recover_grace_secs; + cfg.general.me_route_backpressure_base_timeout_ms = + new.general.me_route_backpressure_base_timeout_ms; + cfg.general.me_route_backpressure_high_timeout_ms = + new.general.me_route_backpressure_high_timeout_ms; + cfg.general.me_route_backpressure_high_watermark_pct = + new.general.me_route_backpressure_high_watermark_pct; + + cfg.access.users = new.access.users.clone(); + cfg.access.user_ad_tags = new.access.user_ad_tags.clone(); + cfg.access.user_max_tcp_conns = new.access.user_max_tcp_conns.clone(); + cfg.access.user_expirations = new.access.user_expirations.clone(); + cfg.access.user_data_quota = new.access.user_data_quota.clone(); + cfg.access.user_max_unique_ips = new.access.user_max_unique_ips.clone(); + cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode; + cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs; + + cfg +} + /// Warn if any non-hot fields changed (require restart). -fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { +fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: bool) { + let mut warned = false; if old.server.port != new.server.port { + warned = true; warn!( "config reload: server.port changed ({} → {}); restart required", old.server.port, new.server.port @@ -125,23 +314,81 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { != new.server.api.minimal_runtime_cache_ttl_ms || old.server.api.read_only != new.server.api.read_only { + warned = true; warn!("config reload: server.api changed; restart required"); } + if old.server.proxy_protocol != new.server.proxy_protocol + || !listeners_equal(&old.server.listeners, &new.server.listeners) + || old.server.listen_addr_ipv4 != new.server.listen_addr_ipv4 + || old.server.listen_addr_ipv6 != new.server.listen_addr_ipv6 + || old.server.listen_tcp != new.server.listen_tcp + || old.server.listen_unix_sock != new.server.listen_unix_sock + || old.server.listen_unix_sock_perm != new.server.listen_unix_sock_perm + { + warned = true; + warn!("config reload: server listener settings changed; restart required"); + } + if old.censorship.tls_domain != new.censorship.tls_domain + || old.censorship.tls_domains != new.censorship.tls_domains + || old.censorship.mask != new.censorship.mask + || old.censorship.mask_host != new.censorship.mask_host + || old.censorship.mask_port != new.censorship.mask_port + || old.censorship.mask_unix_sock != new.censorship.mask_unix_sock + || old.censorship.fake_cert_len != new.censorship.fake_cert_len + || old.censorship.tls_emulation != new.censorship.tls_emulation + || old.censorship.tls_front_dir != new.censorship.tls_front_dir + || old.censorship.server_hello_delay_min_ms != new.censorship.server_hello_delay_min_ms + || old.censorship.server_hello_delay_max_ms != new.censorship.server_hello_delay_max_ms + || old.censorship.tls_new_session_tickets != new.censorship.tls_new_session_tickets + || old.censorship.tls_full_cert_ttl_secs != new.censorship.tls_full_cert_ttl_secs + || old.censorship.alpn_enforce != new.censorship.alpn_enforce + || old.censorship.mask_proxy_protocol != new.censorship.mask_proxy_protocol + { + warned = true; + warn!("config reload: censorship settings changed; restart required"); + } if old.censorship.tls_domain != new.censorship.tls_domain { + warned = true; warn!( "config reload: censorship.tls_domain changed ('{}' → '{}'); restart required", old.censorship.tls_domain, new.censorship.tls_domain ); } if old.network.ipv4 != new.network.ipv4 || old.network.ipv6 != new.network.ipv6 { + warned = true; warn!("config reload: network.ipv4/ipv6 changed; restart required"); } + if old.network.prefer != new.network.prefer + || old.network.multipath != new.network.multipath + || old.network.stun_use != new.network.stun_use + || old.network.stun_servers != new.network.stun_servers + || old.network.stun_tcp_fallback != new.network.stun_tcp_fallback + || old.network.http_ip_detect_urls != new.network.http_ip_detect_urls + || old.network.cache_public_ip_path != new.network.cache_public_ip_path + { + warned = true; + warn!("config reload: non-hot network settings changed; restart required"); + } if old.general.use_middle_proxy != new.general.use_middle_proxy { + warned = true; warn!("config reload: use_middle_proxy changed; restart required"); } if old.general.stun_nat_probe_concurrency != new.general.stun_nat_probe_concurrency { + warned = true; warn!("config reload: general.stun_nat_probe_concurrency changed; restart required"); } + if old.general.middle_proxy_pool_size != new.general.middle_proxy_pool_size { + warned = true; + warn!("config reload: general.middle_proxy_pool_size changed; restart required"); + } + if old.general.me_keepalive_enabled != new.general.me_keepalive_enabled + || old.general.me_keepalive_interval_secs != new.general.me_keepalive_interval_secs + || old.general.me_keepalive_jitter_secs != new.general.me_keepalive_jitter_secs + || old.general.me_keepalive_payload_random != new.general.me_keepalive_payload_random + { + warned = true; + warn!("config reload: general.me_keepalive_* changed; restart required"); + } if old.general.upstream_connect_retry_attempts != new.general.upstream_connect_retry_attempts || old.general.upstream_connect_retry_backoff_ms != new.general.upstream_connect_retry_backoff_ms @@ -151,8 +398,12 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig) { != new.general.upstream_connect_failfast_hard_errors || old.general.rpc_proxy_req_every != new.general.rpc_proxy_req_every { + warned = true; warn!("config reload: general.upstream_* changed; restart required"); } + if non_hot_changed && !warned { + warn!("config reload: one or more non-hot fields changed; restart required"); + } } /// Resolve the public host for link generation — mirrors the logic in main.rs. @@ -235,10 +486,10 @@ fn log_changes( log_tx.send(new_hot.log_level.clone()).ok(); } - if old_hot.access.user_ad_tags != new_hot.access.user_ad_tags { + if old_hot.user_ad_tags != new_hot.user_ad_tags { info!( "config reload: user_ad_tags updated ({} entries)", - new_hot.access.user_ad_tags.len(), + new_hot.user_ad_tags.len(), ); } @@ -253,13 +504,6 @@ fn log_changes( ); } - if old_hot.middle_proxy_pool_size != new_hot.middle_proxy_pool_size { - info!( - "config reload: middle_proxy_pool_size: {} → {}", - old_hot.middle_proxy_pool_size, new_hot.middle_proxy_pool_size, - ); - } - if old_hot.desync_all_full != new_hot.desync_all_full { info!( "config reload: desync_all_full: {} → {}", @@ -273,6 +517,17 @@ fn log_changes( old_hot.update_every_secs, new_hot.update_every_secs, ); } + if old_hot.me_reinit_every_secs != new_hot.me_reinit_every_secs + || old_hot.me_reinit_singleflight != new_hot.me_reinit_singleflight + || old_hot.me_reinit_coalesce_window_ms != new_hot.me_reinit_coalesce_window_ms + { + info!( + "config reload: me_reinit: interval={}s singleflight={} coalesce={}ms", + new_hot.me_reinit_every_secs, + new_hot.me_reinit_singleflight, + new_hot.me_reinit_coalesce_window_ms + ); + } if old_hot.hardswap != new_hot.hardswap { info!( @@ -301,18 +556,84 @@ fn log_changes( old_hot.me_reinit_drain_timeout_secs, new_hot.me_reinit_drain_timeout_secs, ); } - - if old_hot.me_keepalive_enabled != new_hot.me_keepalive_enabled - || old_hot.me_keepalive_interval_secs != new_hot.me_keepalive_interval_secs - || old_hot.me_keepalive_jitter_secs != new_hot.me_keepalive_jitter_secs - || old_hot.me_keepalive_payload_random != new_hot.me_keepalive_payload_random + if old_hot.me_hardswap_warmup_delay_min_ms != new_hot.me_hardswap_warmup_delay_min_ms + || old_hot.me_hardswap_warmup_delay_max_ms != new_hot.me_hardswap_warmup_delay_max_ms + || old_hot.me_hardswap_warmup_extra_passes != new_hot.me_hardswap_warmup_extra_passes + || old_hot.me_hardswap_warmup_pass_backoff_base_ms + != new_hot.me_hardswap_warmup_pass_backoff_base_ms { info!( - "config reload: me_keepalive: enabled={} interval={}s jitter={}s random_payload={}", - new_hot.me_keepalive_enabled, - new_hot.me_keepalive_interval_secs, - new_hot.me_keepalive_jitter_secs, - new_hot.me_keepalive_payload_random, + "config reload: me_hardswap_warmup: min={}ms max={}ms extra_passes={} pass_backoff={}ms", + new_hot.me_hardswap_warmup_delay_min_ms, + new_hot.me_hardswap_warmup_delay_max_ms, + new_hot.me_hardswap_warmup_extra_passes, + new_hot.me_hardswap_warmup_pass_backoff_base_ms + ); + } + if old_hot.me_bind_stale_mode != new_hot.me_bind_stale_mode + || old_hot.me_bind_stale_ttl_secs != new_hot.me_bind_stale_ttl_secs + { + info!( + "config reload: me_bind_stale: mode={:?} ttl={}s", + new_hot.me_bind_stale_mode, + new_hot.me_bind_stale_ttl_secs + ); + } + if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot + || old_hot.me_deterministic_writer_sort != new_hot.me_deterministic_writer_sort + { + info!( + "config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={}", + new_hot.me_secret_atomic_snapshot, + new_hot.me_deterministic_writer_sort + ); + } + if old_hot.me_single_endpoint_shadow_writers != new_hot.me_single_endpoint_shadow_writers + || old_hot.me_single_endpoint_outage_mode_enabled + != new_hot.me_single_endpoint_outage_mode_enabled + || old_hot.me_single_endpoint_outage_disable_quarantine + != new_hot.me_single_endpoint_outage_disable_quarantine + || old_hot.me_single_endpoint_outage_backoff_min_ms + != new_hot.me_single_endpoint_outage_backoff_min_ms + || old_hot.me_single_endpoint_outage_backoff_max_ms + != new_hot.me_single_endpoint_outage_backoff_max_ms + || old_hot.me_single_endpoint_shadow_rotate_every_secs + != new_hot.me_single_endpoint_shadow_rotate_every_secs + { + info!( + "config reload: me_single_endpoint: shadow={} outage_enabled={} disable_quarantine={} backoff=[{}..{}]ms rotate={}s", + new_hot.me_single_endpoint_shadow_writers, + new_hot.me_single_endpoint_outage_mode_enabled, + new_hot.me_single_endpoint_outage_disable_quarantine, + new_hot.me_single_endpoint_outage_backoff_min_ms, + new_hot.me_single_endpoint_outage_backoff_max_ms, + new_hot.me_single_endpoint_shadow_rotate_every_secs + ); + } + if old_hot.me_config_stable_snapshots != new_hot.me_config_stable_snapshots + || old_hot.me_config_apply_cooldown_secs != new_hot.me_config_apply_cooldown_secs + || old_hot.me_snapshot_require_http_2xx != new_hot.me_snapshot_require_http_2xx + || old_hot.me_snapshot_reject_empty_map != new_hot.me_snapshot_reject_empty_map + || old_hot.me_snapshot_min_proxy_for_lines != new_hot.me_snapshot_min_proxy_for_lines + { + info!( + "config reload: me_snapshot_guard: stable={} cooldown={}s require_2xx={} reject_empty={} min_proxy_for={}", + new_hot.me_config_stable_snapshots, + new_hot.me_config_apply_cooldown_secs, + new_hot.me_snapshot_require_http_2xx, + new_hot.me_snapshot_reject_empty_map, + new_hot.me_snapshot_min_proxy_for_lines + ); + } + if old_hot.proxy_secret_stable_snapshots != new_hot.proxy_secret_stable_snapshots + || old_hot.proxy_secret_rotate_runtime != new_hot.proxy_secret_rotate_runtime + || old_hot.proxy_secret_len_max != new_hot.proxy_secret_len_max + { + info!( + "config reload: proxy_secret_runtime: stable={} rotate={} len_max={}", + new_hot.proxy_secret_stable_snapshots, + new_hot.proxy_secret_rotate_runtime, + new_hot.proxy_secret_len_max ); } @@ -367,21 +688,21 @@ fn log_changes( ); } - if old_hot.access.users != new_hot.access.users { - let mut added: Vec<&String> = new_hot.access.users.keys() - .filter(|u| !old_hot.access.users.contains_key(*u)) + if old_hot.users != new_hot.users { + let mut added: Vec<&String> = new_hot.users.keys() + .filter(|u| !old_hot.users.contains_key(*u)) .collect(); added.sort(); - let mut removed: Vec<&String> = old_hot.access.users.keys() - .filter(|u| !new_hot.access.users.contains_key(*u)) + let mut removed: Vec<&String> = old_hot.users.keys() + .filter(|u| !new_hot.users.contains_key(*u)) .collect(); removed.sort(); - let mut changed: Vec<&String> = new_hot.access.users.keys() + let mut changed: Vec<&String> = new_hot.users.keys() .filter(|u| { - old_hot.access.users.get(*u) - .map(|s| s != &new_hot.access.users[*u]) + old_hot.users.get(*u) + .map(|s| s != &new_hot.users[*u]) .unwrap_or(false) }) .collect(); @@ -395,7 +716,7 @@ fn log_changes( let host = resolve_link_host(new_cfg, detected_ip_v4, detected_ip_v6); let port = new_cfg.general.links.public_port.unwrap_or(new_cfg.server.port); for user in &added { - if let Some(secret) = new_hot.access.users.get(*user) { + if let Some(secret) = new_hot.users.get(*user) { print_user_links(user, secret, &host, port, new_cfg); } } @@ -414,38 +735,38 @@ fn log_changes( } } - if old_hot.access.user_max_tcp_conns != new_hot.access.user_max_tcp_conns { + if old_hot.user_max_tcp_conns != new_hot.user_max_tcp_conns { info!( "config reload: user_max_tcp_conns updated ({} entries)", - new_hot.access.user_max_tcp_conns.len() + new_hot.user_max_tcp_conns.len() ); } - if old_hot.access.user_expirations != new_hot.access.user_expirations { + if old_hot.user_expirations != new_hot.user_expirations { info!( "config reload: user_expirations updated ({} entries)", - new_hot.access.user_expirations.len() + new_hot.user_expirations.len() ); } - if old_hot.access.user_data_quota != new_hot.access.user_data_quota { + if old_hot.user_data_quota != new_hot.user_data_quota { info!( "config reload: user_data_quota updated ({} entries)", - new_hot.access.user_data_quota.len() + new_hot.user_data_quota.len() ); } - if old_hot.access.user_max_unique_ips != new_hot.access.user_max_unique_ips { + if old_hot.user_max_unique_ips != new_hot.user_max_unique_ips { info!( "config reload: user_max_unique_ips updated ({} entries)", - new_hot.access.user_max_unique_ips.len() + new_hot.user_max_unique_ips.len() ); } - if old_hot.access.user_max_unique_ips_mode != new_hot.access.user_max_unique_ips_mode - || old_hot.access.user_max_unique_ips_window_secs - != new_hot.access.user_max_unique_ips_window_secs + if old_hot.user_max_unique_ips_mode != new_hot.user_max_unique_ips_mode + || old_hot.user_max_unique_ips_window_secs + != new_hot.user_max_unique_ips_window_secs { info!( "config reload: user_max_unique_ips policy mode={:?} window={}s", - new_hot.access.user_max_unique_ips_mode, - new_hot.access.user_max_unique_ips_window_secs + new_hot.user_max_unique_ips_mode, + new_hot.user_max_unique_ips_window_secs ); } } @@ -472,15 +793,22 @@ fn reload_config( } let old_cfg = config_tx.borrow().clone(); + let applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg); let old_hot = HotFields::from_config(&old_cfg); - let new_hot = HotFields::from_config(&new_cfg); + let applied_hot = HotFields::from_config(&applied_cfg); + let non_hot_changed = !config_equal(&applied_cfg, &new_cfg); + let hot_changed = old_hot != applied_hot; - if old_hot == new_hot { + if non_hot_changed { + warn_non_hot_changes(&old_cfg, &new_cfg, non_hot_changed); + } + + if !hot_changed { return; } - if old_hot.dns_overrides != new_hot.dns_overrides - && let Err(e) = crate::network::dns_overrides::install_entries(&new_hot.dns_overrides) + if old_hot.dns_overrides != applied_hot.dns_overrides + && let Err(e) = crate::network::dns_overrides::install_entries(&applied_hot.dns_overrides) { error!( "config reload: invalid network.dns_overrides: {}; keeping old config", @@ -489,9 +817,15 @@ fn reload_config( return; } - warn_non_hot_changes(&old_cfg, &new_cfg); - log_changes(&old_hot, &new_hot, &new_cfg, log_tx, detected_ip_v4, detected_ip_v6); - config_tx.send(Arc::new(new_cfg)).ok(); + log_changes( + &old_hot, + &applied_hot, + &applied_cfg, + log_tx, + detected_ip_v4, + detected_ip_v6, + ); + config_tx.send(Arc::new(applied_cfg)).ok(); } // ── Public API ──────────────────────────────────────────────────────────────── @@ -617,3 +951,80 @@ pub fn spawn_config_watcher( (config_rx, log_rx) } + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_config() -> ProxyConfig { + ProxyConfig::default() + } + + #[test] + fn overlay_applies_hot_and_preserves_non_hot() { + let old = sample_config(); + let mut new = old.clone(); + new.general.hardswap = !old.general.hardswap; + new.server.port = old.server.port.saturating_add(1); + + let applied = overlay_hot_fields(&old, &new); + assert_eq!(applied.general.hardswap, new.general.hardswap); + assert_eq!(applied.server.port, old.server.port); + } + + #[test] + fn non_hot_only_change_does_not_change_hot_snapshot() { + let old = sample_config(); + let mut new = old.clone(); + new.server.port = old.server.port.saturating_add(1); + + let applied = overlay_hot_fields(&old, &new); + assert_eq!(HotFields::from_config(&old), HotFields::from_config(&applied)); + assert_eq!(applied.server.port, old.server.port); + } + + #[test] + fn bind_stale_mode_is_hot() { + let old = sample_config(); + let mut new = old.clone(); + new.general.me_bind_stale_mode = match old.general.me_bind_stale_mode { + MeBindStaleMode::Never => MeBindStaleMode::Ttl, + MeBindStaleMode::Ttl => MeBindStaleMode::Always, + MeBindStaleMode::Always => MeBindStaleMode::Never, + }; + + let applied = overlay_hot_fields(&old, &new); + assert_eq!( + applied.general.me_bind_stale_mode, + new.general.me_bind_stale_mode + ); + assert_ne!(HotFields::from_config(&old), HotFields::from_config(&applied)); + } + + #[test] + fn keepalive_is_not_hot() { + let old = sample_config(); + let mut new = old.clone(); + new.general.me_keepalive_interval_secs = old.general.me_keepalive_interval_secs + 5; + + let applied = overlay_hot_fields(&old, &new); + assert_eq!( + applied.general.me_keepalive_interval_secs, + old.general.me_keepalive_interval_secs + ); + assert_eq!(HotFields::from_config(&old), HotFields::from_config(&applied)); + } + + #[test] + fn mixed_hot_and_non_hot_change_applies_only_hot_subset() { + let old = sample_config(); + let mut new = old.clone(); + new.general.hardswap = !old.general.hardswap; + new.general.use_middle_proxy = !old.general.use_middle_proxy; + + let applied = overlay_hot_fields(&old, &new); + assert_eq!(applied.general.hardswap, new.general.hardswap); + assert_eq!(applied.general.use_middle_proxy, old.general.use_middle_proxy); + assert!(!config_equal(&applied, &new)); + } +} From 651f257a5db66c5aa5bdb432d60c076f8d157a4c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:30:29 +0300 Subject: [PATCH 065/261] Update API.md Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- docs/API.md | 137 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 8 deletions(-) diff --git a/docs/API.md b/docs/API.md index bd8f892..50cfb4e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -13,13 +13,18 @@ API runtime is configured in `[server.api]`. | `listen` | `string` (`IP:PORT`) | `127.0.0.1:9091` | API bind address. | | `whitelist` | `CIDR[]` | `127.0.0.1/32, ::1/128` | Source IP allowlist. Empty list means allow all. | | `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. | -| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. | +| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. | | `minimal_runtime_enabled` | `bool` | `false` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. | -| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache. | +| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. | | `read_only` | `bool` | `false` | Disables mutating endpoints. | `server.admin_api` is accepted as an alias for backward compatibility. +Runtime validation for API config: +- `server.api.listen` must be a valid `IP:PORT`. +- `server.api.request_body_limit_bytes` must be `> 0`. +- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`. + ## Protocol Contract | Item | Value | @@ -51,6 +56,21 @@ API runtime is configured in `[server.api]`. } ``` +## Request Processing Order + +Requests are processed in this order: +1. `api_enabled` gate (`503 api_disabled` if disabled). +2. Source IP whitelist gate (`403 forbidden`). +3. `Authorization` header gate when configured (`401 unauthorized`). +4. Route and method matching (`404 not_found` or `405 method_not_allowed`). +5. `read_only` gate for mutating routes (`403 read_only`). +6. Request body read/limit/JSON decode (`413 payload_too_large`, `400 bad_request`). +7. Business validation and config write path. + +Notes: +- Whitelist is evaluated against the direct TCP peer IP (`SocketAddr::ip`), without `X-Forwarded-For` support. +- `Authorization` check is exact string equality against configured `auth_header`. + ## Endpoint Matrix | Method | Path | Body | Success | `data` contract | @@ -58,6 +78,7 @@ API runtime is configured in `[server.api]`. | `GET` | `/v1/health` | none | `200` | `HealthData` | | `GET` | `/v1/stats/summary` | none | `200` | `SummaryData` | | `GET` | `/v1/stats/zero/all` | none | `200` | `ZeroAllData` | +| `GET` | `/v1/stats/upstreams` | none | `200` | `UpstreamsData` | | `GET` | `/v1/stats/minimal/all` | none | `200` | `MinimalAllData` | | `GET` | `/v1/stats/me-writers` | none | `200` | `MeWritersData` | | `GET` | `/v1/stats/dcs` | none | `200` | `DcStatusData` | @@ -67,7 +88,7 @@ API runtime is configured in `[server.api]`. | `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` | | `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` | `UserInfo` | | `DELETE` | `/v1/users/{username}` | none | `200` | `string` (deleted username) | -| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` | `CreateUserResponse` | +| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `404` | `ErrorResponse` (`not_found`, current runtime behavior) | ## Common Error Codes @@ -77,8 +98,8 @@ API runtime is configured in `[server.api]`. | `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. | | `403` | `forbidden` | Source IP is not allowed by whitelist. | | `403` | `read_only` | Mutating endpoint called while `read_only=true`. | -| `404` | `not_found` | Unknown route or unknown user. | -| `405` | `method_not_allowed` | Unsupported method for an existing user route. | +| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route (including current `rotate-secret` route). | +| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. | | `409` | `revision_conflict` | `If-Match` revision mismatch. | | `409` | `user_exists` | User already exists on create. | | `409` | `last_user_forbidden` | Attempt to delete last configured user. | @@ -86,6 +107,28 @@ API runtime is configured in `[server.api]`. | `500` | `internal_error` | Internal error (I/O, serialization, config load/save). | | `503` | `api_disabled` | API disabled in config. | +## Routing and Method Edge Cases + +| Case | Behavior | +| --- | --- | +| Path matching | Exact match on `req.uri().path()`. Query string does not affect route matching. | +| Trailing slash | Not normalized. Example: `/v1/users/` is `404`. | +| Username route with extra slash | `/v1/users/{username}/...` is not treated as user route and returns `404`. | +| `PUT /v1/users/{username}` | `405 method_not_allowed`. | +| `POST /v1/users/{username}` | `404 not_found`. | +| `POST /v1/users/{username}/rotate-secret` | `404 not_found` in current release due route matcher limitation. | + +## Body and JSON Semantics + +- Request body is read only for mutating routes that define a body contract. +- Body size limit is enforced during streaming read (`413 payload_too_large`). +- Invalid transport body frame returns `400 bad_request` (`Invalid request body`). +- Invalid JSON returns `400 bad_request` (`Invalid JSON body`). +- `Content-Type` is not required for JSON parsing. +- Unknown JSON fields are ignored by deserialization. +- `PATCH` updates only provided fields and does not support explicit clearing of optional fields. +- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed. + ## Request Contracts ### `CreateUserRequest` @@ -114,6 +157,8 @@ API runtime is configured in `[server.api]`. | --- | --- | --- | --- | | `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. | +Note: the request contract is defined, but the corresponding route currently returns `404` (see routing edge cases). + ## Response Data Contracts ### `HealthData` @@ -173,6 +218,47 @@ API runtime is configured in `[server.api]`. | `connect_duration_fail_bucket_501_1000ms` | `u64` | Failed connects 501-1000 ms. | | `connect_duration_fail_bucket_gt_1000ms` | `u64` | Failed connects >1000 ms. | +### `UpstreamsData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Runtime upstream snapshot availability according to API config. | +| `reason` | `string?` | `feature_disabled` or `source_unavailable` when runtime snapshot is unavailable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation time. | +| `zero` | `ZeroUpstreamData` | Always available zero-cost upstream counters block. | +| `summary` | `UpstreamSummaryData?` | Runtime upstream aggregate view, null when unavailable. | +| `upstreams` | `UpstreamStatus[]?` | Per-upstream runtime status rows, null when unavailable. | + +#### `UpstreamSummaryData` +| Field | Type | Description | +| --- | --- | --- | +| `configured_total` | `usize` | Total configured upstream entries. | +| `healthy_total` | `usize` | Upstreams currently marked healthy. | +| `unhealthy_total` | `usize` | Upstreams currently marked unhealthy. | +| `direct_total` | `usize` | Number of direct upstream entries. | +| `socks4_total` | `usize` | Number of SOCKS4 upstream entries. | +| `socks5_total` | `usize` | Number of SOCKS5 upstream entries. | + +#### `UpstreamStatus` +| Field | Type | Description | +| --- | --- | --- | +| `upstream_id` | `usize` | Runtime upstream index. | +| `route_kind` | `string` | Upstream route kind: `direct`, `socks4`, `socks5`. | +| `address` | `string` | Upstream address (`direct` for direct route kind). Authentication fields are intentionally omitted. | +| `weight` | `u16` | Selection weight. | +| `scopes` | `string` | Configured scope selector string. | +| `healthy` | `bool` | Current health flag. | +| `fails` | `u32` | Consecutive fail counter. | +| `last_check_age_secs` | `u64` | Seconds since the last health-check update. | +| `effective_latency_ms` | `f64?` | Effective upstream latency used by selector. | +| `dc` | `UpstreamDcStatus[]` | Per-DC latency/IP preference snapshot. | + +#### `UpstreamDcStatus` +| Field | Type | Description | +| --- | --- | --- | +| `dc` | `i16` | Telegram DC id. | +| `latency_ema_ms` | `f64?` | Per-DC latency EMA value. | +| `ip_preference` | `string` | Per-DC IP family preference: `unknown`, `prefer_v4`, `prefer_v6`, `both_work`, `unavailable`. | + #### `ZeroMiddleProxyData` | Field | Type | Description | | --- | --- | --- | @@ -392,8 +478,11 @@ API runtime is configured in `[server.api]`. Link generation uses active config and enabled modes: - `[general.links].public_host/public_port` have priority. +- If `public_host` is not set, startup-detected public IPs are used (`IPv4`, `IPv6`, or both when available). - Fallback host sources: listener `announce`, `announce_ip`, explicit listener `ip`. - Legacy fallback: `listen_addr_ipv4` and `listen_addr_ipv6` when routable. +- Startup-detected IPs are fixed for process lifetime and refreshed on restart. +- User rows are sorted by `username` in ascending lexical order. ### `CreateUserResponse` | Field | Type | Description | @@ -407,21 +496,53 @@ Link generation uses active config and enabled modes: | --- | --- | | `POST /v1/users` | Creates user and validates resulting config before atomic save. | | `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. | -| `POST /v1/users/{username}/rotate-secret` | Replaces secret. Empty body is allowed and auto-generates secret. | +| `POST /v1/users/{username}/rotate-secret` | Currently returns `404` in runtime route matcher; request schema is reserved for intended behavior. | | `DELETE /v1/users/{username}` | Deletes user and related optional settings. Last user deletion is blocked. | All mutating endpoints: - Respect `read_only` mode. - Accept optional `If-Match` for optimistic concurrency. - Return new `revision` after successful write. +- Use process-local mutation lock + atomic write (`tmp + rename`) for config persistence. + +## Runtime State Matrix + +| Endpoint | `minimal_runtime_enabled=false` | `minimal_runtime_enabled=true` + source unavailable | `minimal_runtime_enabled=true` + source available | +| --- | --- | --- | --- | +| `/v1/stats/minimal/all` | `enabled=false`, `reason=feature_disabled`, `data=null` | `enabled=true`, `reason=source_unavailable`, fallback `data` with disabled ME blocks | `enabled=true`, `reason` omitted, full payload | +| `/v1/stats/me-writers` | `middle_proxy_enabled=false`, `reason=feature_disabled` | `middle_proxy_enabled=false`, `reason=source_unavailable` | `middle_proxy_enabled=true`, runtime snapshot | +| `/v1/stats/dcs` | `middle_proxy_enabled=false`, `reason=feature_disabled` | `middle_proxy_enabled=false`, `reason=source_unavailable` | `middle_proxy_enabled=true`, runtime snapshot | +| `/v1/stats/upstreams` | `enabled=false`, `reason=feature_disabled`, `summary/upstreams` omitted, `zero` still present | `enabled=true`, `reason=source_unavailable`, `summary/upstreams` omitted, `zero` present | `enabled=true`, `reason` omitted, `summary/upstreams` present, `zero` present | + +`source_unavailable` conditions: +- ME endpoints: ME pool is absent (for example direct-only mode or failed ME initialization). +- Upstreams endpoint: non-blocking upstream snapshot lock is unavailable at request time. + +## Serialization Rules + +- Success responses always include `revision`. +- Error responses never include `revision`; they include `request_id`. +- Optional fields with `skip_serializing_if` are omitted when absent. +- Nullable payload fields may still be `null` where contract uses `?` (for example `UserInfo` option fields). +- For `/v1/stats/upstreams`, authentication details of SOCKS upstreams are intentionally omitted. ## Operational Notes | Topic | Details | | --- | --- | -| API startup | API binds only when `[server.api].enabled=true`. | -| Restart requirements | Changes in `server.api` settings require process restart. | +| API startup | API listener is spawned only when `[server.api].enabled=true`. | +| `listen` port `0` | API spawn is skipped when parsed listen port is `0` (treated as disabled bind target). | +| Bind failure | Failed API bind logs warning and API task exits (no auto-retry loop). | +| ME runtime status endpoints | `/v1/stats/me-writers`, `/v1/stats/dcs`, `/v1/stats/minimal/all` require `[server.api].minimal_runtime_enabled=true`; otherwise they return disabled payload with `reason=feature_disabled`. | +| Upstream runtime endpoint | `/v1/stats/upstreams` always returns `zero`, but runtime fields (`summary`, `upstreams`) require `[server.api].minimal_runtime_enabled=true`. | +| Restart requirements | `server.api` changes are restart-required for predictable behavior. | +| Hot-reload nuance | A pure `server.api`-only config change may not propagate through watcher broadcast; a mixed change (with hot fields) may propagate API flags while still warning that restart is required. | | Runtime apply path | Successful writes are picked up by existing config watcher/hot-reload path. | | Exposure | Built-in TLS/mTLS is not provided. Use loopback bind + reverse proxy if needed. | | Pagination | User list currently has no pagination/filtering. | | Serialization side effect | Config comments/manual formatting are not preserved on write. | + +## Known Limitations (Current Release) + +- `POST /v1/users/{username}/rotate-secret` is currently unreachable in route matcher and returns `404`. +- API runtime controls under `server.api` are documented as restart-required; hot-reload behavior for these fields is not strictly uniform in all change combinations. From 30b22fe2bf7ad486723b05b974acf3a1f1185632 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:40:04 +0300 Subject: [PATCH 066/261] Update README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 75feb38..8dfa659 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,12 @@ ***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist*** -**Telemt** is a fast, secure, and feature-rich server written in Rust: it fully implements the official Telegram proxy algo and adds many production-ready improvements such as connection pooling, replay protection, detailed statistics, masking from "prying" eyes +**Telemt** is a fast, secure, and feature-rich server written in Rust: it fully implements the official Telegram proxy algo and adds many production-ready improvements such as: +- ME Pool + Reader/Writer + Registry + Refill + Adaptive Floor + Trio-State + Generation Lifecycle +- [Full-covered API w/ management](https://github.com/telemt/telemt/blob/main/docs/API.md) +- Anti-Replay on Sliding Window +- Prometheus-format Metrics +- TLS-Fronting and TCP-Splicing for masking from "prying" eyes [**Telemt Chat in Telegram**](https://t.me/telemtrs) From ccfda10713796cdd70a67237c5d404547b51a956 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:43:07 +0300 Subject: [PATCH 067/261] ME2DC Fallback + ME Init Retries Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 8 + src/config/hot_reload.rs | 8 + src/config/load.rs | 19 +++ src/config/types.rs | 11 ++ src/main.rs | 325 +++++++++++++++++++++++---------------- 5 files changed, 236 insertions(+), 135 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 9ac6c53..b73013a 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -153,6 +153,14 @@ pub(crate) fn default_middle_proxy_warm_standby() -> usize { DEFAULT_MIDDLE_PROXY_WARM_STANDBY } +pub(crate) fn default_me_init_retry_attempts() -> u32 { + 0 +} + +pub(crate) fn default_me2dc_fallback() -> bool { + true +} + pub(crate) fn default_keepalive_interval() -> u64 { 8 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 6d88800..b03f83e 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -381,6 +381,14 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b warned = true; warn!("config reload: general.middle_proxy_pool_size changed; restart required"); } + if old.general.me_init_retry_attempts != new.general.me_init_retry_attempts { + warned = true; + warn!("config reload: general.me_init_retry_attempts changed; restart required"); + } + if old.general.me2dc_fallback != new.general.me2dc_fallback { + warned = true; + warn!("config reload: general.me2dc_fallback changed; restart required"); + } if old.general.me_keepalive_enabled != new.general.me_keepalive_enabled || old.general.me_keepalive_interval_secs != new.general.me_keepalive_interval_secs || old.general.me_keepalive_jitter_secs != new.general.me_keepalive_jitter_secs diff --git a/src/config/load.rs b/src/config/load.rs index 666c938..a2ee5f0 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -237,6 +237,12 @@ impl ProxyConfig { )); } + if config.general.me_init_retry_attempts > 1_000_000 { + return Err(ProxyError::Config( + "general.me_init_retry_attempts must be within [0, 1000000]".to_string(), + )); + } + if config.general.upstream_connect_retry_attempts == 0 { return Err(ProxyError::Config( "general.upstream_connect_retry_attempts must be > 0".to_string(), @@ -659,6 +665,14 @@ mod tests { cfg.general.me_reconnect_fast_retry_count, default_me_reconnect_fast_retry_count() ); + assert_eq!( + cfg.general.me_init_retry_attempts, + default_me_init_retry_attempts() + ); + assert_eq!( + cfg.general.me2dc_fallback, + default_me2dc_fallback() + ); assert_eq!( cfg.general.me_single_endpoint_shadow_writers, default_me_single_endpoint_shadow_writers() @@ -764,6 +778,11 @@ mod tests { general.me_reconnect_fast_retry_count, default_me_reconnect_fast_retry_count() ); + assert_eq!( + general.me_init_retry_attempts, + default_me_init_retry_attempts() + ); + assert_eq!(general.me2dc_fallback, default_me2dc_fallback()); assert_eq!( general.me_single_endpoint_shadow_writers, default_me_single_endpoint_shadow_writers() diff --git a/src/config/types.rs b/src/config/types.rs index 00260a8..5dc1c87 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -353,6 +353,15 @@ pub struct GeneralConfig { #[serde(default = "default_middle_proxy_warm_standby")] pub middle_proxy_warm_standby: usize, + /// Startup retries for Middle-End pool initialization before ME→Direct fallback. + /// 0 means unlimited retries. + #[serde(default = "default_me_init_retry_attempts")] + pub me_init_retry_attempts: u32, + + /// Allow fallback from Middle-End mode to direct DC when ME startup cannot be initialized. + #[serde(default = "default_me2dc_fallback")] + pub me2dc_fallback: bool, + /// Enable ME keepalive padding frames. #[serde(default = "default_true")] pub me_keepalive_enabled: bool, @@ -680,6 +689,8 @@ impl Default for GeneralConfig { stun_nat_probe_concurrency: default_stun_nat_probe_concurrency(), middle_proxy_pool_size: default_pool_size(), middle_proxy_warm_standby: default_middle_proxy_warm_standby(), + me_init_retry_attempts: default_me_init_retry_attempts(), + me2dc_fallback: default_me2dc_fallback(), me_keepalive_enabled: default_true(), me_keepalive_interval_secs: default_keepalive_interval(), me_keepalive_jitter_secs: default_keepalive_jitter(), diff --git a/src/main.rs b/src/main.rs index 6bff525..c4a9c37 100644 --- a/src/main.rs +++ b/src/main.rs @@ -416,7 +416,7 @@ async fn main() -> std::result::Result<(), Box> { log_probe_result(&probe, &decision); let prefer_ipv6 = decision.prefer_ipv6(); - let mut use_middle_proxy = config.general.use_middle_proxy && (decision.ipv4_me || decision.ipv6_me); + let mut use_middle_proxy = config.general.use_middle_proxy; let beobachten = Arc::new(BeobachtenStore::new()); let rng = Arc::new(SecureRandom::new()); @@ -443,9 +443,17 @@ async fn main() -> std::result::Result<(), Box> { // Connection concurrency limit let max_connections = Arc::new(Semaphore::new(10_000)); + let me2dc_fallback = config.general.me2dc_fallback; + let me_init_retry_attempts = config.general.me_init_retry_attempts; if use_middle_proxy && !decision.ipv4_me && !decision.ipv6_me { - warn!("No usable IP family for Middle Proxy detected; falling back to direct DC"); - use_middle_proxy = false; + if me2dc_fallback { + warn!("No usable IP family for Middle Proxy detected; falling back to direct DC"); + use_middle_proxy = false; + } else { + warn!( + "No usable IP family for Middle Proxy detected; me2dc_fallback=false, ME init retries stay active" + ); + } } // ===================================================================== @@ -475,142 +483,189 @@ async fn main() -> std::result::Result<(), Box> { // proxy-secret is from: https://core.telegram.org/getProxySecret // ============================================================= let proxy_secret_path = config.general.proxy_secret_path.as_deref(); - match crate::transport::middle_proxy::fetch_proxy_secret( - proxy_secret_path, - config.general.proxy_secret_len_max, - ) - .await - { - Ok(proxy_secret) => { - info!( - secret_len = proxy_secret.len(), - key_sig = format_args!( - "0x{:08x}", - if proxy_secret.len() >= 4 { - u32::from_le_bytes([ - proxy_secret[0], - proxy_secret[1], - proxy_secret[2], - proxy_secret[3], - ]) - } else { - 0 - } - ), - "Proxy-secret loaded" - ); + let pool_size = config.general.middle_proxy_pool_size.max(1); + let mut init_attempt: u32 = 0; + loop { + init_attempt = init_attempt.saturating_add(1); - // Load ME config (v4/v6) + default DC - let mut cfg_v4 = fetch_proxy_config( - "https://core.telegram.org/getProxyConfig", - ) - .await - .unwrap_or_default(); - let mut cfg_v6 = fetch_proxy_config( - "https://core.telegram.org/getProxyConfigV6", - ) - .await - .unwrap_or_default(); - - if cfg_v4.map.is_empty() { - cfg_v4.map = crate::protocol::constants::TG_MIDDLE_PROXIES_V4.clone(); - } - if cfg_v6.map.is_empty() { - cfg_v6.map = crate::protocol::constants::TG_MIDDLE_PROXIES_V6.clone(); - } - - let pool = MePool::new( - proxy_tag, - proxy_secret, - config.general.middle_proxy_nat_ip, - me_nat_probe, - None, - config.network.stun_servers.clone(), - config.general.stun_nat_probe_concurrency, - probe.detected_ipv6, - config.timeouts.me_one_retry, - config.timeouts.me_one_timeout_ms, - cfg_v4.map.clone(), - cfg_v6.map.clone(), - cfg_v4.default_dc.or(cfg_v6.default_dc), - decision.clone(), - Some(upstream_manager.clone()), - rng.clone(), - stats.clone(), - config.general.me_keepalive_enabled, - config.general.me_keepalive_interval_secs, - config.general.me_keepalive_jitter_secs, - config.general.me_keepalive_payload_random, - config.general.rpc_proxy_req_every, - config.general.me_warmup_stagger_enabled, - config.general.me_warmup_step_delay_ms, - config.general.me_warmup_step_jitter_ms, - config.general.me_reconnect_max_concurrent_per_dc, - config.general.me_reconnect_backoff_base_ms, - config.general.me_reconnect_backoff_cap_ms, - config.general.me_reconnect_fast_retry_count, - config.general.me_single_endpoint_shadow_writers, - config.general.me_single_endpoint_outage_mode_enabled, - config.general.me_single_endpoint_outage_disable_quarantine, - config.general.me_single_endpoint_outage_backoff_min_ms, - config.general.me_single_endpoint_outage_backoff_max_ms, - config.general.me_single_endpoint_shadow_rotate_every_secs, - config.general.me_floor_mode, - config.general.me_adaptive_floor_idle_secs, - config.general.me_adaptive_floor_min_writers_single_endpoint, - config.general.me_adaptive_floor_recover_grace_secs, - config.general.hardswap, - config.general.me_pool_drain_ttl_secs, - config.general.effective_me_pool_force_close_secs(), - config.general.me_pool_min_fresh_ratio, - config.general.me_hardswap_warmup_delay_min_ms, - config.general.me_hardswap_warmup_delay_max_ms, - config.general.me_hardswap_warmup_extra_passes, - config.general.me_hardswap_warmup_pass_backoff_base_ms, - config.general.me_bind_stale_mode, - config.general.me_bind_stale_ttl_secs, - config.general.me_secret_atomic_snapshot, - config.general.me_deterministic_writer_sort, - config.general.me_socks_kdf_policy, - config.general.me_route_backpressure_base_timeout_ms, - config.general.me_route_backpressure_high_timeout_ms, - config.general.me_route_backpressure_high_watermark_pct, - ); - - let pool_size = config.general.middle_proxy_pool_size.max(1); - loop { - match pool.init(pool_size, &rng).await { - Ok(()) => { - info!("Middle-End pool initialized successfully"); - - // Phase 4: Start health monitor - let pool_clone = pool.clone(); - let rng_clone = rng.clone(); - let min_conns = pool_size; - tokio::spawn(async move { - crate::transport::middle_proxy::me_health_monitor( - pool_clone, rng_clone, min_conns, - ) - .await; - }); - - break Some(pool); - } - Err(e) => { - warn!( - error = %e, - retry_in_secs = 2, - "ME pool is not ready yet; retrying startup initialization" - ); - pool.reset_stun_state(); - tokio::time::sleep(Duration::from_secs(2)).await; - } + let proxy_secret = match crate::transport::middle_proxy::fetch_proxy_secret( + proxy_secret_path, + config.general.proxy_secret_len_max, + ) + .await + { + Ok(proxy_secret) => proxy_secret, + Err(e) => { + let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; + if retries_limited && init_attempt >= me_init_retry_attempts { + error!( + error = %e, + attempt = init_attempt, + retry_limit = me_init_retry_attempts, + "ME startup retries exhausted while loading proxy-secret; falling back to direct mode" + ); + break None; } + + warn!( + error = %e, + attempt = init_attempt, + retry_limit = if me_init_retry_attempts == 0 { + String::from("unlimited") + } else { + me_init_retry_attempts.to_string() + }, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "Failed to fetch proxy-secret; retrying ME startup" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + continue; } + }; + + info!( + secret_len = proxy_secret.len(), + key_sig = format_args!( + "0x{:08x}", + if proxy_secret.len() >= 4 { + u32::from_le_bytes([ + proxy_secret[0], + proxy_secret[1], + proxy_secret[2], + proxy_secret[3], + ]) + } else { + 0 + } + ), + "Proxy-secret loaded" + ); + + // Load ME config (v4/v6) + default DC + let mut cfg_v4 = fetch_proxy_config( + "https://core.telegram.org/getProxyConfig", + ) + .await + .unwrap_or_default(); + let mut cfg_v6 = fetch_proxy_config( + "https://core.telegram.org/getProxyConfigV6", + ) + .await + .unwrap_or_default(); + + if cfg_v4.map.is_empty() { + cfg_v4.map = crate::protocol::constants::TG_MIDDLE_PROXIES_V4.clone(); } - Err(e) => { - error!(error = %e, "Failed to fetch proxy-secret. Falling back to direct mode."); - None + if cfg_v6.map.is_empty() { + cfg_v6.map = crate::protocol::constants::TG_MIDDLE_PROXIES_V6.clone(); + } + + let pool = MePool::new( + proxy_tag.clone(), + proxy_secret, + config.general.middle_proxy_nat_ip, + me_nat_probe, + None, + config.network.stun_servers.clone(), + config.general.stun_nat_probe_concurrency, + probe.detected_ipv6, + config.timeouts.me_one_retry, + config.timeouts.me_one_timeout_ms, + cfg_v4.map.clone(), + cfg_v6.map.clone(), + cfg_v4.default_dc.or(cfg_v6.default_dc), + decision.clone(), + Some(upstream_manager.clone()), + rng.clone(), + stats.clone(), + config.general.me_keepalive_enabled, + config.general.me_keepalive_interval_secs, + config.general.me_keepalive_jitter_secs, + config.general.me_keepalive_payload_random, + config.general.rpc_proxy_req_every, + config.general.me_warmup_stagger_enabled, + config.general.me_warmup_step_delay_ms, + config.general.me_warmup_step_jitter_ms, + config.general.me_reconnect_max_concurrent_per_dc, + config.general.me_reconnect_backoff_base_ms, + config.general.me_reconnect_backoff_cap_ms, + config.general.me_reconnect_fast_retry_count, + config.general.me_single_endpoint_shadow_writers, + config.general.me_single_endpoint_outage_mode_enabled, + config.general.me_single_endpoint_outage_disable_quarantine, + config.general.me_single_endpoint_outage_backoff_min_ms, + config.general.me_single_endpoint_outage_backoff_max_ms, + config.general.me_single_endpoint_shadow_rotate_every_secs, + config.general.me_floor_mode, + config.general.me_adaptive_floor_idle_secs, + config.general.me_adaptive_floor_min_writers_single_endpoint, + config.general.me_adaptive_floor_recover_grace_secs, + config.general.hardswap, + config.general.me_pool_drain_ttl_secs, + config.general.effective_me_pool_force_close_secs(), + config.general.me_pool_min_fresh_ratio, + config.general.me_hardswap_warmup_delay_min_ms, + config.general.me_hardswap_warmup_delay_max_ms, + config.general.me_hardswap_warmup_extra_passes, + config.general.me_hardswap_warmup_pass_backoff_base_ms, + config.general.me_bind_stale_mode, + config.general.me_bind_stale_ttl_secs, + config.general.me_secret_atomic_snapshot, + config.general.me_deterministic_writer_sort, + config.general.me_socks_kdf_policy, + config.general.me_route_backpressure_base_timeout_ms, + config.general.me_route_backpressure_high_timeout_ms, + config.general.me_route_backpressure_high_watermark_pct, + ); + + match pool.init(pool_size, &rng).await { + Ok(()) => { + info!( + attempt = init_attempt, + "Middle-End pool initialized successfully" + ); + + // Phase 4: Start health monitor + let pool_clone = pool.clone(); + let rng_clone = rng.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + pool_clone, rng_clone, min_conns, + ) + .await; + }); + + break Some(pool); + } + Err(e) => { + let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; + if retries_limited && init_attempt >= me_init_retry_attempts { + error!( + error = %e, + attempt = init_attempt, + retry_limit = me_init_retry_attempts, + "ME pool init retries exhausted; falling back to direct mode" + ); + break None; + } + + warn!( + error = %e, + attempt = init_attempt, + retry_limit = if me_init_retry_attempts == 0 { + String::from("unlimited") + } else { + me_init_retry_attempts.to_string() + }, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool is not ready yet; retrying startup initialization" + ); + pool.reset_stun_state(); + tokio::time::sleep(Duration::from_secs(2)).await; + } } } } else { From 02c6af49120cc20fa89097c4425a586de920e597 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:46:57 +0300 Subject: [PATCH 068/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d38431d..3a221f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.2.1" +version = "3.2.2" edition = "2024" [dependencies] From 565b4ee923be5821f1c69624d5316c5423174d57 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:21:11 +0300 Subject: [PATCH 069/261] Unique IP always in Metrics+API Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/model.rs | 1 + src/api/users.rs | 7 +++-- src/ip_tracker.rs | 77 ++++++++++++++++++++++++----------------------- src/metrics.rs | 21 ++++++++++++- 4 files changed, 66 insertions(+), 40 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index efe8ebb..09eebdc 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -369,6 +369,7 @@ pub(super) struct UserInfo { pub(super) max_unique_ips: Option, pub(super) current_connections: u64, pub(super) active_unique_ips: usize, + pub(super) recent_unique_ips: usize, pub(super) total_octets: u64, pub(super) links: UserLinks, } diff --git a/src/api/users.rs b/src/api/users.rs index 8e90c7f..32823f1 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -112,6 +112,7 @@ pub(super) async fn create_user( max_unique_ips: updated_limit, current_connections: 0, active_unique_ips: 0, + recent_unique_ips: 0, total_octets: 0, links: build_user_links( &cfg, @@ -300,7 +301,7 @@ pub(super) async fn users_from_config( startup_detected_ip_v4: Option, startup_detected_ip_v6: Option, ) -> Vec { - let ip_counts = ip_tracker + let active_ip_counts = ip_tracker .get_stats() .await .into_iter() @@ -309,6 +310,7 @@ pub(super) async fn users_from_config( let mut names = cfg.access.users.keys().cloned().collect::>(); names.sort(); + let recent_ip_counts = ip_tracker.get_recent_counts_for_users(&names).await; let mut users = Vec::with_capacity(names.len()); for username in names { @@ -340,7 +342,8 @@ pub(super) async fn users_from_config( data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(), max_unique_ips: cfg.access.user_max_unique_ips.get(&username).copied(), current_connections: stats.get_user_curr_connects(&username), - active_unique_ips: ip_counts.get(&username).copied().unwrap_or(0), + active_unique_ips: active_ip_counts.get(&username).copied().unwrap_or(0), + recent_unique_ips: recent_ip_counts.get(&username).copied().unwrap_or(0), total_octets: stats.get_user_total_octets(&username), links, username, diff --git a/src/ip_tracker.rs b/src/ip_tracker.rs index 626d591..3ff6042 100644 --- a/src/ip_tracker.rs +++ b/src/ip_tracker.rs @@ -67,22 +67,15 @@ impl UserIpTracker { let max_ips = self.max_ips.read().await; max_ips.get(username).copied() }; + let mode = *self.limit_mode.read().await; + let window = *self.limit_window.read().await; + let now = Instant::now(); let mut active_ips = self.active_ips.write().await; let user_active = active_ips .entry(username.to_string()) .or_insert_with(HashSet::new); - if limit.is_none() { - user_active.insert(ip); - return Ok(()); - } - - let limit = limit.unwrap_or_default(); - let mode = *self.limit_mode.read().await; - let window = *self.limit_window.read().await; - let now = Instant::now(); - let mut recent_ips = self.recent_ips.write().await; let user_recent = recent_ips .entry(username.to_string()) @@ -94,24 +87,26 @@ impl UserIpTracker { return Ok(()); } - let active_limit_reached = user_active.len() >= limit; - let recent_limit_reached = user_recent.len() >= limit; - let deny = match mode { - UserMaxUniqueIpsMode::ActiveWindow => active_limit_reached, - UserMaxUniqueIpsMode::TimeWindow => recent_limit_reached, - UserMaxUniqueIpsMode::Combined => active_limit_reached || recent_limit_reached, - }; + if let Some(limit) = limit { + let active_limit_reached = user_active.len() >= limit; + let recent_limit_reached = user_recent.len() >= limit; + let deny = match mode { + UserMaxUniqueIpsMode::ActiveWindow => active_limit_reached, + UserMaxUniqueIpsMode::TimeWindow => recent_limit_reached, + UserMaxUniqueIpsMode::Combined => active_limit_reached || recent_limit_reached, + }; - if deny { - return Err(format!( - "IP limit reached for user '{}': active={}/{} recent={}/{} mode={:?}", - username, - user_active.len(), - limit, - user_recent.len(), - limit, - mode - )); + if deny { + return Err(format!( + "IP limit reached for user '{}': active={}/{} recent={}/{} mode={:?}", + username, + user_active.len(), + limit, + user_recent.len(), + limit, + mode + )); + } } user_active.insert(ip); @@ -127,18 +122,26 @@ impl UserIpTracker { active_ips.remove(username); } } - drop(active_ips); + } - let mode = *self.limit_mode.read().await; - if matches!(mode, UserMaxUniqueIpsMode::ActiveWindow) { - let mut recent_ips = self.recent_ips.write().await; - if let Some(user_recent) = recent_ips.get_mut(username) { - user_recent.remove(&ip); - if user_recent.is_empty() { - recent_ips.remove(username); - } - } + pub async fn get_recent_counts_for_users(&self, users: &[String]) -> HashMap { + let window = *self.limit_window.read().await; + let now = Instant::now(); + let mut recent_ips = self.recent_ips.write().await; + + let mut counts = HashMap::with_capacity(users.len()); + for user in users { + let count = if let Some(user_recent) = recent_ips.get_mut(user) { + Self::prune_recent(user_recent, now, window); + user_recent.len() + } else { + 0 + }; + counts.insert(user.clone(), count); } + + recent_ips.retain(|_, user_recent| !user_recent.is_empty()); + counts } pub async fn get_active_ip_count(&self, username: &str) -> usize { diff --git a/src/metrics.rs b/src/metrics.rs index eae69d1..1595445 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1267,11 +1267,21 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp .collect(); let mut unique_users = BTreeSet::new(); + unique_users.extend(config.access.users.keys().cloned()); unique_users.extend(config.access.user_max_unique_ips.keys().cloned()); unique_users.extend(ip_counts.keys().cloned()); + let unique_users_vec: Vec = unique_users.iter().cloned().collect(); + let recent_counts = ip_tracker + .get_recent_counts_for_users(&unique_users_vec) + .await; let _ = writeln!(out, "# HELP telemt_user_unique_ips_current Per-user current number of unique active IPs"); let _ = writeln!(out, "# TYPE telemt_user_unique_ips_current gauge"); + let _ = writeln!( + out, + "# HELP telemt_user_unique_ips_recent_window Per-user unique IPs seen in configured observation window" + ); + let _ = writeln!(out, "# TYPE telemt_user_unique_ips_recent_window gauge"); let _ = writeln!(out, "# HELP telemt_user_unique_ips_limit Per-user configured unique IP limit (0 means unlimited)"); let _ = writeln!(out, "# TYPE telemt_user_unique_ips_limit gauge"); let _ = writeln!(out, "# HELP telemt_user_unique_ips_utilization Per-user unique IP usage ratio (0 for unlimited)"); @@ -1286,6 +1296,12 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp 0.0 }; let _ = writeln!(out, "telemt_user_unique_ips_current{{user=\"{}\"}} {}", user, current); + let _ = writeln!( + out, + "telemt_user_unique_ips_recent_window{{user=\"{}\"}} {}", + user, + recent_counts.get(&user).copied().unwrap_or(0) + ); let _ = writeln!(out, "telemt_user_unique_ips_limit{{user=\"{}\"}} {}", user, limit); let _ = writeln!( out, @@ -1378,6 +1394,7 @@ mod tests { assert!(output.contains("telemt_user_msgs_from_client{user=\"alice\"} 1")); assert!(output.contains("telemt_user_msgs_to_client{user=\"alice\"} 2")); assert!(output.contains("telemt_user_unique_ips_current{user=\"alice\"} 1")); + assert!(output.contains("telemt_user_unique_ips_recent_window{user=\"alice\"} 1")); assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 4")); assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.250000")); } @@ -1391,7 +1408,8 @@ mod tests { assert!(output.contains("telemt_connections_total 0")); assert!(output.contains("telemt_connections_bad_total 0")); assert!(output.contains("telemt_handshake_timeouts_total 0")); - assert!(!output.contains("user=")); + assert!(output.contains("telemt_user_unique_ips_current{user=")); + assert!(output.contains("telemt_user_unique_ips_recent_window{user=")); } #[tokio::test] @@ -1412,6 +1430,7 @@ mod tests { "# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge" )); assert!(output.contains("# TYPE telemt_user_unique_ips_current gauge")); + assert!(output.contains("# TYPE telemt_user_unique_ips_recent_window gauge")); assert!(output.contains("# TYPE telemt_user_unique_ips_limit gauge")); assert!(output.contains("# TYPE telemt_user_unique_ips_utilization gauge")); } From 0b1a8cd3f82dec0b20c01b2fbdd24244264ec0b2 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:41:41 +0300 Subject: [PATCH 070/261] IP Limit fixes --- src/api/model.rs | 4 ++ src/api/users.rs | 27 +++++++------ src/ip_tracker.rs | 96 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 108 insertions(+), 19 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index 09eebdc..2f6c58e 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -1,3 +1,5 @@ +use std::net::IpAddr; + use chrono::{DateTime, Utc}; use hyper::StatusCode; use rand::Rng; @@ -369,7 +371,9 @@ pub(super) struct UserInfo { pub(super) max_unique_ips: Option, pub(super) current_connections: u64, pub(super) active_unique_ips: usize, + pub(super) active_unique_ips_list: Vec, pub(super) recent_unique_ips: usize, + pub(super) recent_unique_ips_list: Vec, pub(super) total_octets: u64, pub(super) links: UserLinks, } diff --git a/src/api/users.rs b/src/api/users.rs index 32823f1..d156896 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::net::IpAddr; use hyper::StatusCode; @@ -112,7 +111,9 @@ pub(super) async fn create_user( max_unique_ips: updated_limit, current_connections: 0, active_unique_ips: 0, + active_unique_ips_list: Vec::new(), recent_unique_ips: 0, + recent_unique_ips_list: Vec::new(), total_octets: 0, links: build_user_links( &cfg, @@ -301,19 +302,21 @@ pub(super) async fn users_from_config( startup_detected_ip_v4: Option, startup_detected_ip_v6: Option, ) -> Vec { - let active_ip_counts = ip_tracker - .get_stats() - .await - .into_iter() - .map(|(user, count, _)| (user, count)) - .collect::>(); - let mut names = cfg.access.users.keys().cloned().collect::>(); names.sort(); - let recent_ip_counts = ip_tracker.get_recent_counts_for_users(&names).await; + let active_ip_lists = ip_tracker.get_active_ips_for_users(&names).await; + let recent_ip_lists = ip_tracker.get_recent_ips_for_users(&names).await; let mut users = Vec::with_capacity(names.len()); for username in names { + let active_ip_list = active_ip_lists + .get(&username) + .cloned() + .unwrap_or_else(Vec::new); + let recent_ip_list = recent_ip_lists + .get(&username) + .cloned() + .unwrap_or_else(Vec::new); let links = cfg .access .users @@ -342,8 +345,10 @@ pub(super) async fn users_from_config( data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(), max_unique_ips: cfg.access.user_max_unique_ips.get(&username).copied(), current_connections: stats.get_user_curr_connects(&username), - active_unique_ips: active_ip_counts.get(&username).copied().unwrap_or(0), - recent_unique_ips: recent_ip_counts.get(&username).copied().unwrap_or(0), + active_unique_ips: active_ip_list.len(), + active_unique_ips_list: active_ip_list, + recent_unique_ips: recent_ip_list.len(), + recent_unique_ips_list: recent_ip_list, total_octets: stats.get_user_total_octets(&username), links, username, diff --git a/src/ip_tracker.rs b/src/ip_tracker.rs index 3ff6042..492d642 100644 --- a/src/ip_tracker.rs +++ b/src/ip_tracker.rs @@ -2,7 +2,7 @@ #![allow(dead_code)] -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::net::IpAddr; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -13,7 +13,7 @@ use crate::config::UserMaxUniqueIpsMode; #[derive(Debug, Clone)] pub struct UserIpTracker { - active_ips: Arc>>>, + active_ips: Arc>>>, recent_ips: Arc>>>, max_ips: Arc>>, limit_mode: Arc>, @@ -74,7 +74,7 @@ impl UserIpTracker { let mut active_ips = self.active_ips.write().await; let user_active = active_ips .entry(username.to_string()) - .or_insert_with(HashSet::new); + .or_insert_with(HashMap::new); let mut recent_ips = self.recent_ips.write().await; let user_recent = recent_ips @@ -82,7 +82,8 @@ impl UserIpTracker { .or_insert_with(HashMap::new); Self::prune_recent(user_recent, now, window); - if user_active.contains(&ip) { + if let Some(count) = user_active.get_mut(&ip) { + *count = count.saturating_add(1); user_recent.insert(ip, now); return Ok(()); } @@ -109,7 +110,7 @@ impl UserIpTracker { } } - user_active.insert(ip); + user_active.insert(ip, 1); user_recent.insert(ip, now); Ok(()) } @@ -117,7 +118,13 @@ impl UserIpTracker { pub async fn remove_ip(&self, username: &str, ip: IpAddr) { let mut active_ips = self.active_ips.write().await; if let Some(user_ips) = active_ips.get_mut(username) { - user_ips.remove(&ip); + if let Some(count) = user_ips.get_mut(&ip) { + if *count > 1 { + *count -= 1; + } else { + user_ips.remove(&ip); + } + } if user_ips.is_empty() { active_ips.remove(username); } @@ -144,6 +151,41 @@ impl UserIpTracker { counts } + pub async fn get_active_ips_for_users(&self, users: &[String]) -> HashMap> { + let active_ips = self.active_ips.read().await; + let mut out = HashMap::with_capacity(users.len()); + for user in users { + let mut ips = active_ips + .get(user) + .map(|per_ip| per_ip.keys().copied().collect::>()) + .unwrap_or_else(Vec::new); + ips.sort(); + out.insert(user.clone(), ips); + } + out + } + + pub async fn get_recent_ips_for_users(&self, users: &[String]) -> HashMap> { + let window = *self.limit_window.read().await; + let now = Instant::now(); + let mut recent_ips = self.recent_ips.write().await; + + let mut out = HashMap::with_capacity(users.len()); + for user in users { + let mut ips = if let Some(user_recent) = recent_ips.get_mut(user) { + Self::prune_recent(user_recent, now, window); + user_recent.keys().copied().collect::>() + } else { + Vec::new() + }; + ips.sort(); + out.insert(user.clone(), ips); + } + + recent_ips.retain(|_, user_recent| !user_recent.is_empty()); + out + } + pub async fn get_active_ip_count(&self, username: &str) -> usize { let active_ips = self.active_ips.read().await; active_ips.get(username).map(|ips| ips.len()).unwrap_or(0) @@ -153,7 +195,7 @@ impl UserIpTracker { let active_ips = self.active_ips.read().await; active_ips .get(username) - .map(|ips| ips.iter().copied().collect()) + .map(|ips| ips.keys().copied().collect()) .unwrap_or_else(Vec::new) } @@ -193,7 +235,7 @@ impl UserIpTracker { let active_ips = self.active_ips.read().await; active_ips .get(username) - .map(|ips| ips.contains(&ip)) + .map(|ips| ips.contains_key(&ip)) .unwrap_or(false) } @@ -269,6 +311,26 @@ mod tests { assert_eq!(tracker.get_active_ip_count("test_user").await, 2); } + #[tokio::test] + async fn test_active_window_rejects_new_ip_and_keeps_existing_session() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("test_user", 1).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::ActiveWindow, 30) + .await; + + let ip1 = test_ipv4(10, 10, 10, 1); + let ip2 = test_ipv4(10, 10, 10, 2); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + assert!(tracker.is_ip_active("test_user", ip1).await); + assert!(tracker.check_and_add("test_user", ip2).await.is_err()); + + // Existing session remains active; only new unique IP is denied. + assert!(tracker.is_ip_active("test_user", ip1).await); + assert_eq!(tracker.get_active_ip_count("test_user").await, 1); + } + #[tokio::test] async fn test_reconnection_from_same_ip() { let tracker = UserIpTracker::new(); @@ -281,6 +343,24 @@ mod tests { assert_eq!(tracker.get_active_ip_count("test_user").await, 1); } + #[tokio::test] + async fn test_same_ip_disconnect_keeps_active_while_other_session_alive() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("test_user", 2).await; + + let ip1 = test_ipv4(192, 168, 1, 1); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + assert_eq!(tracker.get_active_ip_count("test_user").await, 1); + + tracker.remove_ip("test_user", ip1).await; + assert_eq!(tracker.get_active_ip_count("test_user").await, 1); + + tracker.remove_ip("test_user", ip1).await; + assert_eq!(tracker.get_active_ip_count("test_user").await, 0); + } + #[tokio::test] async fn test_ip_removal() { let tracker = UserIpTracker::new(); From 83cadc0bf348aeaebc06322af8b7a092c4984826 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:52:27 +0300 Subject: [PATCH 071/261] No lock-contention in ip-tracker --- src/ip_tracker.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ip_tracker.rs b/src/ip_tracker.rs index 492d642..5da8222 100644 --- a/src/ip_tracker.rs +++ b/src/ip_tracker.rs @@ -134,20 +134,20 @@ impl UserIpTracker { pub async fn get_recent_counts_for_users(&self, users: &[String]) -> HashMap { let window = *self.limit_window.read().await; let now = Instant::now(); - let mut recent_ips = self.recent_ips.write().await; + let recent_ips = self.recent_ips.read().await; let mut counts = HashMap::with_capacity(users.len()); for user in users { - let count = if let Some(user_recent) = recent_ips.get_mut(user) { - Self::prune_recent(user_recent, now, window); - user_recent.len() + let count = if let Some(user_recent) = recent_ips.get(user) { + user_recent + .values() + .filter(|seen_at| now.duration_since(**seen_at) <= window) + .count() } else { 0 }; counts.insert(user.clone(), count); } - - recent_ips.retain(|_, user_recent| !user_recent.is_empty()); counts } @@ -168,21 +168,22 @@ impl UserIpTracker { pub async fn get_recent_ips_for_users(&self, users: &[String]) -> HashMap> { let window = *self.limit_window.read().await; let now = Instant::now(); - let mut recent_ips = self.recent_ips.write().await; + let recent_ips = self.recent_ips.read().await; let mut out = HashMap::with_capacity(users.len()); for user in users { - let mut ips = if let Some(user_recent) = recent_ips.get_mut(user) { - Self::prune_recent(user_recent, now, window); - user_recent.keys().copied().collect::>() + let mut ips = if let Some(user_recent) = recent_ips.get(user) { + user_recent + .iter() + .filter(|(_, seen_at)| now.duration_since(**seen_at) <= window) + .map(|(ip, _)| *ip) + .collect::>() } else { Vec::new() }; ips.sort(); out.insert(user.clone(), ips); } - - recent_ips.retain(|_, user_recent| !user_recent.is_empty()); out } From 456495fd62ae481824e873591f08279b71ac07e6 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Thu, 5 Mar 2026 13:59:58 +0300 Subject: [PATCH 072/261] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8dfa659..2ae0ce6 100644 --- a/README.md +++ b/README.md @@ -118,8 +118,8 @@ We welcome ideas, architectural feedback, and pull requests. ## Quick Start Guide -### [Quick Start Guid RU](docs/QUICK_START_GUIDE.ru.md) -### [Quick Start Guid EN](docs/QUICK_START_GUIDE.en.md) +### [Quick Start Guide RU](docs/QUICK_START_GUIDE.ru.md) +### [Quick Start Guide EN](docs/QUICK_START_GUIDE.en.md) ### Advanced From 09bdafa718db0af448f597250e83cff273298413 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:39:32 +0300 Subject: [PATCH 073/261] Performance improvements --- src/config/defaults.rs | 16 ++ src/config/hot_reload.rs | 16 ++ src/config/load.rs | 61 ++++++ src/config/types.rs | 50 +++++ src/main.rs | 4 + src/metrics.rs | 65 +++++++ src/proxy/client.rs | 113 +++++------ src/proxy/direct_relay.rs | 12 +- src/stats/mod.rs | 50 +++++ src/transport/middle_proxy/pool.rs | 14 +- src/transport/middle_proxy/send.rs | 300 +++++++++++++++++++++++------ 11 files changed, 584 insertions(+), 117 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index b73013a..15be561 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -129,6 +129,10 @@ pub(crate) fn default_unknown_dc_log_path() -> Option { Some("unknown-dc.txt".to_string()) } +pub(crate) fn default_unknown_dc_file_log_enabled() -> bool { + false +} + pub(crate) fn default_pool_size() -> usize { 8 } @@ -273,6 +277,18 @@ pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 { 80 } +pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 { + 250 +} + +pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 { + 3 +} + +pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 { + 3000 +} + pub(crate) fn default_beobachten_minutes() -> u64 { 10 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index b03f83e..14939a0 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -381,6 +381,22 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b warned = true; warn!("config reload: general.middle_proxy_pool_size changed; restart required"); } + if old.general.me_route_no_writer_mode != new.general.me_route_no_writer_mode + || old.general.me_route_no_writer_wait_ms != new.general.me_route_no_writer_wait_ms + || old.general.me_route_inline_recovery_attempts + != new.general.me_route_inline_recovery_attempts + || old.general.me_route_inline_recovery_wait_ms + != new.general.me_route_inline_recovery_wait_ms + { + warned = true; + warn!("config reload: general.me_route_no_writer_* changed; restart required"); + } + if old.general.unknown_dc_log_path != new.general.unknown_dc_log_path + || old.general.unknown_dc_file_log_enabled != new.general.unknown_dc_file_log_enabled + { + warned = true; + warn!("config reload: general.unknown_dc_* changed; restart required"); + } if old.general.me_init_retry_attempts != new.general.me_init_retry_attempts { warned = true; warn!("config reload: general.me_init_retry_attempts changed; restart required"); diff --git a/src/config/load.rs b/src/config/load.rs index a2ee5f0..2954f04 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -410,6 +410,24 @@ impl ProxyConfig { )); } + if !(10..=5000).contains(&config.general.me_route_no_writer_wait_ms) { + return Err(ProxyError::Config( + "general.me_route_no_writer_wait_ms must be within [10, 5000]".to_string(), + )); + } + + if config.general.me_route_inline_recovery_attempts == 0 { + return Err(ProxyError::Config( + "general.me_route_inline_recovery_attempts must be > 0".to_string(), + )); + } + + if !(10..=30000).contains(&config.general.me_route_inline_recovery_wait_ms) { + return Err(ProxyError::Config( + "general.me_route_inline_recovery_wait_ms must be within [10, 30000]".to_string(), + )); + } + if config.server.api.request_body_limit_bytes == 0 { return Err(ProxyError::Config( "server.api.request_body_limit_bytes must be > 0".to_string(), @@ -1206,6 +1224,49 @@ mod tests { let _ = std::fs::remove_file(path_valid); } + #[test] + fn me_route_no_writer_wait_ms_out_of_range_is_rejected() { + let toml = r#" + [general] + me_route_no_writer_wait_ms = 5 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_route_no_writer_wait_ms_out_of_range_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.me_route_no_writer_wait_ms must be within [10, 5000]")); + let _ = std::fs::remove_file(path); + } + + #[test] + fn me_route_no_writer_mode_is_parsed() { + let toml = r#" + [general] + me_route_no_writer_mode = "inline_recovery_legacy" + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_route_no_writer_mode_parse_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert_eq!( + cfg.general.me_route_no_writer_mode, + crate::config::MeRouteNoWriterMode::InlineRecoveryLegacy + ); + let _ = std::fs::remove_file(path); + } + #[test] fn me_hardswap_warmup_defaults_are_set() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index 5dc1c87..a9eaeae 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -183,6 +183,31 @@ impl MeFloorMode { } } +/// Middle-End route behavior when no writer is immediately available. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum MeRouteNoWriterMode { + #[default] + AsyncRecoveryFailfast, + InlineRecoveryLegacy, +} + +impl MeRouteNoWriterMode { + pub fn as_u8(self) -> u8 { + match self { + MeRouteNoWriterMode::AsyncRecoveryFailfast => 0, + MeRouteNoWriterMode::InlineRecoveryLegacy => 1, + } + } + + pub fn from_u8(raw: u8) -> Self { + match raw { + 1 => MeRouteNoWriterMode::InlineRecoveryLegacy, + _ => MeRouteNoWriterMode::AsyncRecoveryFailfast, + } + } +} + /// Per-user unique source IP limit mode. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] @@ -511,6 +536,10 @@ pub struct GeneralConfig { #[serde(default = "default_unknown_dc_log_path")] pub unknown_dc_log_path: Option, + /// Enable unknown-DC file logging. + #[serde(default = "default_unknown_dc_file_log_enabled")] + pub unknown_dc_file_log_enabled: bool, + #[serde(default)] pub log_level: LogLevel, @@ -538,6 +567,22 @@ pub struct GeneralConfig { #[serde(default = "default_me_route_backpressure_high_watermark_pct")] pub me_route_backpressure_high_watermark_pct: u8, + /// ME route behavior when no writer is immediately available. + #[serde(default)] + pub me_route_no_writer_mode: MeRouteNoWriterMode, + + /// Maximum wait time in milliseconds for async-recovery failfast mode. + #[serde(default = "default_me_route_no_writer_wait_ms")] + pub me_route_no_writer_wait_ms: u64, + + /// Number of inline recovery attempts in legacy mode. + #[serde(default = "default_me_route_inline_recovery_attempts")] + pub me_route_inline_recovery_attempts: u32, + + /// Maximum wait time in milliseconds for inline recovery in legacy mode. + #[serde(default = "default_me_route_inline_recovery_wait_ms")] + pub me_route_inline_recovery_wait_ms: u64, + /// [general.links] — proxy link generation overrides. #[serde(default)] pub links: LinksConfig, @@ -719,6 +764,7 @@ impl Default for GeneralConfig { upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(), stun_iface_mismatch_ignore: false, unknown_dc_log_path: default_unknown_dc_log_path(), + unknown_dc_file_log_enabled: default_unknown_dc_file_log_enabled(), log_level: LogLevel::Normal, disable_colors: false, telemetry: TelemetryConfig::default(), @@ -726,6 +772,10 @@ impl Default for GeneralConfig { me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(), me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(), me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(), + me_route_no_writer_mode: MeRouteNoWriterMode::default(), + me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(), + me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(), + me_route_inline_recovery_wait_ms: default_me_route_inline_recovery_wait_ms(), links: LinksConfig::default(), crypto_pending_buffer: default_crypto_pending_buffer(), max_client_frame: default_max_client_frame(), diff --git a/src/main.rs b/src/main.rs index c4a9c37..fc7cf0e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -617,6 +617,10 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_route_backpressure_base_timeout_ms, config.general.me_route_backpressure_high_timeout_ms, config.general.me_route_backpressure_high_watermark_pct, + config.general.me_route_no_writer_mode, + config.general.me_route_no_writer_wait_ms, + config.general.me_route_inline_recovery_attempts, + config.general.me_route_inline_recovery_wait_ms, ); match pool.init(pool_size, &rng).await { diff --git a/src/metrics.rs b/src/metrics.rs index 1595445..0ccec94 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1199,6 +1199,48 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp 0 } ); + let _ = writeln!( + out, + "# HELP telemt_me_no_writer_failfast_total ME route failfast errors due to missing writer in bounded wait window" + ); + let _ = writeln!(out, "# TYPE telemt_me_no_writer_failfast_total counter"); + let _ = writeln!( + out, + "telemt_me_no_writer_failfast_total {}", + if me_allows_normal { + stats.get_me_no_writer_failfast_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_async_recovery_trigger_total Async ME recovery trigger attempts from route path" + ); + let _ = writeln!(out, "# TYPE telemt_me_async_recovery_trigger_total counter"); + let _ = writeln!( + out, + "telemt_me_async_recovery_trigger_total {}", + if me_allows_normal { + stats.get_me_async_recovery_trigger_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_inline_recovery_total Legacy inline ME recovery attempts from route path" + ); + let _ = writeln!(out, "# TYPE telemt_me_inline_recovery_total counter"); + let _ = writeln!( + out, + "telemt_me_inline_recovery_total {}", + if me_allows_normal { + stats.get_me_inline_recovery_total() + } else { + 0 + } + ); let unresolved_writer_losses = if me_allows_normal { stats @@ -1237,6 +1279,29 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp let _ = writeln!(out, "# TYPE telemt_user_msgs_from_client counter"); let _ = writeln!(out, "# HELP telemt_user_msgs_to_client Per-user messages sent"); let _ = writeln!(out, "# TYPE telemt_user_msgs_to_client counter"); + let _ = writeln!( + out, + "# HELP telemt_ip_reservation_rollback_total IP reservation rollbacks caused by later limit checks" + ); + let _ = writeln!(out, "# TYPE telemt_ip_reservation_rollback_total counter"); + let _ = writeln!( + out, + "telemt_ip_reservation_rollback_total{{reason=\"tcp_limit\"}} {}", + if core_enabled { + stats.get_ip_reservation_rollback_tcp_limit_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_ip_reservation_rollback_total{{reason=\"quota_limit\"}} {}", + if core_enabled { + stats.get_ip_reservation_rollback_quota_limit_total() + } else { + 0 + } + ); let _ = writeln!( out, "# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag" diff --git a/src/proxy/client.rs b/src/proxy/client.rs index 4bc4b65..2c9fa0c 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -672,42 +672,16 @@ impl RunningClientHandler { R: AsyncRead + Unpin + Send + 'static, W: AsyncWrite + Unpin + Send + 'static, { - let user = &success.user; + let user = success.user.clone(); - if let Err(e) = Self::check_user_limits_static(user, &config, &stats, peer_addr, &ip_tracker).await { + if let Err(e) = Self::check_user_limits_static(&user, &config, &stats, peer_addr, &ip_tracker).await { warn!(user = %user, error = %e, "User limit exceeded"); return Err(e); } - // IP Cleanup Guard: автоматически удаляет IP при выходе из scope - struct IpCleanupGuard { - tracker: Arc, - user: String, - ip: std::net::IpAddr, - } - - impl Drop for IpCleanupGuard { - fn drop(&mut self) { - let tracker = self.tracker.clone(); - let user = self.user.clone(); - let ip = self.ip; - tokio::spawn(async move { - tracker.remove_ip(&user, ip).await; - debug!(user = %user, ip = %ip, "IP cleaned up on disconnect"); - }); - } - } - - let _cleanup = IpCleanupGuard { - tracker: ip_tracker, - user: user.clone(), - ip: peer_addr.ip(), - }; - - // Decide: middle proxy or direct - if config.general.use_middle_proxy { + let relay_result = if config.general.use_middle_proxy { if let Some(ref pool) = me_pool { - return handle_via_middle_proxy( + handle_via_middle_proxy( client_reader, client_writer, success, @@ -718,23 +692,38 @@ impl RunningClientHandler { local_addr, rng, ) - .await; + .await + } else { + warn!("use_middle_proxy=true but MePool not initialized, falling back to direct"); + handle_via_direct( + client_reader, + client_writer, + success, + upstream_manager, + stats, + config, + buffer_pool, + rng, + ) + .await } - warn!("use_middle_proxy=true but MePool not initialized, falling back to direct"); - } + } else { + // Direct mode (original behavior) + handle_via_direct( + client_reader, + client_writer, + success, + upstream_manager, + stats, + config, + buffer_pool, + rng, + ) + .await + }; - // Direct mode (original behavior) - handle_via_direct( - client_reader, - client_writer, - success, - upstream_manager, - stats, - config, - buffer_pool, - rng, - ) - .await + ip_tracker.remove_ip(&user, peer_addr.ip()).await; + relay_result } async fn check_user_limits_static( @@ -752,22 +741,32 @@ impl RunningClientHandler { }); } + let mut ip_reserved = false; // IP limit check - if let Err(reason) = ip_tracker.check_and_add(user, peer_addr.ip()).await { - warn!( - user = %user, - ip = %peer_addr.ip(), - reason = %reason, - "IP limit exceeded" - ); - return Err(ProxyError::ConnectionLimitExceeded { - user: user.to_string(), - }); + match ip_tracker.check_and_add(user, peer_addr.ip()).await { + Ok(()) => { + ip_reserved = true; + } + Err(reason) => { + warn!( + user = %user, + ip = %peer_addr.ip(), + reason = %reason, + "IP limit exceeded" + ); + return Err(ProxyError::ConnectionLimitExceeded { + user: user.to_string(), + }); + } } if let Some(limit) = config.access.user_max_tcp_conns.get(user) && stats.get_user_curr_connects(user) >= *limit as u64 { + if ip_reserved { + ip_tracker.remove_ip(user, peer_addr.ip()).await; + stats.increment_ip_reservation_rollback_tcp_limit_total(); + } return Err(ProxyError::ConnectionLimitExceeded { user: user.to_string(), }); @@ -776,6 +775,10 @@ impl RunningClientHandler { if let Some(quota) = config.access.user_data_quota.get(user) && stats.get_user_total_octets(user) >= *quota { + if ip_reserved { + ip_tracker.remove_ip(user, peer_addr.ip()).await; + stats.increment_ip_reservation_rollback_quota_limit_total(); + } return Err(ProxyError::DataQuotaExceeded { user: user.to_string(), }); diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index e50623d..a1f4945 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -118,10 +118,16 @@ fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result { // Unknown DC requested by client without override: log and fall back. if !config.dc_overrides.contains_key(&dc_key) { warn!(dc_idx = dc_idx, "Requested non-standard DC with no override; falling back to default cluster"); - if let Some(path) = &config.general.unknown_dc_log_path - && let Ok(mut file) = OpenOptions::new().create(true).append(true).open(path) + if config.general.unknown_dc_file_log_enabled + && let Some(path) = &config.general.unknown_dc_log_path + && let Ok(handle) = tokio::runtime::Handle::try_current() { - let _ = writeln!(file, "dc_idx={dc_idx}"); + let path = path.clone(); + handle.spawn_blocking(move || { + if let Ok(mut file) = OpenOptions::new().create(true).append(true).open(path) { + let _ = writeln!(file, "dc_idx={dc_idx}"); + } + }); } } diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 29d7f45..eedc7f6 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -100,6 +100,11 @@ pub struct Stats { me_refill_failed_total: AtomicU64, me_writer_restored_same_endpoint_total: AtomicU64, me_writer_restored_fallback_total: AtomicU64, + me_no_writer_failfast_total: AtomicU64, + me_async_recovery_trigger_total: AtomicU64, + me_inline_recovery_total: AtomicU64, + ip_reservation_rollback_tcp_limit_total: AtomicU64, + ip_reservation_rollback_quota_limit_total: AtomicU64, telemetry_core_enabled: AtomicBool, telemetry_user_enabled: AtomicBool, telemetry_me_level: AtomicU8, @@ -522,6 +527,34 @@ impl Stats { .fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_no_writer_failfast_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_no_writer_failfast_total.fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_async_recovery_trigger_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_async_recovery_trigger_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_inline_recovery_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_inline_recovery_total.fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_ip_reservation_rollback_tcp_limit_total(&self) { + if self.telemetry_core_enabled() { + self.ip_reservation_rollback_tcp_limit_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_ip_reservation_rollback_quota_limit_total(&self) { + if self.telemetry_core_enabled() { + self.ip_reservation_rollback_quota_limit_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_endpoint_quarantine_total(&self) { if self.telemetry_me_allows_normal() { self.me_endpoint_quarantine_total @@ -791,6 +824,23 @@ impl Stats { pub fn get_me_writer_restored_fallback_total(&self) -> u64 { self.me_writer_restored_fallback_total.load(Ordering::Relaxed) } + pub fn get_me_no_writer_failfast_total(&self) -> u64 { + self.me_no_writer_failfast_total.load(Ordering::Relaxed) + } + pub fn get_me_async_recovery_trigger_total(&self) -> u64 { + self.me_async_recovery_trigger_total.load(Ordering::Relaxed) + } + pub fn get_me_inline_recovery_total(&self) -> u64 { + self.me_inline_recovery_total.load(Ordering::Relaxed) + } + pub fn get_ip_reservation_rollback_tcp_limit_total(&self) -> u64 { + self.ip_reservation_rollback_tcp_limit_total + .load(Ordering::Relaxed) + } + pub fn get_ip_reservation_rollback_quota_limit_total(&self) -> u64 { + self.ip_reservation_rollback_quota_limit_total + .load(Ordering::Relaxed) + } pub fn increment_user_connects(&self, user: &str) { if !self.telemetry_user_enabled() { diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 8c185be..d553944 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, Notify, RwLock, mpsc}; use tokio_util::sync::CancellationToken; -use crate::config::{MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy}; +use crate::config::{MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy}; use crate::crypto::SecureRandom; use crate::network::IpFamily; use crate::network::probe::NetworkDecision; @@ -145,6 +145,10 @@ pub struct MePool { pub(super) secret_atomic_snapshot: AtomicBool, pub(super) me_deterministic_writer_sort: AtomicBool, pub(super) me_socks_kdf_policy: AtomicU8, + pub(super) me_route_no_writer_mode: AtomicU8, + pub(super) me_route_no_writer_wait: Duration, + pub(super) me_route_inline_recovery_attempts: u32, + pub(super) me_route_inline_recovery_wait: Duration, pool_size: usize, } @@ -227,6 +231,10 @@ impl MePool { me_route_backpressure_base_timeout_ms: u64, me_route_backpressure_high_timeout_ms: u64, me_route_backpressure_high_watermark_pct: u8, + me_route_no_writer_mode: MeRouteNoWriterMode, + me_route_no_writer_wait_ms: u64, + me_route_inline_recovery_attempts: u32, + me_route_inline_recovery_wait_ms: u64, ) -> Arc { let registry = Arc::new(ConnRegistry::new()); registry.update_route_backpressure_policy( @@ -343,6 +351,10 @@ impl MePool { secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot), me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort), me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()), + me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()), + me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), + me_route_inline_recovery_attempts, + me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms), }) } diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index ba4a419..8bd21ee 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -1,13 +1,14 @@ use std::cmp::Reverse; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::Ordering; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::sync::mpsc::error::TrySendError; use tracing::{debug, warn}; +use crate::config::MeRouteNoWriterMode; use crate::error::{ProxyError, Result}; use crate::network::IpFamily; use crate::protocol::constants::RPC_CLOSE_EXT_U32; @@ -49,7 +50,11 @@ impl MePool { our_addr, proto_flags, }; - let mut emergency_attempts = 0; + let no_writer_mode = + MeRouteNoWriterMode::from_u8(self.me_route_no_writer_mode.load(Ordering::Relaxed)); + let mut no_writer_deadline: Option = None; + let mut emergency_attempts = 0u32; + let mut async_recovery_triggered = false; loop { if let Some(current) = self.registry.get_writer(conn_id).await { @@ -74,34 +79,66 @@ impl MePool { let mut writers_snapshot = { let ws = self.writers.read().await; if ws.is_empty() { - // Create waiter before recovery attempts so notify_one permits are not missed. - let waiter = self.writer_available.notified(); drop(ws); - for family in self.family_order() { - let map = match family { - IpFamily::V4 => self.proxy_map_v4.read().await.clone(), - IpFamily::V6 => self.proxy_map_v6.read().await.clone(), - }; - for (_dc, addrs) in map.iter() { - for (ip, port) in addrs { - let addr = SocketAddr::new(*ip, *port); - if self.connect_one(addr, self.rng.as_ref()).await.is_ok() { - self.writer_available.notify_one(); + match no_writer_mode { + MeRouteNoWriterMode::AsyncRecoveryFailfast => { + let deadline = *no_writer_deadline.get_or_insert_with(|| { + Instant::now() + self.me_route_no_writer_wait + }); + if !async_recovery_triggered { + let triggered = + self.trigger_async_recovery_for_target_dc(target_dc).await; + if !triggered { + self.trigger_async_recovery_global().await; + } + async_recovery_triggered = true; + } + if self.wait_for_writer_until(deadline).await { + continue; + } + self.stats.increment_me_no_writer_failfast_total(); + return Err(ProxyError::Proxy( + "No ME writer available in failfast window".into(), + )); + } + MeRouteNoWriterMode::InlineRecoveryLegacy => { + self.stats.increment_me_inline_recovery_total(); + for _ in 0..self.me_route_inline_recovery_attempts.max(1) { + for family in self.family_order() { + let map = match family { + IpFamily::V4 => self.proxy_map_v4.read().await.clone(), + IpFamily::V6 => self.proxy_map_v6.read().await.clone(), + }; + for (_dc, addrs) in &map { + for (ip, port) in addrs { + let addr = SocketAddr::new(*ip, *port); + let _ = self.connect_one(addr, self.rng.as_ref()).await; + } + } + } + if !self.writers.read().await.is_empty() { break; } } - } - } - if !self.writers.read().await.is_empty() { - continue; - } - if tokio::time::timeout(Duration::from_secs(3), waiter).await.is_err() { - if !self.writers.read().await.is_empty() { + if !self.writers.read().await.is_empty() { + continue; + } + let waiter = self.writer_available.notified(); + if tokio::time::timeout(self.me_route_inline_recovery_wait, waiter) + .await + .is_err() + { + if !self.writers.read().await.is_empty() { + continue; + } + self.stats.increment_me_no_writer_failfast_total(); + return Err(ProxyError::Proxy( + "All ME connections dead (legacy wait timeout)".into(), + )); + } continue; } - return Err(ProxyError::Proxy("All ME connections dead (waited 3s)".into())); } - continue; } ws.clone() }; @@ -115,46 +152,70 @@ impl MePool { .await; } if candidate_indices.is_empty() { - // Emergency connect-on-demand - if emergency_attempts >= 3 { - return Err(ProxyError::Proxy("No ME writers available for target DC".into())); - } - emergency_attempts += 1; - for family in self.family_order() { - let map_guard = match family { - IpFamily::V4 => self.proxy_map_v4.read().await, - IpFamily::V6 => self.proxy_map_v6.read().await, - }; - if let Some(addrs) = map_guard.get(&(target_dc as i32)) { - let mut shuffled = addrs.clone(); - shuffled.shuffle(&mut rand::rng()); - drop(map_guard); - for (ip, port) in shuffled { - let addr = SocketAddr::new(ip, port); - if self.connect_one(addr, self.rng.as_ref()).await.is_ok() { - break; + match no_writer_mode { + MeRouteNoWriterMode::AsyncRecoveryFailfast => { + let deadline = *no_writer_deadline.get_or_insert_with(|| { + Instant::now() + self.me_route_no_writer_wait + }); + if !async_recovery_triggered { + let triggered = self.trigger_async_recovery_for_target_dc(target_dc).await; + if !triggered { + self.trigger_async_recovery_global().await; + } + async_recovery_triggered = true; + } + if self.wait_for_candidate_until(target_dc, deadline).await { + continue; + } + self.stats.increment_me_no_writer_failfast_total(); + return Err(ProxyError::Proxy( + "No ME writers available for target DC in failfast window".into(), + )); + } + MeRouteNoWriterMode::InlineRecoveryLegacy => { + self.stats.increment_me_inline_recovery_total(); + if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) { + self.stats.increment_me_no_writer_failfast_total(); + return Err(ProxyError::Proxy("No ME writers available for target DC".into())); + } + emergency_attempts += 1; + for family in self.family_order() { + let map_guard = match family { + IpFamily::V4 => self.proxy_map_v4.read().await, + IpFamily::V6 => self.proxy_map_v6.read().await, + }; + if let Some(addrs) = map_guard.get(&(target_dc as i32)) { + let mut shuffled = addrs.clone(); + shuffled.shuffle(&mut rand::rng()); + drop(map_guard); + for (ip, port) in shuffled { + let addr = SocketAddr::new(ip, port); + if self.connect_one(addr, self.rng.as_ref()).await.is_ok() { + break; + } + } + tokio::time::sleep(Duration::from_millis(100 * emergency_attempts as u64)).await; + let ws2 = self.writers.read().await; + writers_snapshot = ws2.clone(); + drop(ws2); + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .await; + if candidate_indices.is_empty() { + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .await; + } + if !candidate_indices.is_empty() { + break; + } } } - tokio::time::sleep(Duration::from_millis(100 * emergency_attempts)).await; - let ws2 = self.writers.read().await; - writers_snapshot = ws2.clone(); - drop(ws2); - candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, false) - .await; if candidate_indices.is_empty() { - candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, true) - .await; - } - if !candidate_indices.is_empty() { - break; + return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } } } - if candidate_indices.is_empty() { - return Err(ProxyError::Proxy("No ME writers available for target DC".into())); - } } let writer_idle_since = self.registry.writer_idle_since_snapshot().await; let now_epoch_secs = Self::now_epoch_secs(); @@ -275,6 +336,129 @@ impl MePool { } } + async fn wait_for_writer_until(&self, deadline: Instant) -> bool { + let waiter = self.writer_available.notified(); + if !self.writers.read().await.is_empty() { + return true; + } + let now = Instant::now(); + if now >= deadline { + return !self.writers.read().await.is_empty(); + } + let timeout = deadline.saturating_duration_since(now); + if tokio::time::timeout(timeout, waiter).await.is_ok() { + return true; + } + !self.writers.read().await.is_empty() + } + + async fn wait_for_candidate_until(&self, target_dc: i16, deadline: Instant) -> bool { + loop { + if self.has_candidate_for_target_dc(target_dc).await { + return true; + } + + let now = Instant::now(); + if now >= deadline { + return self.has_candidate_for_target_dc(target_dc).await; + } + + let remaining = deadline.saturating_duration_since(now); + let sleep_for = remaining.min(Duration::from_millis(25)); + let waiter = self.writer_available.notified(); + tokio::select! { + _ = waiter => {} + _ = tokio::time::sleep(sleep_for) => {} + } + } + } + + async fn has_candidate_for_target_dc(&self, target_dc: i16) -> bool { + let writers_snapshot = { + let ws = self.writers.read().await; + if ws.is_empty() { + return false; + } + ws.clone() + }; + let mut candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .await; + if candidate_indices.is_empty() { + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .await; + } + !candidate_indices.is_empty() + } + + async fn trigger_async_recovery_for_target_dc(self: &Arc, target_dc: i16) -> bool { + let endpoints = self.endpoint_candidates_for_target_dc(target_dc).await; + if endpoints.is_empty() { + return false; + } + self.stats.increment_me_async_recovery_trigger_total(); + for addr in endpoints.into_iter().take(8) { + self.trigger_immediate_refill(addr); + } + true + } + + async fn trigger_async_recovery_global(self: &Arc) { + self.stats.increment_me_async_recovery_trigger_total(); + let mut seen = HashSet::::new(); + for family in self.family_order() { + let map = match family { + IpFamily::V4 => self.proxy_map_v4.read().await.clone(), + IpFamily::V6 => self.proxy_map_v6.read().await.clone(), + }; + for addrs in map.values() { + for (ip, port) in addrs { + let addr = SocketAddr::new(*ip, *port); + if seen.insert(addr) { + self.trigger_immediate_refill(addr); + } + if seen.len() >= 8 { + return; + } + } + } + } + } + + async fn endpoint_candidates_for_target_dc(&self, target_dc: i16) -> Vec { + let key = target_dc as i32; + let mut preferred = Vec::::new(); + let mut seen = HashSet::::new(); + + for family in self.family_order() { + let map = match family { + IpFamily::V4 => self.proxy_map_v4.read().await.clone(), + IpFamily::V6 => self.proxy_map_v6.read().await.clone(), + }; + let mut lookup_keys = vec![key, key.abs(), -key.abs()]; + let def = self.default_dc.load(Ordering::Relaxed); + if def != 0 { + lookup_keys.push(def); + } + for lookup in lookup_keys { + if let Some(addrs) = map.get(&lookup) { + for (ip, port) in addrs { + let addr = SocketAddr::new(*ip, *port); + if seen.insert(addr) { + preferred.push(addr); + } + } + } + } + if !preferred.is_empty() && !self.decision.effective_multipath { + break; + } + } + + preferred + } + pub async fn send_close(self: &Arc, conn_id: u64) -> Result<()> { if let Some(w) = self.registry.get_writer(conn_id).await { let mut p = Vec::with_capacity(12); From 8066ea2163e8ae6720cce2151366a47b79357221 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:31:36 +0300 Subject: [PATCH 074/261] ME Pool Init fixes --- src/config/defaults.rs | 8 + src/config/hot_reload.rs | 6 + src/config/load.rs | 68 +++ src/config/types.rs | 10 + src/main.rs | 450 ++++++++++++------- src/transport/middle_proxy/config_updater.rs | 138 +++--- src/transport/middle_proxy/mod.rs | 6 +- 7 files changed, 461 insertions(+), 225 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 15be561..4b94be6 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -141,6 +141,14 @@ pub(crate) fn default_proxy_secret_path() -> Option { Some("proxy-secret".to_string()) } +pub(crate) fn default_proxy_config_v4_cache_path() -> Option { + Some("cache/proxy-config-v4.txt".to_string()) +} + +pub(crate) fn default_proxy_config_v6_cache_path() -> Option { + Some("cache/proxy-config-v6.txt".to_string()) +} + pub(crate) fn default_middle_proxy_nat_stun() -> Option { None } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 14939a0..97d5e4e 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -405,6 +405,12 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b warned = true; warn!("config reload: general.me2dc_fallback changed; restart required"); } + if old.general.proxy_config_v4_cache_path != new.general.proxy_config_v4_cache_path + || old.general.proxy_config_v6_cache_path != new.general.proxy_config_v6_cache_path + { + warned = true; + warn!("config reload: general.proxy_config_*_cache_path changed; restart required"); + } if old.general.me_keepalive_enabled != new.general.me_keepalive_enabled || old.general.me_keepalive_interval_secs != new.general.me_keepalive_interval_secs || old.general.me_keepalive_jitter_secs != new.general.me_keepalive_jitter_secs diff --git a/src/config/load.rs b/src/config/load.rs index 2954f04..9abab30 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -203,6 +203,22 @@ impl ProxyConfig { sanitize_ad_tag(&mut config.general.ad_tag); + if let Some(path) = &config.general.proxy_config_v4_cache_path + && path.trim().is_empty() + { + return Err(ProxyError::Config( + "general.proxy_config_v4_cache_path cannot be empty when provided".to_string(), + )); + } + + if let Some(path) = &config.general.proxy_config_v6_cache_path + && path.trim().is_empty() + { + return Err(ProxyError::Config( + "general.proxy_config_v6_cache_path cannot be empty when provided".to_string(), + )); + } + if let Some(update_every) = config.general.update_every { if update_every == 0 { return Err(ProxyError::Config( @@ -691,6 +707,14 @@ mod tests { cfg.general.me2dc_fallback, default_me2dc_fallback() ); + assert_eq!( + cfg.general.proxy_config_v4_cache_path, + default_proxy_config_v4_cache_path() + ); + assert_eq!( + cfg.general.proxy_config_v6_cache_path, + default_proxy_config_v6_cache_path() + ); assert_eq!( cfg.general.me_single_endpoint_shadow_writers, default_me_single_endpoint_shadow_writers() @@ -801,6 +825,14 @@ mod tests { default_me_init_retry_attempts() ); assert_eq!(general.me2dc_fallback, default_me2dc_fallback()); + assert_eq!( + general.proxy_config_v4_cache_path, + default_proxy_config_v4_cache_path() + ); + assert_eq!( + general.proxy_config_v6_cache_path, + default_proxy_config_v6_cache_path() + ); assert_eq!( general.me_single_endpoint_shadow_writers, default_me_single_endpoint_shadow_writers() @@ -1267,6 +1299,42 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn proxy_config_cache_paths_empty_are_rejected() { + let toml = r#" + [general] + proxy_config_v4_cache_path = " " + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_proxy_config_v4_cache_path_empty_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.proxy_config_v4_cache_path cannot be empty")); + let _ = std::fs::remove_file(path); + + let toml_v6 = r#" + [general] + proxy_config_v6_cache_path = "" + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let path_v6 = dir.join("telemt_proxy_config_v6_cache_path_empty_test.toml"); + std::fs::write(&path_v6, toml_v6).unwrap(); + let err_v6 = ProxyConfig::load(&path_v6).unwrap_err().to_string(); + assert!(err_v6.contains("general.proxy_config_v6_cache_path cannot be empty")); + let _ = std::fs::remove_file(path_v6); + } + #[test] fn me_hardswap_warmup_defaults_are_set() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index a9eaeae..4c19f8a 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -343,6 +343,14 @@ pub struct GeneralConfig { #[serde(default = "default_proxy_secret_path")] pub proxy_secret_path: Option, + /// Optional path to cache raw getProxyConfig (IPv4) snapshot for startup fallback. + #[serde(default = "default_proxy_config_v4_cache_path")] + pub proxy_config_v4_cache_path: Option, + + /// Optional path to cache raw getProxyConfigV6 snapshot for startup fallback. + #[serde(default = "default_proxy_config_v6_cache_path")] + pub proxy_config_v6_cache_path: Option, + /// Global ad_tag (32 hex chars from @MTProxybot). Fallback when user has no per-user tag in access.user_ad_tags. #[serde(default)] pub ad_tag: Option, @@ -727,6 +735,8 @@ impl Default for GeneralConfig { use_middle_proxy: default_true(), ad_tag: None, proxy_secret_path: default_proxy_secret_path(), + proxy_config_v4_cache_path: default_proxy_config_v4_cache_path(), + proxy_config_v6_cache_path: default_proxy_config_v6_cache_path(), middle_proxy_nat_ip: None, middle_proxy_nat_probe: default_true(), middle_proxy_nat_stun: default_middle_proxy_nat_stun(), diff --git a/src/main.rs b/src/main.rs index fc7cf0e..1da8123 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,8 +41,9 @@ use crate::stats::telemetry::TelemetryPolicy; use crate::stats::{ReplayChecker, Stats}; use crate::stream::BufferPool; use crate::transport::middle_proxy::{ - MePool, fetch_proxy_config, run_me_ping, MePingFamily, MePingSample, MeReinitTrigger, format_sample_line, - format_me_route, + MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line, + load_proxy_config_cache, run_me_ping, save_proxy_config_cache, MePingFamily, MePingSample, + MeReinitTrigger, }; use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes}; use crate::tls_front::TlsFrontCache; @@ -172,6 +173,120 @@ async fn write_beobachten_snapshot(path: &str, payload: &str) -> std::io::Result tokio::fs::write(path, payload).await } +async fn load_startup_proxy_config_snapshot( + url: &str, + cache_path: Option<&str>, + me2dc_fallback: bool, + label: &'static str, +) -> Option { + loop { + match fetch_proxy_config_with_raw(url).await { + Ok((cfg, raw)) => { + if !cfg.map.is_empty() { + if let Some(path) = cache_path + && let Err(e) = save_proxy_config_cache(path, &raw).await + { + warn!(error = %e, path, snapshot = label, "Failed to store startup proxy-config cache"); + } + return Some(cfg); + } + + warn!(snapshot = label, url, "Startup proxy-config is empty; trying disk cache"); + if let Some(path) = cache_path { + match load_proxy_config_cache(path).await { + Ok(cached) if !cached.map.is_empty() => { + info!( + snapshot = label, + path, + proxy_for_lines = cached.proxy_for_lines, + "Loaded startup proxy-config from disk cache" + ); + return Some(cached); + } + Ok(_) => { + warn!( + snapshot = label, + path, + "Startup proxy-config cache is empty; ignoring cache file" + ); + } + Err(cache_err) => { + debug!( + snapshot = label, + path, + error = %cache_err, + "Startup proxy-config cache unavailable" + ); + } + } + } + + if me2dc_fallback { + error!( + snapshot = label, + "Startup proxy-config unavailable and no saved config found; falling back to direct mode" + ); + return None; + } + + warn!( + snapshot = label, + retry_in_secs = 2, + "Startup proxy-config unavailable and no saved config found; retrying because me2dc_fallback=false" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + } + Err(fetch_err) => { + if let Some(path) = cache_path { + match load_proxy_config_cache(path).await { + Ok(cached) if !cached.map.is_empty() => { + info!( + snapshot = label, + path, + proxy_for_lines = cached.proxy_for_lines, + "Loaded startup proxy-config from disk cache" + ); + return Some(cached); + } + Ok(_) => { + warn!( + snapshot = label, + path, + "Startup proxy-config cache is empty; ignoring cache file" + ); + } + Err(cache_err) => { + debug!( + snapshot = label, + path, + error = %cache_err, + "Startup proxy-config cache unavailable" + ); + } + } + } + + if me2dc_fallback { + error!( + snapshot = label, + error = %fetch_err, + "Startup proxy-config unavailable and no cached data; falling back to direct mode" + ); + return None; + } + + warn!( + snapshot = label, + error = %fetch_err, + retry_in_secs = 2, + "Startup proxy-config unavailable; retrying because me2dc_fallback=false" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } +} + #[tokio::main] async fn main() -> std::result::Result<(), Box> { let (config_path, cli_silent, cli_log_level) = parse_cli(); @@ -484,193 +599,188 @@ async fn main() -> std::result::Result<(), Box> { // ============================================================= let proxy_secret_path = config.general.proxy_secret_path.as_deref(); let pool_size = config.general.middle_proxy_pool_size.max(1); - let mut init_attempt: u32 = 0; - loop { - init_attempt = init_attempt.saturating_add(1); - - let proxy_secret = match crate::transport::middle_proxy::fetch_proxy_secret( + let proxy_secret = loop { + match crate::transport::middle_proxy::fetch_proxy_secret( proxy_secret_path, config.general.proxy_secret_len_max, ) .await { - Ok(proxy_secret) => proxy_secret, + Ok(proxy_secret) => break Some(proxy_secret), Err(e) => { - let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; - if retries_limited && init_attempt >= me_init_retry_attempts { + if me2dc_fallback { error!( error = %e, - attempt = init_attempt, - retry_limit = me_init_retry_attempts, - "ME startup retries exhausted while loading proxy-secret; falling back to direct mode" + "ME startup failed: proxy-secret is unavailable and no saved secret found; falling back to direct mode" ); break None; } warn!( error = %e, - attempt = init_attempt, - retry_limit = if me_init_retry_attempts == 0 { - String::from("unlimited") - } else { - me_init_retry_attempts.to_string() - }, - me2dc_fallback = me2dc_fallback, retry_in_secs = 2, - "Failed to fetch proxy-secret; retrying ME startup" + "ME startup failed: proxy-secret is unavailable and no saved secret found; retrying because me2dc_fallback=false" ); tokio::time::sleep(Duration::from_secs(2)).await; - continue; - } - }; - - info!( - secret_len = proxy_secret.len(), - key_sig = format_args!( - "0x{:08x}", - if proxy_secret.len() >= 4 { - u32::from_le_bytes([ - proxy_secret[0], - proxy_secret[1], - proxy_secret[2], - proxy_secret[3], - ]) - } else { - 0 - } - ), - "Proxy-secret loaded" - ); - - // Load ME config (v4/v6) + default DC - let mut cfg_v4 = fetch_proxy_config( - "https://core.telegram.org/getProxyConfig", - ) - .await - .unwrap_or_default(); - let mut cfg_v6 = fetch_proxy_config( - "https://core.telegram.org/getProxyConfigV6", - ) - .await - .unwrap_or_default(); - - if cfg_v4.map.is_empty() { - cfg_v4.map = crate::protocol::constants::TG_MIDDLE_PROXIES_V4.clone(); - } - if cfg_v6.map.is_empty() { - cfg_v6.map = crate::protocol::constants::TG_MIDDLE_PROXIES_V6.clone(); - } - - let pool = MePool::new( - proxy_tag.clone(), - proxy_secret, - config.general.middle_proxy_nat_ip, - me_nat_probe, - None, - config.network.stun_servers.clone(), - config.general.stun_nat_probe_concurrency, - probe.detected_ipv6, - config.timeouts.me_one_retry, - config.timeouts.me_one_timeout_ms, - cfg_v4.map.clone(), - cfg_v6.map.clone(), - cfg_v4.default_dc.or(cfg_v6.default_dc), - decision.clone(), - Some(upstream_manager.clone()), - rng.clone(), - stats.clone(), - config.general.me_keepalive_enabled, - config.general.me_keepalive_interval_secs, - config.general.me_keepalive_jitter_secs, - config.general.me_keepalive_payload_random, - config.general.rpc_proxy_req_every, - config.general.me_warmup_stagger_enabled, - config.general.me_warmup_step_delay_ms, - config.general.me_warmup_step_jitter_ms, - config.general.me_reconnect_max_concurrent_per_dc, - config.general.me_reconnect_backoff_base_ms, - config.general.me_reconnect_backoff_cap_ms, - config.general.me_reconnect_fast_retry_count, - config.general.me_single_endpoint_shadow_writers, - config.general.me_single_endpoint_outage_mode_enabled, - config.general.me_single_endpoint_outage_disable_quarantine, - config.general.me_single_endpoint_outage_backoff_min_ms, - config.general.me_single_endpoint_outage_backoff_max_ms, - config.general.me_single_endpoint_shadow_rotate_every_secs, - config.general.me_floor_mode, - config.general.me_adaptive_floor_idle_secs, - config.general.me_adaptive_floor_min_writers_single_endpoint, - config.general.me_adaptive_floor_recover_grace_secs, - config.general.hardswap, - config.general.me_pool_drain_ttl_secs, - config.general.effective_me_pool_force_close_secs(), - config.general.me_pool_min_fresh_ratio, - config.general.me_hardswap_warmup_delay_min_ms, - config.general.me_hardswap_warmup_delay_max_ms, - config.general.me_hardswap_warmup_extra_passes, - config.general.me_hardswap_warmup_pass_backoff_base_ms, - config.general.me_bind_stale_mode, - config.general.me_bind_stale_ttl_secs, - config.general.me_secret_atomic_snapshot, - config.general.me_deterministic_writer_sort, - config.general.me_socks_kdf_policy, - config.general.me_route_backpressure_base_timeout_ms, - config.general.me_route_backpressure_high_timeout_ms, - config.general.me_route_backpressure_high_watermark_pct, - config.general.me_route_no_writer_mode, - config.general.me_route_no_writer_wait_ms, - config.general.me_route_inline_recovery_attempts, - config.general.me_route_inline_recovery_wait_ms, - ); - - match pool.init(pool_size, &rng).await { - Ok(()) => { - info!( - attempt = init_attempt, - "Middle-End pool initialized successfully" - ); - - // Phase 4: Start health monitor - let pool_clone = pool.clone(); - let rng_clone = rng.clone(); - let min_conns = pool_size; - tokio::spawn(async move { - crate::transport::middle_proxy::me_health_monitor( - pool_clone, rng_clone, min_conns, - ) - .await; - }); - - break Some(pool); - } - Err(e) => { - let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; - if retries_limited && init_attempt >= me_init_retry_attempts { - error!( - error = %e, - attempt = init_attempt, - retry_limit = me_init_retry_attempts, - "ME pool init retries exhausted; falling back to direct mode" - ); - break None; - } - - warn!( - error = %e, - attempt = init_attempt, - retry_limit = if me_init_retry_attempts == 0 { - String::from("unlimited") - } else { - me_init_retry_attempts.to_string() - }, - me2dc_fallback = me2dc_fallback, - retry_in_secs = 2, - "ME pool is not ready yet; retrying startup initialization" - ); - pool.reset_stun_state(); - tokio::time::sleep(Duration::from_secs(2)).await; } } + }; + match proxy_secret { + Some(proxy_secret) => { + info!( + secret_len = proxy_secret.len(), + key_sig = format_args!( + "0x{:08x}", + if proxy_secret.len() >= 4 { + u32::from_le_bytes([ + proxy_secret[0], + proxy_secret[1], + proxy_secret[2], + proxy_secret[3], + ]) + } else { + 0 + } + ), + "Proxy-secret loaded" + ); + + let cfg_v4 = load_startup_proxy_config_snapshot( + "https://core.telegram.org/getProxyConfig", + config.general.proxy_config_v4_cache_path.as_deref(), + me2dc_fallback, + "getProxyConfig", + ) + .await; + let cfg_v6 = load_startup_proxy_config_snapshot( + "https://core.telegram.org/getProxyConfigV6", + config.general.proxy_config_v6_cache_path.as_deref(), + me2dc_fallback, + "getProxyConfigV6", + ) + .await; + + if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) { + let pool = MePool::new( + proxy_tag.clone(), + proxy_secret, + config.general.middle_proxy_nat_ip, + me_nat_probe, + None, + config.network.stun_servers.clone(), + config.general.stun_nat_probe_concurrency, + probe.detected_ipv6, + config.timeouts.me_one_retry, + config.timeouts.me_one_timeout_ms, + cfg_v4.map.clone(), + cfg_v6.map.clone(), + cfg_v4.default_dc.or(cfg_v6.default_dc), + decision.clone(), + Some(upstream_manager.clone()), + rng.clone(), + stats.clone(), + config.general.me_keepalive_enabled, + config.general.me_keepalive_interval_secs, + config.general.me_keepalive_jitter_secs, + config.general.me_keepalive_payload_random, + config.general.rpc_proxy_req_every, + config.general.me_warmup_stagger_enabled, + config.general.me_warmup_step_delay_ms, + config.general.me_warmup_step_jitter_ms, + config.general.me_reconnect_max_concurrent_per_dc, + config.general.me_reconnect_backoff_base_ms, + config.general.me_reconnect_backoff_cap_ms, + config.general.me_reconnect_fast_retry_count, + config.general.me_single_endpoint_shadow_writers, + config.general.me_single_endpoint_outage_mode_enabled, + config.general.me_single_endpoint_outage_disable_quarantine, + config.general.me_single_endpoint_outage_backoff_min_ms, + config.general.me_single_endpoint_outage_backoff_max_ms, + config.general.me_single_endpoint_shadow_rotate_every_secs, + config.general.me_floor_mode, + config.general.me_adaptive_floor_idle_secs, + config.general.me_adaptive_floor_min_writers_single_endpoint, + config.general.me_adaptive_floor_recover_grace_secs, + config.general.hardswap, + config.general.me_pool_drain_ttl_secs, + config.general.effective_me_pool_force_close_secs(), + config.general.me_pool_min_fresh_ratio, + config.general.me_hardswap_warmup_delay_min_ms, + config.general.me_hardswap_warmup_delay_max_ms, + config.general.me_hardswap_warmup_extra_passes, + config.general.me_hardswap_warmup_pass_backoff_base_ms, + config.general.me_bind_stale_mode, + config.general.me_bind_stale_ttl_secs, + config.general.me_secret_atomic_snapshot, + config.general.me_deterministic_writer_sort, + config.general.me_socks_kdf_policy, + config.general.me_route_backpressure_base_timeout_ms, + config.general.me_route_backpressure_high_timeout_ms, + config.general.me_route_backpressure_high_watermark_pct, + config.general.me_route_no_writer_mode, + config.general.me_route_no_writer_wait_ms, + config.general.me_route_inline_recovery_attempts, + config.general.me_route_inline_recovery_wait_ms, + ); + + let mut init_attempt: u32 = 0; + loop { + init_attempt = init_attempt.saturating_add(1); + match pool.init(pool_size, &rng).await { + Ok(()) => { + info!( + attempt = init_attempt, + "Middle-End pool initialized successfully" + ); + + // Phase 4: Start health monitor + let pool_clone = pool.clone(); + let rng_clone = rng.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + pool_clone, rng_clone, min_conns, + ) + .await; + }); + + break Some(pool); + } + Err(e) => { + let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; + if retries_limited && init_attempt >= me_init_retry_attempts { + error!( + error = %e, + attempt = init_attempt, + retry_limit = me_init_retry_attempts, + "ME pool init retries exhausted; falling back to direct mode" + ); + break None; + } + + let retry_limit = if !me2dc_fallback || me_init_retry_attempts == 0 { + String::from("unlimited") + } else { + me_init_retry_attempts.to_string() + }; + warn!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool is not ready yet; retrying startup initialization" + ); + pool.reset_stun_state(); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } + } else { + None + } + } + None => None, } } else { None diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index a9c50ab..072c1f6 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::net::IpAddr; +use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -42,6 +43,87 @@ pub struct ProxyConfigData { pub proxy_for_lines: u32, } +pub fn parse_proxy_config_text(text: &str, http_status: u16) -> ProxyConfigData { + let mut map: HashMap> = HashMap::new(); + let mut proxy_for_lines: u32 = 0; + for line in text.lines() { + if let Some((dc, ip, port)) = parse_proxy_line(line) { + map.entry(dc).or_default().push((ip, port)); + proxy_for_lines = proxy_for_lines.saturating_add(1); + } + } + + let default_dc = text.lines().find_map(|l| { + let t = l.trim(); + if let Some(rest) = t.strip_prefix("default") { + return rest.trim().trim_end_matches(';').parse::().ok(); + } + None + }); + + ProxyConfigData { + map, + default_dc, + http_status, + proxy_for_lines, + } +} + +pub async fn load_proxy_config_cache(path: &str) -> Result { + let text = tokio::fs::read_to_string(path).await.map_err(|e| { + crate::error::ProxyError::Proxy(format!("read proxy-config cache '{path}' failed: {e}")) + })?; + Ok(parse_proxy_config_text(&text, 200)) +} + +pub async fn save_proxy_config_cache(path: &str, raw_text: &str) -> Result<()> { + if let Some(parent) = Path::new(path).parent() + && !parent.as_os_str().is_empty() + { + tokio::fs::create_dir_all(parent).await.map_err(|e| { + crate::error::ProxyError::Proxy(format!( + "create proxy-config cache dir '{}' failed: {e}", + parent.display() + )) + })?; + } + + tokio::fs::write(path, raw_text).await.map_err(|e| { + crate::error::ProxyError::Proxy(format!("write proxy-config cache '{path}' failed: {e}")) + })?; + Ok(()) +} + +pub async fn fetch_proxy_config_with_raw(url: &str) -> Result<(ProxyConfigData, String)> { + let resp = reqwest::get(url) + .await + .map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config GET failed: {e}")))? + ; + let http_status = resp.status().as_u16(); + + if let Some(date) = resp.headers().get(reqwest::header::DATE) + && let Ok(date_str) = date.to_str() + && let Ok(server_time) = httpdate::parse_http_date(date_str) + && let Ok(skew) = SystemTime::now().duration_since(server_time).or_else(|e| { + server_time.duration_since(SystemTime::now()).map_err(|_| e) + }) + { + let skew_secs = skew.as_secs(); + if skew_secs > 60 { + warn!(skew_secs, "Time skew >60s detected from fetch_proxy_config Date header"); + } else if skew_secs > 30 { + warn!(skew_secs, "Time skew >30s detected from fetch_proxy_config Date header"); + } + } + + let text = resp + .text() + .await + .map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config read failed: {e}")))?; + let parsed = parse_proxy_config_text(&text, http_status); + Ok((parsed, text)) +} + #[derive(Debug, Default)] struct StableSnapshot { candidate_hash: Option, @@ -170,61 +252,9 @@ fn parse_proxy_line(line: &str) -> Option<(i32, IpAddr, u16)> { } pub async fn fetch_proxy_config(url: &str) -> Result { - let resp = reqwest::get(url) + fetch_proxy_config_with_raw(url) .await - .map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config GET failed: {e}")))? - ; - let http_status = resp.status().as_u16(); - - if let Some(date) = resp.headers().get(reqwest::header::DATE) - && let Ok(date_str) = date.to_str() - && let Ok(server_time) = httpdate::parse_http_date(date_str) - && let Ok(skew) = SystemTime::now().duration_since(server_time).or_else(|e| { - server_time.duration_since(SystemTime::now()).map_err(|_| e) - }) - { - let skew_secs = skew.as_secs(); - if skew_secs > 60 { - warn!(skew_secs, "Time skew >60s detected from fetch_proxy_config Date header"); - } else if skew_secs > 30 { - warn!(skew_secs, "Time skew >30s detected from fetch_proxy_config Date header"); - } - } - - let text = resp - .text() - .await - .map_err(|e| crate::error::ProxyError::Proxy(format!("fetch_proxy_config read failed: {e}")))?; - - let mut map: HashMap> = HashMap::new(); - let mut proxy_for_lines: u32 = 0; - for line in text.lines() { - if let Some((dc, ip, port)) = parse_proxy_line(line) { - map.entry(dc).or_default().push((ip, port)); - proxy_for_lines = proxy_for_lines.saturating_add(1); - } - } - - let default_dc = text - .lines() - .find_map(|l| { - let t = l.trim(); - if let Some(rest) = t.strip_prefix("default") { - return rest - .trim() - .trim_end_matches(';') - .parse::() - .ok(); - } - None - }); - - Ok(ProxyConfigData { - map, - default_dc, - http_status, - proxy_for_lines, - }) + .map(|(parsed, _raw)| parsed) } fn snapshot_passes_guards( diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index e7c7957..131e215 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -30,7 +30,11 @@ pub use pool::MePool; pub use pool_nat::{stun_probe, detect_public_ip}; pub use registry::ConnRegistry; pub use secret::fetch_proxy_secret; -pub use config_updater::{fetch_proxy_config, me_config_updater}; +#[allow(unused_imports)] +pub use config_updater::{ + ProxyConfigData, fetch_proxy_config, fetch_proxy_config_with_raw, load_proxy_config_cache, + me_config_updater, save_proxy_config_cache, +}; pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task}; pub use wire::proto_flags_for_tag; From 3260746785d57284f0a0c24c6ed8b16f5879c706 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:48:09 +0300 Subject: [PATCH 075/261] Init + Uptime timers --- src/main.rs | 104 ++++++++++++++++++++++++- src/transport/middle_proxy/registry.rs | 5 ++ src/transport/middle_proxy/send.rs | 33 +++++++- 3 files changed, 139 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1da8123..fe001c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use std::net::SocketAddr; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use rand::Rng; use tokio::net::TcpListener; use tokio::signal; @@ -173,6 +173,74 @@ async fn write_beobachten_snapshot(path: &str, payload: &str) -> std::io::Result tokio::fs::write(path, payload).await } +fn unit_label(value: u64, singular: &'static str, plural: &'static str) -> &'static str { + if value == 1 { singular } else { plural } +} + +fn format_uptime(total_secs: u64) -> String { + const SECS_PER_MINUTE: u64 = 60; + const SECS_PER_HOUR: u64 = 60 * SECS_PER_MINUTE; + const SECS_PER_DAY: u64 = 24 * SECS_PER_HOUR; + const SECS_PER_MONTH: u64 = 30 * SECS_PER_DAY; + const SECS_PER_YEAR: u64 = 365 * SECS_PER_DAY; + + let mut remaining = total_secs; + let years = remaining / SECS_PER_YEAR; + remaining %= SECS_PER_YEAR; + let months = remaining / SECS_PER_MONTH; + remaining %= SECS_PER_MONTH; + let days = remaining / SECS_PER_DAY; + remaining %= SECS_PER_DAY; + let hours = remaining / SECS_PER_HOUR; + remaining %= SECS_PER_HOUR; + let minutes = remaining / SECS_PER_MINUTE; + let seconds = remaining % SECS_PER_MINUTE; + + let mut parts = Vec::new(); + if years > 0 { + parts.push(format!( + "{} {}", + years, + unit_label(years, "year", "years") + )); + } + if total_secs >= SECS_PER_YEAR { + parts.push(format!( + "{} {}", + months, + unit_label(months, "month", "months") + )); + } + if total_secs >= SECS_PER_MONTH { + parts.push(format!( + "{} {}", + days, + unit_label(days, "day", "days") + )); + } + if total_secs >= SECS_PER_DAY { + parts.push(format!( + "{} {}", + hours, + unit_label(hours, "hour", "hours") + )); + } + if total_secs >= SECS_PER_HOUR { + parts.push(format!( + "{} {}", + minutes, + unit_label(minutes, "minute", "minutes") + )); + } + parts.push(format!( + "{} {}", + seconds, + unit_label(seconds, "second", "seconds") + )); + + format!("{} / {} seconds", parts.join(", "), total_secs) +} + async fn load_startup_proxy_config_snapshot( url: &str, cache_path: Option<&str>, @@ -289,6 +357,7 @@ async fn load_startup_proxy_config_snapshot( #[tokio::main] async fn main() -> std::result::Result<(), Box> { + let process_started_at = Instant::now(); let (config_path, cli_silent, cli_log_level) = parse_cli(); let mut config = match ProxyConfig::load(&config_path) { @@ -961,6 +1030,15 @@ async fn main() -> std::result::Result<(), Box> { } } + let initialized_secs = process_started_at.elapsed().as_secs(); + let second_suffix = if initialized_secs == 1 { "" } else { "s" }; + info!("================= Telegram Startup ================="); + info!( + " DC/ME Initialized in {} second{}", + initialized_secs, second_suffix + ); + info!("============================================================"); + // Background tasks let um_clone = upstream_manager.clone(); let decision_clone = decision.clone(); @@ -1514,7 +1592,29 @@ async fn main() -> std::result::Result<(), Box> { } match signal::ctrl_c().await { - Ok(()) => info!("Shutting down..."), + Ok(()) => { + let uptime_secs = process_started_at.elapsed().as_secs(); + info!("Uptime: {}", format_uptime(uptime_secs)); + info!("Shutting down..."); + if let Some(pool) = &me_pool { + match tokio::time::timeout( + Duration::from_secs(2), + pool.shutdown_send_close_conn_all(), + ) + .await + { + Ok(total) => { + info!( + close_conn_sent = total, + "ME shutdown: RPC_CLOSE_CONN broadcast completed" + ); + } + Err(_) => { + warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out"); + } + } + } + } Err(e) => error!("Signal error: {}", e), } diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index 869030a..e4d0031 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -278,6 +278,11 @@ impl ConnRegistry { Some(ConnWriter { writer_id, tx: writer }) } + pub async fn active_conn_ids(&self) -> Vec { + let inner = self.inner.read().await; + inner.writer_for_conn.keys().copied().collect() + } + pub async fn writer_lost(&self, writer_id: u64) -> Vec { let mut inner = self.inner.write().await; inner.writers.remove(&writer_id); diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 8bd21ee..c6db028 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -11,7 +11,7 @@ use tracing::{debug, warn}; use crate::config::MeRouteNoWriterMode; use crate::error::{ProxyError, Result}; use crate::network::IpFamily; -use crate::protocol::constants::RPC_CLOSE_EXT_U32; +use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32}; use super::MePool; use super::codec::WriterCommand; @@ -476,6 +476,37 @@ impl MePool { Ok(()) } + pub async fn send_close_conn(self: &Arc, conn_id: u64) -> Result<()> { + if let Some(w) = self.registry.get_writer(conn_id).await { + let mut p = Vec::with_capacity(12); + p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes()); + p.extend_from_slice(&conn_id.to_le_bytes()); + match w.tx.try_send(WriterCommand::DataAndFlush(p)) { + Ok(()) => {} + Err(TrySendError::Full(cmd)) => { + let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await; + } + Err(TrySendError::Closed(_)) => { + debug!(conn_id, "ME close_conn skipped: writer channel closed"); + } + } + } else { + debug!(conn_id, "ME close_conn skipped (writer missing)"); + } + + self.registry.unregister(conn_id).await; + Ok(()) + } + + pub async fn shutdown_send_close_conn_all(self: &Arc) -> usize { + let conn_ids = self.registry.active_conn_ids().await; + let total = conn_ids.len(); + for conn_id in conn_ids { + let _ = self.send_close_conn(conn_id).await; + } + total + } + pub fn connection_count(&self) -> usize { self.conn_count.load(Ordering::Relaxed) } From d62a6e041738d05073a1a0f874caeced4d568893 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:04:32 +0300 Subject: [PATCH 076/261] Shutdown Timer fixes --- src/main.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index fe001c3..84e61a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -182,7 +182,7 @@ fn format_uptime(total_secs: u64) -> String { const SECS_PER_HOUR: u64 = 60 * SECS_PER_MINUTE; const SECS_PER_DAY: u64 = 24 * SECS_PER_HOUR; const SECS_PER_MONTH: u64 = 30 * SECS_PER_DAY; - const SECS_PER_YEAR: u64 = 365 * SECS_PER_DAY; + const SECS_PER_YEAR: u64 = 12 * SECS_PER_MONTH; let mut remaining = total_secs; let years = remaining / SECS_PER_YEAR; @@ -197,35 +197,35 @@ fn format_uptime(total_secs: u64) -> String { let seconds = remaining % SECS_PER_MINUTE; let mut parts = Vec::new(); - if years > 0 { + if total_secs > SECS_PER_YEAR { parts.push(format!( "{} {}", years, unit_label(years, "year", "years") )); } - if total_secs >= SECS_PER_YEAR { + if total_secs > SECS_PER_MONTH { parts.push(format!( "{} {}", months, unit_label(months, "month", "months") )); } - if total_secs >= SECS_PER_MONTH { + if total_secs > SECS_PER_DAY { parts.push(format!( "{} {}", days, unit_label(days, "day", "days") )); } - if total_secs >= SECS_PER_DAY { + if total_secs > SECS_PER_HOUR { parts.push(format!( "{} {}", hours, unit_label(hours, "hour", "hours") )); } - if total_secs >= SECS_PER_HOUR { + if total_secs > SECS_PER_MINUTE { parts.push(format!( "{} {}", minutes, @@ -1032,7 +1032,7 @@ async fn main() -> std::result::Result<(), Box> { let initialized_secs = process_started_at.elapsed().as_secs(); let second_suffix = if initialized_secs == 1 { "" } else { "s" }; - info!("================= Telegram Startup ================="); + info!("===================== Telegram Startup ====================="); info!( " DC/ME Initialized in {} second{}", initialized_secs, second_suffix @@ -1593,9 +1593,10 @@ async fn main() -> std::result::Result<(), Box> { match signal::ctrl_c().await { Ok(()) => { + let shutdown_started_at = Instant::now(); + info!("Shutting down..."); let uptime_secs = process_started_at.elapsed().as_secs(); info!("Uptime: {}", format_uptime(uptime_secs)); - info!("Shutting down..."); if let Some(pool) = &me_pool { match tokio::time::timeout( Duration::from_secs(2), @@ -1614,6 +1615,12 @@ async fn main() -> std::result::Result<(), Box> { } } } + let shutdown_secs = shutdown_started_at.elapsed().as_secs(); + info!( + "Shutdown completed successfully in {} {}.", + shutdown_secs, + unit_label(shutdown_secs, "second", "seconds") + ); } Err(e) => error!("Signal error: {}", e), } From 64130dd02e2c843f76b01707cfa26408bec96a02 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:13:40 +0300 Subject: [PATCH 077/261] MEP not ready only after 3 attempts --- src/main.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 84e61a4..e10c18b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -629,6 +629,7 @@ async fn main() -> std::result::Result<(), Box> { let me2dc_fallback = config.general.me2dc_fallback; let me_init_retry_attempts = config.general.me_init_retry_attempts; + let me_init_warn_after_attempts: u32 = 3; if use_middle_proxy && !decision.ipv4_me && !decision.ipv6_me { if me2dc_fallback { warn!("No usable IP family for Middle Proxy detected; falling back to direct DC"); @@ -832,14 +833,25 @@ async fn main() -> std::result::Result<(), Box> { } else { me_init_retry_attempts.to_string() }; - warn!( - error = %e, - attempt = init_attempt, - retry_limit = retry_limit, - me2dc_fallback = me2dc_fallback, - retry_in_secs = 2, - "ME pool is not ready yet; retrying startup initialization" - ); + if init_attempt >= me_init_warn_after_attempts { + warn!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool is not ready yet; retrying startup initialization" + ); + } else { + info!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool startup warmup: retrying initialization" + ); + } pool.reset_stun_state(); tokio::time::sleep(Duration::from_secs(2)).await; } From a80be783450f68f2f56d111e5534ce3aa550fea4 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:32:31 +0300 Subject: [PATCH 078/261] DC writer floor is below required only in runtime --- src/main.rs | 4 ++++ src/transport/middle_proxy/health.rs | 30 +++++++++++++++++++--------- src/transport/middle_proxy/pool.rs | 10 ++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index e10c18b..798790a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1051,6 +1051,10 @@ async fn main() -> std::result::Result<(), Box> { ); info!("============================================================"); + if let Some(ref pool) = me_pool { + pool.set_runtime_ready(true); + } + // Background tasks let um_clone = upstream_manager.clone(); let decision_clone = decision.clone(); diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index c9ad34c..1cc8d8a 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -295,15 +295,27 @@ async fn check_family( let wait = Duration::from_millis(next_ms) + Duration::from_millis(rand::rng().random_range(0..=jitter.max(1))); next_attempt.insert(key, now + wait); - warn!( - dc = %dc, - ?family, - alive = now_alive, - required, - endpoint_count = endpoints.len(), - backoff_ms = next_ms, - "DC writer floor is below required level, scheduled reconnect" - ); + if pool.is_runtime_ready() { + warn!( + dc = %dc, + ?family, + alive = now_alive, + required, + endpoint_count = endpoints.len(), + backoff_ms = next_ms, + "DC writer floor is below required level, scheduled reconnect" + ); + } else { + info!( + dc = %dc, + ?family, + alive = now_alive, + required, + endpoint_count = endpoints.len(), + backoff_ms = next_ms, + "DC writer floor is below required level during startup, scheduled reconnect" + ); + } } if let Some(v) = inflight.get_mut(&key) { *v = v.saturating_sub(1); diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index d553944..8cc078e 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -149,6 +149,7 @@ pub struct MePool { pub(super) me_route_no_writer_wait: Duration, pub(super) me_route_inline_recovery_attempts: u32, pub(super) me_route_inline_recovery_wait: Duration, + pub(super) runtime_ready: AtomicBool, pool_size: usize, } @@ -355,6 +356,7 @@ impl MePool { me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), me_route_inline_recovery_attempts, me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms), + runtime_ready: AtomicBool::new(false), }) } @@ -362,6 +364,14 @@ impl MePool { self.active_generation.load(Ordering::Relaxed) } + pub fn set_runtime_ready(&self, ready: bool) { + self.runtime_ready.store(ready, Ordering::Relaxed); + } + + pub fn is_runtime_ready(&self) -> bool { + self.runtime_ready.load(Ordering::Relaxed) + } + pub fn update_runtime_reinit_policy( &self, hardswap: bool, From 48ce59900e100f915a1a43aa67b0d8a717cd22d5 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:40:28 +0300 Subject: [PATCH 079/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3a221f2..53e1e5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.2.2" +version = "3.3.0" edition = "2024" [dependencies] From d683faf922e5318def495006a27fb0dee827715a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 22:31:01 +0300 Subject: [PATCH 080/261] HybridAsyncPersistent - new ME Route NoWriter Mode Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/types.rs | 8 ++- src/main.rs | 77 ++++++++++++++++++++++- src/transport/middle_proxy/pool_status.rs | 66 +++++++++++++++++++ src/transport/middle_proxy/send.rs | 50 +++++++++++++++ 4 files changed, 198 insertions(+), 3 deletions(-) diff --git a/src/config/types.rs b/src/config/types.rs index 4c19f8a..88bf8d3 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -187,9 +187,10 @@ impl MeFloorMode { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum MeRouteNoWriterMode { - #[default] AsyncRecoveryFailfast, InlineRecoveryLegacy, + #[default] + HybridAsyncPersistent, } impl MeRouteNoWriterMode { @@ -197,13 +198,16 @@ impl MeRouteNoWriterMode { match self { MeRouteNoWriterMode::AsyncRecoveryFailfast => 0, MeRouteNoWriterMode::InlineRecoveryLegacy => 1, + MeRouteNoWriterMode::HybridAsyncPersistent => 2, } } pub fn from_u8(raw: u8) -> Self { match raw { + 0 => MeRouteNoWriterMode::AsyncRecoveryFailfast, 1 => MeRouteNoWriterMode::InlineRecoveryLegacy, - _ => MeRouteNoWriterMode::AsyncRecoveryFailfast, + 2 => MeRouteNoWriterMode::HybridAsyncPersistent, + _ => MeRouteNoWriterMode::HybridAsyncPersistent, } } } diff --git a/src/main.rs b/src/main.rs index 798790a..9f81edf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; use rand::Rng; use tokio::net::TcpListener; use tokio::signal; -use tokio::sync::{Semaphore, mpsc}; +use tokio::sync::{Semaphore, mpsc, watch}; use tracing::{debug, error, info, warn}; use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; #[cfg(unix)] @@ -241,6 +241,17 @@ fn format_uptime(total_secs: u64) -> String { format!("{} / {} seconds", parts.join(", "), total_secs) } +async fn wait_until_admission_open(admission_rx: &mut watch::Receiver) -> bool { + loop { + if *admission_rx.borrow() { + return true; + } + if admission_rx.changed().await.is_err() { + return *admission_rx.borrow(); + } + } +} + async fn load_startup_proxy_config_snapshot( url: &str, cache_path: Option<&str>, @@ -1325,6 +1336,60 @@ async fn main() -> std::result::Result<(), Box> { print_proxy_links(&host, port, &config); } + let (admission_tx, admission_rx) = watch::channel(true); + if config.general.use_middle_proxy { + if let Some(pool) = me_pool.as_ref() { + let initial_open = pool.admission_ready_full_floor().await; + admission_tx.send_replace(initial_open); + if initial_open { + info!("Conditional-admission gate: open (ME pool ready)"); + } else { + warn!("Conditional-admission gate: closed (ME pool is not ready)"); + } + + let pool_for_gate = pool.clone(); + let admission_tx_gate = admission_tx.clone(); + tokio::spawn(async move { + let mut gate_open = initial_open; + let mut open_streak = if initial_open { 1u32 } else { 0u32 }; + let mut close_streak = if initial_open { 0u32 } else { 1u32 }; + loop { + let ready = pool_for_gate.admission_ready_full_floor().await; + if ready { + open_streak = open_streak.saturating_add(1); + close_streak = 0; + if !gate_open && open_streak >= 2 { + gate_open = true; + admission_tx_gate.send_replace(true); + info!( + open_streak, + "Conditional-admission gate opened (ME pool recovered)" + ); + } + } else { + close_streak = close_streak.saturating_add(1); + open_streak = 0; + if gate_open && close_streak >= 2 { + gate_open = false; + admission_tx_gate.send_replace(false); + warn!( + close_streak, + "Conditional-admission gate closed (ME pool below required floor)" + ); + } + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }); + } else { + admission_tx.send_replace(false); + warn!("Conditional-admission gate: closed (ME pool is unavailable)"); + } + } else { + admission_tx.send_replace(true); + } + let _admission_tx_hold = admission_tx; + // Unix socket setup (before listeners check so unix-only config works) let mut has_unix_listener = false; #[cfg(unix)] @@ -1358,6 +1423,7 @@ async fn main() -> std::result::Result<(), Box> { has_unix_listener = true; let mut config_rx_unix: tokio::sync::watch::Receiver> = config_rx.clone(); + let mut admission_rx_unix = admission_rx.clone(); let stats = stats.clone(); let upstream_manager = upstream_manager.clone(); let replay_checker = replay_checker.clone(); @@ -1373,6 +1439,10 @@ async fn main() -> std::result::Result<(), Box> { let unix_conn_counter = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)); loop { + if !wait_until_admission_open(&mut admission_rx_unix).await { + warn!("Conditional-admission gate channel closed for unix listener"); + break; + } match unix_listener.accept().await { Ok((stream, _)) => { let permit = match max_connections_unix.clone().acquire_owned().await { @@ -1507,6 +1577,7 @@ async fn main() -> std::result::Result<(), Box> { for (listener, listener_proxy_protocol) in listeners { let mut config_rx: tokio::sync::watch::Receiver> = config_rx.clone(); + let mut admission_rx_tcp = admission_rx.clone(); let stats = stats.clone(); let upstream_manager = upstream_manager.clone(); let replay_checker = replay_checker.clone(); @@ -1520,6 +1591,10 @@ async fn main() -> std::result::Result<(), Box> { tokio::spawn(async move { loop { + if !wait_until_admission_open(&mut admission_rx_tcp).await { + warn!("Conditional-admission gate channel closed for tcp listener"); + break; + } match listener.accept().await { Ok((stream, peer_addr)) => { let permit = match max_connections_tcp.clone().acquire_owned().await { diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index c01f74b..9dd3d07 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -100,6 +100,72 @@ pub(crate) struct MeApiRuntimeSnapshot { } impl MePool { + pub(crate) async fn admission_ready_full_floor(&self) -> bool { + let mut endpoints_by_dc = BTreeMap::>::new(); + if self.decision.ipv4_me { + let map = self.proxy_map_v4.read().await.clone(); + for (dc, addrs) in map { + let abs_dc = dc.abs(); + if abs_dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(abs_dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + if self.decision.ipv6_me { + let map = self.proxy_map_v6.read().await.clone(); + for (dc, addrs) in map { + let abs_dc = dc.abs(); + if abs_dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(abs_dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + + if endpoints_by_dc.is_empty() { + return false; + } + + let writers = self.writers.read().await.clone(); + let mut live_writers_by_endpoint = HashMap::::new(); + for writer in writers { + if writer.draining.load(Ordering::Relaxed) { + continue; + } + *live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1; + } + + for endpoints in endpoints_by_dc.values() { + let endpoint_count = endpoints.len(); + if endpoint_count == 0 { + return false; + } + let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false); + let alive: usize = endpoints + .iter() + .map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0)) + .sum(); + if alive < required { + return false; + } + } + + true + } + pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot { let now_epoch_secs = Self::now_epoch_secs(); diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index c6db028..b442a8a 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -22,6 +22,7 @@ use super::registry::ConnMeta; const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45; const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55; +const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4; impl MePool { /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default. @@ -55,6 +56,9 @@ impl MePool { let mut no_writer_deadline: Option = None; let mut emergency_attempts = 0u32; let mut async_recovery_triggered = false; + let mut hybrid_recovery_round = 0u32; + let mut hybrid_last_recovery_at: Option = None; + let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50)); loop { if let Some(current) = self.registry.get_writer(conn_id).await { @@ -138,6 +142,18 @@ impl MePool { } continue; } + MeRouteNoWriterMode::HybridAsyncPersistent => { + self.maybe_trigger_hybrid_recovery( + target_dc, + &mut hybrid_recovery_round, + &mut hybrid_last_recovery_at, + hybrid_wait_step, + ) + .await; + let deadline = Instant::now() + hybrid_wait_step; + let _ = self.wait_for_writer_until(deadline).await; + continue; + } } } ws.clone() @@ -215,6 +231,18 @@ impl MePool { return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } } + MeRouteNoWriterMode::HybridAsyncPersistent => { + self.maybe_trigger_hybrid_recovery( + target_dc, + &mut hybrid_recovery_round, + &mut hybrid_last_recovery_at, + hybrid_wait_step, + ) + .await; + let deadline = Instant::now() + hybrid_wait_step; + let _ = self.wait_for_candidate_until(target_dc, deadline).await; + continue; + } } } let writer_idle_since = self.registry.writer_idle_since_snapshot().await; @@ -459,6 +487,28 @@ impl MePool { preferred } + async fn maybe_trigger_hybrid_recovery( + self: &Arc, + target_dc: i16, + hybrid_recovery_round: &mut u32, + hybrid_last_recovery_at: &mut Option, + hybrid_wait_step: Duration, + ) { + if let Some(last) = *hybrid_last_recovery_at + && last.elapsed() < hybrid_wait_step + { + return; + } + + let round = *hybrid_recovery_round; + let target_triggered = self.trigger_async_recovery_for_target_dc(target_dc).await; + if !target_triggered || round % HYBRID_GLOBAL_BURST_PERIOD_ROUNDS == 0 { + self.trigger_async_recovery_global().await; + } + *hybrid_recovery_round = round.saturating_add(1); + *hybrid_last_recovery_at = Some(Instant::now()); + } + pub async fn send_close(self: &Arc, conn_id: u64) -> Result<()> { if let Some(w) = self.registry.get_writer(conn_id).await { let mut p = Vec::with_capacity(12); From 0e2cbe61789a0a6064eda69a1dfaba159c796d6f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 22:32:08 +0300 Subject: [PATCH 081/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 53e1e5d..f96c85b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.0" +version = "3.3.1" edition = "2024" [dependencies] From 9b84fc7a5b2056166156736050d940bff271d760 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 23:18:26 +0300 Subject: [PATCH 082/261] Secret Atomic Snapshot + KDF Fingerprint on RwLock Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/load.rs | 6 + src/main.rs | 6 +- src/transport/middle_proxy/handshake.rs | 9 +- src/transport/middle_proxy/pool.rs | 4 +- src/transport/middle_proxy/pool_init.rs | 179 +++++++++++++--------- src/transport/middle_proxy/pool_status.rs | 62 ++++++++ 6 files changed, 184 insertions(+), 82 deletions(-) diff --git a/src/config/load.rs b/src/config/load.rs index 9abab30..dcca2a0 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -548,6 +548,12 @@ impl ProxyConfig { config.general.middle_proxy_nat_probe = true; warn!("Auto-enabled middle_proxy_nat_probe for middle proxy mode"); } + if config.general.use_middle_proxy && !config.general.me_secret_atomic_snapshot { + config.general.me_secret_atomic_snapshot = true; + warn!( + "Auto-enabled me_secret_atomic_snapshot for middle proxy mode to keep KDF key_selector/secret coherent" + ); + } validate_network_cfg(&mut config.network)?; crate::network::dns_overrides::validate_entries(&config.network.dns_overrides)?; diff --git a/src/main.rs b/src/main.rs index 9f81edf..064df16 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1339,7 +1339,7 @@ async fn main() -> std::result::Result<(), Box> { let (admission_tx, admission_rx) = watch::channel(true); if config.general.use_middle_proxy { if let Some(pool) = me_pool.as_ref() { - let initial_open = pool.admission_ready_full_floor().await; + let initial_open = pool.admission_ready_conditional_cast().await; admission_tx.send_replace(initial_open); if initial_open { info!("Conditional-admission gate: open (ME pool ready)"); @@ -1354,7 +1354,7 @@ async fn main() -> std::result::Result<(), Box> { let mut open_streak = if initial_open { 1u32 } else { 0u32 }; let mut close_streak = if initial_open { 0u32 } else { 1u32 }; loop { - let ready = pool_for_gate.admission_ready_full_floor().await; + let ready = pool_for_gate.admission_ready_conditional_cast().await; if ready { open_streak = open_streak.saturating_add(1); close_streak = 0; @@ -1374,7 +1374,7 @@ async fn main() -> std::result::Result<(), Box> { admission_tx_gate.send_replace(false); warn!( close_streak, - "Conditional-admission gate closed (ME pool below required floor)" + "Conditional-admission gate closed (ME pool has uncovered DC groups)" ); } } diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 1016c6b..77634a6 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -387,9 +387,11 @@ impl MePool { socks_bound_addr.map(|value| value.ip()), client_port_source, ); - let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.lock().await; - if let Some((prev_fingerprint, prev_client_port)) = + let previous_kdf_fingerprint = { + let kdf_fingerprint_guard = self.kdf_material_fingerprint.read().await; kdf_fingerprint_guard.get(&peer_addr_nat).copied() + }; + if let Some((prev_fingerprint, prev_client_port)) = previous_kdf_fingerprint { if prev_fingerprint != kdf_fingerprint { self.stats.increment_me_kdf_drift_total(); @@ -416,6 +418,9 @@ impl MePool { ); } } + // Keep fingerprint updates eventually consistent for diagnostics while avoiding + // serializing all concurrent handshakes on a single async mutex. + let mut kdf_fingerprint_guard = self.kdf_material_fingerprint.write().await; kdf_fingerprint_guard.insert(peer_addr_nat, (kdf_fingerprint, client_port_for_kdf)); drop(kdf_fingerprint_guard); diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 8cc078e..1dab2f4 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -132,7 +132,7 @@ pub struct MePool { pub(super) pending_hardswap_map_hash: AtomicU64, pub(super) hardswap: AtomicBool, pub(super) endpoint_quarantine: Arc>>, - pub(super) kdf_material_fingerprint: Arc>>, + pub(super) kdf_material_fingerprint: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, pub(super) me_pool_force_close_secs: AtomicU64, pub(super) me_pool_min_fresh_ratio_permille: AtomicU32, @@ -335,7 +335,7 @@ impl MePool { pending_hardswap_map_hash: AtomicU64::new(0), hardswap: AtomicBool::new(hardswap), endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())), - kdf_material_fingerprint: Arc::new(Mutex::new(HashMap::new())), + kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())), me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs), me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs), me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille( diff --git a/src/transport/middle_proxy/pool_init.rs b/src/transport/middle_proxy/pool_init.rs index fef1553..fbb5c64 100644 --- a/src/transport/middle_proxy/pool_init.rs +++ b/src/transport/middle_proxy/pool_init.rs @@ -14,10 +14,12 @@ use super::pool::MePool; impl MePool { pub async fn init(self: &Arc, pool_size: usize, rng: &Arc) -> Result<()> { let family_order = self.family_order(); + let connect_concurrency = self.me_reconnect_max_concurrent_per_dc.max(1) as usize; let ks = self.key_selector().await; info!( me_servers = self.proxy_map_v4.read().await.len(), pool_size, + connect_concurrency, key_selector = format_args!("0x{ks:08x}"), secret_len = self.proxy_secret.read().await.secret.len(), "Initializing ME pool" @@ -41,23 +43,39 @@ impl MePool { }) .collect(); dc_addrs.sort_unstable_by_key(|(dc, _)| *dc); + dc_addrs.sort_by_key(|(_, addrs)| (addrs.len() != 1, addrs.len())); - // Ensure at least one live writer per DC group; run missing DCs in parallel. + // Stage 1: build base coverage for conditional-cast. + // Single-endpoint DCs are prefilled first; multi-endpoint DCs require one live writer. let mut join = tokio::task::JoinSet::new(); for (dc, addrs) in dc_addrs.iter().cloned() { if addrs.is_empty() { continue; } + let target_writers = if addrs.len() == 1 { + self.required_writers_for_dc_with_floor_mode(addrs.len(), false) + } else { + 1usize + }; let endpoints: HashSet = addrs .iter() .map(|(ip, port)| SocketAddr::new(*ip, *port)) .collect(); - if self.active_writer_count_for_endpoints(&endpoints).await > 0 { + if self.active_writer_count_for_endpoints(&endpoints).await >= target_writers { continue; } let pool = Arc::clone(self); let rng_clone = Arc::clone(rng); - join.spawn(async move { pool.connect_primary_for_dc(dc, addrs, rng_clone).await }); + join.spawn(async move { + pool.connect_primary_for_dc( + dc, + addrs, + target_writers, + rng_clone, + connect_concurrency, + ) + .await + }); } while join.join_next().await.is_some() {} @@ -77,47 +95,35 @@ impl MePool { ))); } - // Warm reserve writers asynchronously so startup does not block after first working pool is ready. + // Stage 2: continue saturating multi-endpoint DC groups in background. let pool = Arc::clone(self); let rng_clone = Arc::clone(rng); let dc_addrs_bg = dc_addrs.clone(); tokio::spawn(async move { - if pool.me_warmup_stagger_enabled { - for (dc, addrs) in &dc_addrs_bg { - for (ip, port) in addrs { - if pool.connection_count() >= pool_size { - break; - } - let addr = SocketAddr::new(*ip, *port); - let jitter = rand::rng() - .random_range(0..=pool.me_warmup_step_jitter.as_millis() as u64); - let delay_ms = pool.me_warmup_step_delay.as_millis() as u64 + jitter; - tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; - if let Err(e) = pool.connect_one(addr, rng_clone.as_ref()).await { - debug!(%addr, dc = %dc, error = %e, "Extra ME connect failed (staggered)"); - } - } - } - } else { - for (dc, addrs) in &dc_addrs_bg { - for (ip, port) in addrs { - if pool.connection_count() >= pool_size { - break; - } - let addr = SocketAddr::new(*ip, *port); - if let Err(e) = pool.connect_one(addr, rng_clone.as_ref()).await { - debug!(%addr, dc = %dc, error = %e, "Extra ME connect failed"); - } - } - if pool.connection_count() >= pool_size { - break; - } + let mut join_bg = tokio::task::JoinSet::new(); + for (dc, addrs) in dc_addrs_bg { + if addrs.len() <= 1 { + continue; } + let target_writers = pool.required_writers_for_dc_with_floor_mode(addrs.len(), false); + let pool_clone = Arc::clone(&pool); + let rng_clone_local = Arc::clone(&rng_clone); + join_bg.spawn(async move { + pool_clone + .connect_primary_for_dc( + dc, + addrs, + target_writers, + rng_clone_local, + connect_concurrency, + ) + .await + }); } + while join_bg.join_next().await.is_some() {} debug!( - target_pool_size = pool_size, current_pool_size = pool.connection_count(), - "Background ME reserve warmup finished" + "Background ME saturation warmup finished" ); }); @@ -140,62 +146,85 @@ impl MePool { self: Arc, dc: i32, mut addrs: Vec<(IpAddr, u16)>, + target_writers: usize, rng: Arc, + connect_concurrency: usize, ) -> bool { if addrs.is_empty() { return false; } + let target_writers = target_writers.max(1); addrs.shuffle(&mut rand::rng()); - if addrs.len() > 1 { - let concurrency = 2usize; + let endpoints: Vec = addrs + .iter() + .map(|(ip, port)| SocketAddr::new(*ip, *port)) + .collect(); + let endpoint_set: HashSet = endpoints.iter().copied().collect(); + + loop { + let alive = self.active_writer_count_for_endpoints(&endpoint_set).await; + if alive >= target_writers { + info!( + dc = %dc, + alive, + target_writers, + "ME connected" + ); + return true; + } + + let missing = target_writers.saturating_sub(alive).max(1); + let concurrency = connect_concurrency.max(1).min(missing); let mut join = tokio::task::JoinSet::new(); - let mut next_idx = 0usize; + for _ in 0..concurrency { + let pool = Arc::clone(&self); + let rng_clone = Arc::clone(&rng); + let endpoints_clone = endpoints.clone(); + join.spawn(async move { + pool.connect_endpoints_round_robin(&endpoints_clone, rng_clone.as_ref()) + .await + }); + } - while next_idx < addrs.len() || !join.is_empty() { - while next_idx < addrs.len() && join.len() < concurrency { - let (ip, port) = addrs[next_idx]; - next_idx += 1; - let addr = SocketAddr::new(ip, port); - let pool = Arc::clone(&self); - let rng_clone = Arc::clone(&rng); - join.spawn(async move { - (addr, pool.connect_one(addr, rng_clone.as_ref()).await) - }); - } - - let Some(res) = join.join_next().await else { - break; - }; + let mut progress = false; + while let Some(res) = join.join_next().await { match res { - Ok((addr, Ok(()))) => { - info!(%addr, dc = %dc, "ME connected"); - join.abort_all(); - while join.join_next().await.is_some() {} - return true; - } - Ok((addr, Err(e))) => { - warn!(%addr, dc = %dc, error = %e, "ME connect failed, trying next"); + Ok(true) => { + progress = true; } + Ok(false) => {} Err(e) => { warn!(dc = %dc, error = %e, "ME connect task failed"); } } } - warn!(dc = %dc, "All ME servers for DC failed at init"); - return false; - } - for (ip, port) in addrs { - let addr = SocketAddr::new(ip, port); - match self.connect_one(addr, rng.as_ref()).await { - Ok(()) => { - info!(%addr, dc = %dc, "ME connected"); - return true; - } - Err(e) => warn!(%addr, dc = %dc, error = %e, "ME connect failed, trying next"), + let alive_after = self.active_writer_count_for_endpoints(&endpoint_set).await; + if alive_after >= target_writers { + info!( + dc = %dc, + alive = alive_after, + target_writers, + "ME connected" + ); + return true; + } + if !progress { + warn!( + dc = %dc, + alive = alive_after, + target_writers, + "All ME servers for DC failed at init" + ); + return false; + } + + if self.me_warmup_stagger_enabled { + let jitter = rand::rng() + .random_range(0..=self.me_warmup_step_jitter.as_millis() as u64); + let delay_ms = self.me_warmup_step_delay.as_millis() as u64 + jitter; + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; } } - warn!(dc = %dc, "All ME servers for DC failed at init"); - false } } diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 9dd3d07..17a418c 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -100,6 +100,68 @@ pub(crate) struct MeApiRuntimeSnapshot { } impl MePool { + pub(crate) async fn admission_ready_conditional_cast(&self) -> bool { + let mut endpoints_by_dc = BTreeMap::>::new(); + if self.decision.ipv4_me { + let map = self.proxy_map_v4.read().await.clone(); + for (dc, addrs) in map { + let abs_dc = dc.abs(); + if abs_dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(abs_dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + if self.decision.ipv6_me { + let map = self.proxy_map_v6.read().await.clone(); + for (dc, addrs) in map { + let abs_dc = dc.abs(); + if abs_dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(abs_dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + + if endpoints_by_dc.is_empty() { + return false; + } + + let writers = self.writers.read().await.clone(); + let mut live_writers_by_endpoint = HashMap::::new(); + for writer in writers { + if writer.draining.load(Ordering::Relaxed) { + continue; + } + *live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1; + } + + for endpoints in endpoints_by_dc.values() { + let alive: usize = endpoints + .iter() + .map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0)) + .sum(); + if alive == 0 { + return false; + } + } + + true + } + + #[allow(dead_code)] pub(crate) async fn admission_ready_full_floor(&self) -> bool { let mut endpoints_by_dc = BTreeMap::>::new(); if self.decision.ipv4_me { From 69b02caf7707369e6799e33230eae8a37a4d4809 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 5 Mar 2026 23:23:24 +0300 Subject: [PATCH 083/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f96c85b..33113d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.1" +version = "3.3.2" edition = "2024" [dependencies] From f32c34f12640b44db43a86d55886458ebd47276f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 03:58:08 +0300 Subject: [PATCH 084/261] ME NoWait Routing + Upstream Connbudget + PROXY Header t/o + allocation cuts --- src/config/defaults.rs | 9 ++++ src/config/load.rs | 12 +++++ src/config/types.rs | 10 ++++ src/crypto/random.rs | 21 ++++++-- src/main.rs | 1 + src/proxy/client.rs | 54 +++++++++++++++++---- src/proxy/direct_relay.rs | 2 +- src/proxy/handshake.rs | 46 +++++++++++++----- src/proxy/middle_relay.rs | 4 +- src/stats/mod.rs | 66 +++++++++++++++++++++----- src/transport/middle_proxy/pool.rs | 4 ++ src/transport/middle_proxy/pool_nat.rs | 37 +++++++++++++++ src/transport/middle_proxy/reader.rs | 4 +- src/transport/middle_proxy/registry.rs | 17 +++++++ src/transport/upstream.rs | 41 +++++++++++++--- 15 files changed, 279 insertions(+), 49 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 4b94be6..9851216 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -15,6 +15,7 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; +const DEFAULT_UPSTREAM_CONNECT_BUDGET_MS: u64 = 3000; const DEFAULT_LISTEN_ADDR_IPV6: &str = "::"; const DEFAULT_ACCESS_USER: &str = "default"; const DEFAULT_ACCESS_SECRET: &str = "00000000000000000000000000000000"; @@ -113,6 +114,10 @@ pub(crate) fn default_api_minimal_runtime_cache_ttl_ms() -> u64 { 1000 } +pub(crate) fn default_proxy_protocol_header_timeout_ms() -> u64 { + 500 +} + pub(crate) fn default_prefer_4() -> u8 { 4 } @@ -253,6 +258,10 @@ pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 { DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD } +pub(crate) fn default_upstream_connect_budget_ms() -> u64 { + DEFAULT_UPSTREAM_CONNECT_BUDGET_MS +} + pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool { false } diff --git a/src/config/load.rs b/src/config/load.rs index dcca2a0..470bc37 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -265,6 +265,12 @@ impl ProxyConfig { )); } + if config.general.upstream_connect_budget_ms == 0 { + return Err(ProxyError::Config( + "general.upstream_connect_budget_ms must be > 0".to_string(), + )); + } + if config.general.upstream_unhealthy_fail_threshold == 0 { return Err(ProxyError::Config( "general.upstream_unhealthy_fail_threshold must be > 0".to_string(), @@ -462,6 +468,12 @@ impl ProxyConfig { )); } + if config.server.proxy_protocol_header_timeout_ms == 0 { + return Err(ProxyError::Config( + "server.proxy_protocol_header_timeout_ms must be > 0".to_string(), + )); + } + if config.general.effective_me_pool_force_close_secs() > 0 && config.general.effective_me_pool_force_close_secs() < config.general.me_pool_drain_ttl_secs diff --git a/src/config/types.rs b/src/config/types.rs index 88bf8d3..be238d3 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -532,6 +532,10 @@ pub struct GeneralConfig { #[serde(default = "default_upstream_connect_retry_backoff_ms")] pub upstream_connect_retry_backoff_ms: u64, + /// Total wall-clock budget in milliseconds for one upstream connect request across retries. + #[serde(default = "default_upstream_connect_budget_ms")] + pub upstream_connect_budget_ms: u64, + /// Consecutive failed requests before upstream is marked unhealthy. #[serde(default = "default_upstream_unhealthy_fail_threshold")] pub upstream_unhealthy_fail_threshold: u32, @@ -774,6 +778,7 @@ impl Default for GeneralConfig { me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), + upstream_connect_budget_ms: default_upstream_connect_budget_ms(), upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(), upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(), stun_iface_mismatch_ignore: false, @@ -962,6 +967,10 @@ pub struct ServerConfig { #[serde(default)] pub proxy_protocol: bool, + /// Timeout in milliseconds for reading and parsing PROXY protocol headers. + #[serde(default = "default_proxy_protocol_header_timeout_ms")] + pub proxy_protocol_header_timeout_ms: u64, + #[serde(default)] pub metrics_port: Option, @@ -985,6 +994,7 @@ impl Default for ServerConfig { listen_unix_sock_perm: None, listen_tcp: None, proxy_protocol: false, + proxy_protocol_header_timeout_ms: default_proxy_protocol_header_timeout_ms(), metrics_port: None, metrics_whitelist: default_metrics_whitelist(), api: ApiConfig::default(), diff --git a/src/crypto/random.rs b/src/crypto/random.rs index 6313610..a88efc6 100644 --- a/src/crypto/random.rs +++ b/src/crypto/random.rs @@ -21,6 +21,7 @@ struct SecureRandomInner { rng: StdRng, cipher: AesCtr, buffer: Vec, + buffer_start: usize, } impl Drop for SecureRandomInner { @@ -48,6 +49,7 @@ impl SecureRandom { rng, cipher, buffer: Vec::with_capacity(1024), + buffer_start: 0, }), } } @@ -59,16 +61,29 @@ impl SecureRandom { let mut written = 0usize; while written < out.len() { + if inner.buffer_start >= inner.buffer.len() { + inner.buffer.clear(); + inner.buffer_start = 0; + } + if inner.buffer.is_empty() { let mut chunk = vec![0u8; CHUNK_SIZE]; inner.rng.fill_bytes(&mut chunk); inner.cipher.apply(&mut chunk); inner.buffer.extend_from_slice(&chunk); + inner.buffer_start = 0; } - let take = (out.len() - written).min(inner.buffer.len()); - out[written..written + take].copy_from_slice(&inner.buffer[..take]); - inner.buffer.drain(..take); + let available = inner.buffer.len().saturating_sub(inner.buffer_start); + let take = (out.len() - written).min(available); + let start = inner.buffer_start; + let end = start + take; + out[written..written + take].copy_from_slice(&inner.buffer[start..end]); + inner.buffer_start = end; + if inner.buffer_start >= inner.buffer.len() { + inner.buffer.clear(); + inner.buffer_start = 0; + } written += take; } } diff --git a/src/main.rs b/src/main.rs index 064df16..7f546d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -464,6 +464,7 @@ async fn main() -> std::result::Result<(), Box> { config.upstreams.clone(), config.general.upstream_connect_retry_attempts, config.general.upstream_connect_retry_backoff_ms, + config.general.upstream_connect_budget_ms, config.general.upstream_unhealthy_fail_threshold, config.general.upstream_connect_failfast_hard_errors, stats.clone(), diff --git a/src/proxy/client.rs b/src/proxy/client.rs index 2c9fa0c..ebfabcb 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -97,8 +97,11 @@ where .unwrap_or_else(|_| "0.0.0.0:443".parse().unwrap()); if proxy_protocol_enabled { - match parse_proxy_protocol(&mut stream, peer).await { - Ok(info) => { + let proxy_header_timeout = Duration::from_millis( + config.server.proxy_protocol_header_timeout_ms.max(1), + ); + match timeout(proxy_header_timeout, parse_proxy_protocol(&mut stream, peer)).await { + Ok(Ok(info)) => { debug!( peer = %peer, client = %info.src_addr, @@ -110,12 +113,18 @@ where local_addr = dst; } } - Err(e) => { + Ok(Err(e)) => { stats.increment_connects_bad(); warn!(peer = %peer, error = %e, "Invalid PROXY protocol header"); record_beobachten_class(&beobachten, &config, peer.ip(), "other"); return Err(e); } + Err(_) => { + stats.increment_connects_bad(); + warn!(peer = %peer, timeout_ms = proxy_header_timeout.as_millis(), "PROXY protocol header timeout"); + record_beobachten_class(&beobachten, &config, peer.ip(), "other"); + return Err(ProxyError::InvalidProxyProtocol); + } } } @@ -161,7 +170,7 @@ where let (read_half, write_half) = tokio::io::split(stream); - let (mut tls_reader, tls_writer, _tls_user) = match handle_tls_handshake( + let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake( &handshake, read_half, write_half, real_peer, &config, &replay_checker, &rng, tls_cache.clone(), ).await { @@ -190,7 +199,7 @@ where let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake( &mtproto_handshake, tls_reader, tls_writer, real_peer, - &config, &replay_checker, true, + &config, &replay_checker, true, Some(tls_user.as_str()), ).await { HandshakeResult::Success(result) => result, HandshakeResult::BadClient { reader: _, writer: _ } => { @@ -234,7 +243,7 @@ where let (crypto_reader, crypto_writer, success) = match handle_mtproto_handshake( &handshake, read_half, write_half, real_peer, - &config, &replay_checker, false, + &config, &replay_checker, false, None, ).await { HandshakeResult::Success(result) => result, HandshakeResult::BadClient { reader, writer } => { @@ -415,8 +424,16 @@ impl RunningClientHandler { let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?; if self.proxy_protocol_enabled { - match parse_proxy_protocol(&mut self.stream, self.peer).await { - Ok(info) => { + let proxy_header_timeout = Duration::from_millis( + self.config.server.proxy_protocol_header_timeout_ms.max(1), + ); + match timeout( + proxy_header_timeout, + parse_proxy_protocol(&mut self.stream, self.peer), + ) + .await + { + Ok(Ok(info)) => { debug!( peer = %self.peer, client = %info.src_addr, @@ -428,7 +445,7 @@ impl RunningClientHandler { local_addr = dst; } } - Err(e) => { + Ok(Err(e)) => { self.stats.increment_connects_bad(); warn!(peer = %self.peer, error = %e, "Invalid PROXY protocol header"); record_beobachten_class( @@ -439,6 +456,21 @@ impl RunningClientHandler { ); return Err(e); } + Err(_) => { + self.stats.increment_connects_bad(); + warn!( + peer = %self.peer, + timeout_ms = proxy_header_timeout.as_millis(), + "PROXY protocol header timeout" + ); + record_beobachten_class( + &self.beobachten, + &self.config, + self.peer.ip(), + "other", + ); + return Err(ProxyError::InvalidProxyProtocol); + } } } @@ -494,7 +526,7 @@ impl RunningClientHandler { let (read_half, write_half) = self.stream.into_split(); - let (mut tls_reader, tls_writer, _tls_user) = match handle_tls_handshake( + let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake( &handshake, read_half, write_half, @@ -538,6 +570,7 @@ impl RunningClientHandler { &config, &replay_checker, true, + Some(tls_user.as_str()), ) .await { @@ -611,6 +644,7 @@ impl RunningClientHandler { &config, &replay_checker, false, + None, ) .await { diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index a1f4945..1245f34 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -34,7 +34,7 @@ where let user = &success.user; let dc_addr = get_dc_addr_static(success.dc_idx, &config)?; - info!( + debug!( user = %user, peer = %success.peer, dc = success.dc_idx, diff --git a/src/proxy/handshake.rs b/src/proxy/handshake.rs index 5c63636..296432f 100644 --- a/src/proxy/handshake.rs +++ b/src/proxy/handshake.rs @@ -6,7 +6,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; -use tracing::{debug, warn, trace, info}; +use tracing::{debug, warn, trace}; use zeroize::Zeroize; use crate::crypto::{sha256, AesCtr, SecureRandom}; @@ -19,6 +19,31 @@ use crate::stats::ReplayChecker; use crate::config::ProxyConfig; use crate::tls_front::{TlsFrontCache, emulator}; +fn decode_user_secrets( + config: &ProxyConfig, + preferred_user: Option<&str>, +) -> Vec<(String, Vec)> { + let mut secrets = Vec::with_capacity(config.access.users.len()); + + if let Some(preferred) = preferred_user + && let Some(secret_hex) = config.access.users.get(preferred) + && let Ok(bytes) = hex::decode(secret_hex) + { + secrets.push((preferred.to_string(), bytes)); + } + + for (name, secret_hex) in &config.access.users { + if preferred_user.is_some_and(|preferred| preferred == name.as_str()) { + continue; + } + if let Ok(bytes) = hex::decode(secret_hex) { + secrets.push((name.clone(), bytes)); + } + } + + secrets +} + /// Result of successful handshake /// /// Key material (`dec_key`, `dec_iv`, `enc_key`, `enc_iv`) is @@ -82,11 +107,7 @@ where return HandshakeResult::BadClient { reader, writer }; } - let secrets: Vec<(String, Vec)> = config.access.users.iter() - .filter_map(|(name, hex)| { - hex::decode(hex).ok().map(|bytes| (name.clone(), bytes)) - }) - .collect(); + let secrets = decode_user_secrets(config, None); let validation = match tls::validate_tls_handshake( handshake, @@ -201,7 +222,7 @@ where return HandshakeResult::Error(ProxyError::Io(e)); } - info!( + debug!( peer = %peer, user = %validation.user, "TLS handshake successful" @@ -223,6 +244,7 @@ pub async fn handle_mtproto_handshake( config: &ProxyConfig, replay_checker: &ReplayChecker, is_tls: bool, + preferred_user: Option<&str>, ) -> HandshakeResult<(CryptoReader, CryptoWriter, HandshakeSuccess), R, W> where R: AsyncRead + Unpin + Send, @@ -239,11 +261,9 @@ where let enc_prekey_iv: Vec = dec_prekey_iv.iter().rev().copied().collect(); - for (user, secret_hex) in &config.access.users { - let secret = match hex::decode(secret_hex) { - Ok(s) => s, - Err(_) => continue, - }; + let decoded_users = decode_user_secrets(config, preferred_user); + + for (user, secret) in decoded_users { let dec_prekey = &dec_prekey_iv[..PREKEY_LEN]; let dec_iv_bytes = &dec_prekey_iv[PREKEY_LEN..]; @@ -311,7 +331,7 @@ where is_tls, }; - info!( + debug!( peer = %peer, user = %user, dc = dc_idx, diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index 0690906..8f5fc36 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::sync::{mpsc, oneshot}; -use tracing::{debug, info, trace, warn}; +use tracing::{debug, trace, warn}; use crate::config::ProxyConfig; use crate::crypto::SecureRandom; @@ -210,7 +210,7 @@ where let proto_tag = success.proto_tag; let pool_generation = me_pool.current_generation(); - info!( + debug!( user = %user, peer = %peer, dc = success.dc_idx, diff --git a/src/stats/mod.rs b/src/stats/mod.rs index eedc7f6..4cc9933 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -846,16 +846,30 @@ impl Stats { if !self.telemetry_user_enabled() { return; } - self.user_stats.entry(user.to_string()).or_default() - .connects.fetch_add(1, Ordering::Relaxed); + if let Some(stats) = self.user_stats.get(user) { + stats.connects.fetch_add(1, Ordering::Relaxed); + return; + } + self.user_stats + .entry(user.to_string()) + .or_default() + .connects + .fetch_add(1, Ordering::Relaxed); } pub fn increment_user_curr_connects(&self, user: &str) { if !self.telemetry_user_enabled() { return; } - self.user_stats.entry(user.to_string()).or_default() - .curr_connects.fetch_add(1, Ordering::Relaxed); + if let Some(stats) = self.user_stats.get(user) { + stats.curr_connects.fetch_add(1, Ordering::Relaxed); + return; + } + self.user_stats + .entry(user.to_string()) + .or_default() + .curr_connects + .fetch_add(1, Ordering::Relaxed); } pub fn decrement_user_curr_connects(&self, user: &str) { @@ -889,32 +903,60 @@ impl Stats { if !self.telemetry_user_enabled() { return; } - self.user_stats.entry(user.to_string()).or_default() - .octets_from_client.fetch_add(bytes, Ordering::Relaxed); + if let Some(stats) = self.user_stats.get(user) { + stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed); + return; + } + self.user_stats + .entry(user.to_string()) + .or_default() + .octets_from_client + .fetch_add(bytes, Ordering::Relaxed); } pub fn add_user_octets_to(&self, user: &str, bytes: u64) { if !self.telemetry_user_enabled() { return; } - self.user_stats.entry(user.to_string()).or_default() - .octets_to_client.fetch_add(bytes, Ordering::Relaxed); + if let Some(stats) = self.user_stats.get(user) { + stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed); + return; + } + self.user_stats + .entry(user.to_string()) + .or_default() + .octets_to_client + .fetch_add(bytes, Ordering::Relaxed); } pub fn increment_user_msgs_from(&self, user: &str) { if !self.telemetry_user_enabled() { return; } - self.user_stats.entry(user.to_string()).or_default() - .msgs_from_client.fetch_add(1, Ordering::Relaxed); + if let Some(stats) = self.user_stats.get(user) { + stats.msgs_from_client.fetch_add(1, Ordering::Relaxed); + return; + } + self.user_stats + .entry(user.to_string()) + .or_default() + .msgs_from_client + .fetch_add(1, Ordering::Relaxed); } pub fn increment_user_msgs_to(&self, user: &str) { if !self.telemetry_user_enabled() { return; } - self.user_stats.entry(user.to_string()).or_default() - .msgs_to_client.fetch_add(1, Ordering::Relaxed); + if let Some(stats) = self.user_stats.get(user) { + stats.msgs_to_client.fetch_add(1, Ordering::Relaxed); + return; + } + self.user_stats + .entry(user.to_string()) + .or_default() + .msgs_to_client + .fetch_add(1, Ordering::Relaxed); } pub fn get_user_total_octets(&self, user: &str) -> u64 { diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 1dab2f4..22f40b5 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -119,6 +119,8 @@ pub struct MePool { pub(super) ping_tracker: Arc>>, pub(super) rtt_stats: Arc>>, pub(super) nat_reflection_cache: Arc>, + pub(super) nat_reflection_singleflight_v4: Arc>, + pub(super) nat_reflection_singleflight_v6: Arc>, pub(super) writer_available: Arc, pub(super) refill_inflight: Arc>>, pub(super) refill_inflight_dc: Arc>>, @@ -323,6 +325,8 @@ impl MePool { ping_tracker: Arc::new(Mutex::new(HashMap::new())), rtt_stats: Arc::new(Mutex::new(HashMap::new())), nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())), + nat_reflection_singleflight_v4: Arc::new(Mutex::new(())), + nat_reflection_singleflight_v6: Arc::new(Mutex::new(())), writer_available: Arc::new(Notify::new()), refill_inflight: Arc::new(Mutex::new(HashSet::new())), refill_inflight_dc: Arc::new(Mutex::new(HashSet::new())), diff --git a/src/transport/middle_proxy/pool_nat.rs b/src/transport/middle_proxy/pool_nat.rs index bfcb0e2..07ae0b8 100644 --- a/src/transport/middle_proxy/pool_nat.rs +++ b/src/transport/middle_proxy/pool_nat.rs @@ -248,6 +248,43 @@ impl MePool { } } + let _singleflight_guard = if use_shared_cache { + Some(match family { + IpFamily::V4 => self.nat_reflection_singleflight_v4.lock().await, + IpFamily::V6 => self.nat_reflection_singleflight_v6.lock().await, + }) + } else { + None + }; + + if use_shared_cache + && let Some(until) = *self.stun_backoff_until.read().await + && Instant::now() < until + { + if let Ok(cache) = self.nat_reflection_cache.try_lock() { + let slot = match family { + IpFamily::V4 => cache.v4, + IpFamily::V6 => cache.v6, + }; + return slot.map(|(_, addr)| addr); + } + return None; + } + + if use_shared_cache + && let Ok(mut cache) = self.nat_reflection_cache.try_lock() + { + let slot = match family { + IpFamily::V4 => &mut cache.v4, + IpFamily::V6 => &mut cache.v6, + }; + if let Some((ts, addr)) = slot + && ts.elapsed() < STUN_CACHE_TTL + { + return Some(*addr); + } + } + let attempt = if use_shared_cache { self.nat_probe_attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed) } else { diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index e907d25..2a99164 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -124,7 +124,7 @@ pub(crate) async fn reader_loop( let data = Bytes::copy_from_slice(&body[12..]); trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS"); - let routed = reg.route(cid, MeResponse::Data { flags, data }).await; + let routed = reg.route_nowait(cid, MeResponse::Data { flags, data }).await; if !matches!(routed, RouteResult::Routed) { match routed { RouteResult::NoConn => stats.increment_me_route_drop_no_conn(), @@ -147,7 +147,7 @@ pub(crate) async fn reader_loop( let cfm = u32::from_le_bytes(body[8..12].try_into().unwrap()); trace!(cid, cfm, "RPC_SIMPLE_ACK"); - let routed = reg.route(cid, MeResponse::Ack(cfm)).await; + let routed = reg.route_nowait(cid, MeResponse::Ack(cfm)).await; if !matches!(routed, RouteResult::Routed) { match routed { RouteResult::NoConn => stats.increment_me_route_drop_no_conn(), diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index e4d0031..66a7f81 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -208,6 +208,23 @@ impl ConnRegistry { } } + pub async fn route_nowait(&self, id: u64, resp: MeResponse) -> RouteResult { + let tx = { + let inner = self.inner.read().await; + inner.map.get(&id).cloned() + }; + + let Some(tx) = tx else { + return RouteResult::NoConn; + }; + + match tx.try_send(resp) { + Ok(()) => RouteResult::Routed, + Err(TrySendError::Closed(_)) => RouteResult::ChannelClosed, + Err(TrySendError::Full(_)) => RouteResult::QueueFullBase, + } + } + pub async fn bind_writer( &self, conn_id: u64, diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index d9f0ede..b9db0eb 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -225,6 +225,7 @@ pub struct UpstreamManager { upstreams: Arc>>, connect_retry_attempts: u32, connect_retry_backoff: Duration, + connect_budget: Duration, unhealthy_fail_threshold: u32, connect_failfast_hard_errors: bool, stats: Arc, @@ -235,6 +236,7 @@ impl UpstreamManager { configs: Vec, connect_retry_attempts: u32, connect_retry_backoff_ms: u64, + connect_budget_ms: u64, unhealthy_fail_threshold: u32, connect_failfast_hard_errors: bool, stats: Arc, @@ -248,6 +250,7 @@ impl UpstreamManager { upstreams: Arc::new(RwLock::new(states)), connect_retry_attempts: connect_retry_attempts.max(1), connect_retry_backoff: Duration::from_millis(connect_retry_backoff_ms), + connect_budget: Duration::from_millis(connect_budget_ms.max(1)), unhealthy_fail_threshold: unhealthy_fail_threshold.max(1), connect_failfast_hard_errors, stats, @@ -593,11 +596,27 @@ impl UpstreamManager { let mut last_error: Option = None; let mut attempts_used = 0u32; for attempt in 1..=self.connect_retry_attempts { + let elapsed = connect_started_at.elapsed(); + if elapsed >= self.connect_budget { + last_error = Some(ProxyError::ConnectionTimeout { + addr: target.to_string(), + }); + break; + } + let remaining_budget = self.connect_budget.saturating_sub(elapsed); + let attempt_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS) + .min(remaining_budget); + if attempt_timeout.is_zero() { + last_error = Some(ProxyError::ConnectionTimeout { + addr: target.to_string(), + }); + break; + } attempts_used = attempt; self.stats.increment_upstream_connect_attempt_total(); let start = Instant::now(); match self - .connect_via_upstream(&upstream, target, bind_rr.clone()) + .connect_via_upstream(&upstream, target, bind_rr.clone(), attempt_timeout) .await { Ok((stream, egress)) => { @@ -707,6 +726,7 @@ impl UpstreamManager { config: &UpstreamConfig, target: SocketAddr, bind_rr: Option>, + connect_timeout: Duration, ) -> Result<(TcpStream, UpstreamEgressInfo)> { match &config.upstream_type { UpstreamType::Direct { interface, bind_addresses } => { @@ -735,7 +755,6 @@ impl UpstreamManager { let std_stream: std::net::TcpStream = socket.into(); let stream = TcpStream::from_std(std_stream)?; - let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS); match tokio::time::timeout(connect_timeout, stream.writable()).await { Ok(Ok(())) => {} Ok(Err(e)) => return Err(ProxyError::Io(e)), @@ -762,7 +781,6 @@ impl UpstreamManager { )) }, UpstreamType::Socks4 { address, interface, user_id } => { - let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS); // Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port let mut stream = if let Ok(proxy_addr) = address.parse::() { // IP:port format - use socket with optional interface binding @@ -841,7 +859,6 @@ impl UpstreamManager { )) }, UpstreamType::Socks5 { address, interface, username, password } => { - let connect_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS); // Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port let mut stream = if let Ok(proxy_addr) = address.parse::() { // IP:port format - use socket with optional interface binding @@ -1165,7 +1182,14 @@ impl UpstreamManager { target: SocketAddr, ) -> Result { let start = Instant::now(); - let _ = self.connect_via_upstream(config, target, bind_rr).await?; + let _ = self + .connect_via_upstream( + config, + target, + bind_rr, + Duration::from_secs(DC_PING_TIMEOUT_SECS), + ) + .await?; Ok(start.elapsed().as_secs_f64() * 1000.0) } @@ -1337,7 +1361,12 @@ impl UpstreamManager { let start = Instant::now(); let result = tokio::time::timeout( Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS), - self.connect_via_upstream(&config, endpoint, Some(bind_rr.clone())), + self.connect_via_upstream( + &config, + endpoint, + Some(bind_rr.clone()), + Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS), + ), ) .await; From 691607f269ba46f65d03c7408141d8e9ac9a1072 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 04:05:35 +0300 Subject: [PATCH 085/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 33113d9..cc6cd3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.2" +version = "3.3.3" edition = "2024" [dependencies] From 99edcbe818c3c5f584d9eeab309a3f84ed738acb Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 11:11:44 +0300 Subject: [PATCH 086/261] Runtime Model --- docs/model/MODEL.en.md | 285 +++++++++++++++++++++++++++++++++++++++++ docs/model/MODEL.ru.md | 285 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 570 insertions(+) create mode 100644 docs/model/MODEL.en.md create mode 100644 docs/model/MODEL.ru.md diff --git a/docs/model/MODEL.en.md b/docs/model/MODEL.en.md new file mode 100644 index 0000000..2d570cd --- /dev/null +++ b/docs/model/MODEL.en.md @@ -0,0 +1,285 @@ +# Telemt Runtime Model + +## Scope +This document defines runtime concepts used by the Middle-End (ME) transport pipeline and the orchestration logic around it. + +It focuses on: +- `ME Pool / Reader / Writer / Refill / Registry` +- `Adaptive Floor` +- `Trio-State` +- `Generation Lifecycle` + +## Core Entities + +### ME Pool +`ME Pool` is the runtime orchestrator for all Middle-End writers. + +Responsibilities: +- Holds writer inventory by DC/family/endpoint. +- Maintains routing primitives and writer selection policy. +- Tracks generation state (`active`, `warm`, `draining` context). +- Applies runtime policies (floor mode, refill, reconnect, reinit, fallback behavior). +- Exposes readiness gates used by admission logic (for conditional accept/cast behavior). + +Non-goals: +- It does not own client protocol decoding. +- It does not own per-client business policy (quotas/limits). + +### ME Writer +`ME Writer` is a long-lived ME RPC tunnel bound to one concrete ME endpoint (`ip:port`), with: +- Outbound command channel (send path). +- Associated reader loop (inbound path). +- Health/degraded flags. +- Contour/state and generation metadata. + +A writer is the actual data plane carrier for client sessions once bound. + +### ME Reader +`ME Reader` is the inbound parser/dispatcher for one writer: +- Reads/decrypts ME RPC frames. +- Validates sequence/checksum. +- Routes payloads to client-connection channels via `Registry`. +- Emits close/ack/data events and updates telemetry. + +Design intent: +- Reader must stay non-blocking as much as possible. +- Backpressure on a single client route must not stall the whole writer stream. + +### Refill +`Refill` is the recovery mechanism that restores writer coverage when capacity drops: +- Per-endpoint restore (same endpoint first). +- Per-DC restore to satisfy required floor. +- Optional outage-mode/shadow behavior for fragile single-endpoint DCs. + +Refill works asynchronously and should not block hot routing paths. + +### Registry +`Registry` is the routing index between ME and client sessions: +- `conn_id -> client response channel` +- `conn_id <-> writer_id` binding map +- writer activity snapshots and idle tracking + +Main invariants: +- A `conn_id` routes to at most one active response channel. +- Writer loss triggers safe unbind/cleanup and close propagation. +- Registry state is the source of truth for active ME-bound session mapping. + +## Adaptive Floor + +### What it is +`Adaptive Floor` is a runtime policy that changes target writer count per DC based on observed activity, instead of always holding static peak floor. + +### Why it exists +Goals: +- Reduce idle writer churn under low traffic. +- Keep enough warm capacity to avoid client-visible stalls on burst recovery. +- Limit needless reconnect storms on unstable endpoints. + +### Behavioral model +- Under activity: floor converges toward configured static requirement. +- Under prolonged idle: floor can shrink to a safe minimum. +- Recovery/grace windows prevent aggressive oscillation. + +### Safety constraints +- Never violate minimal survivability floor for a DC group. +- Refill must still restore quickly on demand. +- Floor adaptation must not force-drop already bound healthy sessions. + +## Trio-State + +`Trio-State` is writer contouring: +- `Warm` +- `Active` +- `Draining` + +### State semantics +- `Warm`: connected and validated, not primary for new binds. +- `Active`: preferred for new binds and normal traffic. +- `Draining`: no new regular binds; existing sessions continue until graceful retirement rules apply. + +### Transition intent +- `Warm -> Active`: when coverage/readiness conditions are satisfied. +- `Active -> Draining`: on generation swap, endpoint replacement, or controlled retirement. +- `Draining -> removed`: after drain TTL/force-close policy (or when naturally empty). + +This separation reduces SPOF and keeps cutovers predictable. + +## Generation Lifecycle + +Generation isolates pool epochs during reinit/reconfiguration. + +### Lifecycle phases +1. `Bootstrap`: initial writers are established. +2. `Warmup`: next generation writers are created and validated. +3. `Activation`: generation promoted to active when coverage gate passes. +4. `Drain`: previous generation becomes draining, existing sessions are allowed to finish. +5. `Retire`: old generation writers are removed after graceful rules. + +### Operational guarantees +- No partial generation activation without minimum coverage. +- Existing healthy client sessions should not be dropped just because a new generation appears. +- Draining generation exists to absorb in-flight traffic during swap. + +### Readiness and admission +Pool readiness is not equivalent to “all endpoints fully saturated”. +Typical gating strategy: +- Open admission when per-DC minimal alive coverage exists. +- Continue background saturation for multi-endpoint DCs. + +This keeps startup latency low while preserving eventual full capacity. + +## Interactions Between Concepts + +- `Generation` defines pool epochs. +- `Trio-State` defines per-writer role inside/around those epochs. +- `Adaptive Floor` defines how much capacity should be maintained right now. +- `Refill` is the actuator that closes the gap between desired and current capacity. +- `Registry` keeps per-session routing correctness while all of the above changes over time. + +## Architectural Approach + +### Layered Design +The runtime is intentionally split into two planes: +- `Control Plane`: decides desired topology and policy (`floor`, `generation swap`, `refill`, `fallback`). +- `Data Plane`: executes packet/session transport (`reader`, `writer`, routing, acks, close propagation). + +Architectural rule: +- Control Plane may change writer inventory and policy. +- Data Plane must remain stable and low-latency while those changes happen. + +### Ownership Model +Ownership is centered around explicit state domains: +- `MePool` owns writer lifecycle and policy state. +- `Registry` owns per-connection routing bindings. +- `Writer task` owns outbound ME socket send progression. +- `Reader task` owns inbound ME socket parsing and event dispatch. + +This prevents accidental cross-layer mutation and keeps invariants local. + +### Control Plane Responsibilities +Control Plane is event-driven and policy-driven: +- Startup initialization and readiness gates. +- Runtime reinit (periodic or config-triggered). +- Coverage checks per DC/family/endpoint group. +- Floor enforcement (static/adaptive). +- Refill scheduling and retry orchestration. +- Generation transition (`warm -> active`, previous `active -> draining`). + +Control Plane must prioritize determinism over short-term aggressiveness. + +### Data Plane Responsibilities +Data Plane is throughput-first and allocation-sensitive: +- Session bind to writer. +- Per-frame parsing/validation and dispatch. +- Ack and close signal propagation. +- Route drop behavior under missing connection or closed channel. +- Minimal critical logging in hot path. + +Data Plane should avoid waiting on operations that are not strictly required for frame correctness. + +## Concurrency and Synchronization + +### Concurrency Principles +- Per-writer isolation: each writer has independent send/read task loops. +- Per-connection isolation: client channel state is scoped by `conn_id`. +- Asynchronous recovery: refill/reconnect runs outside the packet hot path. + +### Synchronization Strategy +- Shared maps use fine-grained, short-lived locking. +- Read-mostly paths avoid broad write-lock windows. +- Backpressure decisions are localized at route/channel boundary. + +Design target: +- A slow consumer should degrade only itself (or its route), not global writer progress. + +### Cancellation and Shutdown +Writer and reader loops are cancellation-aware: +- explicit cancel token / close command support; +- safe unbind and cleanup via registry; +- deterministic order: stop admission -> drain/close -> release resources. + +## Consistency Model + +### Session Consistency +For one `conn_id`: +- exactly one active route target at a time; +- close and unbind must be idempotent; +- writer loss must not leave dangling bindings. + +### Generation Consistency +Generational consistency guarantees: +- New generation is not promoted before minimum coverage gate. +- Previous generation remains available in `draining` state during handover. +- Forced retirement is policy-bound (`drain ttl`, optional force-close), not immediate. + +### Policy Consistency +Policy changes (`adaptive/static floor`, fallback mode, retries) should apply without violating established active-session routing invariants. + +## Backpressure and Flow Control + +### Route-Level Backpressure +Route channels are bounded by design. +When pressure increases: +- short burst absorption is allowed; +- prolonged congestion triggers controlled drop semantics; +- drop accounting is explicit via metrics/counters. + +### Reader Non-Blocking Priority +Inbound ME reader path should never be serialized behind one congested client route. +Practical implication: +- prefer non-blocking route attempt in the parser loop; +- move heavy recovery to async side paths. + +## Failure Domain Strategy + +### Endpoint-Level Failure +Failure of one endpoint should trigger endpoint-scoped recovery first: +- same endpoint reconnect; +- endpoint replacement within same DC group if applicable. + +### DC-Level Degradation +If a DC group cannot satisfy floor: +- keep service via remaining coverage if policy allows; +- continue asynchronous refill saturation in background. + +### Whole-Pool Readiness Loss +If no sufficient ME coverage exists: +- admission gate can hold new accepts (conditional policy); +- existing sessions should continue when their path remains healthy. + +## Performance Architecture Notes + +### Hotpath Discipline +Allowed in hotpath: +- fixed-size parsing and cheap validation; +- bounded channel operations; +- precomputed or low-allocation access patterns. + +Avoid in hotpath: +- repeated expensive decoding; +- broad locks with awaits inside critical sections; +- verbose high-frequency logging. + +### Throughput Stability Over Peak Spikes +Architecture prefers stable throughput and predictable latency over short peak gains that increase churn or long-tail reconnect times. + +## Evolution and Extension Rules + +To evolve this model safely: +- Add new policy knobs in Control Plane first. +- Keep Data Plane contracts stable (`conn_id`, route semantics, close semantics). +- Validate generation and registry invariants before enabling by default. +- Introduce new retry/recovery strategies behind explicit config. + +## Failure and Recovery Notes + +- Single-endpoint DC failure is a normal degraded mode case; policy should prioritize fast reconnect and optional shadow/probing strategies. +- Idle close by peer should be treated as expected when upstream enforces idle timeout. +- Reconnect backoff must protect against synchronized churn while still allowing fast first retries. +- Fallback (`ME -> direct DC`) is a policy switch, not a transport bug by itself. + +## Terminology Summary +- `Coverage`: enough live writers to satisfy per-DC acceptance policy. +- `Floor`: target minimum writer count policy. +- `Churn`: frequent writer reconnect/remove cycles. +- `Hotpath`: per-packet/per-connection data path where extra waits/allocations are expensive. diff --git a/docs/model/MODEL.ru.md b/docs/model/MODEL.ru.md new file mode 100644 index 0000000..2f19efe --- /dev/null +++ b/docs/model/MODEL.ru.md @@ -0,0 +1,285 @@ +# Runtime-модель Telemt + +## Область описания +Документ фиксирует ключевые runtime-понятия пайплайна Middle-End (ME) и оркестрации вокруг него. + +Фокус: +- `ME Pool / Reader / Writer / Refill / Registry` +- `Adaptive Floor` +- `Trio-State` +- `Generation Lifecycle` + +## Базовые сущности + +### ME Pool +`ME Pool` — центральный оркестратор всех Middle-End writer-ов. + +Зона ответственности: +- хранит инвентарь writer-ов по DC/family/endpoint; +- управляет выбором writer-а и маршрутизацией; +- ведёт состояние поколений (`active`, `warm`, `draining` контекст); +- применяет runtime-политики (floor, refill, reconnect, reinit, fallback); +- отдаёт сигналы готовности для admission-логики (conditional accept/cast). + +Что не делает: +- не декодирует клиентский протокол; +- не реализует бизнес-политику пользователя (квоты/лимиты). + +### ME Writer +`ME Writer` — долгоживущий ME RPC-канал к конкретному endpoint (`ip:port`), у которого есть: +- канал команд на отправку; +- связанный reader loop для входящего потока; +- флаги состояния/деградации; +- метаданные contour/state и generation. + +Writer — это фактический data-plane носитель клиентских сессий после бинда. + +### ME Reader +`ME Reader` — входной parser/dispatcher одного writer-а: +- читает и расшифровывает ME RPC-фреймы; +- проверяет sequence/checksum; +- маршрутизирует payload в client-каналы через `Registry`; +- обрабатывает close/ack/data и обновляет телеметрию. + +Инженерный принцип: +- Reader должен оставаться неблокирующим. +- Backpressure одной клиентской сессии не должен останавливать весь поток writer-а. + +### Refill +`Refill` — механизм восстановления покрытия writer-ов при просадке: +- восстановление на том же endpoint в первую очередь; +- восстановление по DC до требуемого floor; +- опциональные outage/shadow-режимы для хрупких single-endpoint DC. + +Refill работает асинхронно и не должен блокировать hotpath. + +### Registry +`Registry` — маршрутизационный индекс между ME и клиентскими сессиями: +- `conn_id -> канал ответа клиенту`; +- map биндов `conn_id <-> writer_id`; +- снимки активности writer-ов и idle-трекинг. + +Ключевые инварианты: +- один `conn_id` маршрутизируется максимум в один активный канал ответа; +- потеря writer-а приводит к безопасному unbind/cleanup и отправке close; +- именно `Registry` является источником истины по активным ME-биндам. + +## Adaptive Floor + +### Что это +`Adaptive Floor` — runtime-политика, которая динамически меняет целевое число writer-ов на DC в зависимости от активности, а не держит всегда фиксированный статический floor. + +### Зачем +Цели: +- уменьшить churn на idle-трафике; +- сохранить достаточную прогретую ёмкость для быстрых всплесков; +- снизить лишние reconnect-штормы на нестабильных endpoint. + +### Модель поведения +- при активности floor стремится к статическому требованию; +- при длительном idle floor может снижаться до безопасного минимума; +- grace/recovery окна не дают системе "флапать" слишком резко. + +### Ограничения безопасности +- нельзя нарушать минимальный floor выживаемости DC-группы; +- refill обязан быстро нарастить покрытие по запросу; +- адаптация не должна принудительно ронять уже привязанные healthy-сессии. + +## Trio-State + +`Trio-State` — контурная роль writer-а: +- `Warm` +- `Active` +- `Draining` + +### Семантика состояний +- `Warm`: writer подключён и валиден, но не основной для новых биндов. +- `Active`: приоритетный для новых биндов и обычного трафика. +- `Draining`: новые обычные бинды не назначаются; текущие сессии живут до правил graceful-вывода. + +### Логика переходов +- `Warm -> Active`: когда достигнуты условия покрытия/готовности. +- `Active -> Draining`: при swap поколения, замене endpoint или контролируемом выводе. +- `Draining -> removed`: после drain TTL/force-close политики (или естественного опустошения). + +Такое разделение снижает SPOF-риски и делает cutover предсказуемым. + +## Generation Lifecycle + +Generation изолирует эпохи пула при reinit/reconfiguration. + +### Фазы жизненного цикла +1. `Bootstrap`: поднимается начальный набор writer-ов. +2. `Warmup`: создаётся и валидируется новое поколение. +3. `Activation`: новое поколение становится active после прохождения coverage-gate. +4. `Drain`: предыдущее поколение переводится в draining, текущим сессиям дают завершиться. +5. `Retire`: старое поколение удаляется по graceful-правилам. + +### Операционные гарантии +- нельзя активировать поколение частично без минимального покрытия; +- healthy-клиенты не должны теряться только из-за появления нового поколения; +- draining-поколение служит буфером для in-flight трафика во время swap. + +### Готовность и приём клиентов +Готовность пула не равна "все endpoint полностью насыщены". +Типичная стратегия: +- открыть admission при минимально достаточном alive-покрытии по DC; +- параллельно продолжать saturation для multi-endpoint DC. + +Это уменьшает startup latency и сохраняет выход на полную ёмкость. + +## Как понятия связаны между собой + +- `Generation` задаёт эпохи пула. +- `Trio-State` задаёт роль каждого writer-а внутри/между эпохами. +- `Adaptive Floor` задаёт, сколько ёмкости нужно сейчас. +- `Refill` — исполнитель, который закрывает разницу между desired и current capacity. +- `Registry` гарантирует корректную маршрутизацию сессий, пока всё выше меняется. + +## Архитектурный подход + +### Слоистая модель +Runtime специально разделён на две плоскости: +- `Control Plane`: принимает решения о целевой топологии и политиках (`floor`, `generation swap`, `refill`, `fallback`). +- `Data Plane`: исполняет транспорт сессий и пакетов (`reader`, `writer`, маршрутизация, ack, close). + +Ключевое правило: +- Control Plane может менять состав writer-ов и policy. +- Data Plane должен оставаться стабильным и низколатентным в момент этих изменений. + +### Модель владения состоянием +Владение разделено по доменам: +- `MePool` владеет жизненным циклом writer-ов и policy-state. +- `Registry` владеет routing-биндами клиентских сессий. +- `Writer task` владеет исходящей прогрессией ME-сокета. +- `Reader task` владеет входящим парсингом и dispatch-событиями. + +Это ограничивает побочные мутации и локализует инварианты. + +### Обязанности Control Plane +Control Plane работает событийно и policy-ориентированно: +- стартовая инициализация и readiness-gate; +- runtime reinit (периодический и/или по изменению конфигурации); +- проверки покрытия по DC/family/endpoint group; +- применение floor-политики (static/adaptive); +- планирование refill и orchestration retry; +- переходы поколений (`warm -> active`, прежний `active -> draining`). + +Для него важнее детерминизм, чем агрессивная краткосрочная реакция. + +### Обязанности Data Plane +Data Plane ориентирован на пропускную способность и предсказуемую задержку: +- bind клиентской сессии к writer-у; +- per-frame parsing/validation/dispatch; +- распространение ack/close; +- корректная реакция на missing conn/closed channel; +- минимальный лог-шум в hotpath. + +Data Plane не должен ждать операций, не критичных для корректности текущего фрейма. + +## Конкурентность и синхронизация + +### Принципы конкурентности +- Изоляция по writer-у: у каждого writer-а независимые send/read loop. +- Изоляция по сессии: состояние канала локально для `conn_id`. +- Асинхронное восстановление: refill/reconnect выполняются вне пакетного hotpath. + +### Стратегия синхронизации +- Для shared map используются короткие и узкие lock-секции. +- Read-heavy пути избегают длительных write-lock окон. +- Решения по backpressure локализованы на границе route/channel. + +Цель: +- медленный consumer должен деградировать локально, не останавливая глобальный прогресс writer-а. + +### Cancellation и shutdown +Reader/Writer loop должны быть cancellation-aware: +- явные cancel token / close command; +- безопасный unbind/cleanup через registry; +- детерминированный порядок: stop admission -> drain/close -> release resources. + +## Модель согласованности + +### Согласованность сессии +Для одного `conn_id`: +- одновременно ровно один активный route-target; +- close/unbind операции идемпотентны; +- потеря writer-а не оставляет dangling-бинды. + +### Согласованность поколения +Гарантии generation: +- новое поколение не активируется до прохождения минимального coverage-gate; +- предыдущее поколение остаётся в `draining` на время handover; +- принудительный вывод writer-ов ограничен policy (`drain ttl`, optional force-close), а не мгновенный. + +### Согласованность политик +Изменение policy (`adaptive/static floor`, fallback mode, retries) не должно ломать инварианты маршрутизации уже активных сессий. + +## Backpressure и управление потоком + +### Route-level backpressure +Route-каналы намеренно bounded. +При росте нагрузки: +- кратковременный burst поглощается; +- длительная перегрузка переходит в контролируемую drop-семантику; +- все drop-сценарии должны быть прозрачно видны в метриках. + +### Приоритет неблокирующего Reader +Входящий ME-reader path не должен сериализоваться из-за одной перегруженной клиентской сессии. +Практически это означает: +- использовать неблокирующую попытку route в parser loop; +- выносить тяжёлое восстановление в асинхронные side-path. + +## Стратегия доменов отказа + +### Отказ отдельного endpoint +Сначала применяется endpoint-local recovery: +- reconnect в тот же endpoint; +- затем замена endpoint внутри той же DC-группы (если доступно). + +### Деградация уровня DC +Если DC-группа не набирает floor: +- сервис сохраняется на остаточном покрытии (если policy разрешает); +- saturation refill продолжается асинхронно в фоне. + +### Потеря готовности всего пула +Если достаточного ME-покрытия нет: +- admission gate может временно закрыть приём новых подключений (conditional policy); +- уже активные сессии продолжают работать, пока их маршрут остаётся healthy. + +## Архитектурные заметки по производительности + +### Дисциплина hotpath +Допустимо в hotpath: +- фиксированный и дешёвый parsing/validation; +- bounded channel operations; +- precomputed/low-allocation доступ к данным. + +Нежелательно в hotpath: +- повторные дорогие decode; +- широкие lock-секции с `await` внутри; +- высокочастотный подробный logging. + +### Стабильность важнее пиков +Архитектура приоритетно выбирает стабильную пропускную способность и предсказуемую latency, а не краткосрочные пики ценой churn и long-tail reconnect. + +## Правила эволюции модели + +Чтобы расширять модель безопасно: +- новые policy knobs сначала внедрять в Control Plane; +- контракты Data Plane (`conn_id`, route/close семантика) держать стабильными; +- перед дефолтным включением проверять generation/registry инварианты; +- новые recovery/retry стратегии вводить через явный config-флаг. + +## Нюансы отказов и восстановления + +- падение single-endpoint DC — штатный деградированный сценарий; приоритет: быстрый reconnect и, при необходимости, shadow/probing; +- idle-close со стороны peer должен считаться нормальным событием при upstream idle-timeout; +- backoff reconnect-логики должен ограничивать синхронный churn, но сохранять быстрые первые попытки; +- fallback (`ME -> direct DC`) — это переключаемая policy-ветка, а не автоматический признак бага транспорта. + +## Краткий словарь +- `Coverage`: достаточное число живых writer-ов для политики приёма по DC. +- `Floor`: целевая минимальная ёмкость writer-ов. +- `Churn`: частые циклы reconnect/remove writer-ов. +- `Hotpath`: пер-пакетный/пер-коннектный путь, где любые лишние ожидания и аллокации особенно дороги. From 4e803b141278ec0998296d6518ee0621b5910052 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 12:08:43 +0300 Subject: [PATCH 087/261] Update load.rs --- src/config/load.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/config/load.rs b/src/config/load.rs index 470bc37..8fec710 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -555,11 +555,6 @@ impl ProxyConfig { warn!("prefer_ipv6 is deprecated, use [network].prefer = 6"); } - // Auto-enable NAT probe when Middle Proxy is requested. - if config.general.use_middle_proxy && !config.general.middle_proxy_nat_probe { - config.general.middle_proxy_nat_probe = true; - warn!("Auto-enabled middle_proxy_nat_probe for middle proxy mode"); - } if config.general.use_middle_proxy && !config.general.me_secret_atomic_snapshot { config.general.me_secret_atomic_snapshot = true; warn!( From 30ef8df1b36a9a090f2972f46e03ec9248e19668 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 12:44:40 +0300 Subject: [PATCH 088/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index cc6cd3d..b2c131f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.3" +version = "3.3.4" edition = "2024" [dependencies] From b3d3bca15a9969eee2c40f2725411b99828c0c86 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 12:46:51 +0300 Subject: [PATCH 089/261] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ae0ce6..37c907e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ***Löst Probleme, bevor andere überhaupt wissen, dass sie existieren*** / ***It solves problems before others even realize they exist*** **Telemt** is a fast, secure, and feature-rich server written in Rust: it fully implements the official Telegram proxy algo and adds many production-ready improvements such as: -- ME Pool + Reader/Writer + Registry + Refill + Adaptive Floor + Trio-State + Generation Lifecycle +- [ME Pool + Reader/Writer + Registry + Refill + Adaptive Floor + Trio-State + Generation Lifecycle](https://github.com/telemt/telemt/blob/main/docs/model/MODEL.en.md) - [Full-covered API w/ management](https://github.com/telemt/telemt/blob/main/docs/API.md) - Anti-Replay on Sliding Window - Prometheus-format Metrics From 92c22ef16d1efde47f163a5625334d37e5480a1b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 13:06:57 +0300 Subject: [PATCH 090/261] API Zero Added new endpoints: - GET /v1/system/info - GET /v1/runtime/gates - GET /v1/limits/effective - GET /v1/security/posture Added API runtime state without impacting the hot path: - config_reload_count - last_config_reload_epoch_secs - admission_open - process_started_at_epoch_secs Added background watcher tasks in api::serve: - configuration reload tracking - admission gate state tracking --- src/api/mod.rs | 84 ++++++++++++++- src/api/runtime_zero.rs | 227 ++++++++++++++++++++++++++++++++++++++++ src/main.rs | 9 +- 3 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 src/api/runtime_zero.rs diff --git a/src/api/mod.rs b/src/api/mod.rs index c01566a..f2d31da 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -2,7 +2,8 @@ use std::convert::Infallible; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; use http_body_util::{BodyExt, Full}; use hyper::body::{Bytes, Incoming}; @@ -25,6 +26,7 @@ use crate::transport::UpstreamManager; mod config_store; mod model; mod runtime_stats; +mod runtime_zero; mod users; use config_store::{current_revision, parse_if_match}; @@ -36,8 +38,19 @@ use runtime_stats::{ MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data, build_upstreams_data, build_zero_all_data, }; +use runtime_zero::{ + build_limits_effective_data, build_runtime_gates_data, build_security_posture_data, + build_system_info_data, +}; use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config}; +pub(super) struct ApiRuntimeState { + pub(super) process_started_at_epoch_secs: u64, + pub(super) config_reload_count: AtomicU64, + pub(super) last_config_reload_epoch_secs: AtomicU64, + pub(super) admission_open: AtomicBool, +} + #[derive(Clone)] pub(super) struct ApiShared { pub(super) stats: Arc, @@ -50,6 +63,7 @@ pub(super) struct ApiShared { pub(super) mutation_lock: Arc>, pub(super) minimal_cache: Arc>>, pub(super) request_id: Arc, + pub(super) runtime_state: Arc, } impl ApiShared { @@ -65,9 +79,11 @@ pub async fn serve( me_pool: Option>, upstream_manager: Arc, config_rx: watch::Receiver>, + admission_rx: watch::Receiver, config_path: PathBuf, startup_detected_ip_v4: Option, startup_detected_ip_v6: Option, + process_started_at_epoch_secs: u64, ) { let listener = match TcpListener::bind(listen).await { Ok(listener) => listener, @@ -83,6 +99,13 @@ pub async fn serve( info!("API endpoint: http://{}/v1/*", listen); + let runtime_state = Arc::new(ApiRuntimeState { + process_started_at_epoch_secs, + config_reload_count: AtomicU64::new(0), + last_config_reload_epoch_secs: AtomicU64::new(0), + admission_open: AtomicBool::new(*admission_rx.borrow()), + }); + let shared = Arc::new(ApiShared { stats, ip_tracker, @@ -94,6 +117,38 @@ pub async fn serve( mutation_lock: Arc::new(Mutex::new(())), minimal_cache: Arc::new(Mutex::new(None)), request_id: Arc::new(AtomicU64::new(1)), + runtime_state: runtime_state.clone(), + }); + + let mut config_rx_reload = config_rx.clone(); + let runtime_state_reload = runtime_state.clone(); + tokio::spawn(async move { + loop { + if config_rx_reload.changed().await.is_err() { + break; + } + runtime_state_reload + .config_reload_count + .fetch_add(1, Ordering::Relaxed); + runtime_state_reload + .last_config_reload_epoch_secs + .store(now_epoch_secs(), Ordering::Relaxed); + } + }); + + let mut admission_rx_watch = admission_rx.clone(); + tokio::spawn(async move { + runtime_state + .admission_open + .store(*admission_rx_watch.borrow(), Ordering::Relaxed); + loop { + if admission_rx_watch.changed().await.is_err() { + break; + } + runtime_state + .admission_open + .store(*admission_rx_watch.borrow(), Ordering::Relaxed); + } }); loop { @@ -189,6 +244,26 @@ async fn handle( }; Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/system/info") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision); + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/runtime/gates") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()); + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/limits/effective") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_limits_effective_data(cfg.as_ref()); + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/security/posture") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_security_posture_data(cfg.as_ref()); + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/summary") => { let revision = current_revision(&shared.config_path).await?; let data = SummaryData { @@ -441,3 +516,10 @@ async fn read_body_with_limit(body: Incoming, limit: usize) -> Result, A } Ok(collected) } + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs new file mode 100644 index 0000000..2c50020 --- /dev/null +++ b/src/api/runtime_zero.rs @@ -0,0 +1,227 @@ +use std::sync::atomic::Ordering; + +use serde::Serialize; + +use crate::config::{MeFloorMode, ProxyConfig, UserMaxUniqueIpsMode}; + +use super::ApiShared; + +#[derive(Serialize)] +pub(super) struct SystemInfoData { + pub(super) version: String, + pub(super) target_arch: String, + pub(super) target_os: String, + pub(super) build_profile: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) git_commit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) build_time_utc: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) rustc_version: Option, + pub(super) process_started_at_epoch_secs: u64, + pub(super) uptime_seconds: f64, + pub(super) config_path: String, + pub(super) config_hash: String, + pub(super) config_reload_count: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_config_reload_epoch_secs: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeGatesData { + pub(super) accepting_new_connections: bool, + pub(super) conditional_cast_enabled: bool, + pub(super) me_runtime_ready: bool, + pub(super) me2dc_fallback_enabled: bool, + pub(super) use_middle_proxy: bool, +} + +#[derive(Serialize)] +pub(super) struct EffectiveTimeoutLimits { + pub(super) client_handshake_secs: u64, + pub(super) tg_connect_secs: u64, + pub(super) client_keepalive_secs: u64, + pub(super) client_ack_secs: u64, + pub(super) me_one_retry: u8, + pub(super) me_one_timeout_ms: u64, +} + +#[derive(Serialize)] +pub(super) struct EffectiveUpstreamLimits { + pub(super) connect_retry_attempts: u32, + pub(super) connect_retry_backoff_ms: u64, + pub(super) connect_budget_ms: u64, + pub(super) unhealthy_fail_threshold: u32, + pub(super) connect_failfast_hard_errors: bool, +} + +#[derive(Serialize)] +pub(super) struct EffectiveMiddleProxyLimits { + pub(super) floor_mode: &'static str, + pub(super) adaptive_floor_idle_secs: u64, + pub(super) adaptive_floor_min_writers_single_endpoint: u8, + pub(super) adaptive_floor_recover_grace_secs: u64, + pub(super) reconnect_max_concurrent_per_dc: u32, + pub(super) reconnect_backoff_base_ms: u64, + pub(super) reconnect_backoff_cap_ms: u64, + pub(super) reconnect_fast_retry_count: u32, + pub(super) me2dc_fallback: bool, +} + +#[derive(Serialize)] +pub(super) struct EffectiveUserIpPolicyLimits { + pub(super) mode: &'static str, + pub(super) window_secs: u64, +} + +#[derive(Serialize)] +pub(super) struct EffectiveLimitsData { + pub(super) update_every_secs: u64, + pub(super) me_reinit_every_secs: u64, + pub(super) me_pool_force_close_secs: u64, + pub(super) timeouts: EffectiveTimeoutLimits, + pub(super) upstream: EffectiveUpstreamLimits, + pub(super) middle_proxy: EffectiveMiddleProxyLimits, + pub(super) user_ip_policy: EffectiveUserIpPolicyLimits, +} + +#[derive(Serialize)] +pub(super) struct SecurityPostureData { + pub(super) api_read_only: bool, + pub(super) api_whitelist_enabled: bool, + pub(super) api_whitelist_entries: usize, + pub(super) api_auth_header_enabled: bool, + pub(super) proxy_protocol_enabled: bool, + pub(super) log_level: String, + pub(super) telemetry_core_enabled: bool, + pub(super) telemetry_user_enabled: bool, + pub(super) telemetry_me_level: String, +} + +pub(super) fn build_system_info_data( + shared: &ApiShared, + _cfg: &ProxyConfig, + revision: &str, +) -> SystemInfoData { + let last_reload_epoch_secs = shared + .runtime_state + .last_config_reload_epoch_secs + .load(Ordering::Relaxed); + let last_config_reload_epoch_secs = (last_reload_epoch_secs > 0).then_some(last_reload_epoch_secs); + + let git_commit = option_env!("TELEMT_GIT_COMMIT") + .or(option_env!("VERGEN_GIT_SHA")) + .or(option_env!("GIT_COMMIT")) + .map(ToString::to_string); + let build_time_utc = option_env!("BUILD_TIME_UTC") + .or(option_env!("VERGEN_BUILD_TIMESTAMP")) + .map(ToString::to_string); + let rustc_version = option_env!("RUSTC_VERSION") + .or(option_env!("VERGEN_RUSTC_SEMVER")) + .map(ToString::to_string); + + SystemInfoData { + version: env!("CARGO_PKG_VERSION").to_string(), + target_arch: std::env::consts::ARCH.to_string(), + target_os: std::env::consts::OS.to_string(), + build_profile: option_env!("PROFILE").unwrap_or("unknown").to_string(), + git_commit, + build_time_utc, + rustc_version, + process_started_at_epoch_secs: shared.runtime_state.process_started_at_epoch_secs, + uptime_seconds: shared.stats.uptime_secs(), + config_path: shared.config_path.display().to_string(), + config_hash: revision.to_string(), + config_reload_count: shared.runtime_state.config_reload_count.load(Ordering::Relaxed), + last_config_reload_epoch_secs, + } +} + +pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) -> RuntimeGatesData { + let me_runtime_ready = if !cfg.general.use_middle_proxy { + true + } else { + shared + .me_pool + .as_ref() + .map(|pool| pool.is_runtime_ready()) + .unwrap_or(false) + }; + + RuntimeGatesData { + accepting_new_connections: shared.runtime_state.admission_open.load(Ordering::Relaxed), + conditional_cast_enabled: cfg.general.use_middle_proxy, + me_runtime_ready, + me2dc_fallback_enabled: cfg.general.me2dc_fallback, + use_middle_proxy: cfg.general.use_middle_proxy, + } +} + +pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsData { + EffectiveLimitsData { + update_every_secs: cfg.general.effective_update_every_secs(), + me_reinit_every_secs: cfg.general.effective_me_reinit_every_secs(), + me_pool_force_close_secs: cfg.general.effective_me_pool_force_close_secs(), + timeouts: EffectiveTimeoutLimits { + client_handshake_secs: cfg.timeouts.client_handshake, + tg_connect_secs: cfg.timeouts.tg_connect, + client_keepalive_secs: cfg.timeouts.client_keepalive, + client_ack_secs: cfg.timeouts.client_ack, + me_one_retry: cfg.timeouts.me_one_retry, + me_one_timeout_ms: cfg.timeouts.me_one_timeout_ms, + }, + upstream: EffectiveUpstreamLimits { + connect_retry_attempts: cfg.general.upstream_connect_retry_attempts, + connect_retry_backoff_ms: cfg.general.upstream_connect_retry_backoff_ms, + connect_budget_ms: cfg.general.upstream_connect_budget_ms, + unhealthy_fail_threshold: cfg.general.upstream_unhealthy_fail_threshold, + connect_failfast_hard_errors: cfg.general.upstream_connect_failfast_hard_errors, + }, + middle_proxy: EffectiveMiddleProxyLimits { + floor_mode: me_floor_mode_label(cfg.general.me_floor_mode), + adaptive_floor_idle_secs: cfg.general.me_adaptive_floor_idle_secs, + adaptive_floor_min_writers_single_endpoint: cfg + .general + .me_adaptive_floor_min_writers_single_endpoint, + adaptive_floor_recover_grace_secs: cfg.general.me_adaptive_floor_recover_grace_secs, + reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc, + reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms, + reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms, + reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count, + me2dc_fallback: cfg.general.me2dc_fallback, + }, + user_ip_policy: EffectiveUserIpPolicyLimits { + mode: user_max_unique_ips_mode_label(cfg.access.user_max_unique_ips_mode), + window_secs: cfg.access.user_max_unique_ips_window_secs, + }, + } +} + +pub(super) fn build_security_posture_data(cfg: &ProxyConfig) -> SecurityPostureData { + SecurityPostureData { + api_read_only: cfg.server.api.read_only, + api_whitelist_enabled: !cfg.server.api.whitelist.is_empty(), + api_whitelist_entries: cfg.server.api.whitelist.len(), + api_auth_header_enabled: !cfg.server.api.auth_header.is_empty(), + proxy_protocol_enabled: cfg.server.proxy_protocol, + log_level: cfg.general.log_level.to_string(), + telemetry_core_enabled: cfg.general.telemetry.core_enabled, + telemetry_user_enabled: cfg.general.telemetry.user_enabled, + telemetry_me_level: cfg.general.telemetry.me_level.to_string(), + } +} + +fn user_max_unique_ips_mode_label(mode: UserMaxUniqueIpsMode) -> &'static str { + match mode { + UserMaxUniqueIpsMode::ActiveWindow => "active_window", + UserMaxUniqueIpsMode::TimeWindow => "time_window", + UserMaxUniqueIpsMode::Combined => "combined", + } +} + +fn me_floor_mode_label(mode: MeFloorMode) -> &'static str { + match mode { + MeFloorMode::Static => "static", + MeFloorMode::Adaptive => "adaptive", + } +} diff --git a/src/main.rs b/src/main.rs index 7f546d8..a9207ac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use std::net::SocketAddr; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use rand::Rng; use tokio::net::TcpListener; use tokio::signal; @@ -369,6 +369,10 @@ async fn load_startup_proxy_config_snapshot( #[tokio::main] async fn main() -> std::result::Result<(), Box> { let process_started_at = Instant::now(); + let process_started_at_epoch_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); let (config_path, cli_silent, cli_log_level) = parse_cli(); let mut config = match ProxyConfig::load(&config_path) { @@ -1556,6 +1560,7 @@ async fn main() -> std::result::Result<(), Box> { let me_pool_api = me_pool.clone(); let upstream_manager_api = upstream_manager.clone(); let config_rx_api = config_rx.clone(); + let admission_rx_api = admission_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); let startup_detected_ip_v4 = detected_ip_v4; let startup_detected_ip_v6 = detected_ip_v6; @@ -1567,9 +1572,11 @@ async fn main() -> std::result::Result<(), Box> { me_pool_api, upstream_manager_api, config_rx_api, + admission_rx_api, config_path_api, startup_detected_ip_v4, startup_detected_ip_v6, + process_started_at_epoch_secs, ) .await; }); From fc160913bf0ec9662a4e018958ceed2414eb0caa Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 13:07:31 +0300 Subject: [PATCH 091/261] Update API.md --- docs/API.md | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/docs/API.md b/docs/API.md index 50cfb4e..cb964d9 100644 --- a/docs/API.md +++ b/docs/API.md @@ -76,6 +76,10 @@ Notes: | Method | Path | Body | Success | `data` contract | | --- | --- | --- | --- | --- | | `GET` | `/v1/health` | none | `200` | `HealthData` | +| `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` | +| `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` | +| `GET` | `/v1/limits/effective` | none | `200` | `EffectiveLimitsData` | +| `GET` | `/v1/security/posture` | none | `200` | `SecurityPostureData` | | `GET` | `/v1/stats/summary` | none | `200` | `SummaryData` | | `GET` | `/v1/stats/zero/all` | none | `200` | `ZeroAllData` | | `GET` | `/v1/stats/upstreams` | none | `200` | `UpstreamsData` | @@ -176,6 +180,94 @@ Note: the request contract is defined, but the corresponding route currently ret | `handshake_timeouts_total` | `u64` | Handshake timeout count. | | `configured_users` | `usize` | Number of configured users in config. | +### `SystemInfoData` +| Field | Type | Description | +| --- | --- | --- | +| `version` | `string` | Binary version (`CARGO_PKG_VERSION`). | +| `target_arch` | `string` | Target architecture (`std::env::consts::ARCH`). | +| `target_os` | `string` | Target OS (`std::env::consts::OS`). | +| `build_profile` | `string` | Build profile (`PROFILE` env when available). | +| `git_commit` | `string?` | Optional commit hash from build env metadata. | +| `build_time_utc` | `string?` | Optional build timestamp from build env metadata. | +| `rustc_version` | `string?` | Optional compiler version from build env metadata. | +| `process_started_at_epoch_secs` | `u64` | Process start time as Unix epoch seconds. | +| `uptime_seconds` | `f64` | Process uptime in seconds. | +| `config_path` | `string` | Active config file path used by runtime. | +| `config_hash` | `string` | SHA-256 hash of current config content (same value as envelope `revision`). | +| `config_reload_count` | `u64` | Number of successfully observed config updates since process start. | +| `last_config_reload_epoch_secs` | `u64?` | Unix epoch seconds of the latest observed config reload; null/absent before first reload. | + +### `RuntimeGatesData` +| Field | Type | Description | +| --- | --- | --- | +| `accepting_new_connections` | `bool` | Current admission-gate state for new listener accepts. | +| `conditional_cast_enabled` | `bool` | Whether conditional ME admission logic is enabled (`general.use_middle_proxy`). | +| `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. | +| `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. | +| `use_middle_proxy` | `bool` | Current transport mode preference. | + +### `EffectiveLimitsData` +| Field | Type | Description | +| --- | --- | --- | +| `update_every_secs` | `u64` | Effective unified updater interval. | +| `me_reinit_every_secs` | `u64` | Effective ME periodic reinit interval. | +| `me_pool_force_close_secs` | `u64` | Effective stale-writer force-close timeout. | +| `timeouts` | `EffectiveTimeoutLimits` | Effective timeout policy snapshot. | +| `upstream` | `EffectiveUpstreamLimits` | Effective upstream connect/retry limits. | +| `middle_proxy` | `EffectiveMiddleProxyLimits` | Effective ME pool/floor/reconnect limits. | +| `user_ip_policy` | `EffectiveUserIpPolicyLimits` | Effective unique-IP policy mode/window. | + +#### `EffectiveTimeoutLimits` +| Field | Type | Description | +| --- | --- | --- | +| `client_handshake_secs` | `u64` | Client handshake timeout. | +| `tg_connect_secs` | `u64` | Upstream Telegram connect timeout. | +| `client_keepalive_secs` | `u64` | Client keepalive interval. | +| `client_ack_secs` | `u64` | ACK timeout. | +| `me_one_retry` | `u8` | Fast retry count for single-endpoint ME DC. | +| `me_one_timeout_ms` | `u64` | Fast retry timeout per attempt for single-endpoint ME DC. | + +#### `EffectiveUpstreamLimits` +| Field | Type | Description | +| --- | --- | --- | +| `connect_retry_attempts` | `u32` | Upstream connect retry attempts. | +| `connect_retry_backoff_ms` | `u64` | Upstream retry backoff delay. | +| `connect_budget_ms` | `u64` | Total connect wall-clock budget across retries. | +| `unhealthy_fail_threshold` | `u32` | Consecutive fail threshold for unhealthy marking. | +| `connect_failfast_hard_errors` | `bool` | Whether hard errors skip additional retries. | + +#### `EffectiveMiddleProxyLimits` +| Field | Type | Description | +| --- | --- | --- | +| `floor_mode` | `string` | Effective floor mode (`static` or `adaptive`). | +| `adaptive_floor_idle_secs` | `u64` | Adaptive floor idle threshold. | +| `adaptive_floor_min_writers_single_endpoint` | `u8` | Adaptive floor minimum for single-endpoint DCs. | +| `adaptive_floor_recover_grace_secs` | `u64` | Adaptive floor recovery grace period. | +| `reconnect_max_concurrent_per_dc` | `u32` | Max concurrent reconnects per DC. | +| `reconnect_backoff_base_ms` | `u64` | Reconnect base backoff. | +| `reconnect_backoff_cap_ms` | `u64` | Reconnect backoff cap. | +| `reconnect_fast_retry_count` | `u32` | Number of fast retries before standard backoff strategy. | +| `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. | + +#### `EffectiveUserIpPolicyLimits` +| Field | Type | Description | +| --- | --- | --- | +| `mode` | `string` | Unique-IP policy mode (`active_window`, `time_window`, `combined`). | +| `window_secs` | `u64` | Time window length used by unique-IP policy. | + +### `SecurityPostureData` +| Field | Type | Description | +| --- | --- | --- | +| `api_read_only` | `bool` | Current API read-only state. | +| `api_whitelist_enabled` | `bool` | Whether whitelist filtering is active. | +| `api_whitelist_entries` | `usize` | Number of configured whitelist CIDRs. | +| `api_auth_header_enabled` | `bool` | Whether `Authorization` header validation is active. | +| `proxy_protocol_enabled` | `bool` | Global PROXY protocol accept setting. | +| `log_level` | `string` | Effective log level (`debug`, `verbose`, `normal`, `silent`). | +| `telemetry_core_enabled` | `bool` | Core telemetry toggle. | +| `telemetry_user_enabled` | `bool` | Per-user telemetry toggle. | +| `telemetry_me_level` | `string` | ME telemetry level (`silent`, `normal`, `debug`). | + ### `ZeroAllData` | Field | Type | Description | | --- | --- | --- | From bdcf110c87291617f729b21759c6b998e53cac6e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 15:01:51 +0300 Subject: [PATCH 092/261] Update README.md --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 37c907e..c4ddc5a 100644 --- a/README.md +++ b/README.md @@ -19,18 +19,18 @@ ### 🇷🇺 RU -#### Релиз 3.0.15 — 25 февраля +#### Релиз 3.3.3 LTS - 6 марта -25 февраля мы выпустили версию **3.0.15** +6 марта мы выпустили Telemt **3.3.3** -Мы предполагаем, что она станет завершающей версией поколения 3.0 и уже сейчас мы рассматриваем её как **LTS-кандидата** для версии **3.1.0**! +Это первая версия telemt работающая в комплексных условиях и при этом предоставляющая API -После нескольких дней детального анализа особенностей работы Middle-End мы спроектировали и реализовали продуманный режим **ротации ME Writer**. Данный режим позволяет поддерживать стабильно высокую производительность в long-run сценариях без возникновения ошибок, связанных с некорректной конфигурацией прокси +В ней используется новый алгоритм - ME NoWait, который вместе с Adaptive Floor и моделью усовершенствованного доступа к KDF Fingerprint на RwLock позволяет достигать максимальную производительность, даже в условиях lossy-сети Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **статистики** и **UX** Релиз: -[3.0.15](https://github.com/telemt/telemt/releases/tag/3.0.15) +[3.3.3](https://github.com/telemt/telemt/releases/tag/3.3.3) --- @@ -47,18 +47,18 @@ ### 🇬🇧 EN -#### Release 3.0.15 — February 25 +#### Release 3.3.3 LTS - March 6 -On February 25, we released version **3.0.15** +On March 6, we released Telemt **3.3.3** -We expect this to become the final release of the 3.0 generation and at this point, we already see it as a strong **LTS candidate** for the upcoming **3.1.0** release! +This is the first telemt's version designed to operate reliably in complex network conditions while also providing a runtime API! -After several days of deep analysis of Middle-End behavior, we designed and implemented a well-engineered **ME Writer rotation mode**. This mode enables sustained high throughput in long-run scenarios while preventing proxy misconfiguration errors +The release introduces a new algorithm — ME NoWait, which combined with Adaptive Floor and an improved KDF Fingerprint access model based on RwLock, it enables the system to achieve maximum performance even in lossy network environments We are looking forward to your feedback and improvement proposals — especially regarding **statistics** and **UX** Release: -[3.0.15](https://github.com/telemt/telemt/releases/tag/3.0.15) +[3.3.3](https://github.com/telemt/telemt/releases/tag/3.3.3) --- @@ -79,7 +79,7 @@ We welcome ideas, architectural feedback, and pull requests. ⚓ Our implementation of **TLS-fronting** is one of the most deeply debugged, focused, advanced and *almost* **"behaviorally consistent to real"**: we are confident we have it right - [see evidence on our validation and traces](#recognizability-for-dpi-and-crawler) -⚓ Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to the Middle-End Proxy: non dramatically, but usual +⚓ Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to he Middle-End Proxy: non dramatically, but usual # GOTO - [Features](#features) From edce194948f7fcbe905195451168f23e7fa372dc Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 15:02:56 +0300 Subject: [PATCH 093/261] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c4ddc5a..a1c2958 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ We welcome ideas, architectural feedback, and pull requests. ⚓ Our implementation of **TLS-fronting** is one of the most deeply debugged, focused, advanced and *almost* **"behaviorally consistent to real"**: we are confident we have it right - [see evidence on our validation and traces](#recognizability-for-dpi-and-crawler) -⚓ Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to he Middle-End Proxy: non dramatically, but usual +⚓ Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to the Middle-End Proxy: non dramatically, but usual # GOTO - [Features](#features) From d7716ad8757eea2fde545d5172110ecebd398233 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:52:17 +0300 Subject: [PATCH 094/261] Upstream API Policy Snapshot --- src/transport/upstream.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index b9db0eb..84c6fdf 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -202,6 +202,15 @@ pub struct UpstreamApiSnapshot { pub upstreams: Vec, } +#[derive(Debug, Clone, Copy)] +pub struct UpstreamApiPolicySnapshot { + pub connect_retry_attempts: u32, + pub connect_retry_backoff_ms: u64, + pub connect_budget_ms: u64, + pub unhealthy_fail_threshold: u32, + pub connect_failfast_hard_errors: bool, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct UpstreamEgressInfo { pub route_kind: UpstreamRouteKind, @@ -315,6 +324,16 @@ impl UpstreamManager { Some(UpstreamApiSnapshot { summary, upstreams }) } + pub fn api_policy_snapshot(&self) -> UpstreamApiPolicySnapshot { + UpstreamApiPolicySnapshot { + connect_retry_attempts: self.connect_retry_attempts, + connect_retry_backoff_ms: self.connect_retry_backoff.as_millis() as u64, + connect_budget_ms: self.connect_budget.as_millis() as u64, + unhealthy_fail_threshold: self.unhealthy_fail_threshold, + connect_failfast_hard_errors: self.connect_failfast_hard_errors, + } + } + #[cfg(unix)] fn resolve_interface_addrs(name: &str, want_ipv6: bool) -> Vec { use nix::ifaddrs::getifaddrs; From c465c200c4478900c7fc1e3a51ee084fa6b98fb9 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:52:31 +0300 Subject: [PATCH 095/261] ME Pool Runtime API --- .../middle_proxy/pool_runtime_api.rs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 src/transport/middle_proxy/pool_runtime_api.rs diff --git a/src/transport/middle_proxy/pool_runtime_api.rs b/src/transport/middle_proxy/pool_runtime_api.rs new file mode 100644 index 0000000..37ef298 --- /dev/null +++ b/src/transport/middle_proxy/pool_runtime_api.rs @@ -0,0 +1,128 @@ +use std::collections::HashMap; +use std::time::Instant; + +use super::pool::{MePool, RefillDcKey}; +use crate::network::IpFamily; + +#[derive(Clone, Debug)] +pub(crate) struct MeApiRefillDcSnapshot { + pub dc: i16, + pub family: &'static str, + pub inflight: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiRefillSnapshot { + pub inflight_endpoints_total: usize, + pub inflight_dc_total: usize, + pub by_dc: Vec, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiNatReflectionSnapshot { + pub addr: std::net::SocketAddr, + pub age_secs: u64, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiNatStunSnapshot { + pub nat_probe_enabled: bool, + pub nat_probe_disabled_runtime: bool, + pub nat_probe_attempts: u8, + pub configured_servers: Vec, + pub live_servers: Vec, + pub reflection_v4: Option, + pub reflection_v6: Option, + pub stun_backoff_remaining_ms: Option, +} + +impl MePool { + pub(crate) async fn api_refill_snapshot(&self) -> MeApiRefillSnapshot { + let inflight_endpoints_total = self.refill_inflight.lock().await.len(); + let inflight_dc_keys = self + .refill_inflight_dc + .lock() + .await + .iter() + .copied() + .collect::>(); + + let mut by_dc_map = HashMap::<(i16, &'static str), usize>::new(); + for key in inflight_dc_keys { + let family = match key.family { + IpFamily::V4 => "v4", + IpFamily::V6 => "v6", + }; + let dc = key.dc as i16; + *by_dc_map.entry((dc, family)).or_insert(0) += 1; + } + + let mut by_dc = by_dc_map + .into_iter() + .map(|((dc, family), inflight)| MeApiRefillDcSnapshot { + dc, + family, + inflight, + }) + .collect::>(); + by_dc.sort_by_key(|entry| (entry.dc, entry.family)); + + MeApiRefillSnapshot { + inflight_endpoints_total, + inflight_dc_total: by_dc.len(), + by_dc, + } + } + + pub(crate) async fn api_nat_stun_snapshot(&self) -> MeApiNatStunSnapshot { + let now = Instant::now(); + let mut configured_servers = if !self.nat_stun_servers.is_empty() { + self.nat_stun_servers.clone() + } else if let Some(stun) = &self.nat_stun { + if stun.trim().is_empty() { + Vec::new() + } else { + vec![stun.clone()] + } + } else { + Vec::new() + }; + configured_servers.sort(); + configured_servers.dedup(); + + let mut live_servers = self.nat_stun_live_servers.read().await.clone(); + live_servers.sort(); + live_servers.dedup(); + + let reflection = self.nat_reflection_cache.lock().await; + let reflection_v4 = reflection.v4.map(|(ts, addr)| MeApiNatReflectionSnapshot { + addr, + age_secs: now.saturating_duration_since(ts).as_secs(), + }); + let reflection_v6 = reflection.v6.map(|(ts, addr)| MeApiNatReflectionSnapshot { + addr, + age_secs: now.saturating_duration_since(ts).as_secs(), + }); + drop(reflection); + + let backoff_until = *self.stun_backoff_until.read().await; + let stun_backoff_remaining_ms = backoff_until.and_then(|until| { + (until > now).then_some(until.duration_since(now).as_millis() as u64) + }); + + MeApiNatStunSnapshot { + nat_probe_enabled: self.nat_probe, + nat_probe_disabled_runtime: self + .nat_probe_disabled + .load(std::sync::atomic::Ordering::Relaxed), + nat_probe_attempts: self + .nat_probe_attempts + .load(std::sync::atomic::Ordering::Relaxed), + configured_servers, + live_servers, + reflection_v4, + reflection_v6, + stun_backoff_remaining_ms, + } + } +} From 487e95a66e5811ba0be02b8ddd0383a84a6c0992 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:52:39 +0300 Subject: [PATCH 096/261] Update mod.rs --- src/transport/middle_proxy/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 131e215..87c012b 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -10,6 +10,7 @@ mod pool_init; mod pool_nat; mod pool_refill; mod pool_reinit; +mod pool_runtime_api; mod pool_writer; mod ping; mod reader; From 44b825edf52339fa84b6068e325ad5e344b5cdab Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:53:13 +0300 Subject: [PATCH 097/261] Atomics in Stats Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/stats/mod.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 4cc9933..4b59367 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -25,6 +25,8 @@ use self::telemetry::TelemetryPolicy; pub struct Stats { connects_all: AtomicU64, connects_bad: AtomicU64, + current_connections_direct: AtomicU64, + current_connections_me: AtomicU64, handshake_timeouts: AtomicU64, upstream_connect_attempt_total: AtomicU64, upstream_connect_success_total: AtomicU64, @@ -150,6 +152,24 @@ impl Stats { self.telemetry_me_level().allows_debug() } + fn decrement_atomic_saturating(counter: &AtomicU64) { + let mut current = counter.load(Ordering::Relaxed); + loop { + if current == 0 { + break; + } + match counter.compare_exchange_weak( + current, + current - 1, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) { self.telemetry_core_enabled .store(policy.core_enabled, Ordering::Relaxed); @@ -177,6 +197,18 @@ impl Stats { self.connects_bad.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_current_connections_direct(&self) { + self.current_connections_direct.fetch_add(1, Ordering::Relaxed); + } + pub fn decrement_current_connections_direct(&self) { + Self::decrement_atomic_saturating(&self.current_connections_direct); + } + pub fn increment_current_connections_me(&self) { + self.current_connections_me.fetch_add(1, Ordering::Relaxed); + } + pub fn decrement_current_connections_me(&self) { + Self::decrement_atomic_saturating(&self.current_connections_me); + } pub fn increment_handshake_timeouts(&self) { if self.telemetry_core_enabled() { self.handshake_timeouts.fetch_add(1, Ordering::Relaxed); @@ -646,6 +678,16 @@ impl Stats { } pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) } pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) } + pub fn get_current_connections_direct(&self) -> u64 { + self.current_connections_direct.load(Ordering::Relaxed) + } + pub fn get_current_connections_me(&self) -> u64 { + self.current_connections_me.load(Ordering::Relaxed) + } + pub fn get_current_connections_total(&self) -> u64 { + self.get_current_connections_direct() + .saturating_add(self.get_current_connections_me()) + } pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) } pub fn get_me_keepalive_failed(&self) -> u64 { self.me_keepalive_failed.load(Ordering::Relaxed) } pub fn get_me_keepalive_pong(&self) -> u64 { self.me_keepalive_pong.load(Ordering::Relaxed) } From 83ed9065b046d102ee1a2eea00e3c37eba6e26c3 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:53:22 +0300 Subject: [PATCH 098/261] Update middle_relay.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/proxy/middle_relay.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index 8f5fc36..8384e32 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -237,6 +237,7 @@ where stats.increment_user_connects(&user); stats.increment_user_curr_connects(&user); + stats.increment_current_connections_me(); // Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable) let user_tag: Option> = config @@ -466,6 +467,7 @@ where "ME relay cleanup" ); me_pool.registry().unregister(conn_id).await; + stats.decrement_current_connections_me(); stats.decrement_user_curr_connects(&user); result } From 2a3b6b917f89eabf4e5ee9569a106d22cd65eae0 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:53:28 +0300 Subject: [PATCH 099/261] Update direct_relay.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/proxy/direct_relay.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index 1245f34..e39e446 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -57,6 +57,7 @@ where stats.increment_user_connects(user); stats.increment_user_curr_connects(user); + stats.increment_current_connections_direct(); let relay_result = relay_bidirectional( client_reader, @@ -69,6 +70,7 @@ where ) .await; + stats.decrement_current_connections_direct(); stats.decrement_user_curr_connects(user); match &relay_result { From 2ea85c00d3ccc30799443d5a8a9aaaec9a8837a0 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:54:00 +0300 Subject: [PATCH 100/261] Runtime API Defaults --- src/config/defaults.rs | 5 ++ src/config/hot_reload.rs | 6 ++ src/config/load.rs | 116 +++++++++++++++++++++++++++++++++++++++ src/config/types.rs | 20 +++++++ 4 files changed, 147 insertions(+) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 9851216..3ba146c 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -114,6 +114,11 @@ pub(crate) fn default_api_minimal_runtime_cache_ttl_ms() -> u64 { 1000 } +pub(crate) fn default_api_runtime_edge_enabled() -> bool { false } +pub(crate) fn default_api_runtime_edge_cache_ttl_ms() -> u64 { 1000 } +pub(crate) fn default_api_runtime_edge_top_n() -> usize { 10 } +pub(crate) fn default_api_runtime_edge_events_capacity() -> usize { 256 } + pub(crate) fn default_proxy_protocol_header_timeout_ms() -> u64 { 500 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 97d5e4e..c39cafa 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -312,6 +312,12 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b || old.server.api.minimal_runtime_enabled != new.server.api.minimal_runtime_enabled || old.server.api.minimal_runtime_cache_ttl_ms != new.server.api.minimal_runtime_cache_ttl_ms + || old.server.api.runtime_edge_enabled != new.server.api.runtime_edge_enabled + || old.server.api.runtime_edge_cache_ttl_ms + != new.server.api.runtime_edge_cache_ttl_ms + || old.server.api.runtime_edge_top_n != new.server.api.runtime_edge_top_n + || old.server.api.runtime_edge_events_capacity + != new.server.api.runtime_edge_events_capacity || old.server.api.read_only != new.server.api.read_only { warned = true; diff --git a/src/config/load.rs b/src/config/load.rs index 8fec710..6ce7b65 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -462,6 +462,24 @@ impl ProxyConfig { )); } + if config.server.api.runtime_edge_cache_ttl_ms > 60_000 { + return Err(ProxyError::Config( + "server.api.runtime_edge_cache_ttl_ms must be within [0, 60000]".to_string(), + )); + } + + if !(1..=1000).contains(&config.server.api.runtime_edge_top_n) { + return Err(ProxyError::Config( + "server.api.runtime_edge_top_n must be within [1, 1000]".to_string(), + )); + } + + if !(16..=4096).contains(&config.server.api.runtime_edge_events_capacity) { + return Err(ProxyError::Config( + "server.api.runtime_edge_events_capacity must be within [16, 4096]".to_string(), + )); + } + if config.server.api.listen.parse::().is_err() { return Err(ProxyError::Config( "server.api.listen must be in IP:PORT format".to_string(), @@ -802,6 +820,22 @@ mod tests { cfg.server.api.minimal_runtime_cache_ttl_ms, default_api_minimal_runtime_cache_ttl_ms() ); + assert_eq!( + cfg.server.api.runtime_edge_enabled, + default_api_runtime_edge_enabled() + ); + assert_eq!( + cfg.server.api.runtime_edge_cache_ttl_ms, + default_api_runtime_edge_cache_ttl_ms() + ); + assert_eq!( + cfg.server.api.runtime_edge_top_n, + default_api_runtime_edge_top_n() + ); + assert_eq!( + cfg.server.api.runtime_edge_events_capacity, + default_api_runtime_edge_events_capacity() + ); assert_eq!(cfg.access.users, default_access_users()); assert_eq!( cfg.access.user_max_unique_ips_mode, @@ -918,6 +952,22 @@ mod tests { server.api.minimal_runtime_cache_ttl_ms, default_api_minimal_runtime_cache_ttl_ms() ); + assert_eq!( + server.api.runtime_edge_enabled, + default_api_runtime_edge_enabled() + ); + assert_eq!( + server.api.runtime_edge_cache_ttl_ms, + default_api_runtime_edge_cache_ttl_ms() + ); + assert_eq!( + server.api.runtime_edge_top_n, + default_api_runtime_edge_top_n() + ); + assert_eq!( + server.api.runtime_edge_events_capacity, + default_api_runtime_edge_events_capacity() + ); let access = AccessConfig::default(); assert_eq!(access.users, default_access_users()); @@ -1565,6 +1615,72 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn api_runtime_edge_cache_ttl_out_of_range_is_rejected() { + let toml = r#" + [server.api] + enabled = true + listen = "127.0.0.1:9091" + runtime_edge_cache_ttl_ms = 70000 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_api_runtime_edge_cache_ttl_invalid_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("server.api.runtime_edge_cache_ttl_ms must be within [0, 60000]")); + let _ = std::fs::remove_file(path); + } + + #[test] + fn api_runtime_edge_top_n_out_of_range_is_rejected() { + let toml = r#" + [server.api] + enabled = true + listen = "127.0.0.1:9091" + runtime_edge_top_n = 0 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_api_runtime_edge_top_n_invalid_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("server.api.runtime_edge_top_n must be within [1, 1000]")); + let _ = std::fs::remove_file(path); + } + + #[test] + fn api_runtime_edge_events_capacity_out_of_range_is_rejected() { + let toml = r#" + [server.api] + enabled = true + listen = "127.0.0.1:9091" + runtime_edge_events_capacity = 8 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_api_runtime_edge_events_capacity_invalid_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("server.api.runtime_edge_events_capacity must be within [16, 4096]")); + let _ = std::fs::remove_file(path); + } + #[test] fn force_close_bumped_when_below_drain_ttl() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index be238d3..4a33b7c 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -918,6 +918,22 @@ pub struct ApiConfig { #[serde(default = "default_api_minimal_runtime_cache_ttl_ms")] pub minimal_runtime_cache_ttl_ms: u64, + /// Enables runtime edge endpoints with optional cached aggregation. + #[serde(default = "default_api_runtime_edge_enabled")] + pub runtime_edge_enabled: bool, + + /// Cache TTL for runtime edge aggregation payloads in milliseconds. + #[serde(default = "default_api_runtime_edge_cache_ttl_ms")] + pub runtime_edge_cache_ttl_ms: u64, + + /// Top-N limit for edge connection leaderboard payloads. + #[serde(default = "default_api_runtime_edge_top_n")] + pub runtime_edge_top_n: usize, + + /// Ring-buffer capacity for runtime edge control-plane events. + #[serde(default = "default_api_runtime_edge_events_capacity")] + pub runtime_edge_events_capacity: usize, + /// Read-only mode: mutating endpoints are rejected. #[serde(default)] pub read_only: bool, @@ -933,6 +949,10 @@ impl Default for ApiConfig { request_body_limit_bytes: default_api_request_body_limit_bytes(), minimal_runtime_enabled: default_api_minimal_runtime_enabled(), minimal_runtime_cache_ttl_ms: default_api_minimal_runtime_cache_ttl_ms(), + runtime_edge_enabled: default_api_runtime_edge_enabled(), + runtime_edge_cache_ttl_ms: default_api_runtime_edge_cache_ttl_ms(), + runtime_edge_top_n: default_api_runtime_edge_top_n(), + runtime_edge_events_capacity: default_api_runtime_edge_events_capacity(), read_only: false, } } From fb5e9947bd46e0eb5cda6dec8c1526529b80e763 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:54:12 +0300 Subject: [PATCH 101/261] Runtime Watch --- src/api/runtime_watch.rs | 66 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/api/runtime_watch.rs diff --git a/src/api/runtime_watch.rs b/src/api/runtime_watch.rs new file mode 100644 index 0000000..0485e55 --- /dev/null +++ b/src/api/runtime_watch.rs @@ -0,0 +1,66 @@ +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::{SystemTime, UNIX_EPOCH}; + +use tokio::sync::watch; + +use crate::config::ProxyConfig; + +use super::ApiRuntimeState; +use super::events::ApiEventStore; + +pub(super) fn spawn_runtime_watchers( + config_rx: watch::Receiver>, + admission_rx: watch::Receiver, + runtime_state: Arc, + runtime_events: Arc, +) { + let mut config_rx_reload = config_rx; + let runtime_state_reload = runtime_state.clone(); + let runtime_events_reload = runtime_events.clone(); + tokio::spawn(async move { + loop { + if config_rx_reload.changed().await.is_err() { + break; + } + runtime_state_reload + .config_reload_count + .fetch_add(1, Ordering::Relaxed); + runtime_state_reload + .last_config_reload_epoch_secs + .store(now_epoch_secs(), Ordering::Relaxed); + runtime_events_reload.record("config.reload.applied", "config receiver updated"); + } + }); + + let mut admission_rx_watch = admission_rx; + tokio::spawn(async move { + runtime_state + .admission_open + .store(*admission_rx_watch.borrow(), Ordering::Relaxed); + runtime_events.record( + "admission.state", + format!("accepting_new_connections={}", *admission_rx_watch.borrow()), + ); + loop { + if admission_rx_watch.changed().await.is_err() { + break; + } + let admission_open = *admission_rx_watch.borrow(); + runtime_state + .admission_open + .store(admission_open, Ordering::Relaxed); + runtime_events.record( + "admission.state", + format!("accepting_new_connections={}", admission_open), + ); + } + }); +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} From 2d98ebf3c3dd39124ad0caa17593b1a8130c08e6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:54:26 +0300 Subject: [PATCH 102/261] Runtime w/ Minimal Overhead --- src/api/runtime_min.rs | 534 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 534 insertions(+) create mode 100644 src/api/runtime_min.rs diff --git a/src/api/runtime_min.rs b/src/api/runtime_min.rs new file mode 100644 index 0000000..96270df --- /dev/null +++ b/src/api/runtime_min.rs @@ -0,0 +1,534 @@ +use std::collections::BTreeSet; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Serialize; + +use crate::config::ProxyConfig; + +use super::ApiShared; + +const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable"; + +#[derive(Serialize)] +pub(super) struct SecurityWhitelistData { + pub(super) generated_at_epoch_secs: u64, + pub(super) enabled: bool, + pub(super) entries_total: usize, + pub(super) entries: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateGenerationData { + pub(super) active_generation: u64, + pub(super) warm_generation: u64, + pub(super) pending_hardswap_generation: u64, + pub(super) pending_hardswap_age_secs: Option, + pub(super) draining_generations: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateHardswapData { + pub(super) enabled: bool, + pub(super) pending: bool, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateWriterContourData { + pub(super) warm: usize, + pub(super) active: usize, + pub(super) draining: usize, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateWriterHealthData { + pub(super) healthy: usize, + pub(super) degraded: usize, + pub(super) draining: usize, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateWriterData { + pub(super) total: usize, + pub(super) alive_non_draining: usize, + pub(super) draining: usize, + pub(super) degraded: usize, + pub(super) contour: RuntimeMePoolStateWriterContourData, + pub(super) health: RuntimeMePoolStateWriterHealthData, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateRefillDcData { + pub(super) dc: i16, + pub(super) family: &'static str, + pub(super) inflight: usize, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateRefillData { + pub(super) inflight_endpoints_total: usize, + pub(super) inflight_dc_total: usize, + pub(super) by_dc: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStatePayload { + pub(super) generations: RuntimeMePoolStateGenerationData, + pub(super) hardswap: RuntimeMePoolStateHardswapData, + pub(super) writers: RuntimeMePoolStateWriterData, + pub(super) refill: RuntimeMePoolStateRefillData, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMePoolStateData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityCountersData { + pub(super) idle_close_by_peer_total: u64, + pub(super) reader_eof_total: u64, + pub(super) kdf_drift_total: u64, + pub(super) kdf_port_only_drift_total: u64, + pub(super) reconnect_attempt_total: u64, + pub(super) reconnect_success_total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityRouteDropData { + pub(super) no_conn_total: u64, + pub(super) channel_closed_total: u64, + pub(super) queue_full_total: u64, + pub(super) queue_full_base_total: u64, + pub(super) queue_full_high_total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityDcRttData { + pub(super) dc: i16, + pub(super) rtt_ema_ms: Option, + pub(super) alive_writers: usize, + pub(super) required_writers: usize, + pub(super) coverage_pct: f64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityPayload { + pub(super) counters: RuntimeMeQualityCountersData, + pub(super) route_drops: RuntimeMeQualityRouteDropData, + pub(super) dc_rtt: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeUpstreamQualityPolicyData { + pub(super) connect_retry_attempts: u32, + pub(super) connect_retry_backoff_ms: u64, + pub(super) connect_budget_ms: u64, + pub(super) unhealthy_fail_threshold: u32, + pub(super) connect_failfast_hard_errors: bool, +} + +#[derive(Serialize)] +pub(super) struct RuntimeUpstreamQualityCountersData { + pub(super) connect_attempt_total: u64, + pub(super) connect_success_total: u64, + pub(super) connect_fail_total: u64, + pub(super) connect_failfast_hard_error_total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeUpstreamQualitySummaryData { + pub(super) configured_total: usize, + pub(super) healthy_total: usize, + pub(super) unhealthy_total: usize, + pub(super) direct_total: usize, + pub(super) socks4_total: usize, + pub(super) socks5_total: usize, +} + +#[derive(Serialize)] +pub(super) struct RuntimeUpstreamQualityDcData { + pub(super) dc: i16, + pub(super) latency_ema_ms: Option, + pub(super) ip_preference: &'static str, +} + +#[derive(Serialize)] +pub(super) struct RuntimeUpstreamQualityUpstreamData { + pub(super) upstream_id: usize, + pub(super) route_kind: &'static str, + pub(super) address: String, + pub(super) weight: u16, + pub(super) scopes: String, + pub(super) healthy: bool, + pub(super) fails: u32, + pub(super) last_check_age_secs: u64, + pub(super) effective_latency_ms: Option, + pub(super) dc: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeUpstreamQualityData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + pub(super) policy: RuntimeUpstreamQualityPolicyData, + pub(super) counters: RuntimeUpstreamQualityCountersData, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) upstreams: Option>, +} + +#[derive(Serialize)] +pub(super) struct RuntimeNatStunReflectionData { + pub(super) addr: String, + pub(super) age_secs: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeNatStunFlagsData { + pub(super) nat_probe_enabled: bool, + pub(super) nat_probe_disabled_runtime: bool, + pub(super) nat_probe_attempts: u8, +} + +#[derive(Serialize)] +pub(super) struct RuntimeNatStunServersData { + pub(super) configured: Vec, + pub(super) live: Vec, + pub(super) live_total: usize, +} + +#[derive(Serialize)] +pub(super) struct RuntimeNatStunReflectionBlockData { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) v4: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) v6: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeNatStunPayload { + pub(super) flags: RuntimeNatStunFlagsData, + pub(super) servers: RuntimeNatStunServersData, + pub(super) reflection: RuntimeNatStunReflectionBlockData, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) stun_backoff_remaining_ms: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeNatStunData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + +pub(super) fn build_security_whitelist_data(cfg: &ProxyConfig) -> SecurityWhitelistData { + let entries = cfg + .server + .api + .whitelist + .iter() + .map(ToString::to_string) + .collect::>(); + SecurityWhitelistData { + generated_at_epoch_secs: now_epoch_secs(), + enabled: !entries.is_empty(), + entries_total: entries.len(), + entries, + } +} + +pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData { + let now_epoch_secs = now_epoch_secs(); + let Some(pool) = &shared.me_pool else { + return RuntimeMePoolStateData { + enabled: false, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + }; + + let status = pool.api_status_snapshot().await; + let runtime = pool.api_runtime_snapshot().await; + let refill = pool.api_refill_snapshot().await; + + let mut draining_generations = BTreeSet::::new(); + let mut contour_warm = 0usize; + let mut contour_active = 0usize; + let mut contour_draining = 0usize; + let mut draining = 0usize; + let mut degraded = 0usize; + let mut healthy = 0usize; + + for writer in &status.writers { + if writer.draining { + draining_generations.insert(writer.generation); + draining += 1; + } + if writer.degraded && !writer.draining { + degraded += 1; + } + if !writer.degraded && !writer.draining { + healthy += 1; + } + match writer.state { + "warm" => contour_warm += 1, + "active" => contour_active += 1, + _ => contour_draining += 1, + } + } + + RuntimeMePoolStateData { + enabled: true, + reason: None, + generated_at_epoch_secs: status.generated_at_epoch_secs, + data: Some(RuntimeMePoolStatePayload { + generations: RuntimeMePoolStateGenerationData { + active_generation: runtime.active_generation, + warm_generation: runtime.warm_generation, + pending_hardswap_generation: runtime.pending_hardswap_generation, + pending_hardswap_age_secs: runtime.pending_hardswap_age_secs, + draining_generations: draining_generations.into_iter().collect(), + }, + hardswap: RuntimeMePoolStateHardswapData { + enabled: runtime.hardswap_enabled, + pending: runtime.pending_hardswap_generation != 0, + }, + writers: RuntimeMePoolStateWriterData { + total: status.writers.len(), + alive_non_draining: status.writers.len().saturating_sub(draining), + draining, + degraded, + contour: RuntimeMePoolStateWriterContourData { + warm: contour_warm, + active: contour_active, + draining: contour_draining, + }, + health: RuntimeMePoolStateWriterHealthData { + healthy, + degraded, + draining, + }, + }, + refill: RuntimeMePoolStateRefillData { + inflight_endpoints_total: refill.inflight_endpoints_total, + inflight_dc_total: refill.inflight_dc_total, + by_dc: refill + .by_dc + .into_iter() + .map(|entry| RuntimeMePoolStateRefillDcData { + dc: entry.dc, + family: entry.family, + inflight: entry.inflight, + }) + .collect(), + }, + }), + } +} + +pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData { + let now_epoch_secs = now_epoch_secs(); + let Some(pool) = &shared.me_pool else { + return RuntimeMeQualityData { + enabled: false, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + }; + + let status = pool.api_status_snapshot().await; + RuntimeMeQualityData { + enabled: true, + reason: None, + generated_at_epoch_secs: status.generated_at_epoch_secs, + data: Some(RuntimeMeQualityPayload { + counters: RuntimeMeQualityCountersData { + idle_close_by_peer_total: shared.stats.get_me_idle_close_by_peer_total(), + reader_eof_total: shared.stats.get_me_reader_eof_total(), + kdf_drift_total: shared.stats.get_me_kdf_drift_total(), + kdf_port_only_drift_total: shared.stats.get_me_kdf_port_only_drift_total(), + reconnect_attempt_total: shared.stats.get_me_reconnect_attempts(), + reconnect_success_total: shared.stats.get_me_reconnect_success(), + }, + route_drops: RuntimeMeQualityRouteDropData { + no_conn_total: shared.stats.get_me_route_drop_no_conn(), + channel_closed_total: shared.stats.get_me_route_drop_channel_closed(), + queue_full_total: shared.stats.get_me_route_drop_queue_full(), + queue_full_base_total: shared.stats.get_me_route_drop_queue_full_base(), + queue_full_high_total: shared.stats.get_me_route_drop_queue_full_high(), + }, + dc_rtt: status + .dcs + .into_iter() + .map(|dc| RuntimeMeQualityDcRttData { + dc: dc.dc, + rtt_ema_ms: dc.rtt_ms, + alive_writers: dc.alive_writers, + required_writers: dc.required_writers, + coverage_pct: dc.coverage_pct, + }) + .collect(), + }), + } +} + +pub(super) async fn build_runtime_upstream_quality_data( + shared: &ApiShared, +) -> RuntimeUpstreamQualityData { + let generated_at_epoch_secs = now_epoch_secs(); + let policy = shared.upstream_manager.api_policy_snapshot(); + let counters = RuntimeUpstreamQualityCountersData { + connect_attempt_total: shared.stats.get_upstream_connect_attempt_total(), + connect_success_total: shared.stats.get_upstream_connect_success_total(), + connect_fail_total: shared.stats.get_upstream_connect_fail_total(), + connect_failfast_hard_error_total: shared.stats.get_upstream_connect_failfast_hard_error_total(), + }; + + let Some(snapshot) = shared.upstream_manager.try_api_snapshot() else { + return RuntimeUpstreamQualityData { + enabled: false, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs, + policy: RuntimeUpstreamQualityPolicyData { + connect_retry_attempts: policy.connect_retry_attempts, + connect_retry_backoff_ms: policy.connect_retry_backoff_ms, + connect_budget_ms: policy.connect_budget_ms, + unhealthy_fail_threshold: policy.unhealthy_fail_threshold, + connect_failfast_hard_errors: policy.connect_failfast_hard_errors, + }, + counters, + summary: None, + upstreams: None, + }; + }; + + RuntimeUpstreamQualityData { + enabled: true, + reason: None, + generated_at_epoch_secs, + policy: RuntimeUpstreamQualityPolicyData { + connect_retry_attempts: policy.connect_retry_attempts, + connect_retry_backoff_ms: policy.connect_retry_backoff_ms, + connect_budget_ms: policy.connect_budget_ms, + unhealthy_fail_threshold: policy.unhealthy_fail_threshold, + connect_failfast_hard_errors: policy.connect_failfast_hard_errors, + }, + counters, + summary: Some(RuntimeUpstreamQualitySummaryData { + configured_total: snapshot.summary.configured_total, + healthy_total: snapshot.summary.healthy_total, + unhealthy_total: snapshot.summary.unhealthy_total, + direct_total: snapshot.summary.direct_total, + socks4_total: snapshot.summary.socks4_total, + socks5_total: snapshot.summary.socks5_total, + }), + upstreams: Some( + snapshot + .upstreams + .into_iter() + .map(|upstream| RuntimeUpstreamQualityUpstreamData { + upstream_id: upstream.upstream_id, + route_kind: match upstream.route_kind { + crate::transport::UpstreamRouteKind::Direct => "direct", + crate::transport::UpstreamRouteKind::Socks4 => "socks4", + crate::transport::UpstreamRouteKind::Socks5 => "socks5", + }, + address: upstream.address, + weight: upstream.weight, + scopes: upstream.scopes, + healthy: upstream.healthy, + fails: upstream.fails, + last_check_age_secs: upstream.last_check_age_secs, + effective_latency_ms: upstream.effective_latency_ms, + dc: upstream + .dc + .into_iter() + .map(|dc| RuntimeUpstreamQualityDcData { + dc: dc.dc, + latency_ema_ms: dc.latency_ema_ms, + ip_preference: match dc.ip_preference { + crate::transport::upstream::IpPreference::Unknown => "unknown", + crate::transport::upstream::IpPreference::PreferV6 => "prefer_v6", + crate::transport::upstream::IpPreference::PreferV4 => "prefer_v4", + crate::transport::upstream::IpPreference::BothWork => "both_work", + crate::transport::upstream::IpPreference::Unavailable => "unavailable", + }, + }) + .collect(), + }) + .collect(), + ), + } +} + +pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData { + let now_epoch_secs = now_epoch_secs(); + let Some(pool) = &shared.me_pool else { + return RuntimeNatStunData { + enabled: false, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + }; + + let snapshot = pool.api_nat_stun_snapshot().await; + RuntimeNatStunData { + enabled: true, + reason: None, + generated_at_epoch_secs: now_epoch_secs, + data: Some(RuntimeNatStunPayload { + flags: RuntimeNatStunFlagsData { + nat_probe_enabled: snapshot.nat_probe_enabled, + nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime, + nat_probe_attempts: snapshot.nat_probe_attempts, + }, + servers: RuntimeNatStunServersData { + configured: snapshot.configured_servers, + live: snapshot.live_servers.clone(), + live_total: snapshot.live_servers.len(), + }, + reflection: RuntimeNatStunReflectionBlockData { + v4: snapshot.reflection_v4.map(|entry| RuntimeNatStunReflectionData { + addr: entry.addr.to_string(), + age_secs: entry.age_secs, + }), + v6: snapshot.reflection_v6.map(|entry| RuntimeNatStunReflectionData { + addr: entry.addr.to_string(), + age_secs: entry.age_secs, + }), + }, + stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms, + }), + } +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} From da89415961f6a133d47b6a455dd4f1ffc09d32ff Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:54:37 +0300 Subject: [PATCH 103/261] Runtime API on Edge --- src/api/runtime_edge.rs | 294 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 src/api/runtime_edge.rs diff --git a/src/api/runtime_edge.rs b/src/api/runtime_edge.rs new file mode 100644 index 0000000..b61f504 --- /dev/null +++ b/src/api/runtime_edge.rs @@ -0,0 +1,294 @@ +use std::cmp::Reverse; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; + +use crate::config::ProxyConfig; + +use super::ApiShared; +use super::events::ApiEventRecord; + +const FEATURE_DISABLED_REASON: &str = "feature_disabled"; +const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable"; +const EVENTS_DEFAULT_LIMIT: usize = 50; +const EVENTS_MAX_LIMIT: usize = 1000; + +#[derive(Clone, Serialize)] +pub(super) struct RuntimeEdgeConnectionUserData { + pub(super) username: String, + pub(super) current_connections: u64, + pub(super) total_octets: u64, +} + +#[derive(Clone, Serialize)] +pub(super) struct RuntimeEdgeConnectionTotalsData { + pub(super) current_connections: u64, + pub(super) current_connections_me: u64, + pub(super) current_connections_direct: u64, + pub(super) active_users: usize, +} + +#[derive(Clone, Serialize)] +pub(super) struct RuntimeEdgeConnectionTopData { + pub(super) limit: usize, + pub(super) by_connections: Vec, + pub(super) by_throughput: Vec, +} + +#[derive(Clone, Serialize)] +pub(super) struct RuntimeEdgeConnectionCacheData { + pub(super) ttl_ms: u64, + pub(super) served_from_cache: bool, + pub(super) stale_cache_used: bool, +} + +#[derive(Clone, Serialize)] +pub(super) struct RuntimeEdgeConnectionTelemetryData { + pub(super) user_enabled: bool, + pub(super) throughput_is_cumulative: bool, +} + +#[derive(Clone, Serialize)] +pub(super) struct RuntimeEdgeConnectionsSummaryPayload { + pub(super) cache: RuntimeEdgeConnectionCacheData, + pub(super) totals: RuntimeEdgeConnectionTotalsData, + pub(super) top: RuntimeEdgeConnectionTopData, + pub(super) telemetry: RuntimeEdgeConnectionTelemetryData, +} + +#[derive(Serialize)] +pub(super) struct RuntimeEdgeConnectionsSummaryData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + +#[derive(Clone)] +pub(crate) struct EdgeConnectionsCacheEntry { + pub(super) expires_at: Instant, + pub(super) payload: RuntimeEdgeConnectionsSummaryPayload, + pub(super) generated_at_epoch_secs: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeEdgeEventsPayload { + pub(super) capacity: usize, + pub(super) dropped_total: u64, + pub(super) events: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeEdgeEventsData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + +pub(super) async fn build_runtime_connections_summary_data( + shared: &ApiShared, + cfg: &ProxyConfig, +) -> RuntimeEdgeConnectionsSummaryData { + let now_epoch_secs = now_epoch_secs(); + let api_cfg = &cfg.server.api; + if !api_cfg.runtime_edge_enabled { + return RuntimeEdgeConnectionsSummaryData { + enabled: false, + reason: Some(FEATURE_DISABLED_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + } + + let (generated_at_epoch_secs, payload) = match get_connections_payload_cached( + shared, + api_cfg.runtime_edge_cache_ttl_ms, + api_cfg.runtime_edge_top_n, + ) + .await + { + Some(v) => v, + None => { + return RuntimeEdgeConnectionsSummaryData { + enabled: true, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + } + }; + + RuntimeEdgeConnectionsSummaryData { + enabled: true, + reason: None, + generated_at_epoch_secs, + data: Some(payload), + } +} + +pub(super) fn build_runtime_events_recent_data( + shared: &ApiShared, + cfg: &ProxyConfig, + query: Option<&str>, +) -> RuntimeEdgeEventsData { + let now_epoch_secs = now_epoch_secs(); + let api_cfg = &cfg.server.api; + if !api_cfg.runtime_edge_enabled { + return RuntimeEdgeEventsData { + enabled: false, + reason: Some(FEATURE_DISABLED_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + } + + let limit = parse_recent_events_limit(query, EVENTS_DEFAULT_LIMIT, EVENTS_MAX_LIMIT); + let snapshot = shared.runtime_events.snapshot(limit); + + RuntimeEdgeEventsData { + enabled: true, + reason: None, + generated_at_epoch_secs: now_epoch_secs, + data: Some(RuntimeEdgeEventsPayload { + capacity: snapshot.capacity, + dropped_total: snapshot.dropped_total, + events: snapshot.events, + }), + } +} + +async fn get_connections_payload_cached( + shared: &ApiShared, + cache_ttl_ms: u64, + top_n: usize, +) -> Option<(u64, RuntimeEdgeConnectionsSummaryPayload)> { + if cache_ttl_ms > 0 { + let now = Instant::now(); + let cached = shared.runtime_edge_connections_cache.lock().await.clone(); + if let Some(entry) = cached + && now < entry.expires_at + { + let mut payload = entry.payload; + payload.cache.served_from_cache = true; + payload.cache.stale_cache_used = false; + return Some((entry.generated_at_epoch_secs, payload)); + } + } + + let Ok(_guard) = shared.runtime_edge_recompute_lock.try_lock() else { + let cached = shared.runtime_edge_connections_cache.lock().await.clone(); + if let Some(entry) = cached { + let mut payload = entry.payload; + payload.cache.served_from_cache = true; + payload.cache.stale_cache_used = true; + return Some((entry.generated_at_epoch_secs, payload)); + } + return None; + }; + + let generated_at_epoch_secs = now_epoch_secs(); + let payload = recompute_connections_payload(shared, cache_ttl_ms, top_n).await; + + if cache_ttl_ms > 0 { + let entry = EdgeConnectionsCacheEntry { + expires_at: Instant::now() + Duration::from_millis(cache_ttl_ms), + payload: payload.clone(), + generated_at_epoch_secs, + }; + *shared.runtime_edge_connections_cache.lock().await = Some(entry); + } + + Some((generated_at_epoch_secs, payload)) +} + +async fn recompute_connections_payload( + shared: &ApiShared, + cache_ttl_ms: u64, + top_n: usize, +) -> RuntimeEdgeConnectionsSummaryPayload { + let mut rows = Vec::::new(); + let mut active_users = 0usize; + for entry in shared.stats.iter_user_stats() { + let user_stats = entry.value(); + let current_connections = user_stats + .curr_connects + .load(std::sync::atomic::Ordering::Relaxed); + let total_octets = user_stats + .octets_from_client + .load(std::sync::atomic::Ordering::Relaxed) + .saturating_add( + user_stats + .octets_to_client + .load(std::sync::atomic::Ordering::Relaxed), + ); + if current_connections > 0 { + active_users = active_users.saturating_add(1); + } + rows.push(RuntimeEdgeConnectionUserData { + username: entry.key().clone(), + current_connections, + total_octets, + }); + } + + let limit = top_n.max(1); + let mut by_connections = rows.clone(); + by_connections.sort_by_key(|row| (Reverse(row.current_connections), row.username.clone())); + by_connections.truncate(limit); + + let mut by_throughput = rows; + by_throughput.sort_by_key(|row| (Reverse(row.total_octets), row.username.clone())); + by_throughput.truncate(limit); + + let telemetry = shared.stats.telemetry_policy(); + RuntimeEdgeConnectionsSummaryPayload { + cache: RuntimeEdgeConnectionCacheData { + ttl_ms: cache_ttl_ms, + served_from_cache: false, + stale_cache_used: false, + }, + totals: RuntimeEdgeConnectionTotalsData { + current_connections: shared.stats.get_current_connections_total(), + current_connections_me: shared.stats.get_current_connections_me(), + current_connections_direct: shared.stats.get_current_connections_direct(), + active_users, + }, + top: RuntimeEdgeConnectionTopData { + limit, + by_connections, + by_throughput, + }, + telemetry: RuntimeEdgeConnectionTelemetryData { + user_enabled: telemetry.user_enabled, + throughput_is_cumulative: true, + }, + } +} + +fn parse_recent_events_limit(query: Option<&str>, default_limit: usize, max_limit: usize) -> usize { + let Some(query) = query else { + return default_limit; + }; + for pair in query.split('&') { + let mut split = pair.splitn(2, '='); + if split.next() == Some("limit") + && let Some(raw) = split.next() + && let Ok(parsed) = raw.parse::() + { + return parsed.clamp(1, max_limit); + } + } + default_limit +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} From d87196c105cd891d7d0d809550764ed0132b54ef Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:55:04 +0300 Subject: [PATCH 104/261] HTTP Utils for API --- src/api/http_utils.rs | 91 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/api/http_utils.rs diff --git a/src/api/http_utils.rs b/src/api/http_utils.rs new file mode 100644 index 0000000..e04bd04 --- /dev/null +++ b/src/api/http_utils.rs @@ -0,0 +1,91 @@ +use http_body_util::{BodyExt, Full}; +use hyper::StatusCode; +use hyper::body::{Bytes, Incoming}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use super::model::{ApiFailure, ErrorBody, ErrorResponse, SuccessResponse}; + +pub(super) fn success_response( + status: StatusCode, + data: T, + revision: String, +) -> hyper::Response> { + let payload = SuccessResponse { + ok: true, + data, + revision, + }; + let body = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{\"ok\":false}".to_vec()); + hyper::Response::builder() + .status(status) + .header("content-type", "application/json; charset=utf-8") + .body(Full::new(Bytes::from(body))) + .unwrap() +} + +pub(super) fn error_response( + request_id: u64, + failure: ApiFailure, +) -> hyper::Response> { + let payload = ErrorResponse { + ok: false, + error: ErrorBody { + code: failure.code, + message: failure.message, + }, + request_id, + }; + let body = serde_json::to_vec(&payload).unwrap_or_else(|_| { + format!( + "{{\"ok\":false,\"error\":{{\"code\":\"internal_error\",\"message\":\"serialization failed\"}},\"request_id\":{}}}", + request_id + ) + .into_bytes() + }); + hyper::Response::builder() + .status(failure.status) + .header("content-type", "application/json; charset=utf-8") + .body(Full::new(Bytes::from(body))) + .unwrap() +} + +pub(super) async fn read_json( + body: Incoming, + limit: usize, +) -> Result { + let bytes = read_body_with_limit(body, limit).await?; + serde_json::from_slice(&bytes).map_err(|_| ApiFailure::bad_request("Invalid JSON body")) +} + +pub(super) async fn read_optional_json( + body: Incoming, + limit: usize, +) -> Result, ApiFailure> { + let bytes = read_body_with_limit(body, limit).await?; + if bytes.is_empty() { + return Ok(None); + } + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|_| ApiFailure::bad_request("Invalid JSON body")) +} + +async fn read_body_with_limit(body: Incoming, limit: usize) -> Result, ApiFailure> { + let mut collected = Vec::new(); + let mut body = body; + while let Some(frame_result) = body.frame().await { + let frame = frame_result.map_err(|_| ApiFailure::bad_request("Invalid request body"))?; + if let Some(chunk) = frame.data_ref() { + if collected.len().saturating_add(chunk.len()) > limit { + return Err(ApiFailure::new( + StatusCode::PAYLOAD_TOO_LARGE, + "payload_too_large", + format!("Body exceeds {} bytes", limit), + )); + } + collected.extend_from_slice(chunk); + } + } + Ok(collected) +} From 42212309698da5fe1f1a0e239b2fb170fd9b99a5 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:55:20 +0300 Subject: [PATCH 105/261] API Events + API as module --- src/api/events.rs | 90 ++++++++++++++++ src/api/mod.rs | 262 +++++++++++++++++++++++----------------------- 2 files changed, 223 insertions(+), 129 deletions(-) create mode 100644 src/api/events.rs diff --git a/src/api/events.rs b/src/api/events.rs new file mode 100644 index 0000000..4ca91e8 --- /dev/null +++ b/src/api/events.rs @@ -0,0 +1,90 @@ +use std::collections::VecDeque; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Serialize; + +#[derive(Clone, Serialize)] +pub(super) struct ApiEventRecord { + pub(super) seq: u64, + pub(super) ts_epoch_secs: u64, + pub(super) event_type: String, + pub(super) context: String, +} + +#[derive(Clone, Serialize)] +pub(super) struct ApiEventSnapshot { + pub(super) capacity: usize, + pub(super) dropped_total: u64, + pub(super) events: Vec, +} + +struct ApiEventsInner { + capacity: usize, + dropped_total: u64, + next_seq: u64, + events: VecDeque, +} + +/// Bounded ring-buffer for control-plane API/runtime events. +pub(crate) struct ApiEventStore { + inner: Mutex, +} + +impl ApiEventStore { + pub(super) fn new(capacity: usize) -> Self { + let bounded = capacity.max(16); + Self { + inner: Mutex::new(ApiEventsInner { + capacity: bounded, + dropped_total: 0, + next_seq: 1, + events: VecDeque::with_capacity(bounded), + }), + } + } + + pub(super) fn record(&self, event_type: &str, context: impl Into) { + let now_epoch_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let mut context = context.into(); + if context.len() > 256 { + context.truncate(256); + } + + let mut guard = self.inner.lock().expect("api event store mutex poisoned"); + if guard.events.len() == guard.capacity { + guard.events.pop_front(); + guard.dropped_total = guard.dropped_total.saturating_add(1); + } + let seq = guard.next_seq; + guard.next_seq = guard.next_seq.saturating_add(1); + guard.events.push_back(ApiEventRecord { + seq, + ts_epoch_secs: now_epoch_secs, + event_type: event_type.to_string(), + context, + }); + } + + pub(super) fn snapshot(&self, limit: usize) -> ApiEventSnapshot { + let guard = self.inner.lock().expect("api event store mutex poisoned"); + let bounded_limit = limit.clamp(1, guard.capacity.max(1)); + let mut items: Vec = guard + .events + .iter() + .rev() + .take(bounded_limit) + .cloned() + .collect(); + items.reverse(); + + ApiEventSnapshot { + capacity: guard.capacity, + dropped_total: guard.dropped_total, + events: items, + } + } +} diff --git a/src/api/mod.rs b/src/api/mod.rs index f2d31da..a705a46 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -3,16 +3,13 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; -use http_body_util::{BodyExt, Full}; +use http_body_util::Full; use hyper::body::{Bytes, Incoming}; use hyper::header::AUTHORIZATION; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper::{Method, Request, Response, StatusCode}; -use serde::Serialize; -use serde::de::DeserializeOwned; use tokio::net::TcpListener; use tokio::sync::{Mutex, watch}; use tracing::{debug, info, warn}; @@ -24,15 +21,29 @@ use crate::transport::middle_proxy::MePool; use crate::transport::UpstreamManager; mod config_store; +mod events; +mod http_utils; mod model; +mod runtime_edge; +mod runtime_min; mod runtime_stats; +mod runtime_watch; mod runtime_zero; mod users; use config_store::{current_revision, parse_if_match}; +use http_utils::{error_response, read_json, read_optional_json, success_response}; +use events::ApiEventStore; use model::{ - ApiFailure, CreateUserRequest, ErrorBody, ErrorResponse, HealthData, PatchUserRequest, - RotateSecretRequest, SuccessResponse, SummaryData, + ApiFailure, CreateUserRequest, HealthData, PatchUserRequest, RotateSecretRequest, SummaryData, +}; +use runtime_edge::{ + EdgeConnectionsCacheEntry, build_runtime_connections_summary_data, + build_runtime_events_recent_data, +}; +use runtime_min::{ + build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data, + build_runtime_upstream_quality_data, build_security_whitelist_data, }; use runtime_stats::{ MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data, @@ -42,6 +53,7 @@ use runtime_zero::{ build_limits_effective_data, build_runtime_gates_data, build_security_posture_data, build_system_info_data, }; +use runtime_watch::spawn_runtime_watchers; use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config}; pub(super) struct ApiRuntimeState { @@ -62,6 +74,9 @@ pub(super) struct ApiShared { pub(super) startup_detected_ip_v6: Option, pub(super) mutation_lock: Arc>, pub(super) minimal_cache: Arc>>, + pub(super) runtime_edge_connections_cache: Arc>>, + pub(super) runtime_edge_recompute_lock: Arc>, + pub(super) runtime_events: Arc, pub(super) request_id: Arc, pub(super) runtime_state: Arc, } @@ -116,40 +131,21 @@ pub async fn serve( startup_detected_ip_v6, mutation_lock: Arc::new(Mutex::new(())), minimal_cache: Arc::new(Mutex::new(None)), + runtime_edge_connections_cache: Arc::new(Mutex::new(None)), + runtime_edge_recompute_lock: Arc::new(Mutex::new(())), + runtime_events: Arc::new(ApiEventStore::new( + config_rx.borrow().server.api.runtime_edge_events_capacity, + )), request_id: Arc::new(AtomicU64::new(1)), runtime_state: runtime_state.clone(), }); - let mut config_rx_reload = config_rx.clone(); - let runtime_state_reload = runtime_state.clone(); - tokio::spawn(async move { - loop { - if config_rx_reload.changed().await.is_err() { - break; - } - runtime_state_reload - .config_reload_count - .fetch_add(1, Ordering::Relaxed); - runtime_state_reload - .last_config_reload_epoch_secs - .store(now_epoch_secs(), Ordering::Relaxed); - } - }); - - let mut admission_rx_watch = admission_rx.clone(); - tokio::spawn(async move { - runtime_state - .admission_open - .store(*admission_rx_watch.borrow(), Ordering::Relaxed); - loop { - if admission_rx_watch.changed().await.is_err() { - break; - } - runtime_state - .admission_open - .store(*admission_rx_watch.borrow(), Ordering::Relaxed); - } - }); + spawn_runtime_watchers( + config_rx.clone(), + admission_rx.clone(), + runtime_state.clone(), + shared.runtime_events.clone(), + ); loop { let (stream, peer) = match listener.accept().await { @@ -232,6 +228,7 @@ async fn handle( let method = req.method().clone(); let path = req.uri().path().to_string(); + let query = req.uri().query().map(str::to_string); let body_limit = api_cfg.request_body_limit_bytes; let result: Result>, ApiFailure> = async { @@ -264,6 +261,11 @@ async fn handle( let data = build_security_posture_data(cfg.as_ref()); Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/security/whitelist") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_security_whitelist_data(cfg.as_ref()); + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/summary") => { let revision = current_revision(&shared.config_path).await?; let data = SummaryData { @@ -300,6 +302,40 @@ async fn handle( let data = build_dcs_data(shared.as_ref(), api_cfg).await; Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/runtime/me_pool_state") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_me_pool_state_data(shared.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/runtime/me_quality") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_me_quality_data(shared.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/runtime/upstream_quality") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_upstream_quality_data(shared.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/runtime/nat_stun") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_nat_stun_data(shared.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/runtime/connections/summary") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_connections_summary_data(shared.as_ref(), cfg.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } + ("GET", "/v1/runtime/events/recent") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_events_recent_data( + shared.as_ref(), + cfg.as_ref(), + query.as_deref(), + ); + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/stats/users") | ("GET", "/v1/users") => { let revision = current_revision(&shared.config_path).await?; let users = users_from_config( @@ -325,7 +361,17 @@ async fn handle( } let expected_revision = parse_if_match(req.headers()); let body = read_json::(req.into_body(), body_limit).await?; - let (data, revision) = create_user(body, expected_revision, &shared).await?; + let result = create_user(body, expected_revision, &shared).await; + let (data, revision) = match result { + Ok(ok) => ok, + Err(error) => { + shared.runtime_events.record("api.user.create.failed", error.code); + return Err(error); + } + }; + shared + .runtime_events + .record("api.user.create.ok", format!("username={}", data.user.username)); Ok(success_response(StatusCode::CREATED, data, revision)) } _ => { @@ -365,8 +411,20 @@ async fn handle( } let expected_revision = parse_if_match(req.headers()); let body = read_json::(req.into_body(), body_limit).await?; - let (data, revision) = - patch_user(user, body, expected_revision, &shared).await?; + let result = patch_user(user, body, expected_revision, &shared).await; + let (data, revision) = match result { + Ok(ok) => ok, + Err(error) => { + shared.runtime_events.record( + "api.user.patch.failed", + format!("username={} code={}", user, error.code), + ); + return Err(error); + } + }; + shared + .runtime_events + .record("api.user.patch.ok", format!("username={}", data.username)); return Ok(success_response(StatusCode::OK, data, revision)); } if method == Method::DELETE { @@ -381,8 +439,21 @@ async fn handle( )); } let expected_revision = parse_if_match(req.headers()); - let (deleted_user, revision) = - delete_user(user, expected_revision, &shared).await?; + let result = delete_user(user, expected_revision, &shared).await; + let (deleted_user, revision) = match result { + Ok(ok) => ok, + Err(error) => { + shared.runtime_events.record( + "api.user.delete.failed", + format!("username={} code={}", user, error.code), + ); + return Err(error); + } + }; + shared.runtime_events.record( + "api.user.delete.ok", + format!("username={}", deleted_user), + ); return Ok(success_response(StatusCode::OK, deleted_user, revision)); } if method == Method::POST @@ -404,9 +475,27 @@ async fn handle( let body = read_optional_json::(req.into_body(), body_limit) .await?; - let (data, revision) = - rotate_secret(base_user, body.unwrap_or_default(), expected_revision, &shared) - .await?; + let result = rotate_secret( + base_user, + body.unwrap_or_default(), + expected_revision, + &shared, + ) + .await; + let (data, revision) = match result { + Ok(ok) => ok, + Err(error) => { + shared.runtime_events.record( + "api.user.rotate_secret.failed", + format!("username={} code={}", base_user, error.code), + ); + return Err(error); + } + }; + shared.runtime_events.record( + "api.user.rotate_secret.ok", + format!("username={}", base_user), + ); return Ok(success_response(StatusCode::OK, data, revision)); } if method == Method::POST { @@ -438,88 +527,3 @@ async fn handle( Err(error) => Ok(error_response(request_id, error)), } } - -fn success_response( - status: StatusCode, - data: T, - revision: String, -) -> Response> { - let payload = SuccessResponse { - ok: true, - data, - revision, - }; - let body = serde_json::to_vec(&payload).unwrap_or_else(|_| b"{\"ok\":false}".to_vec()); - Response::builder() - .status(status) - .header("content-type", "application/json; charset=utf-8") - .body(Full::new(Bytes::from(body))) - .unwrap() -} - -fn error_response(request_id: u64, failure: ApiFailure) -> Response> { - let payload = ErrorResponse { - ok: false, - error: ErrorBody { - code: failure.code, - message: failure.message, - }, - request_id, - }; - let body = serde_json::to_vec(&payload).unwrap_or_else(|_| { - format!( - "{{\"ok\":false,\"error\":{{\"code\":\"internal_error\",\"message\":\"serialization failed\"}},\"request_id\":{}}}", - request_id - ) - .into_bytes() - }); - Response::builder() - .status(failure.status) - .header("content-type", "application/json; charset=utf-8") - .body(Full::new(Bytes::from(body))) - .unwrap() -} - -async fn read_json(body: Incoming, limit: usize) -> Result { - let bytes = read_body_with_limit(body, limit).await?; - serde_json::from_slice(&bytes).map_err(|_| ApiFailure::bad_request("Invalid JSON body")) -} - -async fn read_optional_json( - body: Incoming, - limit: usize, -) -> Result, ApiFailure> { - let bytes = read_body_with_limit(body, limit).await?; - if bytes.is_empty() { - return Ok(None); - } - serde_json::from_slice(&bytes) - .map(Some) - .map_err(|_| ApiFailure::bad_request("Invalid JSON body")) -} - -async fn read_body_with_limit(body: Incoming, limit: usize) -> Result, ApiFailure> { - let mut collected = Vec::new(); - let mut body = body; - while let Some(frame_result) = body.frame().await { - let frame = frame_result.map_err(|_| ApiFailure::bad_request("Invalid request body"))?; - if let Some(chunk) = frame.data_ref() { - if collected.len().saturating_add(chunk.len()) > limit { - return Err(ApiFailure::new( - StatusCode::PAYLOAD_TOO_LARGE, - "payload_too_large", - format!("Body exceeds {} bytes", limit), - )); - } - collected.extend_from_slice(chunk); - } - } - Ok(collected) -} - -fn now_epoch_secs() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} From e9f8c7949876e6c64a67788f080fb481db6f860e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:58:57 +0300 Subject: [PATCH 106/261] ME Pool w/ Strict-Index --- src/transport/middle_proxy/pool.rs | 54 +++++++++++++- src/transport/middle_proxy/send.rs | 109 +++++++++++------------------ 2 files changed, 92 insertions(+), 71 deletions(-) diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 22f40b5..b0ae394 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -320,7 +320,7 @@ impl MePool { pool_size: 2, proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), - default_dc: AtomicI32::new(default_dc.unwrap_or(0)), + default_dc: AtomicI32::new(default_dc.unwrap_or(2)), next_writer_id: AtomicU64::new(1), ping_tracker: Arc::new(Mutex::new(HashMap::new())), rtt_stats: Arc::new(Mutex::new(HashMap::new())), @@ -625,6 +625,58 @@ impl MePool { order } + pub(super) fn default_dc_for_routing(&self) -> i32 { + let dc = self.default_dc.load(Ordering::Relaxed); + if dc == 0 { 2 } else { dc } + } + + pub(super) fn dc_lookup_chain_for_target(&self, target_dc: i32) -> Vec { + let mut out = Vec::with_capacity(1); + if target_dc != 0 { + out.push(target_dc); + } else { + // Use default DC only when target DC is unknown and pinning is not established. + let fallback_dc = self.default_dc_for_routing(); + out.push(fallback_dc); + } + out + } + + pub(super) async fn resolve_dc_for_endpoint(&self, addr: SocketAddr) -> i32 { + let map_guard = if addr.is_ipv4() { + self.proxy_map_v4.read().await + } else { + self.proxy_map_v6.read().await + }; + + let mut matched_dc: Option = None; + let mut ambiguous = false; + for (dc, addrs) in map_guard.iter() { + if addrs + .iter() + .any(|(ip, port)| SocketAddr::new(*ip, *port) == addr) + { + match matched_dc { + None => matched_dc = Some(*dc), + Some(prev_dc) if prev_dc == *dc => {} + Some(_) => { + ambiguous = true; + break; + } + } + } + } + drop(map_guard); + + if !ambiguous + && let Some(dc) = matched_dc + { + return dc; + } + + self.default_dc_for_routing() + } + pub(super) async fn proxy_map_for_family( &self, family: IpFamily, diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index b442a8a..b9b1fd5 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -195,38 +195,25 @@ impl MePool { return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } emergency_attempts += 1; - for family in self.family_order() { - let map_guard = match family { - IpFamily::V4 => self.proxy_map_v4.read().await, - IpFamily::V6 => self.proxy_map_v6.read().await, - }; - if let Some(addrs) = map_guard.get(&(target_dc as i32)) { - let mut shuffled = addrs.clone(); - shuffled.shuffle(&mut rand::rng()); - drop(map_guard); - for (ip, port) in shuffled { - let addr = SocketAddr::new(ip, port); - if self.connect_one(addr, self.rng.as_ref()).await.is_ok() { - break; - } - } - tokio::time::sleep(Duration::from_millis(100 * emergency_attempts as u64)).await; - let ws2 = self.writers.read().await; - writers_snapshot = ws2.clone(); - drop(ws2); - candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, false) - .await; - if candidate_indices.is_empty() { - candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, true) - .await; - } - if !candidate_indices.is_empty() { - break; - } + let mut endpoints = self.endpoint_candidates_for_target_dc(target_dc).await; + endpoints.shuffle(&mut rand::rng()); + for addr in endpoints { + if self.connect_one(addr, self.rng.as_ref()).await.is_ok() { + break; } } + tokio::time::sleep(Duration::from_millis(100 * emergency_attempts as u64)).await; + let ws2 = self.writers.read().await; + writers_snapshot = ws2.clone(); + drop(ws2); + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .await; + if candidate_indices.is_empty() { + candidate_indices = self + .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .await; + } if candidate_indices.is_empty() { return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } @@ -458,26 +445,28 @@ impl MePool { let key = target_dc as i32; let mut preferred = Vec::::new(); let mut seen = HashSet::::new(); + let lookup_keys = self.dc_lookup_chain_for_target(key); for family in self.family_order() { let map = match family { IpFamily::V4 => self.proxy_map_v4.read().await.clone(), IpFamily::V6 => self.proxy_map_v6.read().await.clone(), }; - let mut lookup_keys = vec![key, key.abs(), -key.abs()]; - let def = self.default_dc.load(Ordering::Relaxed); - if def != 0 { - lookup_keys.push(def); - } - for lookup in lookup_keys { + let mut family_selected = Vec::::new(); + for lookup in lookup_keys.iter().copied() { if let Some(addrs) = map.get(&lookup) { for (ip, port) in addrs { - let addr = SocketAddr::new(*ip, *port); - if seen.insert(addr) { - preferred.push(addr); - } + family_selected.push(SocketAddr::new(*ip, *port)); } } + if !family_selected.is_empty() { + break; + } + } + for addr in family_selected { + if seen.insert(addr) { + preferred.push(addr); + } } if !preferred.is_empty() && !self.decision.effective_multipath { break; @@ -569,36 +558,23 @@ impl MePool { ) -> Vec { let key = target_dc as i32; let mut preferred = Vec::::new(); + let lookup_keys = self.dc_lookup_chain_for_target(key); for family in self.family_order() { let map_guard = match family { IpFamily::V4 => self.proxy_map_v4.read().await, IpFamily::V6 => self.proxy_map_v6.read().await, }; - - if let Some(v) = map_guard.get(&key) { - preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); - } - if preferred.is_empty() { - let abs = key.abs(); - if let Some(v) = map_guard.get(&abs) { - preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); - } - } - if preferred.is_empty() { - let abs = key.abs(); - if let Some(v) = map_guard.get(&-abs) { - preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); - } - } - if preferred.is_empty() { - let def = self.default_dc.load(Ordering::Relaxed); - if def != 0 - && let Some(v) = map_guard.get(&def) - { - preferred.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); + let mut family_selected = Vec::::new(); + for lookup in lookup_keys.iter().copied() { + if let Some(v) = map_guard.get(&lookup) { + family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); + } + if !family_selected.is_empty() { + break; } } + preferred.extend(family_selected); drop(map_guard); @@ -608,9 +584,7 @@ impl MePool { } if preferred.is_empty() { - return (0..writers.len()) - .filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm)) - .collect(); + return Vec::new(); } let mut out = Vec::new(); @@ -622,11 +596,6 @@ impl MePool { out.push(idx); } } - if out.is_empty() { - return (0..writers.len()) - .filter(|i| self.writer_eligible_for_selection(&writers[*i], include_warm)) - .collect(); - } out } From 24df8655035c5401cfecf7891adc2e7a507371d3 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:59:23 +0300 Subject: [PATCH 107/261] Session by Target-DC-ID Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/pool_init.rs | 12 +-- src/transport/middle_proxy/pool_refill.rs | 55 ++-------- src/transport/middle_proxy/pool_reinit.rs | 4 +- src/transport/middle_proxy/pool_status.rs | 116 +++++++--------------- src/transport/middle_proxy/registry.rs | 12 +-- 5 files changed, 56 insertions(+), 143 deletions(-) diff --git a/src/transport/middle_proxy/pool_init.rs b/src/transport/middle_proxy/pool_init.rs index fbb5c64..668cfda 100644 --- a/src/transport/middle_proxy/pool_init.rs +++ b/src/transport/middle_proxy/pool_init.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; @@ -27,20 +27,14 @@ impl MePool { for family in family_order { let map = self.proxy_map_for_family(family).await; - let mut grouped_dc_addrs: HashMap> = HashMap::new(); - for (dc, addrs) in map { - if addrs.is_empty() { - continue; - } - grouped_dc_addrs.entry(dc.abs()).or_default().extend(addrs); - } - let mut dc_addrs: Vec<(i32, Vec<(IpAddr, u16)>)> = grouped_dc_addrs + let mut dc_addrs: Vec<(i32, Vec<(IpAddr, u16)>)> = map .into_iter() .map(|(dc, mut addrs)| { addrs.sort_unstable(); addrs.dedup(); (dc, addrs) }) + .filter(|(_, addrs)| !addrs.is_empty()) .collect(); dc_addrs.sort_unstable_by_key(|(dc, _)| *dc); dc_addrs.sort_by_key(|(_, addrs)| (addrs.len() != 1, addrs.len())); diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 6e14617..87b87d5 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -108,19 +108,10 @@ impl MePool { } else { IpFamily::V6 }; - let map = self.proxy_map_for_family(family).await; - for (dc, endpoints) in map { - if endpoints - .into_iter() - .any(|(ip, port)| SocketAddr::new(ip, port) == addr) - { - return Some(RefillDcKey { - dc: dc.abs(), - family, - }); - } - } - None + Some(RefillDcKey { + dc: self.resolve_dc_for_endpoint(addr).await, + family, + }) } async fn resolve_refill_dc_keys_for_endpoints( @@ -177,47 +168,23 @@ impl MePool { } async fn endpoints_for_same_dc(&self, addr: SocketAddr) -> Vec { - let mut target_dc = HashSet::::new(); let mut endpoints = HashSet::::new(); + let target_dc = self.resolve_dc_for_endpoint(addr).await; if self.decision.ipv4_me { let map = self.proxy_map_v4.read().await.clone(); - for (dc, addrs) in &map { - if addrs - .iter() - .any(|(ip, port)| SocketAddr::new(*ip, *port) == addr) - { - target_dc.insert(dc.abs()); - } - } - for dc in &target_dc { - for key in [*dc, -*dc] { - if let Some(addrs) = map.get(&key) { - for (ip, port) in addrs { - endpoints.insert(SocketAddr::new(*ip, *port)); - } - } + if let Some(addrs) = map.get(&target_dc) { + for (ip, port) in addrs { + endpoints.insert(SocketAddr::new(*ip, *port)); } } } if self.decision.ipv6_me { let map = self.proxy_map_v6.read().await.clone(); - for (dc, addrs) in &map { - if addrs - .iter() - .any(|(ip, port)| SocketAddr::new(*ip, *port) == addr) - { - target_dc.insert(dc.abs()); - } - } - for dc in &target_dc { - for key in [*dc, -*dc] { - if let Some(addrs) = map.get(&key) { - for (ip, port) in addrs { - endpoints.insert(SocketAddr::new(*ip, *port)); - } - } + if let Some(addrs) = map.get(&target_dc) { + for (ip, port) in addrs { + endpoints.insert(SocketAddr::new(*ip, *port)); } } } diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index d5242b7..39944ba 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -128,7 +128,7 @@ impl MePool { if self.decision.ipv4_me { let map_v4 = self.proxy_map_v4.read().await.clone(); for (dc, addrs) in map_v4 { - let entry = out.entry(dc.abs()).or_default(); + let entry = out.entry(dc).or_default(); for (ip, port) in addrs { entry.insert(SocketAddr::new(ip, port)); } @@ -138,7 +138,7 @@ impl MePool { if self.decision.ipv6_me { let map_v6 = self.proxy_map_v6.read().await.clone(); for (dc, addrs) in map_v6 { - let entry = out.entry(dc.abs()).or_default(); + let entry = out.entry(dc).or_default(); for (ip, port) in addrs { entry.insert(SocketAddr::new(ip, port)); } diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 17a418c..d9898b1 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -1,5 +1,5 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::sync::atomic::Ordering; use std::time::Instant; @@ -104,35 +104,11 @@ impl MePool { let mut endpoints_by_dc = BTreeMap::>::new(); if self.decision.ipv4_me { let map = self.proxy_map_v4.read().await.clone(); - for (dc, addrs) in map { - let abs_dc = dc.abs(); - if abs_dc == 0 { - continue; - } - let Ok(dc_idx) = i16::try_from(abs_dc) else { - continue; - }; - let entry = endpoints_by_dc.entry(dc_idx).or_default(); - for (ip, port) in addrs { - entry.insert(SocketAddr::new(ip, port)); - } - } + extend_signed_endpoints(&mut endpoints_by_dc, map); } if self.decision.ipv6_me { let map = self.proxy_map_v6.read().await.clone(); - for (dc, addrs) in map { - let abs_dc = dc.abs(); - if abs_dc == 0 { - continue; - } - let Ok(dc_idx) = i16::try_from(abs_dc) else { - continue; - }; - let entry = endpoints_by_dc.entry(dc_idx).or_default(); - for (ip, port) in addrs { - entry.insert(SocketAddr::new(ip, port)); - } - } + extend_signed_endpoints(&mut endpoints_by_dc, map); } if endpoints_by_dc.is_empty() { @@ -166,35 +142,11 @@ impl MePool { let mut endpoints_by_dc = BTreeMap::>::new(); if self.decision.ipv4_me { let map = self.proxy_map_v4.read().await.clone(); - for (dc, addrs) in map { - let abs_dc = dc.abs(); - if abs_dc == 0 { - continue; - } - let Ok(dc_idx) = i16::try_from(abs_dc) else { - continue; - }; - let entry = endpoints_by_dc.entry(dc_idx).or_default(); - for (ip, port) in addrs { - entry.insert(SocketAddr::new(ip, port)); - } - } + extend_signed_endpoints(&mut endpoints_by_dc, map); } if self.decision.ipv6_me { let map = self.proxy_map_v6.read().await.clone(); - for (dc, addrs) in map { - let abs_dc = dc.abs(); - if abs_dc == 0 { - continue; - } - let Ok(dc_idx) = i16::try_from(abs_dc) else { - continue; - }; - let entry = endpoints_by_dc.entry(dc_idx).or_default(); - for (ip, port) in addrs { - entry.insert(SocketAddr::new(ip, port)); - } - } + extend_signed_endpoints(&mut endpoints_by_dc, map); } if endpoints_by_dc.is_empty() { @@ -234,41 +186,17 @@ impl MePool { let mut endpoints_by_dc = BTreeMap::>::new(); if self.decision.ipv4_me { let map = self.proxy_map_v4.read().await.clone(); - for (dc, addrs) in map { - let abs_dc = dc.abs(); - if abs_dc == 0 { - continue; - } - let Ok(dc_idx) = i16::try_from(abs_dc) else { - continue; - }; - let entry = endpoints_by_dc.entry(dc_idx).or_default(); - for (ip, port) in addrs { - entry.insert(SocketAddr::new(ip, port)); - } - } + extend_signed_endpoints(&mut endpoints_by_dc, map); } if self.decision.ipv6_me { let map = self.proxy_map_v6.read().await.clone(); - for (dc, addrs) in map { - let abs_dc = dc.abs(); - if abs_dc == 0 { - continue; - } - let Ok(dc_idx) = i16::try_from(abs_dc) else { - continue; - }; - let entry = endpoints_by_dc.entry(dc_idx).or_default(); - for (ip, port) in addrs { - entry.insert(SocketAddr::new(ip, port)); - } - } + extend_signed_endpoints(&mut endpoints_by_dc, map); } - let mut endpoint_to_dc = HashMap::::new(); + let mut endpoint_to_dc = HashMap::>::new(); for (dc, endpoints) in &endpoints_by_dc { for endpoint in endpoints { - endpoint_to_dc.entry(*endpoint).or_insert(*dc); + endpoint_to_dc.entry(*endpoint).or_default().insert(*dc); } } @@ -292,7 +220,13 @@ impl MePool { for writer in writers { let endpoint = writer.addr; - let dc = endpoint_to_dc.get(&endpoint).copied(); + let dc = endpoint_to_dc.get(&endpoint).and_then(|dcs| { + if dcs.len() == 1 { + dcs.iter().next().copied() + } else { + None + } + }); let draining = writer.draining.load(Ordering::Relaxed); let degraded = writer.degraded.load(Ordering::Relaxed); let bound_clients = activity @@ -499,6 +433,24 @@ fn ratio_pct(part: usize, total: usize) -> f64 { pct.clamp(0.0, 100.0) } +fn extend_signed_endpoints( + endpoints_by_dc: &mut BTreeMap>, + map: HashMap>, +) { + for (dc, addrs) in map { + if dc == 0 { + continue; + } + let Ok(dc_idx) = i16::try_from(dc) else { + continue; + }; + let entry = endpoints_by_dc.entry(dc_idx).or_default(); + for (ip, port) in addrs { + entry.insert(SocketAddr::new(ip, port)); + } + } +} + fn floor_mode_label(mode: MeFloorMode) -> &'static str { match mode { MeFloorMode::Static => "static", diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index 66a7f81..b437885 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -273,13 +273,12 @@ impl ConnRegistry { bound_clients_by_writer.insert(*writer_id, conn_ids.len()); } for conn_meta in inner.meta.values() { - let dc_u16 = conn_meta.target_dc.unsigned_abs(); - if dc_u16 == 0 { + if conn_meta.target_dc == 0 { continue; } - if let Ok(dc) = i16::try_from(dc_u16) { - *active_sessions_by_target_dc.entry(dc).or_insert(0) += 1; - } + *active_sessions_by_target_dc + .entry(conn_meta.target_dc) + .or_insert(0) += 1; } WriterActivitySnapshot { @@ -402,7 +401,8 @@ mod tests { let snapshot = registry.writer_activity_snapshot().await; assert_eq!(snapshot.bound_clients_by_writer.get(&10), Some(&2)); assert_eq!(snapshot.bound_clients_by_writer.get(&20), Some(&1)); - assert_eq!(snapshot.active_sessions_by_target_dc.get(&2), Some(&2)); + assert_eq!(snapshot.active_sessions_by_target_dc.get(&2), Some(&1)); + assert_eq!(snapshot.active_sessions_by_target_dc.get(&-2), Some(&1)); assert_eq!(snapshot.active_sessions_by_target_dc.get(&4), Some(&1)); } } From 02fe89f7d0a61d8fe08301411ead6030e442f5c4 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:00:32 +0300 Subject: [PATCH 108/261] DC Endpoints on default Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/main.rs | 15 ++++++----- src/transport/middle_proxy/handshake.rs | 33 +------------------------ src/transport/middle_proxy/health.rs | 2 +- 3 files changed, 9 insertions(+), 41 deletions(-) diff --git a/src/main.rs b/src/main.rs index a9207ac..ee5aaad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -942,22 +942,21 @@ async fn main() -> std::result::Result<(), Box> { let mut grouped: BTreeMap> = BTreeMap::new(); for report in me_results { for s in report.samples { - let key = s.dc.abs(); - grouped.entry(key).or_default().push(s); + grouped.entry(s.dc).or_default().push(s); } } let family_order = if prefer_ipv6 { - vec![(MePingFamily::V6, true), (MePingFamily::V6, false), (MePingFamily::V4, true), (MePingFamily::V4, false)] + vec![MePingFamily::V6, MePingFamily::V4] } else { - vec![(MePingFamily::V4, true), (MePingFamily::V4, false), (MePingFamily::V6, true), (MePingFamily::V6, false)] + vec![MePingFamily::V4, MePingFamily::V6] }; - for (dc_abs, samples) in grouped { - for (family, is_pos) in &family_order { + for (dc, samples) in grouped { + for family in &family_order { let fam_samples: Vec<&MePingSample> = samples .iter() - .filter(|s| matches!(s.family, f if &f == family) && (s.dc >= 0) == *is_pos) + .filter(|s| matches!(s.family, f if &f == family)) .collect(); if fam_samples.is_empty() { continue; @@ -967,7 +966,7 @@ async fn main() -> std::result::Result<(), Box> { MePingFamily::V4 => "IPv4", MePingFamily::V6 => "IPv6", }; - info!(" DC{} [{}]", dc_abs, fam_label); + info!(" DC{} [{}]", dc, fam_label); for sample in fam_samples { let line = format_sample_line(sample); info!("{}", line); diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 77634a6..948c999 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -84,38 +84,7 @@ impl MePool { } async fn resolve_dc_idx_for_endpoint(&self, addr: SocketAddr) -> Option { - if addr.is_ipv4() { - let map = self.proxy_map_v4.read().await; - for (dc, addrs) in map.iter() { - if addrs - .iter() - .any(|(ip, port)| SocketAddr::new(*ip, *port) == addr) - { - let abs_dc = dc.abs(); - if abs_dc > 0 - && let Ok(dc_idx) = i16::try_from(abs_dc) - { - return Some(dc_idx); - } - } - } - } else { - let map = self.proxy_map_v6.read().await; - for (dc, addrs) in map.iter() { - if addrs - .iter() - .any(|(ip, port)| SocketAddr::new(*ip, *port) == addr) - { - let abs_dc = dc.abs(); - if abs_dc > 0 - && let Ok(dc_idx) = i16::try_from(abs_dc) - { - return Some(dc_idx); - } - } - } - } - None + i16::try_from(self.resolve_dc_for_endpoint(addr).await).ok() } fn direct_bind_ip_for_stun( diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 1cc8d8a..9a54e32 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -102,7 +102,7 @@ async fn check_family( let mut dc_endpoints = HashMap::>::new(); for (dc, addrs) in map { - let entry = dc_endpoints.entry(dc.abs()).or_default(); + let entry = dc_endpoints.entry(dc).or_default(); for (ip, port) in addrs { entry.push(SocketAddr::new(ip, port)); } From 640468d4e707edd17ecd86a8a8fd08717dd5201c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:01:12 +0300 Subject: [PATCH 109/261] Update API.md Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- docs/API.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/API.md b/docs/API.md index cb964d9..2f98b62 100644 --- a/docs/API.md +++ b/docs/API.md @@ -16,6 +16,10 @@ API runtime is configured in `[server.api]`. | `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. | | `minimal_runtime_enabled` | `bool` | `false` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. | | `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. | +| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. | +| `runtime_edge_cache_ttl_ms` | `u64` | `1000` | Cache TTL for runtime edge summary payloads. `0` disables cache. | +| `runtime_edge_top_n` | `usize` | `10` | Top-N rows for runtime edge leaderboard payloads. | +| `runtime_edge_events_capacity` | `usize` | `256` | Ring-buffer size for `/v1/runtime/events/recent`. | | `read_only` | `bool` | `false` | Disables mutating endpoints. | `server.admin_api` is accepted as an alias for backward compatibility. @@ -24,6 +28,9 @@ Runtime validation for API config: - `server.api.listen` must be a valid `IP:PORT`. - `server.api.request_body_limit_bytes` must be `> 0`. - `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`. +- `server.api.runtime_edge_cache_ttl_ms` must be within `[0, 60000]`. +- `server.api.runtime_edge_top_n` must be within `[1, 1000]`. +- `server.api.runtime_edge_events_capacity` must be within `[16, 4096]`. ## Protocol Contract @@ -80,12 +87,19 @@ Notes: | `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` | | `GET` | `/v1/limits/effective` | none | `200` | `EffectiveLimitsData` | | `GET` | `/v1/security/posture` | none | `200` | `SecurityPostureData` | +| `GET` | `/v1/security/whitelist` | none | `200` | `SecurityWhitelistData` | | `GET` | `/v1/stats/summary` | none | `200` | `SummaryData` | | `GET` | `/v1/stats/zero/all` | none | `200` | `ZeroAllData` | | `GET` | `/v1/stats/upstreams` | none | `200` | `UpstreamsData` | | `GET` | `/v1/stats/minimal/all` | none | `200` | `MinimalAllData` | | `GET` | `/v1/stats/me-writers` | none | `200` | `MeWritersData` | | `GET` | `/v1/stats/dcs` | none | `200` | `DcStatusData` | +| `GET` | `/v1/runtime/me_pool_state` | none | `200` | `RuntimeMePoolStateData` | +| `GET` | `/v1/runtime/me_quality` | none | `200` | `RuntimeMeQualityData` | +| `GET` | `/v1/runtime/upstream_quality` | none | `200` | `RuntimeUpstreamQualityData` | +| `GET` | `/v1/runtime/nat_stun` | none | `200` | `RuntimeNatStunData` | +| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` | +| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` | | `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` | | `GET` | `/v1/users` | none | `200` | `UserInfo[]` | | `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` | @@ -268,6 +282,25 @@ Note: the request contract is defined, but the corresponding route currently ret | `telemetry_user_enabled` | `bool` | Per-user telemetry toggle. | | `telemetry_me_level` | `string` | ME telemetry level (`silent`, `normal`, `debug`). | +### `SecurityWhitelistData` +| Field | Type | Description | +| --- | --- | --- | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `enabled` | `bool` | `true` when whitelist has at least one CIDR entry. | +| `entries_total` | `usize` | Number of whitelist CIDR entries. | +| `entries` | `string[]` | Whitelist CIDR entries as strings. | + +### Runtime Min Endpoints +- `/v1/runtime/me_pool_state`: generations, hardswap state, writer contour/health counts, refill inflight snapshot. +- `/v1/runtime/me_quality`: ME error/drift/reconnect counters and per-DC RTT coverage snapshot. +- `/v1/runtime/upstream_quality`: upstream runtime policy, connect counters, health summary and per-upstream DC latency/IP preference. +- `/v1/runtime/nat_stun`: NAT/STUN runtime flags, server lists, reflection cache state and backoff remaining. + +### Runtime Edge Endpoints +- `/v1/runtime/connections/summary`: cached connection totals (`total/me/direct`), active users and top-N users by connections/traffic. +- `/v1/runtime/events/recent?limit=N`: bounded control-plane ring-buffer events (`limit` clamped to `[1, 1000]`). +- If `server.api.runtime_edge_enabled=false`, runtime edge endpoints return `enabled=false` with `reason=feature_disabled`. + ### `ZeroAllData` | Field | Type | Description | | --- | --- | --- | From 6fd9f0595d438f22115038c88c010ed1ee369eb7 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Fri, 6 Mar 2026 20:24:17 +0300 Subject: [PATCH 110/261] Update FAQ.ru.md --- docs/FAQ.ru.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index de8a7b8..bec98f6 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -26,6 +26,13 @@ use_middle_proxy = true > [!WARNING] > У вас не будет отображаться "спонсор прокси" если вы уже подписаны на канал. +**Также вы можете настроить разные каналы для разных пользователей.** +```toml +[access.user_ad_tags] +hello = "ad_tag" +hello2 = "ad_tag2" +``` + ## Сколько человек может пользоваться 1 ссылкой По умолчанию 1 ссылкой может пользоваться сколько угодно человек. From d752a440e5c05e5fa1a8b6016ccdd6f5d9611542 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:38:17 +0300 Subject: [PATCH 111/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b2c131f..338d2f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.4" +version = "3.3.5" edition = "2024" [dependencies] From 95971ac62cb17b14e62344358b90ae9e769d0cb5 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Fri, 6 Mar 2026 21:03:45 +0300 Subject: [PATCH 112/261] Update QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index 0169953..d82e33e 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -67,6 +67,12 @@ classic = false secure = false tls = true +[server.api] +enabled = true +# listen = "127.0.0.1:9091" +# whitelist = ["127.0.0.1/32"] +# read_only = true + # === Anti-Censorship & Masking === [censorship] tls_domain = "petrovich.ru" @@ -75,6 +81,7 @@ tls_domain = "petrovich.ru" # format: "username" = "32_hex_chars_secret" hello = "00000000000000000000000000000000" ``` + Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить > [!WARNING] @@ -115,9 +122,14 @@ WantedBy=multi-user.target **5.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt` -**6.** Для получения ссылки введите `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` +**6.** Для получения ссылки/ссылок введите +```bash +curl -s http://127.0.0.1:9091/v1/users | jq +``` +> Одной ссылкой модет пользоваться сколько угодно человек. + > [!WARNING] -> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо! +> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете! --- From ce375b62e458b23e28da7b91dea582c415ab7945 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Fri, 6 Mar 2026 21:04:50 +0300 Subject: [PATCH 113/261] Update QUICK_START_GUIDE.en.md --- docs/QUICK_START_GUIDE.en.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/QUICK_START_GUIDE.en.md b/docs/QUICK_START_GUIDE.en.md index 6d94242..52ce66a 100644 --- a/docs/QUICK_START_GUIDE.en.md +++ b/docs/QUICK_START_GUIDE.en.md @@ -67,6 +67,12 @@ classic = false secure = false tls = true +[server.api] +enabled = true +# listen = "127.0.0.1:9091" +# whitelist = ["127.0.0.1/32"] +# read_only = true + # === Anti-Censorship & Masking === [censorship] tls_domain = "petrovich.ru" @@ -75,6 +81,7 @@ tls_domain = "petrovich.ru" # format: "username" = "32_hex_chars_secret" hello = "00000000000000000000000000000000" ``` + then Ctrl+S -> Ctrl+X to save > [!WARNING] @@ -115,7 +122,12 @@ then Ctrl+S -> Ctrl+X to save **5.** For automatic startup at system boot, enter `systemctl enable telemt` -**6.** To get the links, enter `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` +**6.** To get the link(s), enter +```bash +curl -s http://127.0.0.1:9091/v1/users | jq +``` + +> Any number of people can use one link. --- From 6739cd8d01baef4041f62f64bdc865d0e371e90f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 22:54:18 +0300 Subject: [PATCH 114/261] Update README.md --- README.md | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a1c2958..eac817b 100644 --- a/README.md +++ b/README.md @@ -19,18 +19,24 @@ ### 🇷🇺 RU -#### Релиз 3.3.3 LTS - 6 марта +#### Релиз 3.3.5 LTS - 6 марта -6 марта мы выпустили Telemt **3.3.3** +6 марта мы выпустили Telemt **3.3.5** -Это первая версия telemt работающая в комплексных условиях и при этом предоставляющая API +Это [3.3.5 - первая LTS-версия telemt](https://github.com/telemt/telemt/releases/tag/3.3.5)! -В ней используется новый алгоритм - ME NoWait, который вместе с Adaptive Floor и моделью усовершенствованного доступа к KDF Fingerprint на RwLock позволяет достигать максимальную производительность, даже в условиях lossy-сети +В ней используется: +- новый алгоритм ME NoWait для непревзойдённо быстрого восстановления пула +- Adaptive Floor, поддерживающий количество ME Writer на оптимальном уровне +- модель усовершенствованного доступа к KDF Fingerprint на RwLock +- строгая привязка Middle-End к DC-ID с предсказуемым алгоритмом деградации и самовосстановления -Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **статистики** и **UX** +Telemt Control API V1 в 3.3.5 включает: +- несколько режимов работы в зависимости от доступных ресурсов +- снапшот-модель для живых метрик без вмешательства в hot-path +- минималистичный набор запросов для управления пользователями -Релиз: -[3.3.3](https://github.com/telemt/telemt/releases/tag/3.3.3) +Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **API**, **статистики**, **UX** --- @@ -47,18 +53,24 @@ ### 🇬🇧 EN -#### Release 3.3.3 LTS - March 6 +#### Release 3.3.5 LTS - March 6 On March 6, we released Telemt **3.3.3** -This is the first telemt's version designed to operate reliably in complex network conditions while also providing a runtime API! +This is [3.3.5 - the first LTS release of telemt](https://github.com/telemt/telemt/releases/tag/3.3.5) -The release introduces a new algorithm — ME NoWait, which combined with Adaptive Floor and an improved KDF Fingerprint access model based on RwLock, it enables the system to achieve maximum performance even in lossy network environments +It introduces: +- the new ME NoWait algorithm for exceptionally fast pool recovery +- Adaptive Floor, which maintains the number of ME Writers at an optimal level +- an improved KDF Fingerprint access model based on RwLock +- strict binding of Middle-End instances to DC-ID with a predictable degradation and self-recovery algorithm -We are looking forward to your feedback and improvement proposals — especially regarding **statistics** and **UX** +Telemt Control API V1 in version 3.3.5 includes: +- multiple operating modes depending on available resources +- a snapshot-based model for live metrics without interfering with the hot path +- a minimalistic request set for user management -Release: -[3.3.3](https://github.com/telemt/telemt/releases/tag/3.3.3) +We are looking forward to your feedback and improvement proposals — especially regarding **API**, **statistics**, **UX** --- From e90c42ae680517e5d669ff768ae031f72962af30 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 6 Mar 2026 23:54:29 +0300 Subject: [PATCH 115/261] Migration aesdiag.py --- tools/aesdiag.py | 403 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 tools/aesdiag.py diff --git a/tools/aesdiag.py b/tools/aesdiag.py new file mode 100644 index 0000000..6de7132 --- /dev/null +++ b/tools/aesdiag.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +""" +AES-CBC validation tool for telemt middle proxy logs with support for noop padding. + +Parses log lines containing: + - "ME diag: derived keys and handshake plaintext" (provides write_key, write_iv, hs_plain) + - "ME diag: handshake ciphertext" (provides hs_cipher) + +For each pair it: + - Decrypts the ciphertext using the provided key and IV. + - Compares the beginning of the decrypted data with hs_plain. + - Attempts to identify the actual padding scheme (PKCS#7, zero padding, noop padding). + - Re-encrypts with different paddings and reports mismatches block by block. + - Accumulates statistics for final summary. +""" + +import sys +import re +from collections import defaultdict +from Crypto.Cipher import AES + +# Constants +NOOP_FRAME = bytes([0x04, 0x00, 0x00, 0x00]) # noop frame used for padding + +def hex_str_to_bytes(hex_str): + """Convert a hex string like 'aa bb cc' to bytes.""" + return bytes.fromhex(hex_str.replace(' ', '')) + +def parse_params(line): + """Extract key=value pairs where value is a space-separated hex string.""" + pattern = r'(\w+)=((?:[0-9a-f]{2} )*[0-9a-f]{2})' + return {key: val for key, val in re.findall(pattern, line)} + +def pkcs7_pad(data, block_size=16): + """Apply PKCS#7 padding to the given data.""" + pad_len = block_size - (len(data) % block_size) + if pad_len == 0: + pad_len = block_size + return data + bytes([pad_len]) * pad_len + +def zero_pad(data, block_size=16): + """Pad with zeros to the next block boundary.""" + pad_len = block_size - (len(data) % block_size) + if pad_len == block_size: + return data # already full blocks, no zero padding needed + return data + bytes(pad_len) + +def noop_pad(data): + """ + Pad with minimal number of noop frames (b'\\x04\\x00\\x00\\x00') + to reach a multiple of 16 bytes. + """ + block_size = 16 + frame_len = len(NOOP_FRAME) # 4 + remainder = len(data) % block_size + if remainder == 0: + return data # no padding needed + # We need to add k frames such that (len(data) + k*frame_len) % block_size == 0 + # => k*frame_len ≡ -remainder (mod block_size) + # Since frame_len=4 and block_size=16, we need k*4 ≡ (16-remainder) mod 16 + # k must be an integer in {1,2,3} (because 4*4=16 ≡0 mod16, so k=4 gives remainder 0, but then total increase=16, + # but if remainder==0 we already handled; if remainder!=0, k=4 gives (len+16)%16 == remainder, not 0, + # so k=4 doesn't solve unless remainder=0. Actually 4*4=16 ≡0, so k=4 gives (len+16)%16 = remainder, so still not 0. + # The equation is k*4 ≡ (16-remainder) mod 16. Let r=16-remainder (1..15). Then k ≡ r*inv(4) mod 4? Since mod 16, + # 4 has no inverse modulo 16 because gcd(4,16)=4. So solutions exist only if r is multiple of 4. + # Therefore remainder must be 4,8,12 (so that r = 12,8,4). This matches the idea that noop padding is only added + # when the plaintext length mod 16 is 4,8,12. In our logs it's always 44 mod16=12, so r=4, so k=1 works. + # For safety, we compute k as (block_size - remainder) // frame_len, but this only works if that value is integer. + need = block_size - remainder + if need % frame_len != 0: + # This shouldn't happen by protocol, but if it does, fall back to adding full blocks of noop until multiple. + # We'll add ceil(need/frame_len) frames. + k = (need + frame_len - 1) // frame_len + else: + k = need // frame_len + return data + NOOP_FRAME * k + +def unpad_pkcs7(data): + """Remove PKCS#7 padding (assumes correct padding).""" + if not data: + return data + pad_len = data[-1] + if pad_len < 1 or pad_len > 16: + return data # not valid PKCS#7, return as is + # Check that all padding bytes are equal to pad_len + if all(b == pad_len for b in data[-pad_len:]): + return data[:-pad_len] + return data + +def is_noop_padded(decrypted, plain_log): + """ + Check if the extra bytes after plain_log in decrypted consist of one or more NOOP_FRAMEs. + Returns True if they do, False otherwise. + """ + extra = decrypted[len(plain_log):] + if len(extra) == 0: + return False + # Split into chunks of 4 + if len(extra) % 4 != 0: + return False + for i in range(0, len(extra), 4): + if extra[i:i+4] != NOOP_FRAME: + return False + return True + +def main(): + derived_list = [] # entries from "derived keys and handshake plaintext" + cipher_list = [] # entries from "handshake ciphertext" + + for line in sys.stdin: + if 'ME diag: derived keys and handshake plaintext' in line: + params = parse_params(line) + if all(k in params for k in ('write_key', 'write_iv', 'hs_plain')): + derived_list.append(params) + elif 'ME diag: handshake ciphertext' in line: + params = parse_params(line) + if 'hs_cipher' in params: + cipher_list.append(params) + + # Warn about count mismatch but process as many pairs as possible + n_pairs = min(len(derived_list), len(cipher_list)) + if len(derived_list) != len(cipher_list): + print(f"\n[WARN] Number of derived entries ({len(derived_list)}) " + f"differs from cipher entries ({len(cipher_list)}). " + f"Processing first {n_pairs} pairs.\n") + + # Statistics accumulators + stats = { + 'total': n_pairs, + 'key_length_ok': 0, + 'iv_length_ok': 0, + 'cipher_aligned': 0, + 'decryption_match_start': 0, # first bytes equal hs_plain + 'pkcs7_after_unpad_matches': 0, # after removing PKCS7, equals hs_plain + 'extra_bytes_all_zero': 0, # extra bytes after hs_plain are zero + 'extra_bytes_noop': 0, # extra bytes are noop frames + 'pkcs7_encrypt_ok': 0, # re-encryption with PKCS7 matches ciphertext + 'zero_encrypt_ok': 0, # re-encryption with zero padding matches + 'noop_encrypt_ok': 0, # re-encryption with noop padding matches + 'no_padding_encrypt_ok': 0, # only if plaintext multiple of 16 and matches + 'no_padding_applicable': 0, # number of tests where plaintext len %16 ==0 + } + + detailed_results = [] # store per-test summary for final heuristic + + for idx, (der, ciph) in enumerate(zip(derived_list[:n_pairs], cipher_list[:n_pairs]), 1): + print(f"\n{'='*60}") + print(f"Test #{idx}") + print(f"{'='*60}") + + # Local stats for this test + test_stats = defaultdict(bool) + + try: + key = hex_str_to_bytes(der['write_key']) + iv = hex_str_to_bytes(der['write_iv']) + plain_log = hex_str_to_bytes(der['hs_plain']) + ciphertext = hex_str_to_bytes(ciph['hs_cipher']) + + # Basic sanity checks + print(f"[INFO] Key length : {len(key)} bytes (expected 32)") + print(f"[INFO] IV length : {len(iv)} bytes (expected 16)") + print(f"[INFO] hs_plain length : {len(plain_log)} bytes") + print(f"[INFO] hs_cipher length : {len(ciphertext)} bytes") + + if len(key) == 32: + stats['key_length_ok'] += 1 + test_stats['key_ok'] = True + else: + print("[WARN] Key length is not 32 bytes – AES-256 requires 32-byte key.") + + if len(iv) == 16: + stats['iv_length_ok'] += 1 + test_stats['iv_ok'] = True + else: + print("[WARN] IV length is not 16 bytes – AES-CBC requires 16-byte IV.") + + if len(ciphertext) % 16 == 0: + stats['cipher_aligned'] += 1 + test_stats['cipher_aligned'] = True + else: + print("[ERROR] Ciphertext length is not a multiple of 16 – invalid AES-CBC block alignment.") + # Skip further processing for this test + detailed_results.append(test_stats) + continue + + # --- Decryption test --- + cipher_dec = AES.new(key, AES.MODE_CBC, iv) + decrypted = cipher_dec.decrypt(ciphertext) + print(f"[INFO] Decrypted ({len(decrypted)} bytes): {decrypted.hex()}") + + # Compare beginning with hs_plain + match_len = min(len(plain_log), len(decrypted)) + if decrypted[:match_len] == plain_log[:match_len]: + print(f"[OK] First {match_len} bytes match hs_plain.") + stats['decryption_match_start'] += 1 + test_stats['decrypt_start_ok'] = True + else: + print(f"[FAIL] First bytes do NOT match hs_plain.") + for i in range(match_len): + if decrypted[i] != plain_log[i]: + print(f" First mismatch at byte {i}: hs_plain={plain_log[i]:02x}, decrypted={decrypted[i]:02x}") + break + test_stats['decrypt_start_ok'] = False + + # --- Try to identify actual padding --- + # Remove possible PKCS#7 padding from decrypted data + decrypted_unpadded = unpad_pkcs7(decrypted) + if decrypted_unpadded != decrypted: + print(f"[INFO] After removing PKCS#7 padding: {len(decrypted_unpadded)} bytes left.") + if decrypted_unpadded == plain_log: + print("[OK] Decrypted data with PKCS#7 removed exactly matches hs_plain.") + stats['pkcs7_after_unpad_matches'] += 1 + test_stats['pkcs7_unpad_matches'] = True + else: + print("[INFO] Decrypted (PKCS#7 removed) does NOT match hs_plain.") + test_stats['pkcs7_unpad_matches'] = False + else: + print("[INFO] No valid PKCS#7 padding detected in decrypted data.") + test_stats['pkcs7_unpad_matches'] = False + + # Check if the extra bytes after hs_plain in decrypted are all zero (zero padding) + extra = decrypted[len(plain_log):] + if extra and all(b == 0 for b in extra): + print("[INFO] Extra bytes after hs_plain are all zeros – likely zero padding.") + stats['extra_bytes_all_zero'] += 1 + test_stats['extra_zero'] = True + else: + test_stats['extra_zero'] = False + + # Check for noop padding in extra bytes + if is_noop_padded(decrypted, plain_log): + print(f"[OK] Extra bytes after hs_plain consist of noop frames ({NOOP_FRAME.hex()}).") + stats['extra_bytes_noop'] += 1 + test_stats['extra_noop'] = True + else: + test_stats['extra_noop'] = False + if extra: + print(f"[INFO] Extra bytes after hs_plain (hex): {extra.hex()}") + + # --- Re-encryption tests --- + # PKCS#7 + padded_pkcs7 = pkcs7_pad(plain_log) + cipher_enc = AES.new(key, AES.MODE_CBC, iv) + computed_pkcs7 = cipher_enc.encrypt(padded_pkcs7) + if computed_pkcs7 == ciphertext: + print("[OK] PKCS#7 padding produces the expected ciphertext.") + stats['pkcs7_encrypt_ok'] += 1 + test_stats['pkcs7_enc_ok'] = True + else: + print("[FAIL] PKCS#7 padding does NOT match the ciphertext.") + test_stats['pkcs7_enc_ok'] = False + # Show block where first difference occurs + block_size = 16 + for blk in range(len(ciphertext)//block_size): + start = blk*block_size + exp = ciphertext[start:start+block_size] + comp = computed_pkcs7[start:start+block_size] + if exp != comp: + print(f" First difference in block {blk}:") + print(f" expected : {exp.hex()}") + print(f" computed : {comp.hex()}") + break + + # Zero padding + padded_zero = zero_pad(plain_log) + # Ensure multiple of 16 + if len(padded_zero) % 16 != 0: + padded_zero += bytes(16 - (len(padded_zero)%16)) + cipher_enc_zero = AES.new(key, AES.MODE_CBC, iv) + computed_zero = cipher_enc_zero.encrypt(padded_zero) + if computed_zero == ciphertext: + print("[OK] Zero padding produces the expected ciphertext.") + stats['zero_encrypt_ok'] += 1 + test_stats['zero_enc_ok'] = True + else: + print("[INFO] Zero padding does NOT match (expected, unless log used PKCS#7).") + test_stats['zero_enc_ok'] = False + + # Noop padding + padded_noop = noop_pad(plain_log) + # Ensure multiple of 16 (noop_pad already returns multiple of 16) + cipher_enc_noop = AES.new(key, AES.MODE_CBC, iv) + computed_noop = cipher_enc_noop.encrypt(padded_noop) + if computed_noop == ciphertext: + print("[OK] Noop padding produces the expected ciphertext.") + stats['noop_encrypt_ok'] += 1 + test_stats['noop_enc_ok'] = True + else: + print("[FAIL] Noop padding does NOT match the ciphertext.") + test_stats['noop_enc_ok'] = False + # Show block difference if needed + for blk in range(len(ciphertext)//16): + start = blk*16 + if computed_noop[start:start+16] != ciphertext[start:start+16]: + print(f" First difference in block {blk}:") + print(f" expected : {ciphertext[start:start+16].hex()}") + print(f" computed : {computed_noop[start:start+16].hex()}") + break + + # No padding (only possible if plaintext is already multiple of 16) + if len(plain_log) % 16 == 0: + stats['no_padding_applicable'] += 1 + cipher_enc_nopad = AES.new(key, AES.MODE_CBC, iv) + computed_nopad = cipher_enc_nopad.encrypt(plain_log) + if computed_nopad == ciphertext: + print("[OK] No padding (plaintext multiple of 16) matches.") + stats['no_padding_encrypt_ok'] += 1 + test_stats['no_pad_enc_ok'] = True + else: + print("[INFO] No padding does NOT match.") + test_stats['no_pad_enc_ok'] = False + else: + print("[INFO] Skipping no‑padding test because plaintext length is not a multiple of 16.") + + except Exception as e: + print(f"[EXCEPTION] {e}") + test_stats['exception'] = True + + detailed_results.append(test_stats) + + # --- Final statistics and heuristic summary --- + print("\n" + "="*60) + print("STATISTICS SUMMARY") + print("="*60) + print(f"Total tests processed : {stats['total']}") + print(f"Key length OK (32) : {stats['key_length_ok']}/{stats['total']}") + print(f"IV length OK (16) : {stats['iv_length_ok']}/{stats['total']}") + print(f"Ciphertext 16-byte aligned : {stats['cipher_aligned']}/{stats['total']}") + print(f"Decryption starts with hs_plain : {stats['decryption_match_start']}/{stats['total']}") + print(f"After PKCS#7 removal matches : {stats['pkcs7_after_unpad_matches']}/{stats['total']}") + print(f"Extra bytes after hs_plain are 0 : {stats['extra_bytes_all_zero']}/{stats['total']}") + print(f"Extra bytes are noop frames : {stats['extra_bytes_noop']}/{stats['total']}") + print(f"PKCS#7 re-encryption OK : {stats['pkcs7_encrypt_ok']}/{stats['total']}") + print(f"Zero padding re-encryption OK : {stats['zero_encrypt_ok']}/{stats['total']}") + print(f"Noop padding re-encryption OK : {stats['noop_encrypt_ok']}/{stats['total']}") + if stats['no_padding_applicable'] > 0: + print(f"No-padding applicable tests : {stats['no_padding_applicable']}") + print(f"No-padding re-encryption OK : {stats['no_padding_encrypt_ok']}/{stats['no_padding_applicable']}") + + # Heuristic: determine most likely padding + print("\n" + "="*60) + print("HEURISTIC CONCLUSION") + print("="*60) + + if stats['decryption_match_start'] == stats['total']: + print("✓ All tests: first bytes of decrypted data match hs_plain → keys and IV are correct.") + else: + print("✗ Some tests: first bytes mismatch → possible key/IV issues or corrupted ciphertext.") + + # Guess padding based on re-encryption success and extra bytes + candidates = [] + if stats['pkcs7_encrypt_ok'] == stats['total']: + candidates.append("PKCS#7") + if stats['zero_encrypt_ok'] == stats['total']: + candidates.append("zero padding") + if stats['noop_encrypt_ok'] == stats['total']: + candidates.append("noop padding") + if stats['no_padding_applicable'] == stats['total'] and stats['no_padding_encrypt_ok'] == stats['total']: + candidates.append("no padding") + + if len(candidates) == 1: + print(f"✓ All tests consistent with padding scheme: {candidates[0]}.") + elif len(candidates) > 1: + print(f"⚠ Multiple padding schemes succeed in all tests: {', '.join(candidates)}. This is unusual.") + else: + # No scheme succeeded in all tests – look at ratios + print("Mixed padding results:") + total = stats['total'] + pkcs7_ratio = stats['pkcs7_encrypt_ok'] / total if total else 0 + zero_ratio = stats['zero_encrypt_ok'] / total if total else 0 + noop_ratio = stats['noop_encrypt_ok'] / total if total else 0 + print(f" PKCS#7 success = {stats['pkcs7_encrypt_ok']}/{total} ({pkcs7_ratio*100:.1f}%)") + print(f" Zero success = {stats['zero_encrypt_ok']}/{total} ({zero_ratio*100:.1f}%)") + print(f" Noop success = {stats['noop_encrypt_ok']}/{total} ({noop_ratio*100:.1f}%)") + + if noop_ratio > max(pkcs7_ratio, zero_ratio): + print("→ Noop padding is most frequent. Check if extra bytes are indeed noop frames.") + elif pkcs7_ratio > zero_ratio: + print("→ PKCS#7 is most frequent, but fails in some tests.") + elif zero_ratio > pkcs7_ratio: + print("→ Zero padding is most frequent, but fails in some tests.") + else: + print("→ No clear winner; possibly a different padding scheme or random data.") + + # Additional heuristics based on extra bytes + if stats['extra_bytes_noop'] == stats['total']: + print("✓ All tests: extra bytes after hs_plain are noop frames → strongly indicates noop padding.") + if stats['extra_bytes_all_zero'] == stats['total']: + print("✓ All tests: extra bytes are zeros → suggests zero padding.") + + # Final health check + if (stats['decryption_match_start'] == stats['total'] and + (stats['pkcs7_encrypt_ok'] == stats['total'] or + stats['zero_encrypt_ok'] == stats['total'] or + stats['noop_encrypt_ok'] == stats['total'] or + stats['no_padding_encrypt_ok'] == stats['no_padding_applicable'] == stats['total'])): + print("\n✅ OVERALL: All tests consistent. The encryption parameters and padding are correct.") + else: + print("\n⚠️ OVERALL: Inconsistencies detected. Review the detailed output for failing tests.") + +if __name__ == '__main__': + main() From 6856466cef86615ee5b465b9eac8db44ed22680c Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Sat, 7 Mar 2026 00:16:03 +0300 Subject: [PATCH 116/261] Update README.md --- README.md | 141 ++++++++++++------------------------------------------ 1 file changed, 31 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index eac817b..6df79fb 100644 --- a/README.md +++ b/README.md @@ -93,31 +93,6 @@ We welcome ideas, architectural feedback, and pull requests. ⚓ Our ***Middle-End Pool*** is fastest by design in standard scenarios, compared to other implementations of connecting to the Middle-End Proxy: non dramatically, but usual -# GOTO -- [Features](#features) -- [Quick Start Guide](#quick-start-guide) -- [How to use?](#how-to-use) - - [Systemd Method](#telemt-via-systemd) -- [Configuration](#configuration) - - [Minimal Configuration](#minimal-configuration-for-first-start) - - [Advanced](#advanced) - - [Adtag](#adtag) - - [Listening and Announce IPs](#listening-and-announce-ips) - - [Upstream Manager](#upstream-manager) - - [IP](#bind-on-ip) - - [SOCKS](#socks45-as-upstream) -- [FAQ](#faq) - - [Recognizability for DPI + crawler](#recognizability-for-dpi-and-crawler) - - [Telegram Calls](#telegram-calls-via-mtproxy) - - [DPI](#how-does-dpi-see-mtproxy-tls) - - [Whitelist on Network Level](#whitelist-on-ip) - - [Too many open files](#too-many-open-files) -- [Build](#build) -- [Docker](#docker) -- [Why Rust?](#why-rust) - -## Features - - Full support for all official MTProto proxy modes: - Classic - Secure - with `dd` prefix @@ -128,59 +103,40 @@ We welcome ideas, architectural feedback, and pull requests. - Graceful shutdown on Ctrl+C - Extensive logging via `trace` and `debug` with `RUST_LOG` method +# GOTO +- [Telemt - MTProxy on Rust + Tokio](#telemt---mtproxy-on-rust--tokio) + - [NEWS and EMERGENCY](#news-and-emergency) + - [✈️ Telemt 3 is released!](#️-telemt-3-is-released) + - [🇷🇺 RU](#-ru) + - [Релиз 3.3.5 LTS - 6 марта](#релиз-335-lts---6-марта) + - [🇬🇧 EN](#-en) + - [Release 3.3.5 LTS - March 6](#release-335-lts---march-6) +- [Features](#features) +- [GOTO](#goto) + - [Quick Start Guide](#quick-start-guide) + - [FAQ](#faq) + - [Recognizability for DPI and crawler](#recognizability-for-dpi-and-crawler) + - [Client WITH secret-key accesses the MTProxy resource:](#client-with-secret-key-accesses-the-mtproxy-resource) + - [Client WITHOUT secret-key gets transparent access to the specified resource:](#client-without-secret-key-gets-transparent-access-to-the-specified-resource) + - [Telegram Calls via MTProxy](#telegram-calls-via-mtproxy) + - [How does DPI see MTProxy TLS?](#how-does-dpi-see-mtproxy-tls) + - [Whitelist on IP](#whitelist-on-ip) + - [Too many open files](#too-many-open-files) + - [Build](#build) + - [Why Rust?](#why-rust) + - [Issues](#issues) + - [Roadmap](#roadmap) + + ## Quick Start Guide - -### [Quick Start Guide RU](docs/QUICK_START_GUIDE.ru.md) -### [Quick Start Guide EN](docs/QUICK_START_GUIDE.en.md) - - -### Advanced -#### Adtag (per-user) -To use channel advertising and usage statistics from Telegram, get an Adtag from [@mtproxybot](https://t.me/mtproxybot). Set it per user in `[access.user_ad_tags]` (32 hex chars): -```toml -[access.user_ad_tags] -username1 = "11111111111111111111111111111111" # Replace with your tag from @mtproxybot -username2 = "22222222222222222222222222222222" -``` -#### Listening and Announce IPs -To specify listening address and/or address in links, add to section `[[server.listeners]]` of config.toml: -```toml -[[server.listeners]] -ip = "0.0.0.0" # 0.0.0.0 = all IPs; your IP = specific listening -announce_ip = "1.2.3.4" # IP in links; comment with # if not used -``` -#### Upstream Manager -To specify upstream, add to section `[[upstreams]]` of config.toml: -##### Bind on IP -```toml -[[upstreams]] -type = "direct" -weight = 1 -enabled = true -interface = "192.168.1.100" # Change to your outgoing IP -``` -##### SOCKS4/5 as Upstream -- Without Auth: -```toml -[[upstreams]] -type = "socks5" # Specify SOCKS4 or SOCKS5 -address = "1.2.3.4:1234" # SOCKS-server Address -weight = 1 # Set Weight for Scenarios -enabled = true -``` - -- With Auth: -```toml -[[upstreams]] -type = "socks5" # Specify SOCKS4 or SOCKS5 -address = "1.2.3.4:1234" # SOCKS-server Address -username = "user" # Username for Auth on SOCKS-server -password = "pass" # Password for Auth on SOCKS-server -weight = 1 # Set Weight for Scenarios -enabled = true -``` +- [Quick Start Guide RU](docs/QUICK_START_GUIDE.ru.md) +- [Quick Start Guide EN](docs/QUICK_START_GUIDE.en.md) ## FAQ + +- [FAQ RU](docs/FAQ.ru.md) +- [FAQ EN](docs/FAQ.en.md) + ### Recognizability for DPI and crawler Since version 1.1.0.0, we have debugged masking perfectly: for all clients without "presenting" a key, we transparently direct traffic to the target host! @@ -325,41 +281,6 @@ chmod +x /bin/telemt telemt config.toml ``` -## Docker -**Quick start (Docker Compose)** - -1. Edit `config.toml` in repo root (at least: port, users secrets, tls_domain) -2. Start container: -```bash -docker compose up -d --build -``` -3. Check logs: -```bash -docker compose logs -f telemt -``` -4. Stop: -```bash -docker compose down -``` - -**Notes** -- `docker-compose.yml` maps `./config.toml` to `/app/config.toml` (read-only) -- By default it publishes `443:443` and runs with dropped capabilities (only `NET_BIND_SERVICE` is added) -- If you really need host networking (usually only for some IPv6 setups) uncomment `network_mode: host` - -**Run without Compose** -```bash -docker build -t telemt:local . -docker run --name telemt --restart unless-stopped \ - -p 443:443 \ - -e RUST_LOG=info \ - -v "$PWD/config.toml:/app/config.toml:ro" \ - --read-only \ - --cap-drop ALL --cap-add NET_BIND_SERVICE \ - --ulimit nofile=65536:65536 \ - telemt:local -``` - ## Why Rust? - Long-running reliability and idempotent behavior - Rust's deterministic resource management - RAII From 8ae741ec724b2e172ad34064dde7ef2ebd4840e7 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Sat, 7 Mar 2026 00:16:46 +0300 Subject: [PATCH 117/261] Update FAQ.ru.md --- docs/FAQ.ru.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index bec98f6..a2fcf1a 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -1,4 +1,4 @@ -## Как настроить канал "спонсор прокси" +## Как настроить канал "спонсор прокси" и статистику через бота @MTProxybot 1. Зайти в бота @MTProxybot. 2. Ввести команду `/newproxy` @@ -70,3 +70,43 @@ metrics_whitelist = ["127.0.0.1/32", "::1/128", "0.0.0.0/0"] 4. Метрики доступны по адресу SERVER_IP:9090/metrics. > [!WARNING] > "0.0.0.0/0" в metrics_whitelist открывает доступ с любого IP. Замените на свой ip. Например "1.2.3.4" + +## Дополнительные параметры + +### Домен в ссылке вместо IP +Чтобы указать домен в ссылках, добавьте в секцию `[general.links]` файла config. +```toml +[general.links] +public_host = "proxy.example.com" +``` + +### Upstream Manager +Чтобы указать апстрим, добавьте в секцию `[[upstreams]]` файла config.toml: +#### Привязка к IP +```toml +[[upstreams]] +type = "direct" +weight = 1 +enabled = true +interface = "192.168.1.100" # Change to your outgoing IP +``` +#### SOCKS4/5 как Upstream +- Без авторизации: +```toml +[[upstreams]] +type = "socks5" # Specify SOCKS4 or SOCKS5 +address = "1.2.3.4:1234" # SOCKS-server Address +weight = 1 # Set Weight for Scenarios +enabled = true +``` + +- С авторизацией: +```toml +[[upstreams]] +type = "socks5" # Specify SOCKS4 or SOCKS5 +address = "1.2.3.4:1234" # SOCKS-server Address +username = "user" # Username for Auth on SOCKS-server +password = "pass" # Password for Auth on SOCKS-server +weight = 1 # Set Weight for Scenarios +enabled = true +``` From 01893f371233ae29e17b019a9b235883db58e18d Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Sat, 7 Mar 2026 00:25:40 +0300 Subject: [PATCH 118/261] Create FAQ.en.md --- docs/FAQ.en.md | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 docs/FAQ.en.md diff --git a/docs/FAQ.en.md b/docs/FAQ.en.md new file mode 100644 index 0000000..49d1592 --- /dev/null +++ b/docs/FAQ.en.md @@ -0,0 +1,112 @@ +## How to set up "proxy sponsor" channel and statistics via @MTProxybot bot + +1. Go to @MTProxybot bot. +2. Enter the command `/newproxy` +3. Send the server IP and port. For example: 1.2.3.4:443 +4. Open the config `nano /etc/telemt.toml`. +5. Copy and send the user secret from the [access.users] section to the bot. +6. Copy the tag received from the bot. For example 1234567890abcdef1234567890abcdef. +> [!WARNING] +> The link provided by the bot will not work. Do not copy or use it! +7. Uncomment the ad_tag parameter and enter the tag received from the bot. +8. Uncomment/add the parameter `use_middle_proxy = true`. + +Config example: +```toml +[general] +ad_tag = "1234567890abcdef1234567890abcdef" +use_middle_proxy = true +``` +9. Save the config. Ctrl+S -> Ctrl+X. +10. Restart telemt `systemctl restart telemt`. +11. In the bot, send the command /myproxies and select the added server. +12. Click the "Set promotion" button. +13. Send a **public link** to the channel. Private channels cannot be added! +14. Wait approximately 1 hour for the information to update on Telegram servers. +> [!WARNING] +> You will not see the "proxy sponsor" if you are already subscribed to the channel. + +**You can also set up different channels for different users.** +```toml +[access.user_ad_tags] +hello = "ad_tag" +hello2 = "ad_tag2" +``` + +## How many people can use 1 link + +By default, 1 link can be used by any number of people. +You can limit the number of IPs using the proxy. +```toml +[access.user_max_unique_ips] +hello = 1 +``` +This parameter limits how many unique IPs can use 1 link simultaneously. If one user disconnects, a second user can connect. Also, multiple users can sit behind the same IP. + +## How to create multiple different links + +1. Generate the required number of secrets `openssl rand -hex 16` +2. Open the config `nano /etc/telemt.toml` +3. Add new users. +```toml +[access.users] +user1 = "00000000000000000000000000000001" +user2 = "00000000000000000000000000000002" +user3 = "00000000000000000000000000000003" +``` +4. Save the config. Ctrl+S -> Ctrl+X. You don't need to restart telemt. +5. Get the links via `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` + +## How to view metrics + +1. Open the config `nano /etc/telemt.toml` +2. Add the following parameters +```toml +[server] +metrics_port = 9090 +metrics_whitelist = ["127.0.0.1/32", "::1/128", "0.0.0.0/0"] +``` +3. Save the config. Ctrl+S -> Ctrl+X. +4. Metrics are available at SERVER_IP:9090/metrics. +> [!WARNING] +> "0.0.0.0/0" in metrics_whitelist opens access from any IP. Replace with your own IP. For example "1.2.3.4" + +## Additional parameters + +### Domain in link instead of IP +To specify a domain in the links, add to the `[general.links]` section of the config file. +```toml +[general.links] +public_host = "proxy.example.com" +``` + +### Upstream Manager +To specify an upstream, add to the `[[upstreams]]` section of the config.toml file: +#### Binding to IP +```toml +[[upstreams]] +type = "direct" +weight = 1 +enabled = true +interface = "192.168.1.100" # Change to your outgoing IP +``` +#### SOCKS4/5 as Upstream +- Without authentication: +```toml +[[upstreams]] +type = "socks5" # Specify SOCKS4 or SOCKS5 +address = "1.2.3.4:1234" # SOCKS-server Address +weight = 1 # Set Weight for Scenarios +enabled = true +``` + +- With authentication: +```toml +[[upstreams]] +type = "socks5" # Specify SOCKS4 or SOCKS5 +address = "1.2.3.4:1234" # SOCKS-server Address +username = "user" # Username for Auth on SOCKS-server +password = "pass" # Password for Auth on SOCKS-server +weight = 1 # Set Weight for Scenarios +enabled = true +``` From ce9698d39b1013e7a0147ed2e0811f5d3d555c94 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 02:50:11 +0300 Subject: [PATCH 119/261] ME Adaptive Floor Planner Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/model.rs | 14 + src/api/runtime_stats.rs | 18 + src/api/runtime_zero.rs | 20 + src/config/defaults.rs | 25 ++ src/config/hot_reload.rs | 47 ++- src/config/load.rs | 15 + src/config/types.rs | 26 ++ src/main.rs | 5 + src/metrics.rs | 127 +++++++ src/stats/mod.rs | 82 +++++ src/transport/middle_proxy/config_updater.rs | 10 + src/transport/middle_proxy/health.rs | 364 ++++++++++++++++++- src/transport/middle_proxy/pool.rs | 156 +++++++- src/transport/middle_proxy/pool_status.rs | 77 ++++ 14 files changed, 963 insertions(+), 23 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index 2f6c58e..88c6ddc 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -269,6 +269,10 @@ pub(super) struct DcStatus { pub(super) available_endpoints: usize, pub(super) available_pct: f64, pub(super) required_writers: usize, + pub(super) floor_min: usize, + pub(super) floor_target: usize, + pub(super) floor_max: usize, + pub(super) floor_capped: bool, pub(super) alive_writers: usize, pub(super) coverage_pct: f64, pub(super) rtt_ms: Option, @@ -308,7 +312,17 @@ pub(super) struct MinimalMeRuntimeData { pub(super) floor_mode: &'static str, pub(super) adaptive_floor_idle_secs: u64, pub(super) adaptive_floor_min_writers_single_endpoint: u8, + pub(super) adaptive_floor_min_writers_multi_endpoint: u8, pub(super) adaptive_floor_recover_grace_secs: u64, + pub(super) adaptive_floor_writers_per_core_total: u16, + pub(super) adaptive_floor_cpu_cores_override: u16, + pub(super) adaptive_floor_max_extra_writers_single_per_core: u16, + pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16, + pub(super) adaptive_floor_cpu_cores_detected: u32, + pub(super) adaptive_floor_cpu_cores_effective: u32, + pub(super) adaptive_floor_global_cap_raw: u64, + pub(super) adaptive_floor_global_cap_effective: u64, + pub(super) adaptive_floor_target_writers_total: u64, pub(super) me_keepalive_enabled: bool, pub(super) me_keepalive_interval_secs: u64, pub(super) me_keepalive_jitter_secs: u64, diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 3019636..c69f817 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -349,6 +349,10 @@ async fn get_minimal_payload_cached( available_endpoints: entry.available_endpoints, available_pct: entry.available_pct, required_writers: entry.required_writers, + floor_min: entry.floor_min, + floor_target: entry.floor_target, + floor_max: entry.floor_max, + floor_capped: entry.floor_capped, alive_writers: entry.alive_writers, coverage_pct: entry.coverage_pct, rtt_ms: entry.rtt_ms, @@ -366,7 +370,21 @@ async fn get_minimal_payload_cached( adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs, adaptive_floor_min_writers_single_endpoint: runtime .adaptive_floor_min_writers_single_endpoint, + adaptive_floor_min_writers_multi_endpoint: runtime + .adaptive_floor_min_writers_multi_endpoint, adaptive_floor_recover_grace_secs: runtime.adaptive_floor_recover_grace_secs, + adaptive_floor_writers_per_core_total: runtime + .adaptive_floor_writers_per_core_total, + adaptive_floor_cpu_cores_override: runtime.adaptive_floor_cpu_cores_override, + adaptive_floor_max_extra_writers_single_per_core: runtime + .adaptive_floor_max_extra_writers_single_per_core, + adaptive_floor_max_extra_writers_multi_per_core: runtime + .adaptive_floor_max_extra_writers_multi_per_core, + adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected, + adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective, + adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw, + adaptive_floor_global_cap_effective: runtime.adaptive_floor_global_cap_effective, + adaptive_floor_target_writers_total: runtime.adaptive_floor_target_writers_total, me_keepalive_enabled: runtime.me_keepalive_enabled, me_keepalive_interval_secs: runtime.me_keepalive_interval_secs, me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs, diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index 2c50020..61b6844 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -60,7 +60,12 @@ pub(super) struct EffectiveMiddleProxyLimits { pub(super) floor_mode: &'static str, pub(super) adaptive_floor_idle_secs: u64, pub(super) adaptive_floor_min_writers_single_endpoint: u8, + pub(super) adaptive_floor_min_writers_multi_endpoint: u8, pub(super) adaptive_floor_recover_grace_secs: u64, + pub(super) adaptive_floor_writers_per_core_total: u16, + pub(super) adaptive_floor_cpu_cores_override: u16, + pub(super) adaptive_floor_max_extra_writers_single_per_core: u16, + pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16, pub(super) reconnect_max_concurrent_per_dc: u32, pub(super) reconnect_backoff_base_ms: u64, pub(super) reconnect_backoff_cap_ms: u64, @@ -183,7 +188,22 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD adaptive_floor_min_writers_single_endpoint: cfg .general .me_adaptive_floor_min_writers_single_endpoint, + adaptive_floor_min_writers_multi_endpoint: cfg + .general + .me_adaptive_floor_min_writers_multi_endpoint, adaptive_floor_recover_grace_secs: cfg.general.me_adaptive_floor_recover_grace_secs, + adaptive_floor_writers_per_core_total: cfg + .general + .me_adaptive_floor_writers_per_core_total, + adaptive_floor_cpu_cores_override: cfg + .general + .me_adaptive_floor_cpu_cores_override, + adaptive_floor_max_extra_writers_single_per_core: cfg + .general + .me_adaptive_floor_max_extra_writers_single_per_core, + adaptive_floor_max_extra_writers_multi_per_core: cfg + .general + .me_adaptive_floor_max_extra_writers_multi_per_core, reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc, reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms, reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms, diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 3ba146c..465cef1 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -11,7 +11,12 @@ const DEFAULT_ME_RECONNECT_FAST_RETRY_COUNT: u32 = 16; const DEFAULT_ME_SINGLE_ENDPOINT_SHADOW_WRITERS: u8 = 2; const DEFAULT_ME_ADAPTIVE_FLOOR_IDLE_SECS: u64 = 90; const DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT: u8 = 1; +const DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_MULTI_ENDPOINT: u8 = 1; const DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS: u64 = 180; +const DEFAULT_ME_ADAPTIVE_FLOOR_WRITERS_PER_CORE_TOTAL: u16 = 48; +const DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE: u16 = 0; +const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE: u16 = 1; +const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE: u16 = 2; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; @@ -247,10 +252,30 @@ pub(crate) fn default_me_adaptive_floor_min_writers_single_endpoint() -> u8 { DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_SINGLE_ENDPOINT } +pub(crate) fn default_me_adaptive_floor_min_writers_multi_endpoint() -> u8 { + DEFAULT_ME_ADAPTIVE_FLOOR_MIN_WRITERS_MULTI_ENDPOINT +} + pub(crate) fn default_me_adaptive_floor_recover_grace_secs() -> u64 { DEFAULT_ME_ADAPTIVE_FLOOR_RECOVER_GRACE_SECS } +pub(crate) fn default_me_adaptive_floor_writers_per_core_total() -> u16 { + DEFAULT_ME_ADAPTIVE_FLOOR_WRITERS_PER_CORE_TOTAL +} + +pub(crate) fn default_me_adaptive_floor_cpu_cores_override() -> u16 { + DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE +} + +pub(crate) fn default_me_adaptive_floor_max_extra_writers_single_per_core() -> u16 { + DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE +} + +pub(crate) fn default_me_adaptive_floor_max_extra_writers_multi_per_core() -> u16 { + DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE +} + pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index c39cafa..a24f9d5 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -78,7 +78,12 @@ pub struct HotFields { pub me_floor_mode: MeFloorMode, pub me_adaptive_floor_idle_secs: u64, pub me_adaptive_floor_min_writers_single_endpoint: u8, + pub me_adaptive_floor_min_writers_multi_endpoint: u8, pub me_adaptive_floor_recover_grace_secs: u64, + pub me_adaptive_floor_writers_per_core_total: u16, + pub me_adaptive_floor_cpu_cores_override: u16, + pub me_adaptive_floor_max_extra_writers_single_per_core: u16, + pub me_adaptive_floor_max_extra_writers_multi_per_core: u16, pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_watermark_pct: u8, @@ -150,9 +155,24 @@ impl HotFields { me_adaptive_floor_min_writers_single_endpoint: cfg .general .me_adaptive_floor_min_writers_single_endpoint, + me_adaptive_floor_min_writers_multi_endpoint: cfg + .general + .me_adaptive_floor_min_writers_multi_endpoint, me_adaptive_floor_recover_grace_secs: cfg .general .me_adaptive_floor_recover_grace_secs, + me_adaptive_floor_writers_per_core_total: cfg + .general + .me_adaptive_floor_writers_per_core_total, + me_adaptive_floor_cpu_cores_override: cfg + .general + .me_adaptive_floor_cpu_cores_override, + me_adaptive_floor_max_extra_writers_single_per_core: cfg + .general + .me_adaptive_floor_max_extra_writers_single_per_core, + me_adaptive_floor_max_extra_writers_multi_per_core: cfg + .general + .me_adaptive_floor_max_extra_writers_multi_per_core, me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, @@ -273,8 +293,18 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.general.me_adaptive_floor_idle_secs = new.general.me_adaptive_floor_idle_secs; cfg.general.me_adaptive_floor_min_writers_single_endpoint = new.general.me_adaptive_floor_min_writers_single_endpoint; + cfg.general.me_adaptive_floor_min_writers_multi_endpoint = + new.general.me_adaptive_floor_min_writers_multi_endpoint; cfg.general.me_adaptive_floor_recover_grace_secs = new.general.me_adaptive_floor_recover_grace_secs; + cfg.general.me_adaptive_floor_writers_per_core_total = + new.general.me_adaptive_floor_writers_per_core_total; + cfg.general.me_adaptive_floor_cpu_cores_override = + new.general.me_adaptive_floor_cpu_cores_override; + cfg.general.me_adaptive_floor_max_extra_writers_single_per_core = + new.general.me_adaptive_floor_max_extra_writers_single_per_core; + cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core = + new.general.me_adaptive_floor_max_extra_writers_multi_per_core; cfg.general.me_route_backpressure_base_timeout_ms = new.general.me_route_backpressure_base_timeout_ms; cfg.general.me_route_backpressure_high_timeout_ms = @@ -697,15 +727,30 @@ fn log_changes( || old_hot.me_adaptive_floor_idle_secs != new_hot.me_adaptive_floor_idle_secs || old_hot.me_adaptive_floor_min_writers_single_endpoint != new_hot.me_adaptive_floor_min_writers_single_endpoint + || old_hot.me_adaptive_floor_min_writers_multi_endpoint + != new_hot.me_adaptive_floor_min_writers_multi_endpoint || old_hot.me_adaptive_floor_recover_grace_secs != new_hot.me_adaptive_floor_recover_grace_secs + || old_hot.me_adaptive_floor_writers_per_core_total + != new_hot.me_adaptive_floor_writers_per_core_total + || old_hot.me_adaptive_floor_cpu_cores_override + != new_hot.me_adaptive_floor_cpu_cores_override + || old_hot.me_adaptive_floor_max_extra_writers_single_per_core + != new_hot.me_adaptive_floor_max_extra_writers_single_per_core + || old_hot.me_adaptive_floor_max_extra_writers_multi_per_core + != new_hot.me_adaptive_floor_max_extra_writers_multi_per_core { info!( - "config reload: me_floor: mode={:?} idle={}s min_single={} recover_grace={}s", + "config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={}", new_hot.me_floor_mode, new_hot.me_adaptive_floor_idle_secs, new_hot.me_adaptive_floor_min_writers_single_endpoint, + new_hot.me_adaptive_floor_min_writers_multi_endpoint, new_hot.me_adaptive_floor_recover_grace_secs, + new_hot.me_adaptive_floor_writers_per_core_total, + new_hot.me_adaptive_floor_cpu_cores_override, + new_hot.me_adaptive_floor_max_extra_writers_single_per_core, + new_hot.me_adaptive_floor_max_extra_writers_multi_per_core, ); } diff --git a/src/config/load.rs b/src/config/load.rs index 6ce7b65..e6dc728 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -312,6 +312,21 @@ impl ProxyConfig { )); } + if config.general.me_adaptive_floor_min_writers_multi_endpoint == 0 + || config.general.me_adaptive_floor_min_writers_multi_endpoint > 32 + { + return Err(ProxyError::Config( + "general.me_adaptive_floor_min_writers_multi_endpoint must be within [1, 32]" + .to_string(), + )); + } + + if config.general.me_adaptive_floor_writers_per_core_total == 0 { + return Err(ProxyError::Config( + "general.me_adaptive_floor_writers_per_core_total must be > 0".to_string(), + )); + } + if config.general.me_single_endpoint_outage_backoff_min_ms == 0 { return Err(ProxyError::Config( "general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(), diff --git a/src/config/types.rs b/src/config/types.rs index 4a33b7c..5a0dbb2 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -520,10 +520,31 @@ pub struct GeneralConfig { #[serde(default = "default_me_adaptive_floor_min_writers_single_endpoint")] pub me_adaptive_floor_min_writers_single_endpoint: u8, + /// Minimum writer target for multi-endpoint DC groups in adaptive floor mode. + #[serde(default = "default_me_adaptive_floor_min_writers_multi_endpoint")] + pub me_adaptive_floor_min_writers_multi_endpoint: u8, + /// Grace period in seconds to hold static floor after activity in adaptive mode. #[serde(default = "default_me_adaptive_floor_recover_grace_secs")] pub me_adaptive_floor_recover_grace_secs: u64, + /// Global ME writer budget per logical CPU core in adaptive mode. + #[serde(default = "default_me_adaptive_floor_writers_per_core_total")] + pub me_adaptive_floor_writers_per_core_total: u16, + + /// Override logical CPU core count for adaptive floor calculations. + /// Set to 0 to use runtime auto-detection. + #[serde(default = "default_me_adaptive_floor_cpu_cores_override")] + pub me_adaptive_floor_cpu_cores_override: u16, + + /// Per-core max extra writers above base required floor for single-endpoint DC groups. + #[serde(default = "default_me_adaptive_floor_max_extra_writers_single_per_core")] + pub me_adaptive_floor_max_extra_writers_single_per_core: u16, + + /// Per-core max extra writers above base required floor for multi-endpoint DC groups. + #[serde(default = "default_me_adaptive_floor_max_extra_writers_multi_per_core")] + pub me_adaptive_floor_max_extra_writers_multi_per_core: u16, + /// Connect attempts for the selected upstream before returning error/fallback. #[serde(default = "default_upstream_connect_retry_attempts")] pub upstream_connect_retry_attempts: u32, @@ -775,7 +796,12 @@ impl Default for GeneralConfig { me_floor_mode: MeFloorMode::default(), me_adaptive_floor_idle_secs: default_me_adaptive_floor_idle_secs(), me_adaptive_floor_min_writers_single_endpoint: default_me_adaptive_floor_min_writers_single_endpoint(), + me_adaptive_floor_min_writers_multi_endpoint: default_me_adaptive_floor_min_writers_multi_endpoint(), me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(), + me_adaptive_floor_writers_per_core_total: default_me_adaptive_floor_writers_per_core_total(), + me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(), + me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(), + me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_connect_budget_ms: default_upstream_connect_budget_ms(), diff --git a/src/main.rs b/src/main.rs index ee5aaad..ca24d0e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -786,7 +786,12 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_floor_mode, config.general.me_adaptive_floor_idle_secs, config.general.me_adaptive_floor_min_writers_single_endpoint, + config.general.me_adaptive_floor_min_writers_multi_endpoint, config.general.me_adaptive_floor_recover_grace_secs, + config.general.me_adaptive_floor_writers_per_core_total, + config.general.me_adaptive_floor_cpu_cores_override, + config.general.me_adaptive_floor_max_extra_writers_single_per_core, + config.general.me_adaptive_floor_max_extra_writers_multi_per_core, config.general.hardswap, config.general.me_pool_drain_ttl_secs, config.general.effective_me_pool_force_close_secs(), diff --git a/src/metrics.rs b/src/metrics.rs index 0ccec94..633a884 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -968,6 +968,133 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp 0 } ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_cpu_cores_detected Runtime detected logical CPU cores for adaptive floor" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_cpu_cores_detected gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_cpu_cores_detected {}", + if me_allows_normal { + stats.get_me_floor_cpu_cores_detected_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_cpu_cores_effective Runtime effective logical CPU cores for adaptive floor" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_cpu_cores_effective gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_cpu_cores_effective {}", + if me_allows_normal { + stats.get_me_floor_cpu_cores_effective_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_global_cap_raw Runtime raw global adaptive floor cap" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_global_cap_raw gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_global_cap_raw {}", + if me_allows_normal { + stats.get_me_floor_global_cap_raw_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_global_cap_effective Runtime effective global adaptive floor cap" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_global_cap_effective gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_global_cap_effective {}", + if me_allows_normal { + stats.get_me_floor_global_cap_effective_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_target_writers_total Runtime adaptive floor target writers total" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_target_writers_total gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_target_writers_total {}", + if me_allows_normal { + stats.get_me_floor_target_writers_total_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps" + ); + let _ = writeln!(out, "# TYPE telemt_me_floor_cap_block_total counter"); + let _ = writeln!( + out, + "telemt_me_floor_cap_block_total {}", + if me_allows_normal { + stats.get_me_floor_cap_block_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_floor_swap_idle_total Adaptive floor cap recovery via idle writer swap" + ); + let _ = writeln!(out, "# TYPE telemt_me_floor_swap_idle_total counter"); + let _ = writeln!( + out, + "telemt_me_floor_swap_idle_total {}", + if me_allows_normal { + stats.get_me_floor_swap_idle_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_floor_swap_idle_failed_total Failed idle swap attempts under adaptive floor caps" + ); + let _ = writeln!(out, "# TYPE telemt_me_floor_swap_idle_failed_total counter"); + let _ = writeln!( + out, + "telemt_me_floor_swap_idle_failed_total {}", + if me_allows_normal { + stats.get_me_floor_swap_idle_failed_total() + } else { + 0 + } + ); let _ = writeln!(out, "# HELP telemt_secure_padding_invalid_total Invalid secure frame lengths"); let _ = writeln!(out, "# TYPE telemt_secure_padding_invalid_total counter"); diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 4b59367..b51c941 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -75,6 +75,14 @@ pub struct Stats { me_floor_mode_switch_total: AtomicU64, me_floor_mode_switch_static_to_adaptive_total: AtomicU64, me_floor_mode_switch_adaptive_to_static_total: AtomicU64, + me_floor_cpu_cores_detected_gauge: AtomicU64, + me_floor_cpu_cores_effective_gauge: AtomicU64, + me_floor_global_cap_raw_gauge: AtomicU64, + me_floor_global_cap_effective_gauge: AtomicU64, + me_floor_target_writers_total_gauge: AtomicU64, + me_floor_cap_block_total: AtomicU64, + me_floor_swap_idle_total: AtomicU64, + me_floor_swap_idle_failed_total: AtomicU64, me_handshake_error_codes: DashMap, me_route_drop_no_conn: AtomicU64, me_route_drop_channel_closed: AtomicU64, @@ -676,6 +684,52 @@ impl Stats { .fetch_add(1, Ordering::Relaxed); } } + pub fn set_me_floor_cpu_cores_detected_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_cpu_cores_detected_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_cpu_cores_effective_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_cpu_cores_effective_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_global_cap_raw_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_global_cap_raw_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_global_cap_effective_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_global_cap_effective_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_target_writers_total_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_target_writers_total_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn increment_me_floor_cap_block_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_floor_cap_block_total.fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_floor_swap_idle_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_floor_swap_idle_total.fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_floor_swap_idle_failed_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_floor_swap_idle_failed_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn get_connects_all(&self) -> u64 { self.connects_all.load(Ordering::Relaxed) } pub fn get_connects_bad(&self) -> u64 { self.connects_bad.load(Ordering::Relaxed) } pub fn get_current_connections_direct(&self) -> u64 { @@ -781,6 +835,34 @@ impl Stats { self.me_floor_mode_switch_adaptive_to_static_total .load(Ordering::Relaxed) } + pub fn get_me_floor_cpu_cores_detected_gauge(&self) -> u64 { + self.me_floor_cpu_cores_detected_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_cpu_cores_effective_gauge(&self) -> u64 { + self.me_floor_cpu_cores_effective_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_global_cap_raw_gauge(&self) -> u64 { + self.me_floor_global_cap_raw_gauge.load(Ordering::Relaxed) + } + pub fn get_me_floor_global_cap_effective_gauge(&self) -> u64 { + self.me_floor_global_cap_effective_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_target_writers_total_gauge(&self) -> u64 { + self.me_floor_target_writers_total_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_cap_block_total(&self) -> u64 { + self.me_floor_cap_block_total.load(Ordering::Relaxed) + } + pub fn get_me_floor_swap_idle_total(&self) -> u64 { + self.me_floor_swap_idle_total.load(Ordering::Relaxed) + } + pub fn get_me_floor_swap_idle_failed_total(&self) -> u64 { + self.me_floor_swap_idle_failed_total.load(Ordering::Relaxed) + } pub fn get_me_handshake_error_code_counts(&self) -> Vec<(i32, u64)> { let mut out: Vec<(i32, u64)> = self .me_handshake_error_codes diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 072c1f6..4bc3ff7 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -315,7 +315,12 @@ async fn run_update_cycle( cfg.general.me_floor_mode, cfg.general.me_adaptive_floor_idle_secs, cfg.general.me_adaptive_floor_min_writers_single_endpoint, + cfg.general.me_adaptive_floor_min_writers_multi_endpoint, cfg.general.me_adaptive_floor_recover_grace_secs, + cfg.general.me_adaptive_floor_writers_per_core_total, + cfg.general.me_adaptive_floor_cpu_cores_override, + cfg.general.me_adaptive_floor_max_extra_writers_single_per_core, + cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core, ); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); @@ -527,7 +532,12 @@ pub async fn me_config_updater( cfg.general.me_floor_mode, cfg.general.me_adaptive_floor_idle_secs, cfg.general.me_adaptive_floor_min_writers_single_endpoint, + cfg.general.me_adaptive_floor_min_writers_multi_endpoint, cfg.general.me_adaptive_floor_recover_grace_secs, + cfg.general.me_adaptive_floor_writers_per_core_total, + cfg.general.me_adaptive_floor_cpu_cores_override, + cfg.general.me_adaptive_floor_max_extra_writers_single_per_core, + cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core, ); let new_secs = cfg.general.effective_update_every_secs().max(1); if new_secs == update_every_secs { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 9a54e32..a594a01 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -23,6 +23,25 @@ const IDLE_REFRESH_TRIGGER_JITTER_SECS: u64 = 5; const IDLE_REFRESH_RETRY_SECS: u64 = 8; const IDLE_REFRESH_SUCCESS_GUARD_SECS: u64 = 5; +#[derive(Debug, Clone)] +struct DcFloorPlanEntry { + dc: i32, + endpoints: Vec, + alive: usize, + min_required: usize, + target_required: usize, + max_required: usize, + has_bound_clients: bool, + floor_capped: bool, +} + +#[derive(Debug, Clone)] +struct FamilyFloorPlan { + by_dc: HashMap, + global_cap_effective_total: usize, + target_writers_total: usize, +} + pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_connections: usize) { let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new(); let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); @@ -129,22 +148,33 @@ async fn check_family( .push(writer.id); } let writer_idle_since = pool.registry.writer_idle_since_snapshot().await; + let floor_plan = build_family_floor_plan( + pool, + family, + &dc_endpoints, + &live_addr_counts, + &live_writer_ids_by_addr, + adaptive_idle_since, + adaptive_recover_until, + ) + .await; + pool.set_adaptive_floor_runtime_caps( + floor_plan.global_cap_effective_total, + floor_plan.target_writers_total, + ); for (dc, endpoints) in dc_endpoints { if endpoints.is_empty() { continue; } let key = (dc, family); - let reduce_for_idle = should_reduce_floor_for_idle( - pool, - key, - &endpoints, - &live_writer_ids_by_addr, - adaptive_idle_since, - adaptive_recover_until, - ) - .await; - let required = pool.required_writers_for_dc_with_floor_mode(endpoints.len(), reduce_for_idle); + let required = floor_plan + .by_dc + .get(&dc) + .map(|entry| entry.target_required) + .unwrap_or_else(|| { + pool.required_writers_for_dc_with_floor_mode(endpoints.len(), false) + }); let alive = endpoints .iter() .map(|addr| *live_addr_counts.get(addr).unwrap_or(&0)) @@ -251,6 +281,36 @@ async fn check_family( let mut restored = 0usize; for _ in 0..missing { + if pool.floor_mode() == MeFloorMode::Adaptive + && pool.active_writer_count_total().await >= floor_plan.global_cap_effective_total + { + let swapped = maybe_swap_idle_writer_for_cap( + pool, + rng, + dc, + family, + &endpoints, + &live_writer_ids_by_addr, + &writer_idle_since, + ) + .await; + if swapped { + pool.stats.increment_me_floor_swap_idle_total(); + restored += 1; + continue; + } + pool.stats.increment_me_floor_cap_block_total(); + pool.stats.increment_me_floor_swap_idle_failed_total(); + debug!( + dc = %dc, + ?family, + alive, + required, + global_cap_effective_total = floor_plan.global_cap_effective_total, + "Adaptive floor cap reached, reconnect attempt blocked" + ); + break; + } let res = tokio::time::timeout( pool.me_one_timeout, pool.connect_endpoints_round_robin(&endpoints, rng.as_ref()), @@ -323,6 +383,280 @@ async fn check_family( } } +fn adaptive_floor_class_min( + pool: &Arc, + endpoint_count: usize, + base_required: usize, +) -> usize { + if endpoint_count <= 1 { + let min_single = (pool + .me_adaptive_floor_min_writers_single_endpoint + .load(std::sync::atomic::Ordering::Relaxed) as usize) + .max(1); + min_single.min(base_required.max(1)) + } else { + pool.adaptive_floor_min_writers_multi_endpoint() + .min(base_required.max(1)) + } +} + +fn adaptive_floor_class_max( + pool: &Arc, + endpoint_count: usize, + base_required: usize, + cpu_cores: usize, +) -> usize { + let extra_per_core = if endpoint_count <= 1 { + pool.adaptive_floor_max_extra_single_per_core() + } else { + pool.adaptive_floor_max_extra_multi_per_core() + }; + base_required.saturating_add(cpu_cores.saturating_mul(extra_per_core)) +} + +fn list_writer_ids_for_endpoints( + endpoints: &[SocketAddr], + live_writer_ids_by_addr: &HashMap>, +) -> Vec { + let mut out = Vec::::new(); + for endpoint in endpoints { + if let Some(ids) = live_writer_ids_by_addr.get(endpoint) { + out.extend(ids.iter().copied()); + } + } + out +} + +async fn build_family_floor_plan( + pool: &Arc, + family: IpFamily, + dc_endpoints: &HashMap>, + live_addr_counts: &HashMap, + live_writer_ids_by_addr: &HashMap>, + adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, + adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, +) -> FamilyFloorPlan { + let mut entries = Vec::::new(); + let mut by_dc = HashMap::::new(); + let mut family_active_total = 0usize; + + let floor_mode = pool.floor_mode(); + let is_adaptive = floor_mode == MeFloorMode::Adaptive; + let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1); + + for (dc, endpoints) in dc_endpoints { + if endpoints.is_empty() { + continue; + } + let key = (*dc, family); + let reduce_for_idle = should_reduce_floor_for_idle( + pool, + key, + endpoints, + live_writer_ids_by_addr, + adaptive_idle_since, + adaptive_recover_until, + ) + .await; + let base_required = pool.required_writers_for_dc(endpoints.len()).max(1); + let min_required = if is_adaptive { + adaptive_floor_class_min(pool, endpoints.len(), base_required) + } else { + base_required + }; + let mut max_required = if is_adaptive { + adaptive_floor_class_max(pool, endpoints.len(), base_required, cpu_cores) + } else { + base_required + }; + if max_required < min_required { + max_required = min_required; + } + let desired_raw = if is_adaptive && reduce_for_idle { + min_required + } else { + base_required + }; + let target_required = desired_raw.clamp(min_required, max_required); + let alive = endpoints + .iter() + .map(|endpoint| live_addr_counts.get(endpoint).copied().unwrap_or(0)) + .sum::(); + family_active_total = family_active_total.saturating_add(alive); + let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr); + let has_bound_clients = has_bound_clients_on_endpoint(pool, &writer_ids).await; + + entries.push(DcFloorPlanEntry { + dc: *dc, + endpoints: endpoints.clone(), + alive, + min_required, + target_required, + max_required, + has_bound_clients, + floor_capped: false, + }); + } + + if entries.is_empty() { + return FamilyFloorPlan { + by_dc, + global_cap_effective_total: 0, + target_writers_total: 0, + }; + } + + if !is_adaptive { + let target_total = entries + .iter() + .map(|entry| entry.target_required) + .sum::(); + let active_total = pool.active_writer_count_total().await; + for entry in entries { + by_dc.insert(entry.dc, entry); + } + return FamilyFloorPlan { + by_dc, + global_cap_effective_total: active_total.max(target_total), + target_writers_total: target_total, + }; + } + + let global_cap_raw = pool.adaptive_floor_global_cap_raw(); + let total_active = pool.active_writer_count_total().await; + let other_active = total_active.saturating_sub(family_active_total); + let min_sum = entries + .iter() + .map(|entry| entry.min_required) + .sum::(); + let mut target_sum = entries + .iter() + .map(|entry| entry.target_required) + .sum::(); + let family_cap = global_cap_raw + .saturating_sub(other_active) + .max(min_sum); + if target_sum > family_cap { + entries.sort_by_key(|entry| { + ( + entry.has_bound_clients, + std::cmp::Reverse(entry.target_required.saturating_sub(entry.min_required)), + std::cmp::Reverse(entry.alive), + entry.dc.abs(), + entry.dc, + entry.endpoints.len(), + entry.max_required, + ) + }); + let mut changed = true; + while target_sum > family_cap && changed { + changed = false; + for entry in &mut entries { + if target_sum <= family_cap { + break; + } + if entry.target_required > entry.min_required { + entry.target_required -= 1; + entry.floor_capped = true; + target_sum -= 1; + changed = true; + } + } + } + } + + for entry in entries { + by_dc.insert(entry.dc, entry); + } + let global_cap_effective_total = global_cap_raw.max(other_active.saturating_add(min_sum)); + let target_writers_total = other_active.saturating_add(target_sum); + FamilyFloorPlan { + by_dc, + global_cap_effective_total, + target_writers_total, + } +} + +async fn maybe_swap_idle_writer_for_cap( + pool: &Arc, + rng: &Arc, + dc: i32, + family: IpFamily, + endpoints: &[SocketAddr], + live_writer_ids_by_addr: &HashMap>, + writer_idle_since: &HashMap, +) -> bool { + let now_epoch_secs = MePool::now_epoch_secs(); + let mut candidate: Option<(u64, SocketAddr, u64)> = None; + for endpoint in endpoints { + let Some(writer_ids) = live_writer_ids_by_addr.get(endpoint) else { + continue; + }; + for writer_id in writer_ids { + if !pool.registry.is_writer_empty(*writer_id).await { + continue; + } + let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else { + continue; + }; + let idle_age_secs = now_epoch_secs.saturating_sub(idle_since_epoch_secs); + if candidate + .as_ref() + .map(|(_, _, age)| idle_age_secs > *age) + .unwrap_or(true) + { + candidate = Some((*writer_id, *endpoint, idle_age_secs)); + } + } + } + + let Some((old_writer_id, endpoint, idle_age_secs)) = candidate else { + return false; + }; + + let connected = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + Ok(Ok(())) => true, + Ok(Err(error)) => { + debug!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + idle_age_secs, + %error, + "Adaptive floor cap swap connect failed" + ); + false + } + Err(_) => { + debug!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + idle_age_secs, + "Adaptive floor cap swap connect timed out" + ); + false + } + }; + if !connected { + return false; + } + + pool.mark_writer_draining_with_timeout(old_writer_id, pool.force_close_timeout(), false) + .await; + info!( + dc = %dc, + ?family, + %endpoint, + old_writer_id, + idle_age_secs, + "Adaptive floor cap swap: idle writer rotated" + ); + true +} + async fn maybe_refresh_idle_writer_for_dc( pool: &Arc, rng: &Arc, @@ -438,19 +772,15 @@ async fn should_reduce_floor_for_idle( adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, ) -> bool { - if endpoints.len() != 1 || pool.floor_mode() != MeFloorMode::Adaptive { + if pool.floor_mode() != MeFloorMode::Adaptive { adaptive_idle_since.remove(&key); adaptive_recover_until.remove(&key); return false; } let now = Instant::now(); - let endpoint = endpoints[0]; - let writer_ids = live_writer_ids_by_addr - .get(&endpoint) - .map(Vec::as_slice) - .unwrap_or(&[]); - let has_bound_clients = has_bound_clients_on_endpoint(pool, writer_ids).await; + let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr); + let has_bound_clients = has_bound_clients_on_endpoint(pool, &writer_ids).await; if has_bound_clients { adaptive_idle_since.remove(&key); adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration()); diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index b0ae394..1145823 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -111,7 +111,17 @@ pub struct MePool { pub(super) me_floor_mode: AtomicU8, pub(super) me_adaptive_floor_idle_secs: AtomicU64, pub(super) me_adaptive_floor_min_writers_single_endpoint: AtomicU8, + pub(super) me_adaptive_floor_min_writers_multi_endpoint: AtomicU8, pub(super) me_adaptive_floor_recover_grace_secs: AtomicU64, + pub(super) me_adaptive_floor_writers_per_core_total: AtomicU32, + pub(super) me_adaptive_floor_cpu_cores_override: AtomicU32, + pub(super) me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32, + pub(super) me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32, + pub(super) me_adaptive_floor_cpu_cores_detected: AtomicU32, + pub(super) me_adaptive_floor_cpu_cores_effective: AtomicU32, + pub(super) me_adaptive_floor_global_cap_raw: AtomicU64, + pub(super) me_adaptive_floor_global_cap_effective: AtomicU64, + pub(super) me_adaptive_floor_target_writers_total: AtomicU64, pub(super) proxy_map_v4: Arc>>>, pub(super) proxy_map_v6: Arc>>>, pub(super) default_dc: AtomicI32, @@ -217,7 +227,12 @@ impl MePool { me_floor_mode: MeFloorMode, me_adaptive_floor_idle_secs: u64, me_adaptive_floor_min_writers_single_endpoint: u8, + me_adaptive_floor_min_writers_multi_endpoint: u8, me_adaptive_floor_recover_grace_secs: u64, + me_adaptive_floor_writers_per_core_total: u16, + me_adaptive_floor_cpu_cores_override: u16, + me_adaptive_floor_max_extra_writers_single_per_core: u16, + me_adaptive_floor_max_extra_writers_multi_per_core: u16, hardswap: bool, me_pool_drain_ttl_secs: u64, me_pool_force_close_secs: u64, @@ -314,9 +329,29 @@ impl MePool { me_adaptive_floor_min_writers_single_endpoint: AtomicU8::new( me_adaptive_floor_min_writers_single_endpoint, ), + me_adaptive_floor_min_writers_multi_endpoint: AtomicU8::new( + me_adaptive_floor_min_writers_multi_endpoint, + ), me_adaptive_floor_recover_grace_secs: AtomicU64::new( me_adaptive_floor_recover_grace_secs, ), + me_adaptive_floor_writers_per_core_total: AtomicU32::new( + me_adaptive_floor_writers_per_core_total as u32, + ), + me_adaptive_floor_cpu_cores_override: AtomicU32::new( + me_adaptive_floor_cpu_cores_override as u32, + ), + me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32::new( + me_adaptive_floor_max_extra_writers_single_per_core as u32, + ), + me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32::new( + me_adaptive_floor_max_extra_writers_multi_per_core as u32, + ), + me_adaptive_floor_cpu_cores_detected: AtomicU32::new(1), + me_adaptive_floor_cpu_cores_effective: AtomicU32::new(1), + me_adaptive_floor_global_cap_raw: AtomicU64::new(0), + me_adaptive_floor_global_cap_effective: AtomicU64::new(0), + me_adaptive_floor_target_writers_total: AtomicU64::new(0), pool_size: 2, proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), @@ -399,7 +434,12 @@ impl MePool { floor_mode: MeFloorMode, adaptive_floor_idle_secs: u64, adaptive_floor_min_writers_single_endpoint: u8, + adaptive_floor_min_writers_multi_endpoint: u8, adaptive_floor_recover_grace_secs: u64, + adaptive_floor_writers_per_core_total: u16, + adaptive_floor_cpu_cores_override: u16, + adaptive_floor_max_extra_writers_single_per_core: u16, + adaptive_floor_max_extra_writers_multi_per_core: u16, ) { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs @@ -443,8 +483,24 @@ impl MePool { .store(adaptive_floor_idle_secs, Ordering::Relaxed); self.me_adaptive_floor_min_writers_single_endpoint .store(adaptive_floor_min_writers_single_endpoint, Ordering::Relaxed); + self.me_adaptive_floor_min_writers_multi_endpoint + .store(adaptive_floor_min_writers_multi_endpoint, Ordering::Relaxed); self.me_adaptive_floor_recover_grace_secs .store(adaptive_floor_recover_grace_secs, Ordering::Relaxed); + self.me_adaptive_floor_writers_per_core_total + .store(adaptive_floor_writers_per_core_total as u32, Ordering::Relaxed); + self.me_adaptive_floor_cpu_cores_override + .store(adaptive_floor_cpu_cores_override as u32, Ordering::Relaxed); + self.me_adaptive_floor_max_extra_writers_single_per_core + .store( + adaptive_floor_max_extra_writers_single_per_core as u32, + Ordering::Relaxed, + ); + self.me_adaptive_floor_max_extra_writers_multi_per_core + .store( + adaptive_floor_max_extra_writers_multi_per_core as u32, + Ordering::Relaxed, + ); if previous_floor_mode != floor_mode { self.stats.increment_me_floor_mode_switch_total(); match (previous_floor_mode, floor_mode) { @@ -515,6 +571,13 @@ impl MePool { self.proxy_secret.read().await.key_selector } + pub(super) async fn active_writer_count_total(&self) -> usize { + let ws = self.writers.read().await; + ws.iter() + .filter(|w| !w.draining.load(Ordering::Relaxed)) + .count() + } + pub(super) async fn secret_snapshot(&self) -> SecretSnapshot { self.proxy_secret.read().await.clone() } @@ -551,6 +614,82 @@ impl MePool { ) } + pub(super) fn adaptive_floor_min_writers_multi_endpoint(&self) -> usize { + (self + .me_adaptive_floor_min_writers_multi_endpoint + .load(Ordering::Relaxed) as usize) + .max(1) + } + + pub(super) fn adaptive_floor_writers_per_core_total(&self) -> usize { + (self + .me_adaptive_floor_writers_per_core_total + .load(Ordering::Relaxed) as usize) + .max(1) + } + + pub(super) fn adaptive_floor_max_extra_single_per_core(&self) -> usize { + self.me_adaptive_floor_max_extra_writers_single_per_core + .load(Ordering::Relaxed) as usize + } + + pub(super) fn adaptive_floor_max_extra_multi_per_core(&self) -> usize { + self.me_adaptive_floor_max_extra_writers_multi_per_core + .load(Ordering::Relaxed) as usize + } + + pub(super) fn adaptive_floor_detected_cpu_cores(&self) -> usize { + std::thread::available_parallelism() + .map(|value| value.get()) + .unwrap_or(1) + .max(1) + } + + pub(super) fn adaptive_floor_effective_cpu_cores(&self) -> usize { + let detected = self.adaptive_floor_detected_cpu_cores(); + let override_cores = self + .me_adaptive_floor_cpu_cores_override + .load(Ordering::Relaxed) as usize; + let effective = if override_cores == 0 { + detected + } else { + override_cores.max(1) + }; + self.me_adaptive_floor_cpu_cores_detected + .store(detected as u32, Ordering::Relaxed); + self.me_adaptive_floor_cpu_cores_effective + .store(effective as u32, Ordering::Relaxed); + self.stats + .set_me_floor_cpu_cores_detected_gauge(detected as u64); + self.stats + .set_me_floor_cpu_cores_effective_gauge(effective as u64); + effective + } + + pub(super) fn adaptive_floor_global_cap_raw(&self) -> usize { + let cores = self.adaptive_floor_effective_cpu_cores(); + let cap = cores.saturating_mul(self.adaptive_floor_writers_per_core_total()); + self.me_adaptive_floor_global_cap_raw + .store(cap as u64, Ordering::Relaxed); + self.stats.set_me_floor_global_cap_raw_gauge(cap as u64); + cap + } + + pub(super) fn set_adaptive_floor_runtime_caps( + &self, + global_cap_effective: usize, + target_writers_total: usize, + ) { + self.me_adaptive_floor_global_cap_effective + .store(global_cap_effective as u64, Ordering::Relaxed); + self.me_adaptive_floor_target_writers_total + .store(target_writers_total as u64, Ordering::Relaxed); + self.stats + .set_me_floor_global_cap_effective_gauge(global_cap_effective as u64); + self.stats + .set_me_floor_target_writers_total_gauge(target_writers_total as u64); + } + pub(super) fn required_writers_for_dc_with_floor_mode( &self, endpoint_count: usize, @@ -560,13 +699,20 @@ impl MePool { if !reduce_for_idle { return base_required; } - if endpoint_count != 1 || self.floor_mode() != MeFloorMode::Adaptive { + if self.floor_mode() != MeFloorMode::Adaptive { return base_required; } - let min_writers = (self - .me_adaptive_floor_min_writers_single_endpoint - .load(Ordering::Relaxed) as usize) - .max(1); + let min_writers = if endpoint_count == 1 { + (self + .me_adaptive_floor_min_writers_single_endpoint + .load(Ordering::Relaxed) as usize) + .max(1) + } else { + (self + .me_adaptive_floor_min_writers_multi_endpoint + .load(Ordering::Relaxed) as usize) + .max(1) + }; base_required.min(min_writers) } diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index d9898b1..46346b5 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -28,6 +28,10 @@ pub(crate) struct MeApiDcStatusSnapshot { pub available_endpoints: usize, pub available_pct: f64, pub required_writers: usize, + pub floor_min: usize, + pub floor_target: usize, + pub floor_max: usize, + pub floor_capped: bool, pub alive_writers: usize, pub coverage_pct: f64, pub rtt_ms: Option, @@ -72,7 +76,17 @@ pub(crate) struct MeApiRuntimeSnapshot { pub floor_mode: &'static str, pub adaptive_floor_idle_secs: u64, pub adaptive_floor_min_writers_single_endpoint: u8, + pub adaptive_floor_min_writers_multi_endpoint: u8, pub adaptive_floor_recover_grace_secs: u64, + pub adaptive_floor_writers_per_core_total: u16, + pub adaptive_floor_cpu_cores_override: u16, + pub adaptive_floor_max_extra_writers_single_per_core: u16, + pub adaptive_floor_max_extra_writers_multi_per_core: u16, + pub adaptive_floor_cpu_cores_detected: u32, + pub adaptive_floor_cpu_cores_effective: u32, + pub adaptive_floor_global_cap_raw: u64, + pub adaptive_floor_global_cap_effective: u64, + pub adaptive_floor_target_writers_total: u64, pub me_keepalive_enabled: bool, pub me_keepalive_interval_secs: u64, pub me_keepalive_jitter_secs: u64, @@ -275,14 +289,43 @@ impl MePool { let mut dcs = Vec::::with_capacity(endpoints_by_dc.len()); let mut available_endpoints = 0usize; let mut alive_writers = 0usize; + let floor_mode = self.floor_mode(); + let adaptive_cpu_cores = (self + .me_adaptive_floor_cpu_cores_effective + .load(Ordering::Relaxed) as usize) + .max(1); for (dc, endpoints) in endpoints_by_dc { let endpoint_count = endpoints.len(); let dc_available_endpoints = endpoints .iter() .filter(|endpoint| live_writers_by_endpoint.contains_key(endpoint)) .count(); + let base_required = self.required_writers_for_dc(endpoint_count); let dc_required_writers = self.required_writers_for_dc_with_floor_mode(endpoint_count, false); + let floor_min = if endpoint_count <= 1 { + (self + .me_adaptive_floor_min_writers_single_endpoint + .load(Ordering::Relaxed) as usize) + .max(1) + .min(base_required.max(1)) + } else { + (self + .me_adaptive_floor_min_writers_multi_endpoint + .load(Ordering::Relaxed) as usize) + .max(1) + .min(base_required.max(1)) + }; + let extra_per_core = if endpoint_count <= 1 { + self.me_adaptive_floor_max_extra_writers_single_per_core + .load(Ordering::Relaxed) as usize + } else { + self.me_adaptive_floor_max_extra_writers_multi_per_core + .load(Ordering::Relaxed) as usize + }; + let floor_max = base_required.saturating_add(adaptive_cpu_cores.saturating_mul(extra_per_core)); + let floor_capped = matches!(floor_mode, MeFloorMode::Adaptive) + && dc_required_writers < base_required; let dc_alive_writers = live_writers_by_dc.get(&dc).copied().unwrap_or(0); let dc_load = activity .active_sessions_by_target_dc @@ -302,6 +345,10 @@ impl MePool { available_endpoints: dc_available_endpoints, available_pct: ratio_pct(dc_available_endpoints, endpoint_count), required_writers: dc_required_writers, + floor_min, + floor_target: dc_required_writers, + floor_max, + floor_capped, alive_writers: dc_alive_writers, coverage_pct: ratio_pct(dc_alive_writers, dc_required_writers), rtt_ms: dc_rtt_ms, @@ -378,9 +425,39 @@ impl MePool { adaptive_floor_min_writers_single_endpoint: self .me_adaptive_floor_min_writers_single_endpoint .load(Ordering::Relaxed), + adaptive_floor_min_writers_multi_endpoint: self + .me_adaptive_floor_min_writers_multi_endpoint + .load(Ordering::Relaxed), adaptive_floor_recover_grace_secs: self .me_adaptive_floor_recover_grace_secs .load(Ordering::Relaxed), + adaptive_floor_writers_per_core_total: self + .me_adaptive_floor_writers_per_core_total + .load(Ordering::Relaxed) as u16, + adaptive_floor_cpu_cores_override: self + .me_adaptive_floor_cpu_cores_override + .load(Ordering::Relaxed) as u16, + adaptive_floor_max_extra_writers_single_per_core: self + .me_adaptive_floor_max_extra_writers_single_per_core + .load(Ordering::Relaxed) as u16, + adaptive_floor_max_extra_writers_multi_per_core: self + .me_adaptive_floor_max_extra_writers_multi_per_core + .load(Ordering::Relaxed) as u16, + adaptive_floor_cpu_cores_detected: self + .me_adaptive_floor_cpu_cores_detected + .load(Ordering::Relaxed), + adaptive_floor_cpu_cores_effective: self + .me_adaptive_floor_cpu_cores_effective + .load(Ordering::Relaxed), + adaptive_floor_global_cap_raw: self + .me_adaptive_floor_global_cap_raw + .load(Ordering::Relaxed), + adaptive_floor_global_cap_effective: self + .me_adaptive_floor_global_cap_effective + .load(Ordering::Relaxed), + adaptive_floor_target_writers_total: self + .me_adaptive_floor_target_writers_total + .load(Ordering::Relaxed), me_keepalive_enabled: self.me_keepalive_enabled, me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(), me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(), From 2468ee15e7a1f0519e46547feeef5bdeb064d2ee Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 03:16:48 +0300 Subject: [PATCH 120/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 338d2f1..e4b3848 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.5" +version = "3.3.6" edition = "2024" [dependencies] From 0ff2e95e49cff9a324618ff0425c3d59e2293fc6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 03:22:01 +0300 Subject: [PATCH 121/261] Event-driven Drafts Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 66 ++++++++++++++++++++--- src/transport/middle_proxy/pool.rs | 3 ++ src/transport/middle_proxy/pool_refill.rs | 4 +- src/transport/middle_proxy/send.rs | 36 +++++++------ 4 files changed, 84 insertions(+), 25 deletions(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index a594a01..4f8370c 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -22,6 +22,10 @@ const IDLE_REFRESH_TRIGGER_BASE_SECS: u64 = 45; const IDLE_REFRESH_TRIGGER_JITTER_SECS: u64 = 5; const IDLE_REFRESH_RETRY_SECS: u64 = 8; const IDLE_REFRESH_SUCCESS_GUARD_SECS: u64 = 5; +const HEALTH_RECONNECT_BUDGET_PER_CORE: usize = 2; +const HEALTH_RECONNECT_BUDGET_PER_DC: usize = 1; +const HEALTH_RECONNECT_BUDGET_MIN: usize = 4; +const HEALTH_RECONNECT_BUDGET_MAX: usize = 128; #[derive(Debug, Clone)] struct DcFloorPlanEntry { @@ -114,22 +118,23 @@ async fn check_family( return; } - let map = match family { - IpFamily::V4 => pool.proxy_map_v4.read().await.clone(), - IpFamily::V6 => pool.proxy_map_v6.read().await.clone(), - }; - let mut dc_endpoints = HashMap::>::new(); - for (dc, addrs) in map { - let entry = dc_endpoints.entry(dc).or_default(); - for (ip, port) in addrs { + let map_guard = match family { + IpFamily::V4 => pool.proxy_map_v4.read().await, + IpFamily::V6 => pool.proxy_map_v6.read().await, + }; + for (dc, addrs) in map_guard.iter() { + let entry = dc_endpoints.entry(*dc).or_default(); + for (ip, port) in addrs.iter().copied() { entry.push(SocketAddr::new(ip, port)); } } + drop(map_guard); for endpoints in dc_endpoints.values_mut() { endpoints.sort_unstable(); endpoints.dedup(); } + let mut reconnect_budget = health_reconnect_budget(pool, dc_endpoints.len()); if pool.floor_mode() == MeFloorMode::Static { adaptive_idle_since.clear(); @@ -200,6 +205,7 @@ async fn check_family( required, outage_backoff, outage_next_attempt, + &mut reconnect_budget, ) .await; continue; @@ -256,6 +262,24 @@ async fn check_family( let missing = required - alive; let now = Instant::now(); + if reconnect_budget == 0 { + let base_ms = pool.me_reconnect_backoff_base.as_millis() as u64; + let next_ms = (*backoff.get(&key).unwrap_or(&base_ms)).max(base_ms); + let jitter = next_ms / JITTER_FRAC_NUM; + let wait = Duration::from_millis(next_ms) + + Duration::from_millis(rand::rng().random_range(0..=jitter.max(1))); + next_attempt.insert(key, now + wait); + debug!( + dc = %dc, + ?family, + alive, + required, + endpoint_count = endpoints.len(), + reconnect_budget, + "Skipping reconnect due to per-tick health reconnect budget" + ); + continue; + } if let Some(ts) = next_attempt.get(&key) && now < *ts { @@ -281,6 +305,10 @@ async fn check_family( let mut restored = 0usize; for _ in 0..missing { + if reconnect_budget == 0 { + break; + } + reconnect_budget = reconnect_budget.saturating_sub(1); if pool.floor_mode() == MeFloorMode::Adaptive && pool.active_writer_count_total().await >= floor_plan.global_cap_effective_total { @@ -383,6 +411,15 @@ async fn check_family( } } +fn health_reconnect_budget(pool: &Arc, dc_groups: usize) -> usize { + let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1); + let by_cpu = cpu_cores.saturating_mul(HEALTH_RECONNECT_BUDGET_PER_CORE); + let by_dc = dc_groups.saturating_mul(HEALTH_RECONNECT_BUDGET_PER_DC); + by_cpu + .saturating_add(by_dc) + .clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX) +} + fn adaptive_floor_class_min( pool: &Arc, endpoint_count: usize, @@ -816,6 +853,7 @@ async fn recover_single_endpoint_outage( required: usize, outage_backoff: &mut HashMap<(i32, IpFamily), u64>, outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, + reconnect_budget: &mut usize, ) { let now = Instant::now(); if let Some(ts) = outage_next_attempt.get(&key) @@ -825,6 +863,18 @@ async fn recover_single_endpoint_outage( } let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms(); + if *reconnect_budget == 0 { + outage_next_attempt.insert(key, now + Duration::from_millis(min_backoff_ms.max(250))); + debug!( + dc = %key.0, + family = ?key.1, + %endpoint, + required, + "Single-endpoint outage reconnect deferred by health reconnect budget" + ); + return; + } + *reconnect_budget = (*reconnect_budget).saturating_sub(1); pool.stats .increment_me_single_endpoint_outage_reconnect_attempt_total(); diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 1145823..8c59b5b 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -124,6 +124,7 @@ pub struct MePool { pub(super) me_adaptive_floor_target_writers_total: AtomicU64, pub(super) proxy_map_v4: Arc>>>, pub(super) proxy_map_v6: Arc>>>, + pub(super) endpoint_dc_map: Arc>>>, pub(super) default_dc: AtomicI32, pub(super) next_writer_id: AtomicU64, pub(super) ping_tracker: Arc>>, @@ -254,6 +255,7 @@ impl MePool { me_route_inline_recovery_attempts: u32, me_route_inline_recovery_wait_ms: u64, ) -> Arc { + let endpoint_dc_map = Self::build_endpoint_dc_map_from_maps(&proxy_map_v4, &proxy_map_v6); let registry = Arc::new(ConnRegistry::new()); registry.update_route_backpressure_policy( me_route_backpressure_base_timeout_ms, @@ -355,6 +357,7 @@ impl MePool { pool_size: 2, proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), + endpoint_dc_map: Arc::new(RwLock::new(endpoint_dc_map)), default_dc: AtomicI32::new(default_dc.unwrap_or(2)), next_writer_id: AtomicU64::new(1), ping_tracker: Arc::new(Mutex::new(HashMap::new())), diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 87b87d5..7da6acc 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -172,7 +172,7 @@ impl MePool { let target_dc = self.resolve_dc_for_endpoint(addr).await; if self.decision.ipv4_me { - let map = self.proxy_map_v4.read().await.clone(); + let map = self.proxy_map_v4.read().await; if let Some(addrs) = map.get(&target_dc) { for (ip, port) in addrs { endpoints.insert(SocketAddr::new(*ip, *port)); @@ -181,7 +181,7 @@ impl MePool { } if self.decision.ipv6_me { - let map = self.proxy_map_v6.read().await.clone(); + let map = self.proxy_map_v6.read().await; if let Some(addrs) = map.get(&target_dc) { for (ip, port) in addrs { endpoints.insert(SocketAddr::new(*ip, *port)); diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index b9b1fd5..9ffcc8e 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -378,12 +378,16 @@ impl MePool { return self.has_candidate_for_target_dc(target_dc).await; } - let remaining = deadline.saturating_duration_since(now); - let sleep_for = remaining.min(Duration::from_millis(25)); let waiter = self.writer_available.notified(); - tokio::select! { - _ = waiter => {} - _ = tokio::time::sleep(sleep_for) => {} + if self.has_candidate_for_target_dc(target_dc).await { + return true; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return self.has_candidate_for_target_dc(target_dc).await; + } + if tokio::time::timeout(remaining, waiter).await.is_err() { + return self.has_candidate_for_target_dc(target_dc).await; } } } @@ -423,11 +427,11 @@ impl MePool { self.stats.increment_me_async_recovery_trigger_total(); let mut seen = HashSet::::new(); for family in self.family_order() { - let map = match family { - IpFamily::V4 => self.proxy_map_v4.read().await.clone(), - IpFamily::V6 => self.proxy_map_v6.read().await.clone(), + let map_guard = match family { + IpFamily::V4 => self.proxy_map_v4.read().await, + IpFamily::V6 => self.proxy_map_v6.read().await, }; - for addrs in map.values() { + for addrs in map_guard.values() { for (ip, port) in addrs { let addr = SocketAddr::new(*ip, *port); if seen.insert(addr) { @@ -448,13 +452,13 @@ impl MePool { let lookup_keys = self.dc_lookup_chain_for_target(key); for family in self.family_order() { - let map = match family { - IpFamily::V4 => self.proxy_map_v4.read().await.clone(), - IpFamily::V6 => self.proxy_map_v6.read().await.clone(), + let map_guard = match family { + IpFamily::V4 => self.proxy_map_v4.read().await, + IpFamily::V6 => self.proxy_map_v6.read().await, }; let mut family_selected = Vec::::new(); for lookup in lookup_keys.iter().copied() { - if let Some(addrs) = map.get(&lookup) { + if let Some(addrs) = map_guard.get(&lookup) { for (ip, port) in addrs { family_selected.push(SocketAddr::new(*ip, *port)); } @@ -557,7 +561,7 @@ impl MePool { include_warm: bool, ) -> Vec { let key = target_dc as i32; - let mut preferred = Vec::::new(); + let mut preferred = HashSet::::new(); let lookup_keys = self.dc_lookup_chain_for_target(key); for family in self.family_order() { @@ -574,7 +578,9 @@ impl MePool { break; } } - preferred.extend(family_selected); + for endpoint in family_selected { + preferred.insert(endpoint); + } drop(map_guard); From 93f58524d12c9b9315b7bdcad64a2e413d6e4d4c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 03:25:26 +0300 Subject: [PATCH 122/261] No busy-poll in ME Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 2 +- src/transport/middle_proxy/pool.rs | 73 ++++++++++++++--------- src/transport/middle_proxy/pool_config.rs | 6 ++ 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 4f8370c..8f4ad95 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -124,7 +124,7 @@ async fn check_family( IpFamily::V6 => pool.proxy_map_v6.read().await, }; for (dc, addrs) in map_guard.iter() { - let entry = dc_endpoints.entry(*dc).or_default(); + let entry = dc_endpoints.entry(dc.abs()).or_default(); for (ip, port) in addrs.iter().copied() { entry.push(SocketAddr::new(ip, port)); } diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 8c59b5b..b3d8dc6 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -792,33 +792,8 @@ impl MePool { } pub(super) async fn resolve_dc_for_endpoint(&self, addr: SocketAddr) -> i32 { - let map_guard = if addr.is_ipv4() { - self.proxy_map_v4.read().await - } else { - self.proxy_map_v6.read().await - }; - - let mut matched_dc: Option = None; - let mut ambiguous = false; - for (dc, addrs) in map_guard.iter() { - if addrs - .iter() - .any(|(ip, port)| SocketAddr::new(*ip, *port) == addr) - { - match matched_dc { - None => matched_dc = Some(*dc), - Some(prev_dc) if prev_dc == *dc => {} - Some(_) => { - ambiguous = true; - break; - } - } - } - } - drop(map_guard); - - if !ambiguous - && let Some(dc) = matched_dc + if let Some(cached) = self.endpoint_dc_map.read().await.get(&addr).copied() + && let Some(dc) = cached { return dc; } @@ -835,4 +810,48 @@ impl MePool { IpFamily::V6 => self.proxy_map_v6.read().await.clone(), } } + + fn merge_endpoint_dc( + endpoint_dc_map: &mut HashMap>, + dc: i32, + ip: IpAddr, + port: u16, + ) { + let endpoint = SocketAddr::new(ip, port); + match endpoint_dc_map.get_mut(&endpoint) { + None => { + endpoint_dc_map.insert(endpoint, Some(dc)); + } + Some(existing) => { + if existing.is_some_and(|existing_dc| existing_dc != dc) { + *existing = None; + } + } + } + } + + fn build_endpoint_dc_map_from_maps( + map_v4: &HashMap>, + map_v6: &HashMap>, + ) -> HashMap> { + let mut endpoint_dc_map = HashMap::>::new(); + for (dc, endpoints) in map_v4 { + for (ip, port) in endpoints { + Self::merge_endpoint_dc(&mut endpoint_dc_map, *dc, *ip, *port); + } + } + for (dc, endpoints) in map_v6 { + for (ip, port) in endpoints { + Self::merge_endpoint_dc(&mut endpoint_dc_map, *dc, *ip, *port); + } + } + endpoint_dc_map + } + + pub(super) async fn rebuild_endpoint_dc_map(&self) { + let map_v4 = self.proxy_map_v4.read().await.clone(); + let map_v6 = self.proxy_map_v6.read().await.clone(); + let rebuilt = Self::build_endpoint_dc_map_from_maps(&map_v4, &map_v6); + *self.endpoint_dc_map.write().await = rebuilt; + } } diff --git a/src/transport/middle_proxy/pool_config.rs b/src/transport/middle_proxy/pool_config.rs index 04e3bb5..a43f9bf 100644 --- a/src/transport/middle_proxy/pool_config.rs +++ b/src/transport/middle_proxy/pool_config.rs @@ -54,6 +54,7 @@ impl MePool { && let Some(addrs) = guard.get(&k).cloned() { guard.insert(-k, addrs); + changed = true; } } } @@ -65,9 +66,14 @@ impl MePool { && let Some(addrs) = guard.get(&k).cloned() { guard.insert(-k, addrs); + changed = true; } } } + if changed { + self.rebuild_endpoint_dc_map().await; + self.writer_available.notify_waiters(); + } if changed { SnapshotApplyOutcome::AppliedChanged } else { From 484137793f61e725a37ce66f14c1fa2436738557 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 03:32:00 +0300 Subject: [PATCH 123/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e4b3848..076f00b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.6" +version = "3.3.7" edition = "2024" [dependencies] From 27e6dec018aa2f7c39381f55036b87094550b527 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:32:02 +0300 Subject: [PATCH 124/261] ME Strict Writers Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/ip_tracker.rs | 52 +++++++++ src/proxy/middle_relay.rs | 19 ++-- src/stats/mod.rs | 114 ++++++++++++++------ src/transport/middle_proxy/codec.rs | 5 +- src/transport/middle_proxy/handshake.rs | 7 +- src/transport/middle_proxy/health.rs | 63 +++++++++-- src/transport/middle_proxy/ping.rs | 2 +- src/transport/middle_proxy/pool.rs | 42 +++++++- src/transport/middle_proxy/pool_init.rs | 18 +++- src/transport/middle_proxy/pool_refill.rs | 122 ++++++++++------------ src/transport/middle_proxy/pool_writer.rs | 114 ++++++++++++++------ src/transport/middle_proxy/reader.rs | 8 +- src/transport/middle_proxy/registry.rs | 14 +++ src/transport/middle_proxy/send.rs | 33 ++++-- src/transport/middle_proxy/wire.rs | 5 +- src/transport/upstream.rs | 43 +++++++- 16 files changed, 487 insertions(+), 174 deletions(-) diff --git a/src/ip_tracker.rs b/src/ip_tracker.rs index 5da8222..d406d51 100644 --- a/src/ip_tracker.rs +++ b/src/ip_tracker.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::net::IpAddr; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tokio::sync::RwLock; @@ -18,6 +19,7 @@ pub struct UserIpTracker { max_ips: Arc>>, limit_mode: Arc>, limit_window: Arc>, + last_compact_epoch_secs: Arc, } impl UserIpTracker { @@ -28,6 +30,54 @@ impl UserIpTracker { max_ips: Arc::new(RwLock::new(HashMap::new())), limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)), limit_window: Arc::new(RwLock::new(Duration::from_secs(30))), + last_compact_epoch_secs: Arc::new(AtomicU64::new(0)), + } + } + + fn now_epoch_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + } + + async fn maybe_compact_empty_users(&self) { + const COMPACT_INTERVAL_SECS: u64 = 60; + let now_epoch_secs = Self::now_epoch_secs(); + let last_compact_epoch_secs = self.last_compact_epoch_secs.load(Ordering::Relaxed); + if now_epoch_secs.saturating_sub(last_compact_epoch_secs) < COMPACT_INTERVAL_SECS { + return; + } + if self + .last_compact_epoch_secs + .compare_exchange( + last_compact_epoch_secs, + now_epoch_secs, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .is_err() + { + return; + } + + let mut active_ips = self.active_ips.write().await; + let mut recent_ips = self.recent_ips.write().await; + let mut users = Vec::::with_capacity(active_ips.len().saturating_add(recent_ips.len())); + users.extend(active_ips.keys().cloned()); + for user in recent_ips.keys() { + if !active_ips.contains_key(user) { + users.push(user.clone()); + } + } + + for user in users { + let active_empty = active_ips.get(&user).map(|ips| ips.is_empty()).unwrap_or(true); + let recent_empty = recent_ips.get(&user).map(|ips| ips.is_empty()).unwrap_or(true); + if active_empty && recent_empty { + active_ips.remove(&user); + recent_ips.remove(&user); + } } } @@ -63,6 +113,7 @@ impl UserIpTracker { } pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> { + self.maybe_compact_empty_users().await; let limit = { let max_ips = self.max_ips.read().await; max_ips.get(username).copied() @@ -116,6 +167,7 @@ impl UserIpTracker { } pub async fn remove_ip(&self, username: &str, ip: IpAddr) { + self.maybe_compact_empty_users().await; let mut active_ips = self.active_ips.write().await; if let Some(user_ips) = active_ips.get_mut(username) { if let Some(count) = user_ips.get_mut(&ip) { diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index 8384e32..707c8af 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -6,6 +6,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; +use bytes::Bytes; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, trace, warn}; @@ -20,7 +21,7 @@ use crate::stream::{BufferPool, CryptoReader, CryptoWriter}; use crate::transport::middle_proxy::{MePool, MeResponse, proto_flags_for_tag}; enum C2MeCommand { - Data { payload: Vec, flags: u32 }, + Data { payload: Bytes, flags: u32 }, Close, } @@ -283,7 +284,7 @@ where success.dc_idx, peer, translated_local_addr, - &payload, + payload.as_ref(), flags, effective_tag.as_deref(), ).await?; @@ -479,7 +480,7 @@ async fn read_client_payload( forensics: &RelayForensicsState, frame_counter: &mut u64, stats: &Stats, -) -> Result, bool)>> +) -> Result> where R: AsyncRead + Unpin + Send + 'static, { @@ -578,7 +579,7 @@ where payload.truncate(secure_payload_len); } *frame_counter += 1; - return Ok(Some((payload, quickack))); + return Ok(Some((Bytes::from(payload), quickack))); } } @@ -715,7 +716,7 @@ mod tests { enqueue_c2me_command( &tx, C2MeCommand::Data { - payload: vec![1, 2, 3], + payload: Bytes::from_static(&[1, 2, 3]), flags: 0, }, ) @@ -728,7 +729,7 @@ mod tests { .unwrap(); match recv { C2MeCommand::Data { payload, flags } => { - assert_eq!(payload, vec![1, 2, 3]); + assert_eq!(payload.as_ref(), &[1, 2, 3]); assert_eq!(flags, 0); } C2MeCommand::Close => panic!("unexpected close command"), @@ -739,7 +740,7 @@ mod tests { async fn enqueue_c2me_command_falls_back_to_send_when_queue_is_full() { let (tx, mut rx) = mpsc::channel::(1); tx.send(C2MeCommand::Data { - payload: vec![9], + payload: Bytes::from_static(&[9]), flags: 9, }) .await @@ -750,7 +751,7 @@ mod tests { enqueue_c2me_command( &tx2, C2MeCommand::Data { - payload: vec![7, 7], + payload: Bytes::from_static(&[7, 7]), flags: 7, }, ) @@ -769,7 +770,7 @@ mod tests { .unwrap(); match recv { C2MeCommand::Data { payload, flags } => { - assert_eq!(payload, vec![7, 7]); + assert_eq!(payload.as_ref(), &[7, 7]); assert_eq!(flags, 7); } C2MeCommand::Close => panic!("unexpected close command"), diff --git a/src/stats/mod.rs b/src/stats/mod.rs index b51c941..fbfc987 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -6,7 +6,7 @@ pub mod beobachten; pub mod telemetry; use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; -use std::time::{Instant, Duration}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use dashmap::DashMap; use parking_lot::Mutex; use lru::LruCache; @@ -119,6 +119,7 @@ pub struct Stats { telemetry_user_enabled: AtomicBool, telemetry_me_level: AtomicU8, user_stats: DashMap, + user_stats_last_cleanup_epoch_secs: AtomicU64, start_time: parking_lot::RwLock>, } @@ -130,6 +131,7 @@ pub struct UserStats { pub octets_to_client: AtomicU64, pub msgs_from_client: AtomicU64, pub msgs_to_client: AtomicU64, + pub last_seen_epoch_secs: AtomicU64, } impl Stats { @@ -178,6 +180,54 @@ impl Stats { } } + fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + } + + fn touch_user_stats(stats: &UserStats) { + stats + .last_seen_epoch_secs + .store(Self::now_epoch_secs(), Ordering::Relaxed); + } + + fn maybe_cleanup_user_stats(&self) { + const USER_STATS_CLEANUP_INTERVAL_SECS: u64 = 60; + const USER_STATS_IDLE_TTL_SECS: u64 = 24 * 60 * 60; + + let now_epoch_secs = Self::now_epoch_secs(); + let last_cleanup_epoch_secs = self + .user_stats_last_cleanup_epoch_secs + .load(Ordering::Relaxed); + if now_epoch_secs.saturating_sub(last_cleanup_epoch_secs) + < USER_STATS_CLEANUP_INTERVAL_SECS + { + return; + } + if self + .user_stats_last_cleanup_epoch_secs + .compare_exchange( + last_cleanup_epoch_secs, + now_epoch_secs, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .is_err() + { + return; + } + + self.user_stats.retain(|_, stats| { + if stats.curr_connects.load(Ordering::Relaxed) > 0 { + return true; + } + let last_seen_epoch_secs = stats.last_seen_epoch_secs.load(Ordering::Relaxed); + now_epoch_secs.saturating_sub(last_seen_epoch_secs) <= USER_STATS_IDLE_TTL_SECS + }); + } + pub fn apply_telemetry_policy(&self, policy: TelemetryPolicy) { self.telemetry_core_enabled .store(policy.core_enabled, Ordering::Relaxed); @@ -970,34 +1020,36 @@ impl Stats { if !self.telemetry_user_enabled() { return; } + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); stats.connects.fetch_add(1, Ordering::Relaxed); return; } - self.user_stats - .entry(user.to_string()) - .or_default() - .connects - .fetch_add(1, Ordering::Relaxed); + let stats = self.user_stats.entry(user.to_string()).or_default(); + Self::touch_user_stats(stats.value()); + stats.connects.fetch_add(1, Ordering::Relaxed); } pub fn increment_user_curr_connects(&self, user: &str) { if !self.telemetry_user_enabled() { return; } + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); stats.curr_connects.fetch_add(1, Ordering::Relaxed); return; } - self.user_stats - .entry(user.to_string()) - .or_default() - .curr_connects - .fetch_add(1, Ordering::Relaxed); + let stats = self.user_stats.entry(user.to_string()).or_default(); + Self::touch_user_stats(stats.value()); + stats.curr_connects.fetch_add(1, Ordering::Relaxed); } pub fn decrement_user_curr_connects(&self, user: &str) { + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); let counter = &stats.curr_connects; let mut current = counter.load(Ordering::Relaxed); loop { @@ -1027,60 +1079,60 @@ impl Stats { if !self.telemetry_user_enabled() { return; } + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed); return; } - self.user_stats - .entry(user.to_string()) - .or_default() - .octets_from_client - .fetch_add(bytes, Ordering::Relaxed); + let stats = self.user_stats.entry(user.to_string()).or_default(); + Self::touch_user_stats(stats.value()); + stats.octets_from_client.fetch_add(bytes, Ordering::Relaxed); } pub fn add_user_octets_to(&self, user: &str, bytes: u64) { if !self.telemetry_user_enabled() { return; } + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed); return; } - self.user_stats - .entry(user.to_string()) - .or_default() - .octets_to_client - .fetch_add(bytes, Ordering::Relaxed); + let stats = self.user_stats.entry(user.to_string()).or_default(); + Self::touch_user_stats(stats.value()); + stats.octets_to_client.fetch_add(bytes, Ordering::Relaxed); } pub fn increment_user_msgs_from(&self, user: &str) { if !self.telemetry_user_enabled() { return; } + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); stats.msgs_from_client.fetch_add(1, Ordering::Relaxed); return; } - self.user_stats - .entry(user.to_string()) - .or_default() - .msgs_from_client - .fetch_add(1, Ordering::Relaxed); + let stats = self.user_stats.entry(user.to_string()).or_default(); + Self::touch_user_stats(stats.value()); + stats.msgs_from_client.fetch_add(1, Ordering::Relaxed); } pub fn increment_user_msgs_to(&self, user: &str) { if !self.telemetry_user_enabled() { return; } + self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { + Self::touch_user_stats(stats.value()); stats.msgs_to_client.fetch_add(1, Ordering::Relaxed); return; } - self.user_stats - .entry(user.to_string()) - .or_default() - .msgs_to_client - .fetch_add(1, Ordering::Relaxed); + let stats = self.user_stats.entry(user.to_string()).or_default(); + Self::touch_user_stats(stats.value()); + stats.msgs_to_client.fetch_add(1, Ordering::Relaxed); } pub fn get_user_total_octets(&self, user: &str) -> u64 { diff --git a/src/transport/middle_proxy/codec.rs b/src/transport/middle_proxy/codec.rs index 6df0466..7f51aaa 100644 --- a/src/transport/middle_proxy/codec.rs +++ b/src/transport/middle_proxy/codec.rs @@ -1,4 +1,5 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use bytes::Bytes; use crate::crypto::{AesCbc, crc32, crc32c}; use crate::error::{ProxyError, Result}; @@ -6,8 +7,8 @@ use crate::protocol::constants::*; /// Commands sent to dedicated writer tasks to avoid mutex contention on TCP writes. pub(crate) enum WriterCommand { - Data(Vec), - DataAndFlush(Vec), + Data(Bytes), + DataAndFlush(Bytes), Close, } diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 948c999..f556b99 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -135,10 +135,15 @@ impl MePool { pub(crate) async fn connect_tcp( &self, addr: SocketAddr, + dc_idx_override: Option, ) -> Result<(TcpStream, f64, Option)> { let start = Instant::now(); let (stream, upstream_egress) = if let Some(upstream) = &self.upstream { - let dc_idx = self.resolve_dc_idx_for_endpoint(addr).await; + let dc_idx = if let Some(dc_idx) = dc_idx_override { + Some(dc_idx) + } else { + self.resolve_dc_idx_for_endpoint(addr).await + }; let (stream, egress) = upstream.connect_with_details(addr, dc_idx, None).await?; (stream, Some(egress)) } else { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 8f4ad95..4fcba39 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -60,6 +60,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c loop { tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await; pool.prune_closed_writers().await; + reap_draining_writers(&pool).await; check_family( IpFamily::V4, &pool, @@ -95,6 +96,28 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c } } +async fn reap_draining_writers(pool: &Arc) { + let now_epoch_secs = MePool::now_epoch_secs(); + let writers = pool.writers.read().await.clone(); + for writer in writers { + if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) { + continue; + } + if pool.registry.is_writer_empty(writer.id).await { + pool.remove_writer_and_close_clients(writer.id).await; + continue; + } + let deadline_epoch_secs = writer + .drain_deadline_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + if deadline_epoch_secs != 0 && now_epoch_secs >= deadline_epoch_secs { + warn!(writer_id = writer.id, "Drain timeout, force-closing"); + pool.stats.increment_pool_force_close_total(); + pool.remove_writer_and_close_clients(writer.id).await; + } + } +} + async fn check_family( family: IpFamily, pool: &Arc, @@ -153,12 +176,18 @@ async fn check_family( .push(writer.id); } let writer_idle_since = pool.registry.writer_idle_since_snapshot().await; + let bound_clients_by_writer = pool + .registry + .writer_activity_snapshot() + .await + .bound_clients_by_writer; let floor_plan = build_family_floor_plan( pool, family, &dc_endpoints, &live_addr_counts, &live_writer_ids_by_addr, + &bound_clients_by_writer, adaptive_idle_since, adaptive_recover_until, ) @@ -241,6 +270,7 @@ async fn check_family( required, &live_writer_ids_by_addr, &writer_idle_since, + &bound_clients_by_writer, idle_refresh_next_attempt, ) .await; @@ -254,6 +284,7 @@ async fn check_family( alive, required, &live_writer_ids_by_addr, + &bound_clients_by_writer, shadow_rotate_deadline, ) .await; @@ -320,6 +351,7 @@ async fn check_family( &endpoints, &live_writer_ids_by_addr, &writer_idle_since, + &bound_clients_by_writer, ) .await; if swapped { @@ -470,6 +502,7 @@ async fn build_family_floor_plan( dc_endpoints: &HashMap>, live_addr_counts: &HashMap, live_writer_ids_by_addr: &HashMap>, + bound_clients_by_writer: &HashMap, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, ) -> FamilyFloorPlan { @@ -491,6 +524,7 @@ async fn build_family_floor_plan( key, endpoints, live_writer_ids_by_addr, + bound_clients_by_writer, adaptive_idle_since, adaptive_recover_until, ) @@ -521,7 +555,7 @@ async fn build_family_floor_plan( .sum::(); family_active_total = family_active_total.saturating_add(alive); let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr); - let has_bound_clients = has_bound_clients_on_endpoint(pool, &writer_ids).await; + let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer); entries.push(DcFloorPlanEntry { dc: *dc, @@ -622,6 +656,7 @@ async fn maybe_swap_idle_writer_for_cap( endpoints: &[SocketAddr], live_writer_ids_by_addr: &HashMap>, writer_idle_since: &HashMap, + bound_clients_by_writer: &HashMap, ) -> bool { let now_epoch_secs = MePool::now_epoch_secs(); let mut candidate: Option<(u64, SocketAddr, u64)> = None; @@ -630,7 +665,7 @@ async fn maybe_swap_idle_writer_for_cap( continue; }; for writer_id in writer_ids { - if !pool.registry.is_writer_empty(*writer_id).await { + if bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0 { continue; } let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else { @@ -705,6 +740,7 @@ async fn maybe_refresh_idle_writer_for_dc( required: usize, live_writer_ids_by_addr: &HashMap>, writer_idle_since: &HashMap, + bound_clients_by_writer: &HashMap, idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, ) { if alive < required { @@ -725,6 +761,9 @@ async fn maybe_refresh_idle_writer_for_dc( continue; }; for writer_id in writer_ids { + if bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0 { + continue; + } let Some(idle_since_epoch_secs) = writer_idle_since.get(writer_id).copied() else { continue; }; @@ -806,6 +845,7 @@ async fn should_reduce_floor_for_idle( key: (i32, IpFamily), endpoints: &[SocketAddr], live_writer_ids_by_addr: &HashMap>, + bound_clients_by_writer: &HashMap, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, ) -> bool { @@ -817,7 +857,7 @@ async fn should_reduce_floor_for_idle( let now = Instant::now(); let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr); - let has_bound_clients = has_bound_clients_on_endpoint(pool, &writer_ids).await; + let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer); if has_bound_clients { adaptive_idle_since.remove(&key); adaptive_recover_until.insert(key, now + pool.adaptive_floor_recover_grace_duration()); @@ -836,13 +876,13 @@ async fn should_reduce_floor_for_idle( now.saturating_duration_since(*idle_since) >= pool.adaptive_floor_idle_duration() } -async fn has_bound_clients_on_endpoint(pool: &Arc, writer_ids: &[u64]) -> bool { - for writer_id in writer_ids { - if !pool.registry.is_writer_empty(*writer_id).await { - return true; - } - } - false +fn has_bound_clients_on_endpoint( + writer_ids: &[u64], + bound_clients_by_writer: &HashMap, +) -> bool { + writer_ids + .iter() + .any(|writer_id| bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0) } async fn recover_single_endpoint_outage( @@ -973,6 +1013,7 @@ async fn maybe_rotate_single_endpoint_shadow( alive: usize, required: usize, live_writer_ids_by_addr: &HashMap>, + bound_clients_by_writer: &HashMap, shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>, ) { if endpoints.len() != 1 || alive < required { @@ -1011,7 +1052,7 @@ async fn maybe_rotate_single_endpoint_shadow( let mut candidate_writer_id = None; for writer_id in writer_ids { - if pool.registry.is_writer_empty(*writer_id).await { + if bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) == 0 { candidate_writer_id = Some(*writer_id); break; } diff --git a/src/transport/middle_proxy/ping.rs b/src/transport/middle_proxy/ping.rs index b9f0836..2c76592 100644 --- a/src/transport/middle_proxy/ping.rs +++ b/src/transport/middle_proxy/ping.rs @@ -331,7 +331,7 @@ pub async fn run_me_ping(pool: &Arc, rng: &SecureRandom) -> Vec { connect_ms = Some(conn_rtt); route = route_from_egress(upstream_egress); diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index b3d8dc6..236a12a 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -22,10 +22,17 @@ pub(super) struct RefillDcKey { pub family: IpFamily, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(super) struct RefillEndpointKey { + pub dc: i32, + pub addr: SocketAddr, +} + #[derive(Clone)] pub struct MeWriter { pub id: u64, pub addr: SocketAddr, + pub writer_dc: i32, pub generation: u64, pub contour: Arc, pub created_at: Instant, @@ -34,6 +41,7 @@ pub struct MeWriter { pub degraded: Arc, pub draining: Arc, pub draining_started_at_epoch_secs: Arc, + pub drain_deadline_epoch_secs: Arc, pub allow_drain_fallback: Arc, } @@ -128,12 +136,13 @@ pub struct MePool { pub(super) default_dc: AtomicI32, pub(super) next_writer_id: AtomicU64, pub(super) ping_tracker: Arc>>, + pub(super) ping_tracker_last_cleanup_epoch_ms: AtomicU64, pub(super) rtt_stats: Arc>>, pub(super) nat_reflection_cache: Arc>, pub(super) nat_reflection_singleflight_v4: Arc>, pub(super) nat_reflection_singleflight_v6: Arc>, pub(super) writer_available: Arc, - pub(super) refill_inflight: Arc>>, + pub(super) refill_inflight: Arc>>, pub(super) refill_inflight_dc: Arc>>, pub(super) conn_count: AtomicUsize, pub(super) stats: Arc, @@ -361,6 +370,7 @@ impl MePool { default_dc: AtomicI32::new(default_dc.unwrap_or(2)), next_writer_id: AtomicU64::new(1), ping_tracker: Arc::new(Mutex::new(HashMap::new())), + ping_tracker_last_cleanup_epoch_ms: AtomicU64::new(0), rtt_stats: Arc::new(Mutex::new(HashMap::new())), nat_reflection_cache: Arc::new(Mutex::new(NatReflectionCache::default())), nat_reflection_singleflight_v4: Arc::new(Mutex::new(())), @@ -779,6 +789,36 @@ impl MePool { if dc == 0 { 2 } else { dc } } + pub(super) async fn has_configured_endpoints_for_dc(&self, dc: i32) -> bool { + if self.decision.ipv4_me { + let map = self.proxy_map_v4.read().await; + if map.get(&dc).is_some_and(|endpoints| !endpoints.is_empty()) { + return true; + } + } + + if self.decision.ipv6_me { + let map = self.proxy_map_v6.read().await; + if map.get(&dc).is_some_and(|endpoints| !endpoints.is_empty()) { + return true; + } + } + + false + } + + pub(super) async fn resolve_target_dc_for_routing(&self, target_dc: i32) -> (i32, bool) { + if target_dc == 0 { + return (self.default_dc_for_routing(), true); + } + + if self.has_configured_endpoints_for_dc(target_dc).await { + return (target_dc, false); + } + + (self.default_dc_for_routing(), true) + } + pub(super) fn dc_lookup_chain_for_target(&self, target_dc: i32) -> Vec { let mut out = Vec::with_capacity(1); if target_dc != 0 { diff --git a/src/transport/middle_proxy/pool_init.rs b/src/transport/middle_proxy/pool_init.rs index 668cfda..52cbc68 100644 --- a/src/transport/middle_proxy/pool_init.rs +++ b/src/transport/middle_proxy/pool_init.rs @@ -55,7 +55,11 @@ impl MePool { .iter() .map(|(ip, port)| SocketAddr::new(*ip, *port)) .collect(); - if self.active_writer_count_for_endpoints(&endpoints).await >= target_writers { + if self + .active_writer_count_for_dc_endpoints(dc, &endpoints) + .await + >= target_writers + { continue; } let pool = Arc::clone(self); @@ -79,7 +83,7 @@ impl MePool { .iter() .map(|(ip, port)| SocketAddr::new(*ip, *port)) .collect(); - if self.active_writer_count_for_endpoints(&endpoints).await == 0 { + if self.active_writer_count_for_dc_endpoints(*dc, &endpoints).await == 0 { missing_dcs.push(*dc); } } @@ -156,7 +160,9 @@ impl MePool { let endpoint_set: HashSet = endpoints.iter().copied().collect(); loop { - let alive = self.active_writer_count_for_endpoints(&endpoint_set).await; + let alive = self + .active_writer_count_for_dc_endpoints(dc, &endpoint_set) + .await; if alive >= target_writers { info!( dc = %dc, @@ -175,7 +181,7 @@ impl MePool { let rng_clone = Arc::clone(&rng); let endpoints_clone = endpoints.clone(); join.spawn(async move { - pool.connect_endpoints_round_robin(&endpoints_clone, rng_clone.as_ref()) + pool.connect_endpoints_round_robin(dc, &endpoints_clone, rng_clone.as_ref()) .await }); } @@ -193,7 +199,9 @@ impl MePool { } } - let alive_after = self.active_writer_count_for_endpoints(&endpoint_set).await; + let alive_after = self + .active_writer_count_for_dc_endpoints(dc, &endpoint_set) + .await; if alive_after >= target_writers { info!( dc = %dc, diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 7da6acc..3c8b0bb 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -9,7 +9,7 @@ use tracing::{debug, info, warn}; use crate::crypto::SecureRandom; use crate::network::IpFamily; -use super::pool::{MePool, RefillDcKey, WriterContour}; +use super::pool::{MePool, RefillDcKey, RefillEndpointKey, WriterContour}; const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20; const ME_FLAP_QUARANTINE_SECS: u64 = 25; @@ -82,57 +82,19 @@ impl MePool { Vec::new() } - pub(super) async fn has_refill_inflight_for_endpoints(&self, endpoints: &[SocketAddr]) -> bool { - if endpoints.is_empty() { - return false; - } - - { - let guard = self.refill_inflight.lock().await; - if endpoints.iter().any(|addr| guard.contains(addr)) { - return true; - } - } - - let dc_keys = self.resolve_refill_dc_keys_for_endpoints(endpoints).await; - if dc_keys.is_empty() { - return false; - } + pub(super) async fn has_refill_inflight_for_dc_key(&self, key: RefillDcKey) -> bool { let guard = self.refill_inflight_dc.lock().await; - dc_keys.iter().any(|key| guard.contains(key)) - } - - async fn resolve_refill_dc_key_for_addr(&self, addr: SocketAddr) -> Option { - let family = if addr.is_ipv4() { - IpFamily::V4 - } else { - IpFamily::V6 - }; - Some(RefillDcKey { - dc: self.resolve_dc_for_endpoint(addr).await, - family, - }) - } - - async fn resolve_refill_dc_keys_for_endpoints( - &self, - endpoints: &[SocketAddr], - ) -> HashSet { - let mut out = HashSet::::new(); - for addr in endpoints { - if let Some(key) = self.resolve_refill_dc_key_for_addr(*addr).await { - out.insert(key); - } - } - out + guard.contains(&key) } pub(super) async fn connect_endpoints_round_robin( self: &Arc, + dc: i32, endpoints: &[SocketAddr], rng: &SecureRandom, ) -> bool { self.connect_endpoints_round_robin_with_generation_contour( + dc, endpoints, rng, self.current_generation(), @@ -143,6 +105,7 @@ impl MePool { pub(super) async fn connect_endpoints_round_robin_with_generation_contour( self: &Arc, + dc: i32, endpoints: &[SocketAddr], rng: &SecureRandom, generation: u64, @@ -157,7 +120,7 @@ impl MePool { let idx = (start + offset) % candidates.len(); let addr = candidates[idx]; match self - .connect_one_with_generation_contour(addr, rng, generation, contour) + .connect_one_with_generation_contour_for_dc(addr, rng, generation, contour, dc) .await { Ok(()) => return true, @@ -167,9 +130,8 @@ impl MePool { false } - async fn endpoints_for_same_dc(&self, addr: SocketAddr) -> Vec { + async fn endpoints_for_dc(&self, target_dc: i32) -> Vec { let mut endpoints = HashSet::::new(); - let target_dc = self.resolve_dc_for_endpoint(addr).await; if self.decision.ipv4_me { let map = self.proxy_map_v4.read().await; @@ -194,14 +156,14 @@ impl MePool { sorted } - async fn refill_writer_after_loss(self: &Arc, addr: SocketAddr) -> bool { + async fn refill_writer_after_loss(self: &Arc, addr: SocketAddr, writer_dc: i32) -> bool { let fast_retries = self.me_reconnect_fast_retry_count.max(1); let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await; if !same_endpoint_quarantined { for attempt in 0..fast_retries { self.stats.increment_me_reconnect_attempt(); - match self.connect_one(addr, self.rng.as_ref()).await { + match self.connect_one_for_dc(addr, writer_dc, self.rng.as_ref()).await { Ok(()) => { self.stats.increment_me_reconnect_success(); self.stats.increment_me_writer_restored_same_endpoint_total(); @@ -229,7 +191,7 @@ impl MePool { ); } - let dc_endpoints = self.endpoints_for_same_dc(addr).await; + let dc_endpoints = self.endpoints_for_dc(writer_dc).await; if dc_endpoints.is_empty() { self.stats.increment_me_refill_failed_total(); return false; @@ -238,7 +200,7 @@ impl MePool { for attempt in 0..fast_retries { self.stats.increment_me_reconnect_attempt(); if self - .connect_endpoints_round_robin(&dc_endpoints, self.rng.as_ref()) + .connect_endpoints_round_robin(writer_dc, &dc_endpoints, self.rng.as_ref()) .await { self.stats.increment_me_reconnect_success(); @@ -259,45 +221,69 @@ impl MePool { pub(crate) fn trigger_immediate_refill(self: &Arc, addr: SocketAddr) { let pool = Arc::clone(self); tokio::spawn(async move { - let dc_endpoints = pool.endpoints_for_same_dc(addr).await; - let dc_keys = pool.resolve_refill_dc_keys_for_endpoints(&dc_endpoints).await; + let writer_dc = pool.resolve_dc_for_endpoint(addr).await; + pool.trigger_immediate_refill_for_dc(addr, writer_dc); + }); + } - { + pub(crate) fn trigger_immediate_refill_for_dc(self: &Arc, addr: SocketAddr, writer_dc: i32) { + let endpoint_key = RefillEndpointKey { + dc: writer_dc, + addr, + }; + let pre_inserted = if let Ok(mut guard) = self.refill_inflight.try_lock() { + if !guard.insert(endpoint_key) { + self.stats.increment_me_refill_skipped_inflight_total(); + return; + } + true + } else { + false + }; + + let pool = Arc::clone(self); + tokio::spawn(async move { + let dc_endpoints = pool.endpoints_for_dc(writer_dc).await; + let dc_key = RefillDcKey { + dc: writer_dc, + family: if addr.is_ipv4() { + IpFamily::V4 + } else { + IpFamily::V6 + }, + }; + + if !pre_inserted { let mut guard = pool.refill_inflight.lock().await; - if !guard.insert(addr) { + if !guard.insert(endpoint_key) { pool.stats.increment_me_refill_skipped_inflight_total(); return; } } - if !dc_keys.is_empty() { + { let mut dc_guard = pool.refill_inflight_dc.lock().await; - if dc_keys.iter().any(|key| dc_guard.contains(key)) { + if dc_guard.contains(&dc_key) { pool.stats.increment_me_refill_skipped_inflight_total(); drop(dc_guard); let mut guard = pool.refill_inflight.lock().await; - guard.remove(&addr); + guard.remove(&endpoint_key); return; } - dc_guard.extend(dc_keys.iter().copied()); + dc_guard.insert(dc_key); } pool.stats.increment_me_refill_triggered_total(); - - let restored = pool.refill_writer_after_loss(addr).await; + let restored = pool.refill_writer_after_loss(addr, writer_dc).await; if !restored { - warn!(%addr, "ME immediate refill failed"); + warn!(%addr, dc = writer_dc, "ME immediate refill failed"); } let mut guard = pool.refill_inflight.lock().await; - guard.remove(&addr); + guard.remove(&endpoint_key); drop(guard); - if !dc_keys.is_empty() { - let mut dc_guard = pool.refill_inflight_dc.lock().await; - for key in &dc_keys { - dc_guard.remove(key); - } - } + let mut dc_guard = pool.refill_inflight_dc.lock().await; + dc_guard.remove(&dc_key); }); } } diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 1e86ea3..17ef331 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use std::io::ErrorKind; +use bytes::Bytes; use bytes::BytesMut; use rand::Rng; use tokio::sync::mpsc; @@ -50,11 +51,22 @@ impl MePool { } pub(crate) async fn connect_one(self: &Arc, addr: SocketAddr, rng: &SecureRandom) -> Result<()> { + let writer_dc = self.resolve_dc_for_endpoint(addr).await; + self.connect_one_for_dc(addr, writer_dc, rng).await + } + + pub(crate) async fn connect_one_for_dc( + self: &Arc, + addr: SocketAddr, + writer_dc: i32, + rng: &SecureRandom, + ) -> Result<()> { self.connect_one_with_generation_contour( addr, rng, self.current_generation(), WriterContour::Active, + writer_dc, ) .await } @@ -65,13 +77,27 @@ impl MePool { rng: &SecureRandom, generation: u64, contour: WriterContour, + writer_dc: i32, + ) -> Result<()> { + self.connect_one_with_generation_contour_for_dc(addr, rng, generation, contour, writer_dc) + .await + } + + pub(super) async fn connect_one_with_generation_contour_for_dc( + self: &Arc, + addr: SocketAddr, + rng: &SecureRandom, + generation: u64, + contour: WriterContour, + writer_dc: i32, ) -> Result<()> { let secret_len = self.proxy_secret.read().await.secret.len(); if secret_len < 32 { return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into())); } - let (stream, _connect_ms, upstream_egress) = self.connect_tcp(addr).await?; + let dc_idx = i16::try_from(writer_dc).ok(); + let (stream, _connect_ms, upstream_egress) = self.connect_tcp(addr, dc_idx).await?; let hs = self.handshake_only(stream, addr, upstream_egress, rng).await?; let writer_id = self.next_writer_id.fetch_add(1, Ordering::Relaxed); @@ -80,6 +106,7 @@ impl MePool { let degraded = Arc::new(AtomicBool::new(false)); let draining = Arc::new(AtomicBool::new(false)); let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0)); + let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0)); let allow_drain_fallback = Arc::new(AtomicBool::new(false)); let (tx, mut rx) = mpsc::channel::(4096); let mut rpc_writer = RpcWriter { @@ -111,6 +138,7 @@ impl MePool { let writer = MeWriter { id: writer_id, addr, + writer_dc, generation, contour: contour.clone(), created_at: Instant::now(), @@ -119,6 +147,7 @@ impl MePool { degraded: degraded.clone(), draining: draining.clone(), draining_started_at_epoch_secs: draining_started_at_epoch_secs.clone(), + drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(), allow_drain_fallback: allow_drain_fallback.clone(), }; self.writers.write().await.push(writer.clone()); @@ -254,17 +283,47 @@ impl MePool { p.extend_from_slice(&sent_id.to_le_bytes()); { let mut tracker = ping_tracker_ping.lock().await; - let before = tracker.len(); - tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120)); - let expired = before.saturating_sub(tracker.len()); - if expired > 0 { - stats_ping.increment_me_keepalive_timeout_by(expired as u64); + let now_epoch_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let mut run_cleanup = false; + if let Some(pool) = pool_ping.upgrade() { + let last_cleanup_ms = pool + .ping_tracker_last_cleanup_epoch_ms + .load(Ordering::Relaxed); + if now_epoch_ms.saturating_sub(last_cleanup_ms) >= 30_000 + && pool + .ping_tracker_last_cleanup_epoch_ms + .compare_exchange( + last_cleanup_ms, + now_epoch_ms, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .is_ok() + { + run_cleanup = true; + } + } + + if run_cleanup { + let before = tracker.len(); + tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120)); + let expired = before.saturating_sub(tracker.len()); + if expired > 0 { + stats_ping.increment_me_keepalive_timeout_by(expired as u64); + } } tracker.insert(sent_id, (std::time::Instant::now(), writer_id)); } ping_id = ping_id.wrapping_add(1); stats_ping.increment_me_keepalive_sent(); - if tx_ping.send(WriterCommand::DataAndFlush(p)).await.is_err() { + if tx_ping + .send(WriterCommand::DataAndFlush(Bytes::from(p))) + .await + .is_err() + { stats_ping.increment_me_keepalive_failed(); debug!("ME ping failed, removing dead writer"); cancel_ping.cancel(); @@ -338,7 +397,11 @@ impl MePool { meta.proto_flags, ); - if tx_signal.send(WriterCommand::DataAndFlush(payload)).await.is_err() { + if tx_signal + .send(WriterCommand::DataAndFlush(payload)) + .await + .is_err() + { stats_signal.increment_me_rpc_proxy_req_signal_failed_total(); let _ = pool.registry.unregister(conn_id).await; cancel_signal.cancel(); @@ -369,7 +432,7 @@ impl MePool { close_payload.extend_from_slice(&conn_id.to_le_bytes()); if tx_signal - .send(WriterCommand::DataAndFlush(close_payload)) + .send(WriterCommand::DataAndFlush(Bytes::from(close_payload))) .await .is_err() { @@ -404,6 +467,7 @@ impl MePool { async fn remove_writer_only(self: &Arc, writer_id: u64) -> Vec { let mut close_tx: Option> = None; let mut removed_addr: Option = None; + let mut removed_dc: Option = None; let mut removed_uptime: Option = None; let mut trigger_refill = false; { @@ -417,6 +481,7 @@ impl MePool { self.stats.increment_me_writer_removed_total(); w.cancel.cancel(); removed_addr = Some(w.addr); + removed_dc = Some(w.writer_dc); removed_uptime = Some(w.created_at.elapsed()); trigger_refill = !was_draining; if trigger_refill { @@ -431,11 +496,12 @@ impl MePool { } if trigger_refill && let Some(addr) = removed_addr + && let Some(writer_dc) = removed_dc { if let Some(uptime) = removed_uptime { self.maybe_quarantine_flapping_endpoint(addr, uptime).await; } - self.trigger_immediate_refill(addr); + self.trigger_immediate_refill_for_dc(addr, writer_dc); } self.rtt_stats.lock().await.remove(&writer_id); self.registry.writer_lost(writer_id).await @@ -454,8 +520,14 @@ impl MePool { let already_draining = w.draining.swap(true, Ordering::Relaxed); w.allow_drain_fallback .store(allow_drain_fallback, Ordering::Relaxed); + let now_epoch_secs = Self::now_epoch_secs(); w.draining_started_at_epoch_secs - .store(Self::now_epoch_secs(), Ordering::Relaxed); + .store(now_epoch_secs, Ordering::Relaxed); + let drain_deadline_epoch_secs = timeout + .map(|duration| now_epoch_secs.saturating_add(duration.as_secs())) + .unwrap_or(0); + w.drain_deadline_epoch_secs + .store(drain_deadline_epoch_secs, Ordering::Relaxed); if !already_draining { self.stats.increment_pool_drain_active(); } @@ -479,26 +551,6 @@ impl MePool { allow_drain_fallback, "ME writer marked draining" ); - - let pool = Arc::downgrade(self); - tokio::spawn(async move { - let deadline = timeout.map(|t| Instant::now() + t); - while let Some(p) = pool.upgrade() { - if let Some(deadline_at) = deadline - && Instant::now() >= deadline_at - { - warn!(writer_id, "Drain timeout, force-closing"); - p.stats.increment_pool_force_close_total(); - let _ = p.remove_writer_and_close_clients(writer_id).await; - break; - } - if p.registry.is_writer_empty(writer_id).await { - let _ = p.remove_writer_only(writer_id).await; - break; - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - }); } pub(crate) async fn mark_writer_draining(self: &Arc, writer_id: u64) { diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index 2a99164..61bd69c 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -181,7 +181,11 @@ pub(crate) async fn reader_loop( let mut pong = Vec::with_capacity(12); pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes()); pong.extend_from_slice(&ping_id.to_le_bytes()); - if tx.send(WriterCommand::DataAndFlush(pong)).await.is_err() { + if tx + .send(WriterCommand::DataAndFlush(Bytes::from(pong))) + .await + .is_err() + { warn!("PONG send failed"); break; } @@ -222,5 +226,5 @@ async fn send_close_conn(tx: &mpsc::Sender, conn_id: u64) { p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes()); p.extend_from_slice(&conn_id.to_le_bytes()); - let _ = tx.send(WriterCommand::DataAndFlush(p)).await; + let _ = tx.send(WriterCommand::DataAndFlush(Bytes::from(p))).await; } diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index b437885..0ee81e0 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -264,6 +264,20 @@ impl ConnRegistry { inner.writer_idle_since_epoch_secs.clone() } + pub async fn writer_idle_since_for_writer_ids( + &self, + writer_ids: &[u64], + ) -> HashMap { + let inner = self.inner.read().await; + let mut out = HashMap::::with_capacity(writer_ids.len()); + for writer_id in writer_ids { + if let Some(idle_since) = inner.writer_idle_since_epoch_secs.get(writer_id).copied() { + out.insert(*writer_id, idle_since); + } + } + out + } + pub(super) async fn writer_activity_snapshot(&self) -> WriterActivitySnapshot { let inner = self.inner.read().await; let mut bound_clients_by_writer = HashMap::::new(); diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 9ffcc8e..ccaad4a 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; +use bytes::Bytes; use tokio::sync::mpsc::error::TrySendError; use tracing::{debug, warn}; @@ -59,6 +60,7 @@ impl MePool { let mut hybrid_recovery_round = 0u32; let mut hybrid_last_recovery_at: Option = None; let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50)); + let mut hybrid_wait_current = hybrid_wait_step; loop { if let Some(current) = self.registry.get_writer(conn_id).await { @@ -147,11 +149,14 @@ impl MePool { target_dc, &mut hybrid_recovery_round, &mut hybrid_last_recovery_at, - hybrid_wait_step, + hybrid_wait_current, ) .await; - let deadline = Instant::now() + hybrid_wait_step; + let deadline = Instant::now() + hybrid_wait_current; let _ = self.wait_for_writer_until(deadline).await; + hybrid_wait_current = + (hybrid_wait_current.saturating_mul(2)) + .min(Duration::from_millis(400)); continue; } } @@ -223,16 +228,26 @@ impl MePool { target_dc, &mut hybrid_recovery_round, &mut hybrid_last_recovery_at, - hybrid_wait_step, + hybrid_wait_current, ) .await; - let deadline = Instant::now() + hybrid_wait_step; + let deadline = Instant::now() + hybrid_wait_current; let _ = self.wait_for_candidate_until(target_dc, deadline).await; + hybrid_wait_current = (hybrid_wait_current.saturating_mul(2)) + .min(Duration::from_millis(400)); continue; } } } - let writer_idle_since = self.registry.writer_idle_since_snapshot().await; + hybrid_wait_current = hybrid_wait_step; + let writer_ids: Vec = candidate_indices + .iter() + .map(|idx| writers_snapshot[*idx].id) + .collect(); + let writer_idle_since = self + .registry + .writer_idle_since_for_writer_ids(&writer_ids) + .await; let now_epoch_secs = Self::now_epoch_secs(); if self.me_deterministic_writer_sort.load(Ordering::Relaxed) { @@ -507,7 +522,11 @@ impl MePool { let mut p = Vec::with_capacity(12); p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes()); p.extend_from_slice(&conn_id.to_le_bytes()); - if w.tx.send(WriterCommand::DataAndFlush(p)).await.is_err() { + if w.tx + .send(WriterCommand::DataAndFlush(Bytes::from(p))) + .await + .is_err() + { debug!("ME close write failed"); self.remove_writer_and_close_clients(w.writer_id).await; } @@ -524,7 +543,7 @@ impl MePool { let mut p = Vec::with_capacity(12); p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes()); p.extend_from_slice(&conn_id.to_le_bytes()); - match w.tx.try_send(WriterCommand::DataAndFlush(p)) { + match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) { Ok(()) => {} Err(TrySendError::Full(cmd)) => { let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await; diff --git a/src/transport/middle_proxy/wire.rs b/src/transport/middle_proxy/wire.rs index 3f78f20..7667646 100644 --- a/src/transport/middle_proxy/wire.rs +++ b/src/transport/middle_proxy/wire.rs @@ -1,4 +1,5 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use bytes::Bytes; use crate::protocol::constants::*; @@ -48,7 +49,7 @@ pub(crate) fn build_proxy_req_payload( data: &[u8], proxy_tag: Option<&[u8]>, proto_flags: u32, -) -> Vec { +) -> Bytes { let mut b = Vec::with_capacity(128 + data.len()); b.extend_from_slice(&RPC_PROXY_REQ_U32.to_le_bytes()); @@ -85,7 +86,7 @@ pub(crate) fn build_proxy_req_payload( } b.extend_from_slice(data); - b + Bytes::from(b) } pub fn proto_flags_for_tag(tag: crate::protocol::constants::ProtoTag, has_proxy_tag: bool) -> u32 { diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index 84c6fdf..2424f9c 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -7,7 +7,7 @@ use std::collections::{BTreeSet, HashMap}; use std::net::{SocketAddr, IpAddr}; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::time::Duration; use tokio::net::TcpStream; use tokio::sync::RwLock; @@ -237,6 +237,8 @@ pub struct UpstreamManager { connect_budget: Duration, unhealthy_fail_threshold: u32, connect_failfast_hard_errors: bool, + no_upstreams_warn_epoch_ms: Arc, + no_healthy_warn_epoch_ms: Arc, stats: Arc, } @@ -262,10 +264,35 @@ impl UpstreamManager { connect_budget: Duration::from_millis(connect_budget_ms.max(1)), unhealthy_fail_threshold: unhealthy_fail_threshold.max(1), connect_failfast_hard_errors, + no_upstreams_warn_epoch_ms: Arc::new(AtomicU64::new(0)), + no_healthy_warn_epoch_ms: Arc::new(AtomicU64::new(0)), stats, } } + fn now_epoch_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 + } + + fn should_emit_warn(last_epoch_ms: &AtomicU64, cooldown_ms: u64) -> bool { + let now_epoch_ms = Self::now_epoch_ms(); + let previous_epoch_ms = last_epoch_ms.load(Ordering::Relaxed); + if now_epoch_ms.saturating_sub(previous_epoch_ms) < cooldown_ms { + return false; + } + last_epoch_ms + .compare_exchange( + previous_epoch_ms, + now_epoch_ms, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .is_ok() + } + pub fn try_api_snapshot(&self) -> Option { let guard = self.upstreams.try_read().ok()?; let now = std::time::Instant::now(); @@ -533,12 +560,22 @@ impl UpstreamManager { .collect(); if filtered_upstreams.is_empty() { - warn!(scope = scope, "No upstreams available! Using first (direct?)"); + if Self::should_emit_warn( + self.no_upstreams_warn_epoch_ms.as_ref(), + 5_000, + ) { + warn!(scope = scope, "No upstreams available! Using first (direct?)"); + } return None; } if healthy.is_empty() { - warn!(scope = scope, "No healthy upstreams available! Using random."); + if Self::should_emit_warn( + self.no_healthy_warn_epoch_ms.as_ref(), + 5_000, + ) { + warn!(scope = scope, "No healthy upstreams available! Using random."); + } return Some(filtered_upstreams[rand::rng().gen_range(0..filtered_upstreams.len())]); } From 258191ab87d3823b79c47cc213ac331ab8fb55f6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:40:57 +0300 Subject: [PATCH 125/261] Routed DC + Strict ME Writers Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 81 ++++++---- src/transport/middle_proxy/pool.rs | 12 -- src/transport/middle_proxy/pool_config.rs | 5 +- src/transport/middle_proxy/pool_refill.rs | 9 -- src/transport/middle_proxy/pool_reinit.rs | 47 +++--- src/transport/middle_proxy/pool_status.rs | 49 +++--- src/transport/middle_proxy/pool_writer.rs | 5 - src/transport/middle_proxy/send.rs | 173 +++++++++++----------- 8 files changed, 188 insertions(+), 193 deletions(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 4fcba39..b0536cc 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -147,7 +147,7 @@ async fn check_family( IpFamily::V6 => pool.proxy_map_v6.read().await, }; for (dc, addrs) in map_guard.iter() { - let entry = dc_endpoints.entry(dc.abs()).or_default(); + let entry = dc_endpoints.entry(*dc).or_default(); for (ip, port) in addrs.iter().copied() { entry.push(SocketAddr::new(ip, port)); } @@ -164,14 +164,15 @@ async fn check_family( adaptive_recover_until.clear(); } - let mut live_addr_counts = HashMap::::new(); - let mut live_writer_ids_by_addr = HashMap::>::new(); + let mut live_addr_counts = HashMap::<(i32, SocketAddr), usize>::new(); + let mut live_writer_ids_by_addr = HashMap::<(i32, SocketAddr), Vec>::new(); for writer in pool.writers.read().await.iter().filter(|w| { !w.draining.load(std::sync::atomic::Ordering::Relaxed) }) { - *live_addr_counts.entry(writer.addr).or_insert(0) += 1; + let key = (writer.writer_dc, writer.addr); + *live_addr_counts.entry(key).or_insert(0) += 1; live_writer_ids_by_addr - .entry(writer.addr) + .entry(key) .or_default() .push(writer.id); } @@ -211,7 +212,7 @@ async fn check_family( }); let alive = endpoints .iter() - .map(|addr| *live_addr_counts.get(addr).unwrap_or(&0)) + .map(|addr| *live_addr_counts.get(&(dc, *addr)).unwrap_or(&0)) .sum::(); if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 { @@ -321,7 +322,10 @@ async fn check_family( if *inflight.get(&key).unwrap_or(&0) >= max_concurrent { continue; } - if pool.has_refill_inflight_for_endpoints(&endpoints).await { + if pool + .has_refill_inflight_for_dc_key(super::pool::RefillDcKey { dc, family }) + .await + { debug!( dc = %dc, ?family, @@ -373,7 +377,7 @@ async fn check_family( } let res = tokio::time::timeout( pool.me_one_timeout, - pool.connect_endpoints_round_robin(&endpoints, rng.as_ref()), + pool.connect_endpoints_round_robin(dc, &endpoints, rng.as_ref()), ) .await; match res { @@ -484,12 +488,13 @@ fn adaptive_floor_class_max( } fn list_writer_ids_for_endpoints( + dc: i32, endpoints: &[SocketAddr], - live_writer_ids_by_addr: &HashMap>, + live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec>, ) -> Vec { let mut out = Vec::::new(); for endpoint in endpoints { - if let Some(ids) = live_writer_ids_by_addr.get(endpoint) { + if let Some(ids) = live_writer_ids_by_addr.get(&(dc, *endpoint)) { out.extend(ids.iter().copied()); } } @@ -500,8 +505,8 @@ async fn build_family_floor_plan( pool: &Arc, family: IpFamily, dc_endpoints: &HashMap>, - live_addr_counts: &HashMap, - live_writer_ids_by_addr: &HashMap>, + live_addr_counts: &HashMap<(i32, SocketAddr), usize>, + live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec>, bound_clients_by_writer: &HashMap, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, @@ -522,6 +527,7 @@ async fn build_family_floor_plan( let reduce_for_idle = should_reduce_floor_for_idle( pool, key, + *dc, endpoints, live_writer_ids_by_addr, bound_clients_by_writer, @@ -551,10 +557,10 @@ async fn build_family_floor_plan( let target_required = desired_raw.clamp(min_required, max_required); let alive = endpoints .iter() - .map(|endpoint| live_addr_counts.get(endpoint).copied().unwrap_or(0)) + .map(|endpoint| live_addr_counts.get(&(*dc, *endpoint)).copied().unwrap_or(0)) .sum::(); family_active_total = family_active_total.saturating_add(alive); - let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr); + let writer_ids = list_writer_ids_for_endpoints(*dc, endpoints, live_writer_ids_by_addr); let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer); entries.push(DcFloorPlanEntry { @@ -654,14 +660,14 @@ async fn maybe_swap_idle_writer_for_cap( dc: i32, family: IpFamily, endpoints: &[SocketAddr], - live_writer_ids_by_addr: &HashMap>, + live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec>, writer_idle_since: &HashMap, bound_clients_by_writer: &HashMap, ) -> bool { let now_epoch_secs = MePool::now_epoch_secs(); let mut candidate: Option<(u64, SocketAddr, u64)> = None; for endpoint in endpoints { - let Some(writer_ids) = live_writer_ids_by_addr.get(endpoint) else { + let Some(writer_ids) = live_writer_ids_by_addr.get(&(dc, *endpoint)) else { continue; }; for writer_id in writer_ids { @@ -686,7 +692,12 @@ async fn maybe_swap_idle_writer_for_cap( return false; }; - let connected = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + let connected = match tokio::time::timeout( + pool.me_one_timeout, + pool.connect_one_for_dc(endpoint, dc, rng.as_ref()), + ) + .await + { Ok(Ok(())) => true, Ok(Err(error)) => { debug!( @@ -738,7 +749,7 @@ async fn maybe_refresh_idle_writer_for_dc( endpoints: &[SocketAddr], alive: usize, required: usize, - live_writer_ids_by_addr: &HashMap>, + live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec>, writer_idle_since: &HashMap, bound_clients_by_writer: &HashMap, idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, @@ -757,7 +768,7 @@ async fn maybe_refresh_idle_writer_for_dc( let now_epoch_secs = MePool::now_epoch_secs(); let mut candidate: Option<(u64, SocketAddr, u64, u64)> = None; for endpoint in endpoints { - let Some(writer_ids) = live_writer_ids_by_addr.get(endpoint) else { + let Some(writer_ids) = live_writer_ids_by_addr.get(&(dc, *endpoint)) else { continue; }; for writer_id in writer_ids { @@ -787,7 +798,12 @@ async fn maybe_refresh_idle_writer_for_dc( return; }; - let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + let rotate_ok = match tokio::time::timeout( + pool.me_one_timeout, + pool.connect_one_for_dc(endpoint, dc, rng.as_ref()), + ) + .await + { Ok(Ok(())) => true, Ok(Err(error)) => { debug!( @@ -843,8 +859,9 @@ async fn maybe_refresh_idle_writer_for_dc( async fn should_reduce_floor_for_idle( pool: &Arc, key: (i32, IpFamily), + dc: i32, endpoints: &[SocketAddr], - live_writer_ids_by_addr: &HashMap>, + live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec>, bound_clients_by_writer: &HashMap, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, @@ -856,7 +873,7 @@ async fn should_reduce_floor_for_idle( } let now = Instant::now(); - let writer_ids = list_writer_ids_for_endpoints(endpoints, live_writer_ids_by_addr); + let writer_ids = list_writer_ids_for_endpoints(dc, endpoints, live_writer_ids_by_addr); let has_bound_clients = has_bound_clients_on_endpoint(&writer_ids, bound_clients_by_writer); if has_bound_clients { adaptive_idle_since.remove(&key); @@ -922,7 +939,12 @@ async fn recover_single_endpoint_outage( let attempt_ok = if bypass_quarantine { pool.stats .increment_me_single_endpoint_quarantine_bypass_total(); - match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + match tokio::time::timeout( + pool.me_one_timeout, + pool.connect_one_for_dc(endpoint, key.0, rng.as_ref()), + ) + .await + { Ok(Ok(())) => true, Ok(Err(e)) => { debug!( @@ -948,7 +970,7 @@ async fn recover_single_endpoint_outage( let one_endpoint = [endpoint]; match tokio::time::timeout( pool.me_one_timeout, - pool.connect_endpoints_round_robin(&one_endpoint, rng.as_ref()), + pool.connect_endpoints_round_robin(key.0, &one_endpoint, rng.as_ref()), ) .await { @@ -1012,7 +1034,7 @@ async fn maybe_rotate_single_endpoint_shadow( endpoints: &[SocketAddr], alive: usize, required: usize, - live_writer_ids_by_addr: &HashMap>, + live_writer_ids_by_addr: &HashMap<(i32, SocketAddr), Vec>, bound_clients_by_writer: &HashMap, shadow_rotate_deadline: &mut HashMap<(i32, IpFamily), Instant>, ) { @@ -1045,7 +1067,7 @@ async fn maybe_rotate_single_endpoint_shadow( return; } - let Some(writer_ids) = live_writer_ids_by_addr.get(&endpoint) else { + let Some(writer_ids) = live_writer_ids_by_addr.get(&(dc, endpoint)) else { shadow_rotate_deadline.insert(key, now + Duration::from_secs(SHADOW_ROTATE_RETRY_SECS)); return; }; @@ -1071,7 +1093,12 @@ async fn maybe_rotate_single_endpoint_shadow( return; }; - let rotate_ok = match tokio::time::timeout(pool.me_one_timeout, pool.connect_one(endpoint, rng.as_ref())).await { + let rotate_ok = match tokio::time::timeout( + pool.me_one_timeout, + pool.connect_one_for_dc(endpoint, dc, rng.as_ref()), + ) + .await + { Ok(Ok(())) => true, Ok(Err(e)) => { debug!( diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 236a12a..13259bb 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -819,18 +819,6 @@ impl MePool { (self.default_dc_for_routing(), true) } - pub(super) fn dc_lookup_chain_for_target(&self, target_dc: i32) -> Vec { - let mut out = Vec::with_capacity(1); - if target_dc != 0 { - out.push(target_dc); - } else { - // Use default DC only when target DC is unknown and pinning is not established. - let fallback_dc = self.default_dc_for_routing(); - out.push(fallback_dc); - } - out - } - pub(super) async fn resolve_dc_for_endpoint(&self, addr: SocketAddr) -> i32 { if let Some(cached) = self.endpoint_dc_map.read().await.get(&addr).copied() && let Some(dc) = cached diff --git a/src/transport/middle_proxy/pool_config.rs b/src/transport/middle_proxy/pool_config.rs index a43f9bf..66752bf 100644 --- a/src/transport/middle_proxy/pool_config.rs +++ b/src/transport/middle_proxy/pool_config.rs @@ -110,7 +110,10 @@ impl MePool { pub async fn reconnect_all(self: &Arc) { let ws = self.writers.read().await.clone(); for w in ws { - if let Ok(()) = self.connect_one(w.addr, self.rng.as_ref()).await { + if let Ok(()) = self + .connect_one_for_dc(w.addr, w.writer_dc, self.rng.as_ref()) + .await + { self.mark_writer_draining(w.id).await; tokio::time::sleep(Duration::from_secs(2)).await; } diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 3c8b0bb..316f3ff 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -218,14 +218,6 @@ impl MePool { false } - pub(crate) fn trigger_immediate_refill(self: &Arc, addr: SocketAddr) { - let pool = Arc::clone(self); - tokio::spawn(async move { - let writer_dc = pool.resolve_dc_for_endpoint(addr).await; - pool.trigger_immediate_refill_for_dc(addr, writer_dc); - }); - } - pub(crate) fn trigger_immediate_refill_for_dc(self: &Arc, addr: SocketAddr, writer_dc: i32) { let endpoint_key = RefillEndpointKey { dc: writer_dc, @@ -243,7 +235,6 @@ impl MePool { let pool = Arc::clone(self); tokio::spawn(async move { - let dc_endpoints = pool.endpoints_for_dc(writer_dc).await; let dc_key = RefillDcKey { dc: writer_dc, family: if addr.is_ipv4() { diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 39944ba..625ccf0 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -62,7 +62,7 @@ impl MePool { fn coverage_ratio( desired_by_dc: &HashMap>, - active_writer_addrs: &HashSet, + active_writer_addrs: &HashSet<(i32, SocketAddr)>, ) -> (f32, Vec) { if desired_by_dc.is_empty() { return (1.0, Vec::new()); @@ -76,7 +76,7 @@ impl MePool { } if endpoints .iter() - .any(|addr| active_writer_addrs.contains(addr)) + .any(|addr| active_writer_addrs.contains(&(*dc, *addr))) { covered += 1; } else { @@ -91,32 +91,25 @@ impl MePool { } pub async fn reconcile_connections(self: &Arc, rng: &SecureRandom) { - let writers = self.writers.read().await; - let current: HashSet = writers - .iter() - .filter(|w| !w.draining.load(Ordering::Relaxed)) - .map(|w| w.addr) - .collect(); - drop(writers); - for family in self.family_order() { let map = self.proxy_map_for_family(family).await; - for (_dc, addrs) in &map { + for (dc, addrs) in &map { let dc_addrs: Vec = addrs .iter() .map(|(ip, port)| SocketAddr::new(*ip, *port)) .collect(); - if !dc_addrs.iter().any(|a| current.contains(a)) { + let dc_endpoints: HashSet = dc_addrs.iter().copied().collect(); + if self.active_writer_count_for_dc_endpoints(*dc, &dc_endpoints).await == 0 { let mut shuffled = dc_addrs.clone(); shuffled.shuffle(&mut rand::rng()); for addr in shuffled { - if self.connect_one(addr, rng).await.is_ok() { + if self.connect_one_for_dc(addr, *dc, rng).await.is_ok() { break; } } } } - if !self.decision.effective_multipath && !current.is_empty() { + if !self.decision.effective_multipath && self.connection_count() > 0 { break; } } @@ -174,26 +167,30 @@ impl MePool { core.saturating_add(rand::rng().random_range(0..=jitter)) } - async fn fresh_writer_count_for_endpoints( + async fn fresh_writer_count_for_dc_endpoints( &self, generation: u64, + dc: i32, endpoints: &HashSet, ) -> usize { let ws = self.writers.read().await; ws.iter() .filter(|w| !w.draining.load(Ordering::Relaxed)) .filter(|w| w.generation == generation) + .filter(|w| w.writer_dc == dc) .filter(|w| endpoints.contains(&w.addr)) .count() } - pub(super) async fn active_writer_count_for_endpoints( + pub(super) async fn active_writer_count_for_dc_endpoints( &self, + dc: i32, endpoints: &HashSet, ) -> usize { let ws = self.writers.read().await; ws.iter() .filter(|w| !w.draining.load(Ordering::Relaxed)) + .filter(|w| w.writer_dc == dc) .filter(|w| endpoints.contains(&w.addr)) .count() } @@ -220,7 +217,7 @@ impl MePool { let required = self.required_writers_for_dc(endpoint_list.len()); let mut completed = false; let mut last_fresh_count = self - .fresh_writer_count_for_endpoints(generation, endpoints) + .fresh_writer_count_for_dc_endpoints(generation, *dc, endpoints) .await; for pass_idx in 0..total_passes { @@ -247,6 +244,7 @@ impl MePool { let connected = self .connect_endpoints_round_robin_with_generation_contour( + *dc, &endpoint_list, rng, generation, @@ -265,7 +263,7 @@ impl MePool { } last_fresh_count = self - .fresh_writer_count_for_endpoints(generation, endpoints) + .fresh_writer_count_for_dc_endpoints(generation, *dc, endpoints) .await; if last_fresh_count >= required { completed = true; @@ -377,10 +375,10 @@ impl MePool { } let writers = self.writers.read().await; - let active_writer_addrs: HashSet = writers + let active_writer_addrs: HashSet<(i32, SocketAddr)> = writers .iter() .filter(|w| !w.draining.load(Ordering::Relaxed)) - .map(|w| w.addr) + .map(|w| (w.writer_dc, w.addr)) .collect(); let min_ratio = Self::permille_to_ratio( self.me_pool_min_fresh_ratio_permille @@ -410,6 +408,7 @@ impl MePool { .iter() .filter(|w| !w.draining.load(Ordering::Relaxed)) .filter(|w| w.generation == generation) + .filter(|w| w.writer_dc == *dc) .filter(|w| endpoints.contains(&w.addr)) .count(); if fresh_count < required { @@ -438,9 +437,9 @@ impl MePool { self.promote_warm_generation_to_active(generation).await; } - let desired_addrs: HashSet = desired_by_dc - .values() - .flat_map(|set| set.iter().copied()) + let desired_addrs: HashSet<(i32, SocketAddr)> = desired_by_dc + .iter() + .flat_map(|(dc, set)| set.iter().copied().map(|addr| (*dc, addr))) .collect(); let stale_writer_ids: Vec = writers @@ -450,7 +449,7 @@ impl MePool { if hardswap { w.generation < generation } else { - !desired_addrs.contains(&w.addr) + !desired_addrs.contains(&(w.writer_dc, w.addr)) } }) .map(|w| w.id) diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 46346b5..2922ed8 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -130,19 +130,18 @@ impl MePool { } let writers = self.writers.read().await.clone(); - let mut live_writers_by_endpoint = HashMap::::new(); + let mut live_writers_by_dc = HashMap::::new(); for writer in writers { if writer.draining.load(Ordering::Relaxed) { continue; } - *live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1; + if let Ok(dc) = i16::try_from(writer.writer_dc) { + *live_writers_by_dc.entry(dc).or_insert(0) += 1; + } } - for endpoints in endpoints_by_dc.values() { - let alive: usize = endpoints - .iter() - .map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0)) - .sum(); + for dc in endpoints_by_dc.keys() { + let alive = live_writers_by_dc.get(dc).copied().unwrap_or(0); if alive == 0 { return false; } @@ -168,24 +167,23 @@ impl MePool { } let writers = self.writers.read().await.clone(); - let mut live_writers_by_endpoint = HashMap::::new(); + let mut live_writers_by_dc = HashMap::::new(); for writer in writers { if writer.draining.load(Ordering::Relaxed) { continue; } - *live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1; + if let Ok(dc) = i16::try_from(writer.writer_dc) { + *live_writers_by_dc.entry(dc).or_insert(0) += 1; + } } - for endpoints in endpoints_by_dc.values() { + for (dc, endpoints) in endpoints_by_dc { let endpoint_count = endpoints.len(); if endpoint_count == 0 { return false; } let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false); - let alive: usize = endpoints - .iter() - .map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0)) - .sum(); + let alive = live_writers_by_dc.get(&dc).copied().unwrap_or(0); if alive < required { return false; } @@ -207,13 +205,6 @@ impl MePool { extend_signed_endpoints(&mut endpoints_by_dc, map); } - let mut endpoint_to_dc = HashMap::>::new(); - for (dc, endpoints) in &endpoints_by_dc { - for endpoint in endpoints { - endpoint_to_dc.entry(*endpoint).or_default().insert(*dc); - } - } - let configured_dc_groups = endpoints_by_dc.len(); let configured_endpoints = endpoints_by_dc.values().map(BTreeSet::len).sum(); @@ -227,20 +218,14 @@ impl MePool { let rtt = self.rtt_stats.lock().await.clone(); let writers = self.writers.read().await.clone(); - let mut live_writers_by_endpoint = HashMap::::new(); + let mut live_writers_by_dc_endpoint = HashMap::<(i16, SocketAddr), usize>::new(); let mut live_writers_by_dc = HashMap::::new(); let mut dc_rtt_agg = HashMap::::new(); let mut writer_rows = Vec::::with_capacity(writers.len()); for writer in writers { let endpoint = writer.addr; - let dc = endpoint_to_dc.get(&endpoint).and_then(|dcs| { - if dcs.len() == 1 { - dcs.iter().next().copied() - } else { - None - } - }); + let dc = i16::try_from(writer.writer_dc).ok(); let draining = writer.draining.load(Ordering::Relaxed); let degraded = writer.degraded.load(Ordering::Relaxed); let bound_clients = activity @@ -259,8 +244,10 @@ impl MePool { }; if !draining { - *live_writers_by_endpoint.entry(endpoint).or_insert(0) += 1; if let Some(dc_idx) = dc { + *live_writers_by_dc_endpoint + .entry((dc_idx, endpoint)) + .or_insert(0) += 1; *live_writers_by_dc.entry(dc_idx).or_insert(0) += 1; if let Some(ema_ms) = rtt_ema_ms { let entry = dc_rtt_agg.entry(dc_idx).or_insert((0.0, 0)); @@ -298,7 +285,7 @@ impl MePool { let endpoint_count = endpoints.len(); let dc_available_endpoints = endpoints .iter() - .filter(|endpoint| live_writers_by_endpoint.contains_key(endpoint)) + .filter(|endpoint| live_writers_by_dc_endpoint.contains_key(&(dc, **endpoint))) .count(); let base_required = self.required_writers_for_dc(endpoint_count); let dc_required_writers = diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 17ef331..244a08e 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -50,11 +50,6 @@ impl MePool { } } - pub(crate) async fn connect_one(self: &Arc, addr: SocketAddr, rng: &SecureRandom) -> Result<()> { - let writer_dc = self.resolve_dc_for_endpoint(addr).await; - self.connect_one_for_dc(addr, writer_dc, rng).await - } - pub(crate) async fn connect_one_for_dc( self: &Arc, addr: SocketAddr, diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index ccaad4a..07d39f6 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -54,6 +54,9 @@ impl MePool { }; let no_writer_mode = MeRouteNoWriterMode::from_u8(self.me_route_no_writer_mode.load(Ordering::Relaxed)); + let (routed_dc, unknown_target_dc) = self + .resolve_target_dc_for_routing(target_dc as i32) + .await; let mut no_writer_deadline: Option = None; let mut emergency_attempts = 0u32; let mut async_recovery_triggered = false; @@ -91,9 +94,9 @@ impl MePool { let deadline = *no_writer_deadline.get_or_insert_with(|| { Instant::now() + self.me_route_no_writer_wait }); - if !async_recovery_triggered { + if !async_recovery_triggered && !unknown_target_dc { let triggered = - self.trigger_async_recovery_for_target_dc(target_dc).await; + self.trigger_async_recovery_for_target_dc(routed_dc).await; if !triggered { self.trigger_async_recovery_global().await; } @@ -109,31 +112,34 @@ impl MePool { } MeRouteNoWriterMode::InlineRecoveryLegacy => { self.stats.increment_me_inline_recovery_total(); - for _ in 0..self.me_route_inline_recovery_attempts.max(1) { - for family in self.family_order() { - let map = match family { - IpFamily::V4 => self.proxy_map_v4.read().await.clone(), - IpFamily::V6 => self.proxy_map_v6.read().await.clone(), - }; - for (_dc, addrs) in &map { - for (ip, port) in addrs { - let addr = SocketAddr::new(*ip, *port); - let _ = self.connect_one(addr, self.rng.as_ref()).await; + if !unknown_target_dc { + for _ in 0..self.me_route_inline_recovery_attempts.max(1) { + for family in self.family_order() { + let map = match family { + IpFamily::V4 => self.proxy_map_v4.read().await.clone(), + IpFamily::V6 => self.proxy_map_v6.read().await.clone(), + }; + for (dc, addrs) in &map { + for (ip, port) in addrs { + let addr = SocketAddr::new(*ip, *port); + let _ = self + .connect_one_for_dc(addr, *dc, self.rng.as_ref()) + .await; + } } } - } - if !self.writers.read().await.is_empty() { - break; + if !self.writers.read().await.is_empty() { + break; + } } } + if !self.writers.read().await.is_empty() { continue; } - let waiter = self.writer_available.notified(); - if tokio::time::timeout(self.me_route_inline_recovery_wait, waiter) - .await - .is_err() - { + let deadline = *no_writer_deadline + .get_or_insert_with(|| Instant::now() + self.me_route_inline_recovery_wait); + if !self.wait_for_writer_until(deadline).await { if !self.writers.read().await.is_empty() { continue; } @@ -145,13 +151,15 @@ impl MePool { continue; } MeRouteNoWriterMode::HybridAsyncPersistent => { - self.maybe_trigger_hybrid_recovery( - target_dc, - &mut hybrid_recovery_round, - &mut hybrid_last_recovery_at, - hybrid_wait_current, - ) - .await; + if !unknown_target_dc { + self.maybe_trigger_hybrid_recovery( + routed_dc, + &mut hybrid_recovery_round, + &mut hybrid_last_recovery_at, + hybrid_wait_current, + ) + .await; + } let deadline = Instant::now() + hybrid_wait_current; let _ = self.wait_for_writer_until(deadline).await; hybrid_wait_current = @@ -165,11 +173,11 @@ impl MePool { }; let mut candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .candidate_indices_for_dc(&writers_snapshot, routed_dc, false) .await; if candidate_indices.is_empty() { candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .candidate_indices_for_dc(&writers_snapshot, routed_dc, true) .await; } if candidate_indices.is_empty() { @@ -178,14 +186,14 @@ impl MePool { let deadline = *no_writer_deadline.get_or_insert_with(|| { Instant::now() + self.me_route_no_writer_wait }); - if !async_recovery_triggered { - let triggered = self.trigger_async_recovery_for_target_dc(target_dc).await; + if !async_recovery_triggered && !unknown_target_dc { + let triggered = self.trigger_async_recovery_for_target_dc(routed_dc).await; if !triggered { self.trigger_async_recovery_global().await; } async_recovery_triggered = true; } - if self.wait_for_candidate_until(target_dc, deadline).await { + if self.wait_for_candidate_until(routed_dc, deadline).await { continue; } self.stats.increment_me_no_writer_failfast_total(); @@ -195,15 +203,24 @@ impl MePool { } MeRouteNoWriterMode::InlineRecoveryLegacy => { self.stats.increment_me_inline_recovery_total(); + if unknown_target_dc { + let deadline = *no_writer_deadline + .get_or_insert_with(|| Instant::now() + self.me_route_inline_recovery_wait); + if self.wait_for_candidate_until(routed_dc, deadline).await { + continue; + } + self.stats.increment_me_no_writer_failfast_total(); + return Err(ProxyError::Proxy("No ME writers available for target DC".into())); + } if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) { self.stats.increment_me_no_writer_failfast_total(); return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } emergency_attempts += 1; - let mut endpoints = self.endpoint_candidates_for_target_dc(target_dc).await; + let mut endpoints = self.endpoint_candidates_for_target_dc(routed_dc).await; endpoints.shuffle(&mut rand::rng()); for addr in endpoints { - if self.connect_one(addr, self.rng.as_ref()).await.is_ok() { + if self.connect_one_for_dc(addr, routed_dc, self.rng.as_ref()).await.is_ok() { break; } } @@ -212,11 +229,11 @@ impl MePool { writers_snapshot = ws2.clone(); drop(ws2); candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .candidate_indices_for_dc(&writers_snapshot, routed_dc, false) .await; if candidate_indices.is_empty() { candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .candidate_indices_for_dc(&writers_snapshot, routed_dc, true) .await; } if candidate_indices.is_empty() { @@ -224,15 +241,17 @@ impl MePool { } } MeRouteNoWriterMode::HybridAsyncPersistent => { - self.maybe_trigger_hybrid_recovery( - target_dc, - &mut hybrid_recovery_round, - &mut hybrid_last_recovery_at, - hybrid_wait_current, - ) - .await; + if !unknown_target_dc { + self.maybe_trigger_hybrid_recovery( + routed_dc, + &mut hybrid_recovery_round, + &mut hybrid_last_recovery_at, + hybrid_wait_current, + ) + .await; + } let deadline = Instant::now() + hybrid_wait_current; - let _ = self.wait_for_candidate_until(target_dc, deadline).await; + let _ = self.wait_for_candidate_until(routed_dc, deadline).await; hybrid_wait_current = (hybrid_wait_current.saturating_mul(2)) .min(Duration::from_millis(400)); continue; @@ -382,32 +401,32 @@ impl MePool { !self.writers.read().await.is_empty() } - async fn wait_for_candidate_until(&self, target_dc: i16, deadline: Instant) -> bool { + async fn wait_for_candidate_until(&self, routed_dc: i32, deadline: Instant) -> bool { loop { - if self.has_candidate_for_target_dc(target_dc).await { + if self.has_candidate_for_target_dc(routed_dc).await { return true; } let now = Instant::now(); if now >= deadline { - return self.has_candidate_for_target_dc(target_dc).await; + return self.has_candidate_for_target_dc(routed_dc).await; } let waiter = self.writer_available.notified(); - if self.has_candidate_for_target_dc(target_dc).await { + if self.has_candidate_for_target_dc(routed_dc).await { return true; } let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { - return self.has_candidate_for_target_dc(target_dc).await; + return self.has_candidate_for_target_dc(routed_dc).await; } if tokio::time::timeout(remaining, waiter).await.is_err() { - return self.has_candidate_for_target_dc(target_dc).await; + return self.has_candidate_for_target_dc(routed_dc).await; } } } - async fn has_candidate_for_target_dc(&self, target_dc: i16) -> bool { + async fn has_candidate_for_target_dc(&self, routed_dc: i32) -> bool { let writers_snapshot = { let ws = self.writers.read().await; if ws.is_empty() { @@ -416,41 +435,41 @@ impl MePool { ws.clone() }; let mut candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, false) + .candidate_indices_for_dc(&writers_snapshot, routed_dc, false) .await; if candidate_indices.is_empty() { candidate_indices = self - .candidate_indices_for_dc(&writers_snapshot, target_dc, true) + .candidate_indices_for_dc(&writers_snapshot, routed_dc, true) .await; } !candidate_indices.is_empty() } - async fn trigger_async_recovery_for_target_dc(self: &Arc, target_dc: i16) -> bool { - let endpoints = self.endpoint_candidates_for_target_dc(target_dc).await; + async fn trigger_async_recovery_for_target_dc(self: &Arc, routed_dc: i32) -> bool { + let endpoints = self.endpoint_candidates_for_target_dc(routed_dc).await; if endpoints.is_empty() { return false; } self.stats.increment_me_async_recovery_trigger_total(); for addr in endpoints.into_iter().take(8) { - self.trigger_immediate_refill(addr); + self.trigger_immediate_refill_for_dc(addr, routed_dc); } true } async fn trigger_async_recovery_global(self: &Arc) { self.stats.increment_me_async_recovery_trigger_total(); - let mut seen = HashSet::::new(); + let mut seen = HashSet::<(i32, SocketAddr)>::new(); for family in self.family_order() { let map_guard = match family { IpFamily::V4 => self.proxy_map_v4.read().await, IpFamily::V6 => self.proxy_map_v6.read().await, }; - for addrs in map_guard.values() { + for (dc, addrs) in map_guard.iter() { for (ip, port) in addrs { let addr = SocketAddr::new(*ip, *port); - if seen.insert(addr) { - self.trigger_immediate_refill(addr); + if seen.insert((*dc, addr)) { + self.trigger_immediate_refill_for_dc(addr, *dc); } if seen.len() >= 8 { return; @@ -460,11 +479,9 @@ impl MePool { } } - async fn endpoint_candidates_for_target_dc(&self, target_dc: i16) -> Vec { - let key = target_dc as i32; + async fn endpoint_candidates_for_target_dc(&self, routed_dc: i32) -> Vec { let mut preferred = Vec::::new(); let mut seen = HashSet::::new(); - let lookup_keys = self.dc_lookup_chain_for_target(key); for family in self.family_order() { let map_guard = match family { @@ -472,14 +489,9 @@ impl MePool { IpFamily::V6 => self.proxy_map_v6.read().await, }; let mut family_selected = Vec::::new(); - for lookup in lookup_keys.iter().copied() { - if let Some(addrs) = map_guard.get(&lookup) { - for (ip, port) in addrs { - family_selected.push(SocketAddr::new(*ip, *port)); - } - } - if !family_selected.is_empty() { - break; + if let Some(addrs) = map_guard.get(&routed_dc) { + for (ip, port) in addrs { + family_selected.push(SocketAddr::new(*ip, *port)); } } for addr in family_selected { @@ -497,7 +509,7 @@ impl MePool { async fn maybe_trigger_hybrid_recovery( self: &Arc, - target_dc: i16, + routed_dc: i32, hybrid_recovery_round: &mut u32, hybrid_last_recovery_at: &mut Option, hybrid_wait_step: Duration, @@ -509,7 +521,7 @@ impl MePool { } let round = *hybrid_recovery_round; - let target_triggered = self.trigger_async_recovery_for_target_dc(target_dc).await; + let target_triggered = self.trigger_async_recovery_for_target_dc(routed_dc).await; if !target_triggered || round % HYBRID_GLOBAL_BURST_PERIOD_ROUNDS == 0 { self.trigger_async_recovery_global().await; } @@ -576,12 +588,10 @@ impl MePool { pub(super) async fn candidate_indices_for_dc( &self, writers: &[super::pool::MeWriter], - target_dc: i16, + routed_dc: i32, include_warm: bool, ) -> Vec { - let key = target_dc as i32; let mut preferred = HashSet::::new(); - let lookup_keys = self.dc_lookup_chain_for_target(key); for family in self.family_order() { let map_guard = match family { @@ -589,13 +599,8 @@ impl MePool { IpFamily::V6 => self.proxy_map_v6.read().await, }; let mut family_selected = Vec::::new(); - for lookup in lookup_keys.iter().copied() { - if let Some(v) = map_guard.get(&lookup) { - family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); - } - if !family_selected.is_empty() { - break; - } + if let Some(v) = map_guard.get(&routed_dc) { + family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); } for endpoint in family_selected { preferred.insert(endpoint); @@ -617,7 +622,7 @@ impl MePool { if !self.writer_eligible_for_selection(w, include_warm) { continue; } - if preferred.contains(&w.addr) { + if w.writer_dc == routed_dc && preferred.contains(&w.addr) { out.push(idx); } } From 9e7f80b9b3de99c19a0d3b664a1b8ee51ff5dc67 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:57:58 +0300 Subject: [PATCH 126/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 076f00b..aa57699 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.7" +version = "3.3.8" edition = "2024" [dependencies] From e1f3efb619e49a42c4b95443fd2046b0a87cfaf0 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 15:37:49 +0300 Subject: [PATCH 127/261] API from main --- src/api/mod.rs | 8 +-- src/api/runtime_min.rs | 6 +- src/api/runtime_stats.rs | 2 +- src/api/runtime_zero.rs | 7 +- src/main.rs | 140 +++++++++++++++++++++------------------ 5 files changed, 91 insertions(+), 72 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index a705a46..e63de29 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -11,7 +11,7 @@ use hyper::server::conn::http1; use hyper::service::service_fn; use hyper::{Method, Request, Response, StatusCode}; use tokio::net::TcpListener; -use tokio::sync::{Mutex, watch}; +use tokio::sync::{Mutex, RwLock, watch}; use tracing::{debug, info, warn}; use crate::config::ProxyConfig; @@ -67,7 +67,7 @@ pub(super) struct ApiRuntimeState { pub(super) struct ApiShared { pub(super) stats: Arc, pub(super) ip_tracker: Arc, - pub(super) me_pool: Option>, + pub(super) me_pool: Arc>>>, pub(super) upstream_manager: Arc, pub(super) config_path: PathBuf, pub(super) startup_detected_ip_v4: Option, @@ -91,7 +91,7 @@ pub async fn serve( listen: SocketAddr, stats: Arc, ip_tracker: Arc, - me_pool: Option>, + me_pool: Arc>>>, upstream_manager: Arc, config_rx: watch::Receiver>, admission_rx: watch::Receiver, @@ -248,7 +248,7 @@ async fn handle( } ("GET", "/v1/runtime/gates") => { let revision = current_revision(&shared.config_path).await?; - let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()); + let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()).await; Ok(success_response(StatusCode::OK, data, revision)) } ("GET", "/v1/limits/effective") => { diff --git a/src/api/runtime_min.rs b/src/api/runtime_min.rs index 96270df..d3066a3 100644 --- a/src/api/runtime_min.rs +++ b/src/api/runtime_min.rs @@ -260,7 +260,7 @@ pub(super) fn build_security_whitelist_data(cfg: &ProxyConfig) -> SecurityWhitel pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> RuntimeMePoolStateData { let now_epoch_secs = now_epoch_secs(); - let Some(pool) = &shared.me_pool else { + let Some(pool) = shared.me_pool.read().await.clone() else { return RuntimeMePoolStateData { enabled: false, reason: Some(SOURCE_UNAVAILABLE_REASON), @@ -350,7 +350,7 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> RuntimeMeQualityData { let now_epoch_secs = now_epoch_secs(); - let Some(pool) = &shared.me_pool else { + let Some(pool) = shared.me_pool.read().await.clone() else { return RuntimeMeQualityData { enabled: false, reason: Some(SOURCE_UNAVAILABLE_REASON), @@ -486,7 +486,7 @@ pub(super) async fn build_runtime_upstream_quality_data( pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData { let now_epoch_secs = now_epoch_secs(); - let Some(pool) = &shared.me_pool else { + let Some(pool) = shared.me_pool.read().await.clone() else { return RuntimeNatStunData { enabled: false, reason: Some(SOURCE_UNAVAILABLE_REASON), diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index c69f817..7fae31d 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -297,7 +297,7 @@ async fn get_minimal_payload_cached( } } - let pool = shared.me_pool.as_ref()?; + let pool = shared.me_pool.read().await.clone()?; let status = pool.api_status_snapshot().await; let runtime = pool.api_runtime_snapshot().await; let generated_at_epoch_secs = status.generated_at_epoch_secs; diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index 61b6844..e184671 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -142,12 +142,17 @@ pub(super) fn build_system_info_data( } } -pub(super) fn build_runtime_gates_data(shared: &ApiShared, cfg: &ProxyConfig) -> RuntimeGatesData { +pub(super) async fn build_runtime_gates_data( + shared: &ApiShared, + cfg: &ProxyConfig, +) -> RuntimeGatesData { let me_runtime_ready = if !cfg.general.use_middle_proxy { true } else { shared .me_pool + .read() + .await .as_ref() .map(|pool| pool.is_runtime_ready()) .unwrap_or(false) diff --git a/src/main.rs b/src/main.rs index ca24d0e..a46be25 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use rand::Rng; use tokio::net::TcpListener; use tokio::signal; -use tokio::sync::{Semaphore, mpsc, watch}; +use tokio::sync::{RwLock, Semaphore, mpsc, watch}; use tracing::{debug, error, info, warn}; use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; #[cfg(unix)] @@ -473,6 +473,70 @@ async fn main() -> std::result::Result<(), Box> { config.general.upstream_connect_failfast_hard_errors, stats.clone(), )); + let ip_tracker = Arc::new(UserIpTracker::new()); + ip_tracker.load_limits(&config.access.user_max_unique_ips).await; + ip_tracker + .set_limit_policy( + config.access.user_max_unique_ips_mode, + config.access.user_max_unique_ips_window_secs, + ) + .await; + if !config.access.user_max_unique_ips.is_empty() { + info!( + "IP limits configured for {} users", + config.access.user_max_unique_ips.len() + ); + } + if !config.network.dns_overrides.is_empty() { + info!( + "Runtime DNS overrides configured: {} entries", + config.network.dns_overrides.len() + ); + } + + let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone())); + let initial_admission_open = !config.general.use_middle_proxy; + let (admission_tx, admission_rx) = watch::channel(initial_admission_open); + let api_me_pool = Arc::new(RwLock::new(None::>)); + + if config.server.api.enabled { + let listen = match config.server.api.listen.parse::() { + Ok(listen) => listen, + Err(error) => { + warn!( + error = %error, + listen = %config.server.api.listen, + "Invalid server.api.listen; API is disabled" + ); + SocketAddr::from(([127, 0, 0, 1], 0)) + } + }; + if listen.port() != 0 { + let stats_api = stats.clone(); + let ip_tracker_api = ip_tracker.clone(); + let me_pool_api = api_me_pool.clone(); + let upstream_manager_api = upstream_manager.clone(); + let config_rx_api = api_config_rx.clone(); + let admission_rx_api = admission_rx.clone(); + let config_path_api = std::path::PathBuf::from(&config_path); + tokio::spawn(async move { + api::serve( + listen, + stats_api, + ip_tracker_api, + me_pool_api, + upstream_manager_api, + config_rx_api, + admission_rx_api, + config_path_api, + None, + None, + process_started_at_epoch_secs, + ) + .await; + }); + } + } let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); tls_domains.push(config.censorship.tls_domain.clone()); @@ -620,26 +684,6 @@ async fn main() -> std::result::Result<(), Box> { let beobachten = Arc::new(BeobachtenStore::new()); let rng = Arc::new(SecureRandom::new()); - // IP Tracker initialization - let ip_tracker = Arc::new(UserIpTracker::new()); - ip_tracker.load_limits(&config.access.user_max_unique_ips).await; - ip_tracker - .set_limit_policy( - config.access.user_max_unique_ips_mode, - config.access.user_max_unique_ips_window_secs, - ) - .await; - - if !config.access.user_max_unique_ips.is_empty() { - info!("IP limits configured for {} users", config.access.user_max_unique_ips.len()); - } - if !config.network.dns_overrides.is_empty() { - info!( - "Runtime DNS overrides configured: {} entries", - config.network.dns_overrides.len() - ); - } - // Connection concurrency limit let max_connections = Arc::new(Semaphore::new(10_000)); @@ -1074,6 +1118,7 @@ async fn main() -> std::result::Result<(), Box> { if let Some(ref pool) = me_pool { pool.set_runtime_ready(true); } + *api_me_pool.write().await = me_pool.clone(); // Background tasks let um_clone = upstream_manager.clone(); @@ -1114,6 +1159,17 @@ async fn main() -> std::result::Result<(), Box> { detected_ip_v4, detected_ip_v6, ); + let mut config_rx_api_bridge = config_rx.clone(); + let api_config_tx_bridge = api_config_tx.clone(); + tokio::spawn(async move { + loop { + if config_rx_api_bridge.changed().await.is_err() { + break; + } + let cfg = config_rx_api_bridge.borrow_and_update().clone(); + api_config_tx_bridge.send_replace(cfg); + } + }); let stats_policy = stats.clone(); let mut config_rx_policy = config_rx.clone(); @@ -1345,7 +1401,6 @@ async fn main() -> std::result::Result<(), Box> { print_proxy_links(&host, port, &config); } - let (admission_tx, admission_rx) = watch::channel(true); if config.general.use_middle_proxy { if let Some(pool) = me_pool.as_ref() { let initial_open = pool.admission_ready_conditional_cast().await; @@ -1546,47 +1601,6 @@ async fn main() -> std::result::Result<(), Box> { }); } - if config.server.api.enabled { - let listen = match config.server.api.listen.parse::() { - Ok(listen) => listen, - Err(error) => { - warn!( - error = %error, - listen = %config.server.api.listen, - "Invalid server.api.listen; API is disabled" - ); - SocketAddr::from(([127, 0, 0, 1], 0)) - } - }; - if listen.port() != 0 { - let stats = stats.clone(); - let ip_tracker_api = ip_tracker.clone(); - let me_pool_api = me_pool.clone(); - let upstream_manager_api = upstream_manager.clone(); - let config_rx_api = config_rx.clone(); - let admission_rx_api = admission_rx.clone(); - let config_path_api = std::path::PathBuf::from(&config_path); - let startup_detected_ip_v4 = detected_ip_v4; - let startup_detected_ip_v6 = detected_ip_v6; - tokio::spawn(async move { - api::serve( - listen, - stats, - ip_tracker_api, - me_pool_api, - upstream_manager_api, - config_rx_api, - admission_rx_api, - config_path_api, - startup_detected_ip_v4, - startup_detected_ip_v6, - process_started_at_epoch_secs, - ) - .await; - }); - } - } - for (listener, listener_proxy_protocol) in listeners { let mut config_rx: tokio::sync::watch::Receiver> = config_rx.clone(); let mut admission_rx_tcp = admission_rx.clone(); From 5ac0ef1ffdca0b000e4dcb2a792ca7223ea7bde8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:18:09 +0300 Subject: [PATCH 128/261] Init in API --- src/api/mod.rs | 11 ++ src/api/runtime_init.rs | 186 ++++++++++++++++++ src/api/runtime_zero.rs | 8 + src/main.rs | 406 +++++++++++++++++++++++++++++++++++++++- src/startup.rs | 373 ++++++++++++++++++++++++++++++++++++ 5 files changed, 983 insertions(+), 1 deletion(-) create mode 100644 src/api/runtime_init.rs create mode 100644 src/startup.rs diff --git a/src/api/mod.rs b/src/api/mod.rs index e63de29..63fafad 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -16,6 +16,7 @@ use tracing::{debug, info, warn}; use crate::config::ProxyConfig; use crate::ip_tracker::UserIpTracker; +use crate::startup::StartupTracker; use crate::stats::Stats; use crate::transport::middle_proxy::MePool; use crate::transport::UpstreamManager; @@ -25,6 +26,7 @@ mod events; mod http_utils; mod model; mod runtime_edge; +mod runtime_init; mod runtime_min; mod runtime_stats; mod runtime_watch; @@ -41,6 +43,7 @@ use runtime_edge::{ EdgeConnectionsCacheEntry, build_runtime_connections_summary_data, build_runtime_events_recent_data, }; +use runtime_init::build_runtime_initialization_data; use runtime_min::{ build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data, build_runtime_upstream_quality_data, build_security_whitelist_data, @@ -79,6 +82,7 @@ pub(super) struct ApiShared { pub(super) runtime_events: Arc, pub(super) request_id: Arc, pub(super) runtime_state: Arc, + pub(super) startup_tracker: Arc, } impl ApiShared { @@ -99,6 +103,7 @@ pub async fn serve( startup_detected_ip_v4: Option, startup_detected_ip_v6: Option, process_started_at_epoch_secs: u64, + startup_tracker: Arc, ) { let listener = match TcpListener::bind(listen).await { Ok(listener) => listener, @@ -138,6 +143,7 @@ pub async fn serve( )), request_id: Arc::new(AtomicU64::new(1)), runtime_state: runtime_state.clone(), + startup_tracker, }); spawn_runtime_watchers( @@ -251,6 +257,11 @@ async fn handle( let data = build_runtime_gates_data(shared.as_ref(), cfg.as_ref()).await; Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/runtime/initialization") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_initialization_data(shared.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/limits/effective") => { let revision = current_revision(&shared.config_path).await?; let data = build_limits_effective_data(cfg.as_ref()); diff --git a/src/api/runtime_init.rs b/src/api/runtime_init.rs new file mode 100644 index 0000000..4bd8943 --- /dev/null +++ b/src/api/runtime_init.rs @@ -0,0 +1,186 @@ +use serde::Serialize; + +use crate::startup::{ + COMPONENT_ME_CONNECTIVITY_PING, COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, + COMPONENT_ME_PROXY_CONFIG_V4, COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, + StartupComponentStatus, StartupMeStatus, compute_progress_pct, +}; + +use super::ApiShared; + +#[derive(Serialize)] +pub(super) struct RuntimeInitializationComponentData { + pub(super) id: &'static str, + pub(super) title: &'static str, + pub(super) status: &'static str, + pub(super) started_at_epoch_ms: Option, + pub(super) finished_at_epoch_ms: Option, + pub(super) duration_ms: Option, + pub(super) attempts: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) details: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeInitializationMeData { + pub(super) status: &'static str, + pub(super) current_stage: String, + pub(super) progress_pct: f64, + pub(super) init_attempt: u32, + pub(super) retry_limit: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_error: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeInitializationData { + pub(super) status: &'static str, + pub(super) degraded: bool, + pub(super) current_stage: String, + pub(super) progress_pct: f64, + pub(super) started_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) ready_at_epoch_secs: Option, + pub(super) total_elapsed_ms: u64, + pub(super) transport_mode: String, + pub(super) me: RuntimeInitializationMeData, + pub(super) components: Vec, +} + +#[derive(Clone)] +pub(super) struct RuntimeStartupSummaryData { + pub(super) status: &'static str, + pub(super) stage: String, + pub(super) progress_pct: f64, +} + +pub(super) async fn build_runtime_startup_summary(shared: &ApiShared) -> RuntimeStartupSummaryData { + let snapshot = shared.startup_tracker.snapshot().await; + let me_pool_progress = current_me_pool_stage_progress(shared).await; + let progress_pct = compute_progress_pct(&snapshot, me_pool_progress); + RuntimeStartupSummaryData { + status: snapshot.status.as_str(), + stage: snapshot.current_stage, + progress_pct, + } +} + +pub(super) async fn build_runtime_initialization_data( + shared: &ApiShared, +) -> RuntimeInitializationData { + let snapshot = shared.startup_tracker.snapshot().await; + let me_pool_progress = current_me_pool_stage_progress(shared).await; + let progress_pct = compute_progress_pct(&snapshot, me_pool_progress); + let me_progress_pct = compute_me_progress_pct(&snapshot, me_pool_progress); + + RuntimeInitializationData { + status: snapshot.status.as_str(), + degraded: snapshot.degraded, + current_stage: snapshot.current_stage, + progress_pct, + started_at_epoch_secs: snapshot.started_at_epoch_secs, + ready_at_epoch_secs: snapshot.ready_at_epoch_secs, + total_elapsed_ms: snapshot.total_elapsed_ms, + transport_mode: snapshot.transport_mode, + me: RuntimeInitializationMeData { + status: snapshot.me.status.as_str(), + current_stage: snapshot.me.current_stage, + progress_pct: me_progress_pct, + init_attempt: snapshot.me.init_attempt, + retry_limit: snapshot.me.retry_limit, + last_error: snapshot.me.last_error, + }, + components: snapshot + .components + .into_iter() + .map(|component| RuntimeInitializationComponentData { + id: component.id, + title: component.title, + status: component.status.as_str(), + started_at_epoch_ms: component.started_at_epoch_ms, + finished_at_epoch_ms: component.finished_at_epoch_ms, + duration_ms: component.duration_ms, + attempts: component.attempts, + details: component.details, + }) + .collect(), + } +} + +fn compute_me_progress_pct( + snapshot: &crate::startup::StartupSnapshot, + me_pool_progress: Option, +) -> f64 { + match snapshot.me.status { + StartupMeStatus::Pending => 0.0, + StartupMeStatus::Ready | StartupMeStatus::Failed | StartupMeStatus::Skipped => 100.0, + StartupMeStatus::Initializing => { + let mut total_weight = 0.0f64; + let mut completed_weight = 0.0f64; + for component in &snapshot.components { + if !is_me_component(component.id) { + continue; + } + total_weight += component.weight; + let unit_progress = match component.status { + StartupComponentStatus::Pending => 0.0, + StartupComponentStatus::Running => { + if component.id == COMPONENT_ME_POOL_INIT_STAGE1 { + me_pool_progress.unwrap_or(0.0).clamp(0.0, 1.0) + } else { + 0.0 + } + } + StartupComponentStatus::Ready + | StartupComponentStatus::Failed + | StartupComponentStatus::Skipped => 1.0, + }; + completed_weight += component.weight * unit_progress; + } + if total_weight <= f64::EPSILON { + 0.0 + } else { + ((completed_weight / total_weight) * 100.0).clamp(0.0, 100.0) + } + } + } +} + +fn is_me_component(component_id: &str) -> bool { + matches!( + component_id, + COMPONENT_ME_SECRET_FETCH + | COMPONENT_ME_PROXY_CONFIG_V4 + | COMPONENT_ME_PROXY_CONFIG_V6 + | COMPONENT_ME_POOL_CONSTRUCT + | COMPONENT_ME_POOL_INIT_STAGE1 + | COMPONENT_ME_CONNECTIVITY_PING + ) +} + +async fn current_me_pool_stage_progress(shared: &ApiShared) -> Option { + let snapshot = shared.startup_tracker.snapshot().await; + if snapshot.me.status != StartupMeStatus::Initializing { + return None; + } + + let pool = shared.me_pool.read().await.clone()?; + let status = pool.api_status_snapshot().await; + let configured_dc_groups = status.configured_dc_groups; + let covered_dc_groups = status + .dcs + .iter() + .filter(|dc| dc.alive_writers > 0) + .count(); + + let dc_coverage = ratio_01(covered_dc_groups, configured_dc_groups); + let writer_coverage = ratio_01(status.alive_writers, status.required_writers); + Some((0.7 * dc_coverage + 0.3 * writer_coverage).clamp(0.0, 1.0)) +} + +fn ratio_01(part: usize, total: usize) -> f64 { + if total == 0 { + return 0.0; + } + ((part as f64) / (total as f64)).clamp(0.0, 1.0) +} diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index e184671..8def2a9 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -5,6 +5,7 @@ use serde::Serialize; use crate::config::{MeFloorMode, ProxyConfig, UserMaxUniqueIpsMode}; use super::ApiShared; +use super::runtime_init::build_runtime_startup_summary; #[derive(Serialize)] pub(super) struct SystemInfoData { @@ -34,6 +35,9 @@ pub(super) struct RuntimeGatesData { pub(super) me_runtime_ready: bool, pub(super) me2dc_fallback_enabled: bool, pub(super) use_middle_proxy: bool, + pub(super) startup_status: &'static str, + pub(super) startup_stage: String, + pub(super) startup_progress_pct: f64, } #[derive(Serialize)] @@ -146,6 +150,7 @@ pub(super) async fn build_runtime_gates_data( shared: &ApiShared, cfg: &ProxyConfig, ) -> RuntimeGatesData { + let startup_summary = build_runtime_startup_summary(shared).await; let me_runtime_ready = if !cfg.general.use_middle_proxy { true } else { @@ -164,6 +169,9 @@ pub(super) async fn build_runtime_gates_data( me_runtime_ready, me2dc_fallback_enabled: cfg.general.me2dc_fallback, use_middle_proxy: cfg.general.use_middle_proxy, + startup_status: startup_summary.status, + startup_stage: startup_summary.stage, + startup_progress_pct: startup_summary.progress_pct, } } diff --git a/src/main.rs b/src/main.rs index a46be25..cc6f8ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,7 @@ mod protocol; mod proxy; mod stats; mod stream; +mod startup; mod transport; mod tls_front; mod util; @@ -39,6 +40,14 @@ use crate::proxy::ClientHandler; use crate::stats::beobachten::BeobachtenStore; use crate::stats::telemetry::TelemetryPolicy; use crate::stats::{ReplayChecker, Stats}; +use crate::startup::{ + COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, COMPONENT_CONFIG_WATCHER_START, + COMPONENT_DC_CONNECTIVITY_PING, COMPONENT_LISTENERS_BIND, COMPONENT_ME_CONNECTIVITY_PING, + COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, COMPONENT_ME_PROXY_CONFIG_V4, + COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, COMPONENT_METRICS_START, + COMPONENT_NETWORK_PROBE, COMPONENT_RUNTIME_READY, COMPONENT_TLS_FRONT_BOOTSTRAP, + COMPONENT_TRACING_INIT, StartupMeStatus, StartupTracker, +}; use crate::stream::BufferPool; use crate::transport::middle_proxy::{ MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line, @@ -373,6 +382,10 @@ async fn main() -> std::result::Result<(), Box> { .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); + let startup_tracker = Arc::new(StartupTracker::new(process_started_at_epoch_secs)); + startup_tracker + .start_component(COMPONENT_CONFIG_LOAD, Some("load and validate config".to_string())) + .await; let (config_path, cli_silent, cli_log_level) = parse_cli(); let mut config = match ProxyConfig::load(&config_path) { @@ -399,6 +412,9 @@ async fn main() -> std::result::Result<(), Box> { eprintln!("[telemt] Invalid network.dns_overrides: {}", e); std::process::exit(1); } + startup_tracker + .complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string())) + .await; let has_rust_log = std::env::var("RUST_LOG").is_ok(); let effective_log_level = if cli_silent { @@ -410,6 +426,9 @@ async fn main() -> std::result::Result<(), Box> { }; let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info")); + startup_tracker + .start_component(COMPONENT_TRACING_INIT, Some("initialize tracing subscriber".to_string())) + .await; // Configure color output based on config let fmt_layer = if config.general.disable_colors { @@ -422,6 +441,9 @@ async fn main() -> std::result::Result<(), Box> { .with(filter_layer) .with(fmt_layer) .init(); + startup_tracker + .complete_component(COMPONENT_TRACING_INIT, Some("tracing initialized".to_string())) + .await; info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION")); info!("Log level: {}", effective_log_level); @@ -498,6 +520,9 @@ async fn main() -> std::result::Result<(), Box> { let initial_admission_open = !config.general.use_middle_proxy; let (admission_tx, admission_rx) = watch::channel(initial_admission_open); let api_me_pool = Arc::new(RwLock::new(None::>)); + startup_tracker + .start_component(COMPONENT_API_BOOTSTRAP, Some("spawn API listener task".to_string())) + .await; if config.server.api.enabled { let listen = match config.server.api.listen.parse::() { @@ -519,6 +544,7 @@ async fn main() -> std::result::Result<(), Box> { let config_rx_api = api_config_rx.clone(); let admission_rx_api = admission_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); + let startup_tracker_api = startup_tracker.clone(); tokio::spawn(async move { api::serve( listen, @@ -532,10 +558,31 @@ async fn main() -> std::result::Result<(), Box> { None, None, process_started_at_epoch_secs, + startup_tracker_api, ) .await; }); + startup_tracker + .complete_component( + COMPONENT_API_BOOTSTRAP, + Some(format!("api task spawned on {}", listen)), + ) + .await; + } else { + startup_tracker + .skip_component( + COMPONENT_API_BOOTSTRAP, + Some("server.api.listen has zero port".to_string()), + ) + .await; } + } else { + startup_tracker + .skip_component( + COMPONENT_API_BOOTSTRAP, + Some("server.api.enabled is false".to_string()), + ) + .await; } let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); @@ -547,6 +594,12 @@ async fn main() -> std::result::Result<(), Box> { } // Start TLS front fetching in background immediately, in parallel with STUN probing. + startup_tracker + .start_component( + COMPONENT_TLS_FRONT_BOOTSTRAP, + Some("initialize TLS front cache/bootstrap tasks".to_string()), + ) + .await; let tls_cache: Option> = if config.censorship.tls_emulation { let cache = Arc::new(TlsFrontCache::new( &tls_domains, @@ -667,9 +720,26 @@ async fn main() -> std::result::Result<(), Box> { Some(cache) } else { + startup_tracker + .skip_component( + COMPONENT_TLS_FRONT_BOOTSTRAP, + Some("censorship.tls_emulation is false".to_string()), + ) + .await; None }; + if tls_cache.is_some() { + startup_tracker + .complete_component( + COMPONENT_TLS_FRONT_BOOTSTRAP, + Some("tls front cache is initialized".to_string()), + ) + .await; + } + startup_tracker + .start_component(COMPONENT_NETWORK_PROBE, Some("probe network capabilities".to_string())) + .await; let probe = run_probe( &config.network, config.general.middle_proxy_nat_probe, @@ -678,6 +748,12 @@ async fn main() -> std::result::Result<(), Box> { .await?; let decision = decide_network_capabilities(&config.network, &probe); log_probe_result(&probe, &decision); + startup_tracker + .complete_component( + COMPONENT_NETWORK_PROBE, + Some("network capabilities determined".to_string()), + ) + .await; let prefer_ipv6 = decision.prefer_ipv6(); let mut use_middle_proxy = config.general.use_middle_proxy; @@ -701,6 +777,59 @@ async fn main() -> std::result::Result<(), Box> { } } + if use_middle_proxy { + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_SECRET_FETCH) + .await; + startup_tracker + .start_component( + COMPONENT_ME_SECRET_FETCH, + Some("fetch proxy-secret from source/cache".to_string()), + ) + .await; + startup_tracker + .set_me_retry_limit(if !me2dc_fallback || me_init_retry_attempts == 0 { + "unlimited".to_string() + } else { + me_init_retry_attempts.to_string() + }) + .await; + } else { + startup_tracker + .set_me_status(StartupMeStatus::Skipped, "skipped") + .await; + startup_tracker + .skip_component( + COMPONENT_ME_SECRET_FETCH, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("middle proxy mode disabled".to_string()), + ) + .await; + } + // ===================================================================== // Middle Proxy initialization (if enabled) // ===================================================================== @@ -738,6 +867,9 @@ async fn main() -> std::result::Result<(), Box> { { Ok(proxy_secret) => break Some(proxy_secret), Err(e) => { + startup_tracker + .set_me_last_error(Some(e.to_string())) + .await; if me2dc_fallback { error!( error = %e, @@ -757,6 +889,12 @@ async fn main() -> std::result::Result<(), Box> { }; match proxy_secret { Some(proxy_secret) => { + startup_tracker + .complete_component( + COMPONENT_ME_SECRET_FETCH, + Some("proxy-secret loaded".to_string()), + ) + .await; info!( secret_len = proxy_secret.len(), key_sig = format_args!( @@ -775,6 +913,15 @@ async fn main() -> std::result::Result<(), Box> { "Proxy-secret loaded" ); + startup_tracker + .start_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("load startup proxy-config v4".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4) + .await; let cfg_v4 = load_startup_proxy_config_snapshot( "https://core.telegram.org/getProxyConfig", config.general.proxy_config_v4_cache_path.as_deref(), @@ -782,6 +929,30 @@ async fn main() -> std::result::Result<(), Box> { "getProxyConfig", ) .await; + if cfg_v4.is_some() { + startup_tracker + .complete_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("proxy-config v4 loaded".to_string()), + ) + .await; + } else { + startup_tracker + .fail_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("proxy-config v4 unavailable".to_string()), + ) + .await; + } + startup_tracker + .start_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("load startup proxy-config v6".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6) + .await; let cfg_v6 = load_startup_proxy_config_snapshot( "https://core.telegram.org/getProxyConfigV6", config.general.proxy_config_v6_cache_path.as_deref(), @@ -789,8 +960,32 @@ async fn main() -> std::result::Result<(), Box> { "getProxyConfigV6", ) .await; + if cfg_v6.is_some() { + startup_tracker + .complete_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("proxy-config v6 loaded".to_string()), + ) + .await; + } else { + startup_tracker + .fail_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("proxy-config v6 unavailable".to_string()), + ) + .await; + } if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) { + startup_tracker + .start_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("construct ME pool".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_CONSTRUCT) + .await; let pool = MePool::new( proxy_tag.clone(), proxy_secret, @@ -857,12 +1052,44 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_route_inline_recovery_attempts, config.general.me_route_inline_recovery_wait_ms, ); + startup_tracker + .complete_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("ME pool object created".to_string()), + ) + .await; + *api_me_pool.write().await = Some(pool.clone()); + startup_tracker + .start_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("initialize ME pool writers".to_string()), + ) + .await; + startup_tracker + .set_me_status( + StartupMeStatus::Initializing, + COMPONENT_ME_POOL_INIT_STAGE1, + ) + .await; let mut init_attempt: u32 = 0; loop { init_attempt = init_attempt.saturating_add(1); + startup_tracker.set_me_init_attempt(init_attempt).await; match pool.init(pool_size, &rng).await { Ok(()) => { + startup_tracker + .set_me_last_error(None) + .await; + startup_tracker + .complete_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME pool initialized".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Ready, "ready") + .await; info!( attempt = init_attempt, "Middle-End pool initialized successfully" @@ -882,8 +1109,20 @@ async fn main() -> std::result::Result<(), Box> { break Some(pool); } Err(e) => { + startup_tracker + .set_me_last_error(Some(e.to_string())) + .await; let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; if retries_limited && init_attempt >= me_init_retry_attempts { + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME init retry budget exhausted".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; error!( error = %e, attempt = init_attempt, @@ -923,10 +1162,60 @@ async fn main() -> std::result::Result<(), Box> { } } } else { + startup_tracker + .skip_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("ME configs are incomplete".to_string()), + ) + .await; + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME configs are incomplete".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; None } } - None => None, + None => { + startup_tracker + .fail_component( + COMPONENT_ME_SECRET_FETCH, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; + None + } } } else { None @@ -934,12 +1223,33 @@ async fn main() -> std::result::Result<(), Box> { // If ME failed to initialize, force direct-only mode. if me_pool.is_some() { + startup_tracker + .set_transport_mode("middle_proxy") + .await; + startup_tracker + .set_degraded(false) + .await; info!("Transport: Middle-End Proxy - all DC-over-RPC"); } else { let _ = use_middle_proxy; use_middle_proxy = false; // Make runtime config reflect direct-only mode for handlers. config.general.use_middle_proxy = false; + startup_tracker + .set_transport_mode("direct") + .await; + startup_tracker + .set_degraded(true) + .await; + if me2dc_fallback { + startup_tracker + .set_me_status(StartupMeStatus::Failed, "fallback_to_direct") + .await; + } else { + startup_tracker + .set_me_status(StartupMeStatus::Skipped, "skipped") + .await; + } info!("Transport: Direct DC - TCP - standard DC-over-TCP"); } @@ -954,6 +1264,21 @@ async fn main() -> std::result::Result<(), Box> { let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096)); // Middle-End ping before DC connectivity + if me_pool.is_some() { + startup_tracker + .start_component( + COMPONENT_ME_CONNECTIVITY_PING, + Some("run startup ME connectivity check".to_string()), + ) + .await; + } else { + startup_tracker + .skip_component( + COMPONENT_ME_CONNECTIVITY_PING, + Some("ME pool is not available".to_string()), + ) + .await; + } if let Some(ref pool) = me_pool { let me_results = run_me_ping(pool, &rng).await; @@ -1023,9 +1348,21 @@ async fn main() -> std::result::Result<(), Box> { } } info!("============================================================"); + startup_tracker + .complete_component( + COMPONENT_ME_CONNECTIVITY_PING, + Some("startup ME connectivity check completed".to_string()), + ) + .await; } info!("================= Telegram DC Connectivity ================="); + startup_tracker + .start_component( + COMPONENT_DC_CONNECTIVITY_PING, + Some("run startup DC connectivity check".to_string()), + ) + .await; let ping_results = upstream_manager .ping_all_dcs( @@ -1105,9 +1442,21 @@ async fn main() -> std::result::Result<(), Box> { info!("============================================================"); } } + startup_tracker + .complete_component( + COMPONENT_DC_CONNECTIVITY_PING, + Some("startup DC connectivity check completed".to_string()), + ) + .await; let initialized_secs = process_started_at.elapsed().as_secs(); let second_suffix = if initialized_secs == 1 { "" } else { "s" }; + startup_tracker + .start_component( + COMPONENT_RUNTIME_READY, + Some("finalize startup runtime state".to_string()), + ) + .await; info!("===================== Telegram Startup ====================="); info!( " DC/ME Initialized in {} second{}", @@ -1150,6 +1499,12 @@ async fn main() -> std::result::Result<(), Box> { // ── Hot-reload watcher ──────────────────────────────────────────────── // Uses inotify to detect file changes instantly (SIGHUP also works). // detected_ip_v4/v6 are passed so newly added users get correct TG links. + startup_tracker + .start_component( + COMPONENT_CONFIG_WATCHER_START, + Some("spawn config hot-reload watcher".to_string()), + ) + .await; let (config_rx, mut log_level_rx): ( tokio::sync::watch::Receiver>, tokio::sync::watch::Receiver, @@ -1159,6 +1514,12 @@ async fn main() -> std::result::Result<(), Box> { detected_ip_v4, detected_ip_v6, ); + startup_tracker + .complete_component( + COMPONENT_CONFIG_WATCHER_START, + Some("config hot-reload watcher started".to_string()), + ) + .await; let mut config_rx_api_bridge = config_rx.clone(); let api_config_tx_bridge = api_config_tx.clone(); tokio::spawn(async move { @@ -1300,6 +1661,12 @@ async fn main() -> std::result::Result<(), Box> { }); } + startup_tracker + .start_component( + COMPONENT_LISTENERS_BIND, + Some("bind TCP/Unix listeners".to_string()), + ) + .await; let mut listeners = Vec::new(); for listener_conf in &config.server.listeners { @@ -1550,6 +1917,16 @@ async fn main() -> std::result::Result<(), Box> { } }); } + startup_tracker + .complete_component( + COMPONENT_LISTENERS_BIND, + Some(format!( + "listeners configured tcp={} unix={}", + listeners.len(), + has_unix_listener + )), + ) + .await; if listeners.is_empty() && !has_unix_listener { error!("No listeners. Exiting."); @@ -1583,6 +1960,12 @@ async fn main() -> std::result::Result<(), Box> { }); if let Some(port) = config.server.metrics_port { + startup_tracker + .start_component( + COMPONENT_METRICS_START, + Some(format!("spawn metrics endpoint on {}", port)), + ) + .await; let stats = stats.clone(); let beobachten = beobachten.clone(); let config_rx_metrics = config_rx.clone(); @@ -1599,8 +1982,29 @@ async fn main() -> std::result::Result<(), Box> { ) .await; }); + startup_tracker + .complete_component( + COMPONENT_METRICS_START, + Some("metrics task spawned".to_string()), + ) + .await; + } else { + startup_tracker + .skip_component( + COMPONENT_METRICS_START, + Some("server.metrics_port is not configured".to_string()), + ) + .await; } + startup_tracker + .complete_component( + COMPONENT_RUNTIME_READY, + Some("startup pipeline is fully initialized".to_string()), + ) + .await; + startup_tracker.mark_ready().await; + for (listener, listener_proxy_protocol) in listeners { let mut config_rx: tokio::sync::watch::Receiver> = config_rx.clone(); let mut admission_rx_tcp = admission_rx.clone(); diff --git a/src/startup.rs b/src/startup.rs new file mode 100644 index 0000000..f6f857c --- /dev/null +++ b/src/startup.rs @@ -0,0 +1,373 @@ +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use tokio::sync::RwLock; + +pub const COMPONENT_CONFIG_LOAD: &str = "config_load"; +pub const COMPONENT_TRACING_INIT: &str = "tracing_init"; +pub const COMPONENT_API_BOOTSTRAP: &str = "api_bootstrap"; +pub const COMPONENT_TLS_FRONT_BOOTSTRAP: &str = "tls_front_bootstrap"; +pub const COMPONENT_NETWORK_PROBE: &str = "network_probe"; +pub const COMPONENT_ME_SECRET_FETCH: &str = "me_secret_fetch"; +pub const COMPONENT_ME_PROXY_CONFIG_V4: &str = "me_proxy_config_fetch_v4"; +pub const COMPONENT_ME_PROXY_CONFIG_V6: &str = "me_proxy_config_fetch_v6"; +pub const COMPONENT_ME_POOL_CONSTRUCT: &str = "me_pool_construct"; +pub const COMPONENT_ME_POOL_INIT_STAGE1: &str = "me_pool_init_stage1"; +pub const COMPONENT_ME_CONNECTIVITY_PING: &str = "me_connectivity_ping"; +pub const COMPONENT_DC_CONNECTIVITY_PING: &str = "dc_connectivity_ping"; +pub const COMPONENT_LISTENERS_BIND: &str = "listeners_bind"; +pub const COMPONENT_CONFIG_WATCHER_START: &str = "config_watcher_start"; +pub const COMPONENT_METRICS_START: &str = "metrics_start"; +pub const COMPONENT_RUNTIME_READY: &str = "runtime_ready"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StartupStatus { + Initializing, + Ready, +} + +impl StartupStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Initializing => "initializing", + Self::Ready => "ready", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StartupComponentStatus { + Pending, + Running, + Ready, + Failed, + Skipped, +} + +impl StartupComponentStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Running => "running", + Self::Ready => "ready", + Self::Failed => "failed", + Self::Skipped => "skipped", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StartupMeStatus { + Pending, + Initializing, + Ready, + Failed, + Skipped, +} + +impl StartupMeStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Initializing => "initializing", + Self::Ready => "ready", + Self::Failed => "failed", + Self::Skipped => "skipped", + } + } +} + +#[derive(Clone, Debug)] +pub struct StartupComponentSnapshot { + pub id: &'static str, + pub title: &'static str, + pub weight: f64, + pub status: StartupComponentStatus, + pub started_at_epoch_ms: Option, + pub finished_at_epoch_ms: Option, + pub duration_ms: Option, + pub attempts: u32, + pub details: Option, +} + +#[derive(Clone, Debug)] +pub struct StartupMeSnapshot { + pub status: StartupMeStatus, + pub current_stage: String, + pub init_attempt: u32, + pub retry_limit: String, + pub last_error: Option, +} + +#[derive(Clone, Debug)] +pub struct StartupSnapshot { + pub status: StartupStatus, + pub degraded: bool, + pub current_stage: String, + pub started_at_epoch_secs: u64, + pub ready_at_epoch_secs: Option, + pub total_elapsed_ms: u64, + pub transport_mode: String, + pub me: StartupMeSnapshot, + pub components: Vec, +} + +#[derive(Clone, Debug)] +struct StartupComponent { + id: &'static str, + title: &'static str, + weight: f64, + status: StartupComponentStatus, + started_at_epoch_ms: Option, + finished_at_epoch_ms: Option, + duration_ms: Option, + attempts: u32, + details: Option, +} + +#[derive(Clone, Debug)] +struct StartupState { + status: StartupStatus, + degraded: bool, + current_stage: String, + started_at_epoch_secs: u64, + ready_at_epoch_secs: Option, + transport_mode: String, + me: StartupMeSnapshot, + components: Vec, +} + +pub struct StartupTracker { + started_at_instant: Instant, + state: RwLock, +} + +impl StartupTracker { + pub fn new(started_at_epoch_secs: u64) -> Self { + Self { + started_at_instant: Instant::now(), + state: RwLock::new(StartupState { + status: StartupStatus::Initializing, + degraded: false, + current_stage: COMPONENT_CONFIG_LOAD.to_string(), + started_at_epoch_secs, + ready_at_epoch_secs: None, + transport_mode: "unknown".to_string(), + me: StartupMeSnapshot { + status: StartupMeStatus::Pending, + current_stage: "pending".to_string(), + init_attempt: 0, + retry_limit: "unlimited".to_string(), + last_error: None, + }, + components: component_blueprint(), + }), + } + } + + pub async fn set_transport_mode(&self, mode: &'static str) { + self.state.write().await.transport_mode = mode.to_string(); + } + + pub async fn set_degraded(&self, degraded: bool) { + self.state.write().await.degraded = degraded; + } + + pub async fn start_component(&self, id: &'static str, details: Option) { + let mut guard = self.state.write().await; + guard.current_stage = id.to_string(); + if let Some(component) = guard.components.iter_mut().find(|component| component.id == id) { + if component.started_at_epoch_ms.is_none() { + component.started_at_epoch_ms = Some(now_epoch_ms()); + } + component.attempts = component.attempts.saturating_add(1); + component.status = StartupComponentStatus::Running; + component.details = normalize_details(details); + } + } + + pub async fn complete_component(&self, id: &'static str, details: Option) { + self.finish_component(id, StartupComponentStatus::Ready, details) + .await; + } + + pub async fn fail_component(&self, id: &'static str, details: Option) { + self.finish_component(id, StartupComponentStatus::Failed, details) + .await; + } + + pub async fn skip_component(&self, id: &'static str, details: Option) { + self.finish_component(id, StartupComponentStatus::Skipped, details) + .await; + } + + async fn finish_component( + &self, + id: &'static str, + status: StartupComponentStatus, + details: Option, + ) { + let mut guard = self.state.write().await; + let finished_at = now_epoch_ms(); + if let Some(component) = guard.components.iter_mut().find(|component| component.id == id) { + if component.started_at_epoch_ms.is_none() { + component.started_at_epoch_ms = Some(finished_at); + component.attempts = component.attempts.saturating_add(1); + } + component.finished_at_epoch_ms = Some(finished_at); + component.duration_ms = component + .started_at_epoch_ms + .map(|started_at| finished_at.saturating_sub(started_at)); + component.status = status; + component.details = normalize_details(details); + } + } + + pub async fn set_me_status(&self, status: StartupMeStatus, stage: &'static str) { + let mut guard = self.state.write().await; + guard.me.status = status; + guard.me.current_stage = stage.to_string(); + } + + pub async fn set_me_retry_limit(&self, retry_limit: String) { + self.state.write().await.me.retry_limit = retry_limit; + } + + pub async fn set_me_init_attempt(&self, attempt: u32) { + self.state.write().await.me.init_attempt = attempt; + } + + pub async fn set_me_last_error(&self, error: Option) { + self.state.write().await.me.last_error = normalize_details(error); + } + + pub async fn mark_ready(&self) { + let mut guard = self.state.write().await; + if guard.status == StartupStatus::Ready { + return; + } + guard.status = StartupStatus::Ready; + guard.current_stage = "ready".to_string(); + guard.ready_at_epoch_secs = Some(now_epoch_secs()); + } + + pub async fn snapshot(&self) -> StartupSnapshot { + let guard = self.state.read().await; + StartupSnapshot { + status: guard.status, + degraded: guard.degraded, + current_stage: guard.current_stage.clone(), + started_at_epoch_secs: guard.started_at_epoch_secs, + ready_at_epoch_secs: guard.ready_at_epoch_secs, + total_elapsed_ms: self.started_at_instant.elapsed().as_millis() as u64, + transport_mode: guard.transport_mode.clone(), + me: guard.me.clone(), + components: guard + .components + .iter() + .map(|component| StartupComponentSnapshot { + id: component.id, + title: component.title, + weight: component.weight, + status: component.status, + started_at_epoch_ms: component.started_at_epoch_ms, + finished_at_epoch_ms: component.finished_at_epoch_ms, + duration_ms: component.duration_ms, + attempts: component.attempts, + details: component.details.clone(), + }) + .collect(), + } + } +} + +pub fn compute_progress_pct(snapshot: &StartupSnapshot, me_stage_progress: Option) -> f64 { + if snapshot.status == StartupStatus::Ready { + return 100.0; + } + + let mut total_weight = 0.0f64; + let mut completed_weight = 0.0f64; + + for component in &snapshot.components { + total_weight += component.weight; + let unit_progress = match component.status { + StartupComponentStatus::Pending => 0.0, + StartupComponentStatus::Running => { + if component.id == COMPONENT_ME_POOL_INIT_STAGE1 { + me_stage_progress.unwrap_or(0.0).clamp(0.0, 1.0) + } else { + 0.0 + } + } + StartupComponentStatus::Ready + | StartupComponentStatus::Failed + | StartupComponentStatus::Skipped => 1.0, + }; + completed_weight += component.weight * unit_progress; + } + + if total_weight <= f64::EPSILON { + 0.0 + } else { + ((completed_weight / total_weight) * 100.0).clamp(0.0, 100.0) + } +} + +fn component_blueprint() -> Vec { + vec![ + component(COMPONENT_CONFIG_LOAD, "Config load", 5.0), + component(COMPONENT_TRACING_INIT, "Tracing init", 3.0), + component(COMPONENT_API_BOOTSTRAP, "API bootstrap", 5.0), + component(COMPONENT_TLS_FRONT_BOOTSTRAP, "TLS front bootstrap", 5.0), + component(COMPONENT_NETWORK_PROBE, "Network probe", 10.0), + component(COMPONENT_ME_SECRET_FETCH, "ME secret fetch", 8.0), + component(COMPONENT_ME_PROXY_CONFIG_V4, "ME config v4 fetch", 4.0), + component(COMPONENT_ME_PROXY_CONFIG_V6, "ME config v6 fetch", 4.0), + component(COMPONENT_ME_POOL_CONSTRUCT, "ME pool construct", 6.0), + component(COMPONENT_ME_POOL_INIT_STAGE1, "ME pool init stage1", 24.0), + component(COMPONENT_ME_CONNECTIVITY_PING, "ME connectivity ping", 6.0), + component(COMPONENT_DC_CONNECTIVITY_PING, "DC connectivity ping", 8.0), + component(COMPONENT_LISTENERS_BIND, "Listener bind", 8.0), + component(COMPONENT_CONFIG_WATCHER_START, "Config watcher start", 2.0), + component(COMPONENT_METRICS_START, "Metrics start", 1.0), + component(COMPONENT_RUNTIME_READY, "Runtime ready", 1.0), + ] +} + +fn component(id: &'static str, title: &'static str, weight: f64) -> StartupComponent { + StartupComponent { + id, + title, + weight, + status: StartupComponentStatus::Pending, + started_at_epoch_ms: None, + finished_at_epoch_ms: None, + duration_ms: None, + attempts: 0, + details: None, + } +} + +fn normalize_details(details: Option) -> Option { + details.map(|detail| { + if detail.len() <= 256 { + detail + } else { + detail[..256].to_string() + } + }) +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +fn now_epoch_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} From 7b745bc7bcfc397aeaa60edeb1495b53812a8639 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:34:32 +0300 Subject: [PATCH 129/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index aa57699..3311a82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.8" +version = "3.3.9" edition = "2024" [dependencies] From d543dbca9276dca918109ca10fded3c5ee31fdbc Mon Sep 17 00:00:00 2001 From: Talya <22233835+hookzof@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:48:02 +0100 Subject: [PATCH 130/261] Fix typo in QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index d82e33e..f893170 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -126,7 +126,7 @@ WantedBy=multi-user.target ```bash curl -s http://127.0.0.1:9091/v1/users | jq ``` -> Одной ссылкой модет пользоваться сколько угодно человек. +> Одной ссылкой может пользоваться сколько угодно человек. > [!WARNING] > Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете! From 5f77f83b488a26ab03a4b03c74efc1b3a6470bdc Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 17:27:56 +0300 Subject: [PATCH 131/261] ME Adaptive Floor Upper-Limit --- src/api/model.rs | 10 + src/api/runtime_stats.rs | 14 ++ src/api/runtime_zero.rs | 16 ++ src/config/defaults.rs | 20 ++ src/config/hot_reload.rs | 38 +++- src/config/load.rs | 64 ++++++ src/config/types.rs | 20 ++ src/main.rs | 4 + src/metrics.rs | 96 ++++++++ src/stats/mod.rs | 66 ++++++ src/transport/middle_proxy/config_updater.rs | 8 + src/transport/middle_proxy/health.rs | 67 ++++-- src/transport/middle_proxy/pool.rs | 224 +++++++++++++++++-- src/transport/middle_proxy/pool_init.rs | 40 +++- src/transport/middle_proxy/pool_refill.rs | 11 +- src/transport/middle_proxy/pool_reinit.rs | 1 + src/transport/middle_proxy/pool_status.rs | 40 ++++ src/transport/middle_proxy/pool_writer.rs | 29 +++ 18 files changed, 724 insertions(+), 44 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index 88c6ddc..fd678f6 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -318,11 +318,21 @@ pub(super) struct MinimalMeRuntimeData { pub(super) adaptive_floor_cpu_cores_override: u16, pub(super) adaptive_floor_max_extra_writers_single_per_core: u16, pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16, + pub(super) adaptive_floor_max_active_writers_per_core: u16, + pub(super) adaptive_floor_max_warm_writers_per_core: u16, + pub(super) adaptive_floor_max_active_writers_global: u32, + pub(super) adaptive_floor_max_warm_writers_global: u32, pub(super) adaptive_floor_cpu_cores_detected: u32, pub(super) adaptive_floor_cpu_cores_effective: u32, pub(super) adaptive_floor_global_cap_raw: u64, pub(super) adaptive_floor_global_cap_effective: u64, pub(super) adaptive_floor_target_writers_total: u64, + pub(super) adaptive_floor_active_cap_configured: u64, + pub(super) adaptive_floor_active_cap_effective: u64, + pub(super) adaptive_floor_warm_cap_configured: u64, + pub(super) adaptive_floor_warm_cap_effective: u64, + pub(super) adaptive_floor_active_writers_current: u64, + pub(super) adaptive_floor_warm_writers_current: u64, pub(super) me_keepalive_enabled: bool, pub(super) me_keepalive_interval_secs: u64, pub(super) me_keepalive_jitter_secs: u64, diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 7fae31d..f90abe3 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -380,11 +380,25 @@ async fn get_minimal_payload_cached( .adaptive_floor_max_extra_writers_single_per_core, adaptive_floor_max_extra_writers_multi_per_core: runtime .adaptive_floor_max_extra_writers_multi_per_core, + adaptive_floor_max_active_writers_per_core: runtime + .adaptive_floor_max_active_writers_per_core, + adaptive_floor_max_warm_writers_per_core: runtime + .adaptive_floor_max_warm_writers_per_core, + adaptive_floor_max_active_writers_global: runtime + .adaptive_floor_max_active_writers_global, + adaptive_floor_max_warm_writers_global: runtime + .adaptive_floor_max_warm_writers_global, adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected, adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective, adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw, adaptive_floor_global_cap_effective: runtime.adaptive_floor_global_cap_effective, adaptive_floor_target_writers_total: runtime.adaptive_floor_target_writers_total, + adaptive_floor_active_cap_configured: runtime.adaptive_floor_active_cap_configured, + adaptive_floor_active_cap_effective: runtime.adaptive_floor_active_cap_effective, + adaptive_floor_warm_cap_configured: runtime.adaptive_floor_warm_cap_configured, + adaptive_floor_warm_cap_effective: runtime.adaptive_floor_warm_cap_effective, + adaptive_floor_active_writers_current: runtime.adaptive_floor_active_writers_current, + adaptive_floor_warm_writers_current: runtime.adaptive_floor_warm_writers_current, me_keepalive_enabled: runtime.me_keepalive_enabled, me_keepalive_interval_secs: runtime.me_keepalive_interval_secs, me_keepalive_jitter_secs: runtime.me_keepalive_jitter_secs, diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index 8def2a9..5020705 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -70,6 +70,10 @@ pub(super) struct EffectiveMiddleProxyLimits { pub(super) adaptive_floor_cpu_cores_override: u16, pub(super) adaptive_floor_max_extra_writers_single_per_core: u16, pub(super) adaptive_floor_max_extra_writers_multi_per_core: u16, + pub(super) adaptive_floor_max_active_writers_per_core: u16, + pub(super) adaptive_floor_max_warm_writers_per_core: u16, + pub(super) adaptive_floor_max_active_writers_global: u32, + pub(super) adaptive_floor_max_warm_writers_global: u32, pub(super) reconnect_max_concurrent_per_dc: u32, pub(super) reconnect_backoff_base_ms: u64, pub(super) reconnect_backoff_cap_ms: u64, @@ -217,6 +221,18 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD adaptive_floor_max_extra_writers_multi_per_core: cfg .general .me_adaptive_floor_max_extra_writers_multi_per_core, + adaptive_floor_max_active_writers_per_core: cfg + .general + .me_adaptive_floor_max_active_writers_per_core, + adaptive_floor_max_warm_writers_per_core: cfg + .general + .me_adaptive_floor_max_warm_writers_per_core, + adaptive_floor_max_active_writers_global: cfg + .general + .me_adaptive_floor_max_active_writers_global, + adaptive_floor_max_warm_writers_global: cfg + .general + .me_adaptive_floor_max_warm_writers_global, reconnect_max_concurrent_per_dc: cfg.general.me_reconnect_max_concurrent_per_dc, reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms, reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms, diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 465cef1..ce55394 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -17,6 +17,10 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_WRITERS_PER_CORE_TOTAL: u16 = 48; const DEFAULT_ME_ADAPTIVE_FLOOR_CPU_CORES_OVERRIDE: u16 = 0; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_SINGLE_PER_CORE: u16 = 1; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE: u16 = 2; +const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64; +const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64; +const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256; +const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; @@ -276,6 +280,22 @@ pub(crate) fn default_me_adaptive_floor_max_extra_writers_multi_per_core() -> u1 DEFAULT_ME_ADAPTIVE_FLOOR_MAX_EXTRA_WRITERS_MULTI_PER_CORE } +pub(crate) fn default_me_adaptive_floor_max_active_writers_per_core() -> u16 { + DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE +} + +pub(crate) fn default_me_adaptive_floor_max_warm_writers_per_core() -> u16 { + DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE +} + +pub(crate) fn default_me_adaptive_floor_max_active_writers_global() -> u32 { + DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL +} + +pub(crate) fn default_me_adaptive_floor_max_warm_writers_global() -> u32 { + DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL +} + pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index a24f9d5..a7ae60c 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -84,6 +84,10 @@ pub struct HotFields { pub me_adaptive_floor_cpu_cores_override: u16, pub me_adaptive_floor_max_extra_writers_single_per_core: u16, pub me_adaptive_floor_max_extra_writers_multi_per_core: u16, + pub me_adaptive_floor_max_active_writers_per_core: u16, + pub me_adaptive_floor_max_warm_writers_per_core: u16, + pub me_adaptive_floor_max_active_writers_global: u32, + pub me_adaptive_floor_max_warm_writers_global: u32, pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_watermark_pct: u8, @@ -173,6 +177,18 @@ impl HotFields { me_adaptive_floor_max_extra_writers_multi_per_core: cfg .general .me_adaptive_floor_max_extra_writers_multi_per_core, + me_adaptive_floor_max_active_writers_per_core: cfg + .general + .me_adaptive_floor_max_active_writers_per_core, + me_adaptive_floor_max_warm_writers_per_core: cfg + .general + .me_adaptive_floor_max_warm_writers_per_core, + me_adaptive_floor_max_active_writers_global: cfg + .general + .me_adaptive_floor_max_active_writers_global, + me_adaptive_floor_max_warm_writers_global: cfg + .general + .me_adaptive_floor_max_warm_writers_global, me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, @@ -305,6 +321,14 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { new.general.me_adaptive_floor_max_extra_writers_single_per_core; cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core = new.general.me_adaptive_floor_max_extra_writers_multi_per_core; + cfg.general.me_adaptive_floor_max_active_writers_per_core = + new.general.me_adaptive_floor_max_active_writers_per_core; + cfg.general.me_adaptive_floor_max_warm_writers_per_core = + new.general.me_adaptive_floor_max_warm_writers_per_core; + cfg.general.me_adaptive_floor_max_active_writers_global = + new.general.me_adaptive_floor_max_active_writers_global; + cfg.general.me_adaptive_floor_max_warm_writers_global = + new.general.me_adaptive_floor_max_warm_writers_global; cfg.general.me_route_backpressure_base_timeout_ms = new.general.me_route_backpressure_base_timeout_ms; cfg.general.me_route_backpressure_high_timeout_ms = @@ -739,9 +763,17 @@ fn log_changes( != new_hot.me_adaptive_floor_max_extra_writers_single_per_core || old_hot.me_adaptive_floor_max_extra_writers_multi_per_core != new_hot.me_adaptive_floor_max_extra_writers_multi_per_core + || old_hot.me_adaptive_floor_max_active_writers_per_core + != new_hot.me_adaptive_floor_max_active_writers_per_core + || old_hot.me_adaptive_floor_max_warm_writers_per_core + != new_hot.me_adaptive_floor_max_warm_writers_per_core + || old_hot.me_adaptive_floor_max_active_writers_global + != new_hot.me_adaptive_floor_max_active_writers_global + || old_hot.me_adaptive_floor_max_warm_writers_global + != new_hot.me_adaptive_floor_max_warm_writers_global { info!( - "config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={}", + "config reload: me_floor: mode={:?} idle={}s min_single={} min_multi={} recover_grace={}s per_core_total={} cores_override={} extra_single_per_core={} extra_multi_per_core={} max_active_per_core={} max_warm_per_core={} max_active_global={} max_warm_global={}", new_hot.me_floor_mode, new_hot.me_adaptive_floor_idle_secs, new_hot.me_adaptive_floor_min_writers_single_endpoint, @@ -751,6 +783,10 @@ fn log_changes( new_hot.me_adaptive_floor_cpu_cores_override, new_hot.me_adaptive_floor_max_extra_writers_single_per_core, new_hot.me_adaptive_floor_max_extra_writers_multi_per_core, + new_hot.me_adaptive_floor_max_active_writers_per_core, + new_hot.me_adaptive_floor_max_warm_writers_per_core, + new_hot.me_adaptive_floor_max_active_writers_global, + new_hot.me_adaptive_floor_max_warm_writers_global, ); } diff --git a/src/config/load.rs b/src/config/load.rs index e6dc728..25a9994 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -327,6 +327,30 @@ impl ProxyConfig { )); } + if config.general.me_adaptive_floor_max_active_writers_per_core == 0 { + return Err(ProxyError::Config( + "general.me_adaptive_floor_max_active_writers_per_core must be > 0".to_string(), + )); + } + + if config.general.me_adaptive_floor_max_warm_writers_per_core == 0 { + return Err(ProxyError::Config( + "general.me_adaptive_floor_max_warm_writers_per_core must be > 0".to_string(), + )); + } + + if config.general.me_adaptive_floor_max_active_writers_global == 0 { + return Err(ProxyError::Config( + "general.me_adaptive_floor_max_active_writers_global must be > 0".to_string(), + )); + } + + if config.general.me_adaptive_floor_max_warm_writers_global == 0 { + return Err(ProxyError::Config( + "general.me_adaptive_floor_max_warm_writers_global must be > 0".to_string(), + )); + } + if config.general.me_single_endpoint_outage_backoff_min_ms == 0 { return Err(ProxyError::Config( "general.me_single_endpoint_outage_backoff_min_ms must be > 0".to_string(), @@ -1238,6 +1262,46 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn me_adaptive_floor_max_active_writers_per_core_zero_is_rejected() { + let toml = r#" + [general] + me_adaptive_floor_max_active_writers_per_core = 0 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_adaptive_floor_max_active_per_core_zero_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.me_adaptive_floor_max_active_writers_per_core must be > 0")); + let _ = std::fs::remove_file(path); + } + + #[test] + fn me_adaptive_floor_max_warm_writers_global_zero_is_rejected() { + let toml = r#" + [general] + me_adaptive_floor_max_warm_writers_global = 0 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_adaptive_floor_max_warm_global_zero_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.me_adaptive_floor_max_warm_writers_global must be > 0")); + let _ = std::fs::remove_file(path); + } + #[test] fn upstream_connect_retry_attempts_zero_is_rejected() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index 5a0dbb2..0c89df8 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -545,6 +545,22 @@ pub struct GeneralConfig { #[serde(default = "default_me_adaptive_floor_max_extra_writers_multi_per_core")] pub me_adaptive_floor_max_extra_writers_multi_per_core: u16, + /// Hard cap for active ME writers per logical CPU core. + #[serde(default = "default_me_adaptive_floor_max_active_writers_per_core")] + pub me_adaptive_floor_max_active_writers_per_core: u16, + + /// Hard cap for warm ME writers per logical CPU core. + #[serde(default = "default_me_adaptive_floor_max_warm_writers_per_core")] + pub me_adaptive_floor_max_warm_writers_per_core: u16, + + /// Hard global cap for active ME writers. + #[serde(default = "default_me_adaptive_floor_max_active_writers_global")] + pub me_adaptive_floor_max_active_writers_global: u32, + + /// Hard global cap for warm ME writers. + #[serde(default = "default_me_adaptive_floor_max_warm_writers_global")] + pub me_adaptive_floor_max_warm_writers_global: u32, + /// Connect attempts for the selected upstream before returning error/fallback. #[serde(default = "default_upstream_connect_retry_attempts")] pub upstream_connect_retry_attempts: u32, @@ -802,6 +818,10 @@ impl Default for GeneralConfig { me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(), me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(), me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(), + me_adaptive_floor_max_active_writers_per_core: default_me_adaptive_floor_max_active_writers_per_core(), + me_adaptive_floor_max_warm_writers_per_core: default_me_adaptive_floor_max_warm_writers_per_core(), + me_adaptive_floor_max_active_writers_global: default_me_adaptive_floor_max_active_writers_global(), + me_adaptive_floor_max_warm_writers_global: default_me_adaptive_floor_max_warm_writers_global(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_connect_budget_ms: default_upstream_connect_budget_ms(), diff --git a/src/main.rs b/src/main.rs index cc6f8ed..58c70cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1031,6 +1031,10 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_adaptive_floor_cpu_cores_override, config.general.me_adaptive_floor_max_extra_writers_single_per_core, config.general.me_adaptive_floor_max_extra_writers_multi_per_core, + config.general.me_adaptive_floor_max_active_writers_per_core, + config.general.me_adaptive_floor_max_warm_writers_per_core, + config.general.me_adaptive_floor_max_active_writers_global, + config.general.me_adaptive_floor_max_warm_writers_global, config.general.hardswap, config.general.me_pool_drain_ttl_secs, config.general.effective_me_pool_force_close_secs(), diff --git a/src/metrics.rs b/src/metrics.rs index 633a884..b338df5 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1053,6 +1053,102 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp 0 } ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_active_cap_configured Runtime configured active writer cap" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_active_cap_configured gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_active_cap_configured {}", + if me_allows_normal { + stats.get_me_floor_active_cap_configured_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_active_cap_effective Runtime effective active writer cap" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_active_cap_effective gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_active_cap_effective {}", + if me_allows_normal { + stats.get_me_floor_active_cap_effective_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_warm_cap_configured Runtime configured warm writer cap" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_warm_cap_configured gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_warm_cap_configured {}", + if me_allows_normal { + stats.get_me_floor_warm_cap_configured_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_adaptive_floor_warm_cap_effective Runtime effective warm writer cap" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_adaptive_floor_warm_cap_effective gauge" + ); + let _ = writeln!( + out, + "telemt_me_adaptive_floor_warm_cap_effective {}", + if me_allows_normal { + stats.get_me_floor_warm_cap_effective_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_writers_active_current Current non-draining active ME writers" + ); + let _ = writeln!(out, "# TYPE telemt_me_writers_active_current gauge"); + let _ = writeln!( + out, + "telemt_me_writers_active_current {}", + if me_allows_normal { + stats.get_me_writers_active_current_gauge() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_me_writers_warm_current Current non-draining warm ME writers" + ); + let _ = writeln!(out, "# TYPE telemt_me_writers_warm_current gauge"); + let _ = writeln!( + out, + "telemt_me_writers_warm_current {}", + if me_allows_normal { + stats.get_me_writers_warm_current_gauge() + } else { + 0 + } + ); let _ = writeln!( out, "# HELP telemt_me_floor_cap_block_total Reconnect attempts blocked by adaptive floor caps" diff --git a/src/stats/mod.rs b/src/stats/mod.rs index fbfc987..10d8882 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -80,6 +80,12 @@ pub struct Stats { me_floor_global_cap_raw_gauge: AtomicU64, me_floor_global_cap_effective_gauge: AtomicU64, me_floor_target_writers_total_gauge: AtomicU64, + me_floor_active_cap_configured_gauge: AtomicU64, + me_floor_active_cap_effective_gauge: AtomicU64, + me_floor_warm_cap_configured_gauge: AtomicU64, + me_floor_warm_cap_effective_gauge: AtomicU64, + me_writers_active_current_gauge: AtomicU64, + me_writers_warm_current_gauge: AtomicU64, me_floor_cap_block_total: AtomicU64, me_floor_swap_idle_total: AtomicU64, me_floor_swap_idle_failed_total: AtomicU64, @@ -764,6 +770,42 @@ impl Stats { .store(value, Ordering::Relaxed); } } + pub fn set_me_floor_active_cap_configured_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_active_cap_configured_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_active_cap_effective_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_active_cap_effective_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_warm_cap_configured_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_warm_cap_configured_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_floor_warm_cap_effective_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_floor_warm_cap_effective_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_writers_active_current_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_writers_active_current_gauge + .store(value, Ordering::Relaxed); + } + } + pub fn set_me_writers_warm_current_gauge(&self, value: u64) { + if self.telemetry_me_allows_normal() { + self.me_writers_warm_current_gauge + .store(value, Ordering::Relaxed); + } + } pub fn increment_me_floor_cap_block_total(&self) { if self.telemetry_me_allows_normal() { self.me_floor_cap_block_total.fetch_add(1, Ordering::Relaxed); @@ -904,6 +946,30 @@ impl Stats { self.me_floor_target_writers_total_gauge .load(Ordering::Relaxed) } + pub fn get_me_floor_active_cap_configured_gauge(&self) -> u64 { + self.me_floor_active_cap_configured_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_active_cap_effective_gauge(&self) -> u64 { + self.me_floor_active_cap_effective_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_warm_cap_configured_gauge(&self) -> u64 { + self.me_floor_warm_cap_configured_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_floor_warm_cap_effective_gauge(&self) -> u64 { + self.me_floor_warm_cap_effective_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_writers_active_current_gauge(&self) -> u64 { + self.me_writers_active_current_gauge + .load(Ordering::Relaxed) + } + pub fn get_me_writers_warm_current_gauge(&self) -> u64 { + self.me_writers_warm_current_gauge + .load(Ordering::Relaxed) + } pub fn get_me_floor_cap_block_total(&self) -> u64 { self.me_floor_cap_block_total.load(Ordering::Relaxed) } diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 4bc3ff7..cccf381 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -321,6 +321,10 @@ async fn run_update_cycle( cfg.general.me_adaptive_floor_cpu_cores_override, cfg.general.me_adaptive_floor_max_extra_writers_single_per_core, cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core, + cfg.general.me_adaptive_floor_max_active_writers_per_core, + cfg.general.me_adaptive_floor_max_warm_writers_per_core, + cfg.general.me_adaptive_floor_max_active_writers_global, + cfg.general.me_adaptive_floor_max_warm_writers_global, ); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); @@ -538,6 +542,10 @@ pub async fn me_config_updater( cfg.general.me_adaptive_floor_cpu_cores_override, cfg.general.me_adaptive_floor_max_extra_writers_single_per_core, cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core, + cfg.general.me_adaptive_floor_max_active_writers_per_core, + cfg.general.me_adaptive_floor_max_warm_writers_per_core, + cfg.general.me_adaptive_floor_max_active_writers_global, + cfg.general.me_adaptive_floor_max_warm_writers_global, ); let new_secs = cfg.general.effective_update_every_secs().max(1); if new_secs == update_every_secs { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index b0536cc..aefeead 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -42,7 +42,12 @@ struct DcFloorPlanEntry { #[derive(Debug, Clone)] struct FamilyFloorPlan { by_dc: HashMap, - global_cap_effective_total: usize, + active_cap_configured_total: usize, + active_cap_effective_total: usize, + warm_cap_configured_total: usize, + warm_cap_effective_total: usize, + active_writers_current: usize, + warm_writers_current: usize, target_writers_total: usize, } @@ -169,6 +174,14 @@ async fn check_family( for writer in pool.writers.read().await.iter().filter(|w| { !w.draining.load(std::sync::atomic::Ordering::Relaxed) }) { + if !matches!( + super::pool::WriterContour::from_u8( + writer.contour.load(std::sync::atomic::Ordering::Relaxed), + ), + super::pool::WriterContour::Active + ) { + continue; + } let key = (writer.writer_dc, writer.addr); *live_addr_counts.entry(key).or_insert(0) += 1; live_writer_ids_by_addr @@ -194,8 +207,13 @@ async fn check_family( ) .await; pool.set_adaptive_floor_runtime_caps( - floor_plan.global_cap_effective_total, + floor_plan.active_cap_configured_total, + floor_plan.active_cap_effective_total, + floor_plan.warm_cap_configured_total, + floor_plan.warm_cap_effective_total, floor_plan.target_writers_total, + floor_plan.active_writers_current, + floor_plan.warm_writers_current, ); for (dc, endpoints) in dc_endpoints { @@ -344,8 +362,8 @@ async fn check_family( break; } reconnect_budget = reconnect_budget.saturating_sub(1); - if pool.floor_mode() == MeFloorMode::Adaptive - && pool.active_writer_count_total().await >= floor_plan.global_cap_effective_total + if pool.active_contour_writer_count_total().await + >= floor_plan.active_cap_effective_total { let swapped = maybe_swap_idle_writer_for_cap( pool, @@ -370,7 +388,7 @@ async fn check_family( ?family, alive, required, - global_cap_effective_total = floor_plan.global_cap_effective_total, + active_cap_effective_total = floor_plan.active_cap_effective_total, "Adaptive floor cap reached, reconnect attempt blocked" ); break; @@ -518,6 +536,8 @@ async fn build_family_floor_plan( let floor_mode = pool.floor_mode(); let is_adaptive = floor_mode == MeFloorMode::Adaptive; let cpu_cores = pool.adaptive_floor_effective_cpu_cores().max(1); + let (active_writers_current, warm_writers_current, _) = + pool.non_draining_writer_counts_by_contour().await; for (dc, endpoints) in dc_endpoints { if endpoints.is_empty() { @@ -576,9 +596,16 @@ async fn build_family_floor_plan( } if entries.is_empty() { + let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total(); + let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total(); return FamilyFloorPlan { by_dc, - global_cap_effective_total: 0, + active_cap_configured_total, + active_cap_effective_total: active_cap_configured_total, + warm_cap_configured_total, + warm_cap_effective_total: warm_cap_configured_total, + active_writers_current, + warm_writers_current, target_writers_total: 0, }; } @@ -588,20 +615,26 @@ async fn build_family_floor_plan( .iter() .map(|entry| entry.target_required) .sum::(); - let active_total = pool.active_writer_count_total().await; + let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total(); + let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total(); for entry in entries { by_dc.insert(entry.dc, entry); } return FamilyFloorPlan { by_dc, - global_cap_effective_total: active_total.max(target_total), + active_cap_configured_total, + active_cap_effective_total: active_cap_configured_total.max(target_total), + warm_cap_configured_total, + warm_cap_effective_total: warm_cap_configured_total, + active_writers_current, + warm_writers_current, target_writers_total: target_total, }; } - let global_cap_raw = pool.adaptive_floor_global_cap_raw(); - let total_active = pool.active_writer_count_total().await; - let other_active = total_active.saturating_sub(family_active_total); + let active_cap_configured_total = pool.adaptive_floor_active_cap_configured_total(); + let warm_cap_configured_total = pool.adaptive_floor_warm_cap_configured_total(); + let other_active = active_writers_current.saturating_sub(family_active_total); let min_sum = entries .iter() .map(|entry| entry.min_required) @@ -610,7 +643,7 @@ async fn build_family_floor_plan( .iter() .map(|entry| entry.target_required) .sum::(); - let family_cap = global_cap_raw + let family_cap = active_cap_configured_total .saturating_sub(other_active) .max(min_sum); if target_sum > family_cap { @@ -645,11 +678,17 @@ async fn build_family_floor_plan( for entry in entries { by_dc.insert(entry.dc, entry); } - let global_cap_effective_total = global_cap_raw.max(other_active.saturating_add(min_sum)); + let active_cap_effective_total = + active_cap_configured_total.max(other_active.saturating_add(min_sum)); let target_writers_total = other_active.saturating_add(target_sum); FamilyFloorPlan { by_dc, - global_cap_effective_total, + active_cap_configured_total, + active_cap_effective_total, + warm_cap_configured_total, + warm_cap_effective_total: warm_cap_configured_total, + active_writers_current, + warm_writers_current, target_writers_total, } } diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 13259bb..5ec512a 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -125,11 +125,21 @@ pub struct MePool { pub(super) me_adaptive_floor_cpu_cores_override: AtomicU32, pub(super) me_adaptive_floor_max_extra_writers_single_per_core: AtomicU32, pub(super) me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32, + pub(super) me_adaptive_floor_max_active_writers_per_core: AtomicU32, + pub(super) me_adaptive_floor_max_warm_writers_per_core: AtomicU32, + pub(super) me_adaptive_floor_max_active_writers_global: AtomicU32, + pub(super) me_adaptive_floor_max_warm_writers_global: AtomicU32, pub(super) me_adaptive_floor_cpu_cores_detected: AtomicU32, pub(super) me_adaptive_floor_cpu_cores_effective: AtomicU32, pub(super) me_adaptive_floor_global_cap_raw: AtomicU64, pub(super) me_adaptive_floor_global_cap_effective: AtomicU64, pub(super) me_adaptive_floor_target_writers_total: AtomicU64, + pub(super) me_adaptive_floor_active_cap_configured: AtomicU64, + pub(super) me_adaptive_floor_active_cap_effective: AtomicU64, + pub(super) me_adaptive_floor_warm_cap_configured: AtomicU64, + pub(super) me_adaptive_floor_warm_cap_effective: AtomicU64, + pub(super) me_adaptive_floor_active_writers_current: AtomicU64, + pub(super) me_adaptive_floor_warm_writers_current: AtomicU64, pub(super) proxy_map_v4: Arc>>>, pub(super) proxy_map_v6: Arc>>>, pub(super) endpoint_dc_map: Arc>>>, @@ -243,6 +253,10 @@ impl MePool { me_adaptive_floor_cpu_cores_override: u16, me_adaptive_floor_max_extra_writers_single_per_core: u16, me_adaptive_floor_max_extra_writers_multi_per_core: u16, + me_adaptive_floor_max_active_writers_per_core: u16, + me_adaptive_floor_max_warm_writers_per_core: u16, + me_adaptive_floor_max_active_writers_global: u32, + me_adaptive_floor_max_warm_writers_global: u32, hardswap: bool, me_pool_drain_ttl_secs: u64, me_pool_force_close_secs: u64, @@ -358,11 +372,29 @@ impl MePool { me_adaptive_floor_max_extra_writers_multi_per_core: AtomicU32::new( me_adaptive_floor_max_extra_writers_multi_per_core as u32, ), + me_adaptive_floor_max_active_writers_per_core: AtomicU32::new( + me_adaptive_floor_max_active_writers_per_core as u32, + ), + me_adaptive_floor_max_warm_writers_per_core: AtomicU32::new( + me_adaptive_floor_max_warm_writers_per_core as u32, + ), + me_adaptive_floor_max_active_writers_global: AtomicU32::new( + me_adaptive_floor_max_active_writers_global, + ), + me_adaptive_floor_max_warm_writers_global: AtomicU32::new( + me_adaptive_floor_max_warm_writers_global, + ), me_adaptive_floor_cpu_cores_detected: AtomicU32::new(1), me_adaptive_floor_cpu_cores_effective: AtomicU32::new(1), me_adaptive_floor_global_cap_raw: AtomicU64::new(0), me_adaptive_floor_global_cap_effective: AtomicU64::new(0), me_adaptive_floor_target_writers_total: AtomicU64::new(0), + me_adaptive_floor_active_cap_configured: AtomicU64::new(0), + me_adaptive_floor_active_cap_effective: AtomicU64::new(0), + me_adaptive_floor_warm_cap_configured: AtomicU64::new(0), + me_adaptive_floor_warm_cap_effective: AtomicU64::new(0), + me_adaptive_floor_active_writers_current: AtomicU64::new(0), + me_adaptive_floor_warm_writers_current: AtomicU64::new(0), pool_size: 2, proxy_map_v4: Arc::new(RwLock::new(proxy_map_v4)), proxy_map_v6: Arc::new(RwLock::new(proxy_map_v6)), @@ -453,6 +485,10 @@ impl MePool { adaptive_floor_cpu_cores_override: u16, adaptive_floor_max_extra_writers_single_per_core: u16, adaptive_floor_max_extra_writers_multi_per_core: u16, + adaptive_floor_max_active_writers_per_core: u16, + adaptive_floor_max_warm_writers_per_core: u16, + adaptive_floor_max_active_writers_global: u32, + adaptive_floor_max_warm_writers_global: u32, ) { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs @@ -514,6 +550,20 @@ impl MePool { adaptive_floor_max_extra_writers_multi_per_core as u32, Ordering::Relaxed, ); + self.me_adaptive_floor_max_active_writers_per_core + .store( + adaptive_floor_max_active_writers_per_core as u32, + Ordering::Relaxed, + ); + self.me_adaptive_floor_max_warm_writers_per_core + .store( + adaptive_floor_max_warm_writers_per_core as u32, + Ordering::Relaxed, + ); + self.me_adaptive_floor_max_active_writers_global + .store(adaptive_floor_max_active_writers_global, Ordering::Relaxed); + self.me_adaptive_floor_max_warm_writers_global + .store(adaptive_floor_max_warm_writers_global, Ordering::Relaxed); if previous_floor_mode != floor_mode { self.stats.increment_me_floor_mode_switch_total(); match (previous_floor_mode, floor_mode) { @@ -584,11 +634,26 @@ impl MePool { self.proxy_secret.read().await.key_selector } - pub(super) async fn active_writer_count_total(&self) -> usize { + pub(super) async fn non_draining_writer_counts_by_contour(&self) -> (usize, usize, usize) { let ws = self.writers.read().await; - ws.iter() - .filter(|w| !w.draining.load(Ordering::Relaxed)) - .count() + let mut active = 0usize; + let mut warm = 0usize; + for writer in ws.iter() { + if writer.draining.load(Ordering::Relaxed) { + continue; + } + match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) { + WriterContour::Active => active = active.saturating_add(1), + WriterContour::Warm => warm = warm.saturating_add(1), + WriterContour::Draining => {} + } + } + (active, warm, active.saturating_add(warm)) + } + + pub(super) async fn active_contour_writer_count_total(&self) -> usize { + let (active, _, _) = self.non_draining_writer_counts_by_contour().await; + active } pub(super) async fn secret_snapshot(&self) -> SecretSnapshot { @@ -634,13 +699,6 @@ impl MePool { .max(1) } - pub(super) fn adaptive_floor_writers_per_core_total(&self) -> usize { - (self - .me_adaptive_floor_writers_per_core_total - .load(Ordering::Relaxed) as usize) - .max(1) - } - pub(super) fn adaptive_floor_max_extra_single_per_core(&self) -> usize { self.me_adaptive_floor_max_extra_writers_single_per_core .load(Ordering::Relaxed) as usize @@ -651,6 +709,34 @@ impl MePool { .load(Ordering::Relaxed) as usize } + pub(super) fn adaptive_floor_max_active_writers_per_core(&self) -> usize { + (self + .me_adaptive_floor_max_active_writers_per_core + .load(Ordering::Relaxed) as usize) + .max(1) + } + + pub(super) fn adaptive_floor_max_warm_writers_per_core(&self) -> usize { + (self + .me_adaptive_floor_max_warm_writers_per_core + .load(Ordering::Relaxed) as usize) + .max(1) + } + + pub(super) fn adaptive_floor_max_active_writers_global(&self) -> usize { + (self + .me_adaptive_floor_max_active_writers_global + .load(Ordering::Relaxed) as usize) + .max(1) + } + + pub(super) fn adaptive_floor_max_warm_writers_global(&self) -> usize { + (self + .me_adaptive_floor_max_warm_writers_global + .load(Ordering::Relaxed) as usize) + .max(1) + } + pub(super) fn adaptive_floor_detected_cpu_cores(&self) -> usize { std::thread::available_parallelism() .map(|value| value.get()) @@ -679,28 +765,126 @@ impl MePool { effective } - pub(super) fn adaptive_floor_global_cap_raw(&self) -> usize { + pub(super) fn adaptive_floor_active_cap_configured_total(&self) -> usize { let cores = self.adaptive_floor_effective_cpu_cores(); - let cap = cores.saturating_mul(self.adaptive_floor_writers_per_core_total()); - self.me_adaptive_floor_global_cap_raw - .store(cap as u64, Ordering::Relaxed); - self.stats.set_me_floor_global_cap_raw_gauge(cap as u64); - cap + let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_active_writers_per_core()); + let configured = per_core_cap.min(self.adaptive_floor_max_active_writers_global()); + self.me_adaptive_floor_active_cap_configured + .store(configured as u64, Ordering::Relaxed); + self.stats + .set_me_floor_active_cap_configured_gauge(configured as u64); + configured + } + + pub(super) fn adaptive_floor_warm_cap_configured_total(&self) -> usize { + let cores = self.adaptive_floor_effective_cpu_cores(); + let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_warm_writers_per_core()); + let configured = per_core_cap.min(self.adaptive_floor_max_warm_writers_global()); + self.me_adaptive_floor_warm_cap_configured + .store(configured as u64, Ordering::Relaxed); + self.stats + .set_me_floor_warm_cap_configured_gauge(configured as u64); + configured } pub(super) fn set_adaptive_floor_runtime_caps( &self, - global_cap_effective: usize, + active_cap_configured: usize, + active_cap_effective: usize, + warm_cap_configured: usize, + warm_cap_effective: usize, target_writers_total: usize, + active_writers_current: usize, + warm_writers_current: usize, ) { + self.me_adaptive_floor_global_cap_raw + .store(active_cap_configured as u64, Ordering::Relaxed); self.me_adaptive_floor_global_cap_effective - .store(global_cap_effective as u64, Ordering::Relaxed); + .store(active_cap_effective as u64, Ordering::Relaxed); self.me_adaptive_floor_target_writers_total .store(target_writers_total as u64, Ordering::Relaxed); + self.me_adaptive_floor_active_cap_configured + .store(active_cap_configured as u64, Ordering::Relaxed); + self.me_adaptive_floor_active_cap_effective + .store(active_cap_effective as u64, Ordering::Relaxed); + self.me_adaptive_floor_warm_cap_configured + .store(warm_cap_configured as u64, Ordering::Relaxed); + self.me_adaptive_floor_warm_cap_effective + .store(warm_cap_effective as u64, Ordering::Relaxed); + self.me_adaptive_floor_active_writers_current + .store(active_writers_current as u64, Ordering::Relaxed); + self.me_adaptive_floor_warm_writers_current + .store(warm_writers_current as u64, Ordering::Relaxed); self.stats - .set_me_floor_global_cap_effective_gauge(global_cap_effective as u64); + .set_me_floor_global_cap_raw_gauge(active_cap_configured as u64); + self.stats + .set_me_floor_global_cap_effective_gauge(active_cap_effective as u64); self.stats .set_me_floor_target_writers_total_gauge(target_writers_total as u64); + self.stats + .set_me_floor_active_cap_configured_gauge(active_cap_configured as u64); + self.stats + .set_me_floor_active_cap_effective_gauge(active_cap_effective as u64); + self.stats + .set_me_floor_warm_cap_configured_gauge(warm_cap_configured as u64); + self.stats + .set_me_floor_warm_cap_effective_gauge(warm_cap_effective as u64); + self.stats + .set_me_writers_active_current_gauge(active_writers_current as u64); + self.stats + .set_me_writers_warm_current_gauge(warm_writers_current as u64); + } + + pub(super) async fn active_coverage_required_total(&self) -> usize { + let mut endpoints_by_dc = HashMap::>::new(); + + if self.decision.ipv4_me { + let map = self.proxy_map_v4.read().await; + for (dc, addrs) in map.iter() { + let entry = endpoints_by_dc.entry(*dc).or_default(); + for (ip, port) in addrs.iter().copied() { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + + if self.decision.ipv6_me { + let map = self.proxy_map_v6.read().await; + for (dc, addrs) in map.iter() { + let entry = endpoints_by_dc.entry(*dc).or_default(); + for (ip, port) in addrs.iter().copied() { + entry.insert(SocketAddr::new(ip, port)); + } + } + } + + endpoints_by_dc + .values() + .map(|endpoints| self.required_writers_for_dc_with_floor_mode(endpoints.len(), false)) + .sum() + } + + pub(super) async fn can_open_writer_for_contour( + &self, + contour: WriterContour, + allow_coverage_override: bool, + ) -> bool { + let (active_writers, warm_writers, _) = self.non_draining_writer_counts_by_contour().await; + match contour { + WriterContour::Active => { + let active_cap = self.adaptive_floor_active_cap_configured_total(); + if active_writers < active_cap { + return true; + } + if !allow_coverage_override { + return false; + } + let coverage_required = self.active_coverage_required_total().await; + active_writers < coverage_required + } + WriterContour::Warm => warm_writers < self.adaptive_floor_warm_cap_configured_total(), + WriterContour::Draining => true, + } } pub(super) fn required_writers_for_dc_with_floor_mode( diff --git a/src/transport/middle_proxy/pool_init.rs b/src/transport/middle_proxy/pool_init.rs index 52cbc68..29a70c5 100644 --- a/src/transport/middle_proxy/pool_init.rs +++ b/src/transport/middle_proxy/pool_init.rs @@ -71,6 +71,7 @@ impl MePool { target_writers, rng_clone, connect_concurrency, + true, ) .await }); @@ -114,6 +115,7 @@ impl MePool { target_writers, rng_clone_local, connect_concurrency, + false, ) .await }); @@ -147,6 +149,7 @@ impl MePool { target_writers: usize, rng: Arc, connect_concurrency: usize, + allow_coverage_override: bool, ) -> bool { if addrs.is_empty() { return false; @@ -180,9 +183,17 @@ impl MePool { let pool = Arc::clone(&self); let rng_clone = Arc::clone(&rng); let endpoints_clone = endpoints.clone(); + let generation = self.current_generation(); join.spawn(async move { - pool.connect_endpoints_round_robin(dc, &endpoints_clone, rng_clone.as_ref()) - .await + pool.connect_endpoints_round_robin_with_generation_contour( + dc, + &endpoints_clone, + rng_clone.as_ref(), + generation, + super::pool::WriterContour::Active, + allow_coverage_override, + ) + .await }); } @@ -212,12 +223,25 @@ impl MePool { return true; } if !progress { - warn!( - dc = %dc, - alive = alive_after, - target_writers, - "All ME servers for DC failed at init" - ); + let active_writers_current = self.active_contour_writer_count_total().await; + let active_cap_configured = self.adaptive_floor_active_cap_configured_total(); + if !allow_coverage_override && active_writers_current >= active_cap_configured { + info!( + dc = %dc, + alive = alive_after, + target_writers, + active_writers_current, + active_cap_configured, + "ME init saturation stopped by active writer cap" + ); + } else { + warn!( + dc = %dc, + alive = alive_after, + target_writers, + "All ME servers for DC failed at init" + ); + } return false; } diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 316f3ff..544d048 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -99,6 +99,7 @@ impl MePool { rng, self.current_generation(), WriterContour::Active, + false, ) .await } @@ -110,6 +111,7 @@ impl MePool { rng: &SecureRandom, generation: u64, contour: WriterContour, + allow_coverage_override: bool, ) -> bool { let candidates = self.connectable_endpoints(endpoints).await; if candidates.is_empty() { @@ -120,7 +122,14 @@ impl MePool { let idx = (start + offset) % candidates.len(); let addr = candidates[idx]; match self - .connect_one_with_generation_contour_for_dc(addr, rng, generation, contour, dc) + .connect_one_with_generation_contour_for_dc_with_cap_policy( + addr, + rng, + generation, + contour, + dc, + allow_coverage_override, + ) .await { Ok(()) => return true, diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 625ccf0..3d9d679 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -249,6 +249,7 @@ impl MePool { rng, generation, WriterContour::Warm, + false, ) .await; debug!( diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 2922ed8..cc1be5b 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -82,11 +82,21 @@ pub(crate) struct MeApiRuntimeSnapshot { pub adaptive_floor_cpu_cores_override: u16, pub adaptive_floor_max_extra_writers_single_per_core: u16, pub adaptive_floor_max_extra_writers_multi_per_core: u16, + pub adaptive_floor_max_active_writers_per_core: u16, + pub adaptive_floor_max_warm_writers_per_core: u16, + pub adaptive_floor_max_active_writers_global: u32, + pub adaptive_floor_max_warm_writers_global: u32, pub adaptive_floor_cpu_cores_detected: u32, pub adaptive_floor_cpu_cores_effective: u32, pub adaptive_floor_global_cap_raw: u64, pub adaptive_floor_global_cap_effective: u64, pub adaptive_floor_target_writers_total: u64, + pub adaptive_floor_active_cap_configured: u64, + pub adaptive_floor_active_cap_effective: u64, + pub adaptive_floor_warm_cap_configured: u64, + pub adaptive_floor_warm_cap_effective: u64, + pub adaptive_floor_active_writers_current: u64, + pub adaptive_floor_warm_writers_current: u64, pub me_keepalive_enabled: bool, pub me_keepalive_interval_secs: u64, pub me_keepalive_jitter_secs: u64, @@ -430,6 +440,18 @@ impl MePool { adaptive_floor_max_extra_writers_multi_per_core: self .me_adaptive_floor_max_extra_writers_multi_per_core .load(Ordering::Relaxed) as u16, + adaptive_floor_max_active_writers_per_core: self + .me_adaptive_floor_max_active_writers_per_core + .load(Ordering::Relaxed) as u16, + adaptive_floor_max_warm_writers_per_core: self + .me_adaptive_floor_max_warm_writers_per_core + .load(Ordering::Relaxed) as u16, + adaptive_floor_max_active_writers_global: self + .me_adaptive_floor_max_active_writers_global + .load(Ordering::Relaxed), + adaptive_floor_max_warm_writers_global: self + .me_adaptive_floor_max_warm_writers_global + .load(Ordering::Relaxed), adaptive_floor_cpu_cores_detected: self .me_adaptive_floor_cpu_cores_detected .load(Ordering::Relaxed), @@ -445,6 +467,24 @@ impl MePool { adaptive_floor_target_writers_total: self .me_adaptive_floor_target_writers_total .load(Ordering::Relaxed), + adaptive_floor_active_cap_configured: self + .me_adaptive_floor_active_cap_configured + .load(Ordering::Relaxed), + adaptive_floor_active_cap_effective: self + .me_adaptive_floor_active_cap_effective + .load(Ordering::Relaxed), + adaptive_floor_warm_cap_configured: self + .me_adaptive_floor_warm_cap_configured + .load(Ordering::Relaxed), + adaptive_floor_warm_cap_effective: self + .me_adaptive_floor_warm_cap_effective + .load(Ordering::Relaxed), + adaptive_floor_active_writers_current: self + .me_adaptive_floor_active_writers_current + .load(Ordering::Relaxed), + adaptive_floor_warm_writers_current: self + .me_adaptive_floor_warm_writers_current + .load(Ordering::Relaxed), me_keepalive_enabled: self.me_keepalive_enabled, me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(), me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(), diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 244a08e..90f8d0a 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -86,6 +86,35 @@ impl MePool { contour: WriterContour, writer_dc: i32, ) -> Result<()> { + self.connect_one_with_generation_contour_for_dc_with_cap_policy( + addr, + rng, + generation, + contour, + writer_dc, + false, + ) + .await + } + + pub(super) async fn connect_one_with_generation_contour_for_dc_with_cap_policy( + self: &Arc, + addr: SocketAddr, + rng: &SecureRandom, + generation: u64, + contour: WriterContour, + writer_dc: i32, + allow_coverage_override: bool, + ) -> Result<()> { + if !self + .can_open_writer_for_contour(contour, allow_coverage_override) + .await + { + return Err(ProxyError::Proxy(format!( + "ME {contour:?} writer cap reached" + ))); + } + let secret_len = self.proxy_secret.read().await.secret.len(); if secret_len < 32 { return Err(ProxyError::Proxy("proxy-secret too short for ME auth".into())); From 80f3661b8eb6607185f25287075b8fb2f126d12b Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Sat, 7 Mar 2026 17:36:44 +0300 Subject: [PATCH 132/261] Modify telemt.service for network dependencies Updated service dependencies and added SELinux context. `network-online.target` is required to get the ip address and check telegram servers --- telemt.service | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/telemt.service b/telemt.service index 4f522a4..55fd9fd 100644 --- a/telemt.service +++ b/telemt.service @@ -1,6 +1,7 @@ [Unit] Description=Telemt -After=network.target +After=network-online.target +Wants=network-online.target [Service] Type=simple @@ -8,6 +9,7 @@ WorkingDirectory=/bin ExecStart=/bin/telemt /etc/telemt.toml Restart=on-failure LimitNOFILE=65536 +SELinuxContext=system_u:system_r:unconfined_service_t:s0 [Install] WantedBy=multi-user.target From 2f47ec5797dbe8dd25f392edb8a8ceaf2883dbc3 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 18:43:48 +0300 Subject: [PATCH 133/261] Update config.toml --- config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.toml b/config.toml index ab35789..63fa4ae 100644 --- a/config.toml +++ b/config.toml @@ -4,7 +4,7 @@ # === General Settings === [general] -use_middle_proxy = false +use_middle_proxy = true # Global ad_tag fallback when user has no per-user tag in [access.user_ad_tags] # ad_tag = "00000000000000000000000000000000" # Per-user ad_tag in [access.user_ad_tags] (32 hex from @MTProxybot) From a0cf4b47136d80ac8b953e63b68722df2a051073 Mon Sep 17 00:00:00 2001 From: Mirotin Artem Date: Sat, 7 Mar 2026 19:07:30 +0300 Subject: [PATCH 134/261] improve install script --- install.sh | 139 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 80 insertions(+), 59 deletions(-) diff --git a/install.sh b/install.sh index 09b160b..006d33c 100644 --- a/install.sh +++ b/install.sh @@ -1,73 +1,94 @@ -sudo bash -c ' -set -e +#!/bin/sh +set -eu -# --- Проверка на существующую установку --- -if systemctl list-unit-files | grep -q telemt.service; then - # --- РЕЖИМ ОБНОВЛЕНИЯ --- - echo "--- Обнаружена существующая установка Telemt. Запускаю обновление... ---" +REPO="${REPO:-telemt/telemt}" +BIN_NAME="${BIN_NAME:-telemt}" +VERSION="${1:-${VERSION:-latest}}" +INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}" - echo "[*] Остановка службы telemt..." - systemctl stop telemt || true # Игнорируем ошибку, если служба уже остановлена +say() { + printf '%s\n' "$*" +} - echo "[1/2] Скачивание последней версии Telemt..." - wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz +die() { + printf 'Error: %s\n' "$*" >&2 + exit 1 +} - echo "[1/2] Замена исполняемого файла в /usr/local/bin..." - mv telemt /usr/local/bin/telemt - chmod +x /usr/local/bin/telemt +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} - echo "[2/2] Запуск службы..." - systemctl start telemt +detect_arch() { + arch="$(uname -m)" + case "$arch" in + x86_64|amd64) printf 'x86_64\n' ;; + aarch64|arm64) printf 'aarch64\n' ;; + *) die "unsupported architecture: $arch" ;; + esac +} - echo "--- Обновление Telemt успешно завершено! ---" - echo - echo "Для проверки статуса службы выполните:" - echo " systemctl status telemt" +detect_libc() { + if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -iq musl; then + printf 'musl\n' + else + printf 'gnu\n' + fi +} -else - # --- РЕЖИМ НОВОЙ УСТАНОВКИ --- - echo "--- Начало автоматической установки Telemt ---" +fetch_to_stdout() { + url="$1" + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$url" + elif command -v wget >/dev/null 2>&1; then + wget -qO- "$url" + else + die "neither curl nor wget is installed" + fi +} - # Шаг 1: Скачивание и установка бинарного файла - echo "[1/5] Скачивание последней версии Telemt..." - wget -qO- "https://github.com/telemt/telemt/releases/latest/download/telemt-$(uname -m)-linux-$(ldd --version 2>&1 | grep -iq musl && echo musl || echo gnu).tar.gz" | tar -xz +install_binary() { + src="$1" + dst="$2" - echo "[1/5] Перемещение исполняемого файла в /usr/local/bin и установка прав..." - mv telemt /usr/local/bin/telemt - chmod +x /usr/local/bin/telemt + if [ -w "$INSTALL_DIR" ] || { [ ! -e "$INSTALL_DIR" ] && [ -w "$(dirname "$INSTALL_DIR")" ]; }; then + mkdir -p "$INSTALL_DIR" + install -m 0755 "$src" "$dst" + elif command -v sudo >/dev/null 2>&1; then + sudo mkdir -p "$INSTALL_DIR" + sudo install -m 0755 "$src" "$dst" + else + die "cannot write to $INSTALL_DIR and sudo is not available" + fi +} - # Шаг 2: Генерация секрета - echo "[2/5] Генерация секретного ключа..." - SECRET=$(openssl rand -hex 16) +need_cmd uname +need_cmd tar +need_cmd mktemp +need_cmd grep +need_cmd install - # Шаг 3: Создание файла конфигурации - echo "[3/5] Создание файла конфигурации /etc/telemt.toml..." - printf "# === General Settings ===\n[general]\n[general.modes]\nclassic = false\nsecure = false\ntls = true\n\n# === Anti-Censorship & Masking ===\n[censorship]\n# !!! ВАЖНО: Замените на ваш домен или домен, который вы хотите использовать для маскировки !!!\ntls_domain = \"petrovich.ru\"\n\n[access.users]\nhello = \"%s\"\n" "$SECRET" > /etc/telemt.toml +ARCH="$(detect_arch)" +LIBC="$(detect_libc)" - # Шаг 4: Создание службы Systemd - echo "[4/5] Создание службы systemd..." - printf "[Unit]\nDescription=Telemt Proxy\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/telemt /etc/telemt.toml\nRestart=on-failure\nRestartSec=5\nLimitNOFILE=65536\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/telemt.service +case "$VERSION" in + latest) + URL="https://github.com/$REPO/releases/latest/download/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz" + ;; + *) + URL="https://github.com/$REPO/releases/download/${VERSION}/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz" + ;; +esac - # Шаг 5: Запуск службы - echo "[5/5] Перезагрузка systemd, запуск и включение службы telemt..." - systemctl daemon-reload - systemctl start telemt - systemctl enable telemt +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT INT TERM - echo "--- Установка и запуск Telemt успешно завершены! ---" - echo - echo "ВАЖНАЯ ИНФОРМАЦИЯ:" - echo "===================" - echo "1. Вам НЕОБХОДИМО отредактировать файл /etc/telemt.toml и заменить '\''petrovich.ru'\'' на другой домен" - echo " с помощью команды:" - echo " nano /etc/telemt.toml" - echo " После редактирования файла перезапустите службу командой:" - echo " sudo systemctl restart telemt" - echo - echo "2. Для проверки статуса службы выполните команду:" - echo " systemctl status telemt" - echo - echo "3. Для получения ссылок на подключение выполните команду:" - echo " journalctl -u telemt -n -g '\''links'\'' --no-pager -o cat | tac" -fi -' +say "Installing $BIN_NAME ($VERSION) for $ARCH-linux-$LIBC..." +fetch_to_stdout "$URL" | tar -xzf - -C "$TMPDIR" + +[ -f "$TMPDIR/$BIN_NAME" ] || die "archive did not contain $BIN_NAME" + +install_binary "$TMPDIR/$BIN_NAME" "$INSTALL_DIR/$BIN_NAME" + +say "Installed: $INSTALL_DIR/$BIN_NAME" +"$INSTALL_DIR/$BIN_NAME" --version 2>/dev/null || true From d2baa8e721f416164edf189bd0dffb46b8860d70 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 19:33:48 +0300 Subject: [PATCH 135/261] CPU/RAM improvements + removing hot-path obstacles --- src/config/defaults.rs | 35 +++++++ src/config/hot_reload.rs | 23 +++- src/config/load.rs | 42 ++++++++ src/config/types.rs | 35 +++++++ src/main.rs | 19 +++- src/proxy/middle_relay.rs | 8 +- src/transport/middle_proxy/config_updater.rs | 6 ++ src/transport/middle_proxy/health.rs | 71 ++++++++++--- src/transport/middle_proxy/pool.rs | 105 ++++++++++++++++++- src/transport/middle_proxy/pool_writer.rs | 2 +- src/transport/middle_proxy/registry.rs | 18 ++-- src/transport/middle_proxy/send.rs | 51 +-------- 12 files changed, 340 insertions(+), 75 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index ce55394..798d881 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -21,6 +21,13 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256; +const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 512; +const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 512; +const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 128; +const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000; +const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000; +const DEFAULT_ME_ADMISSION_POLL_MS: u64 = 1000; +const DEFAULT_ME_WARN_RATE_LIMIT_MS: u64 = 5000; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; @@ -296,6 +303,34 @@ pub(crate) fn default_me_adaptive_floor_max_warm_writers_global() -> u32 { DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL } +pub(crate) fn default_me_writer_cmd_channel_capacity() -> usize { + DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY +} + +pub(crate) fn default_me_route_channel_capacity() -> usize { + DEFAULT_ME_ROUTE_CHANNEL_CAPACITY +} + +pub(crate) fn default_me_c2me_channel_capacity() -> usize { + DEFAULT_ME_C2ME_CHANNEL_CAPACITY +} + +pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 { + DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY +} + +pub(crate) fn default_me_health_interval_ms_healthy() -> u64 { + DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY +} + +pub(crate) fn default_me_admission_poll_ms() -> u64 { + DEFAULT_ME_ADMISSION_POLL_MS +} + +pub(crate) fn default_me_warn_rate_limit_ms() -> u64 { + DEFAULT_ME_WARN_RATE_LIMIT_MS +} + pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index a7ae60c..e7029a3 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -91,6 +91,10 @@ pub struct HotFields { pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_watermark_pct: u8, + pub me_health_interval_ms_unhealthy: u64, + pub me_health_interval_ms_healthy: u64, + pub me_admission_poll_ms: u64, + pub me_warn_rate_limit_ms: u64, pub users: std::collections::HashMap, pub user_ad_tags: std::collections::HashMap, pub user_max_tcp_conns: std::collections::HashMap, @@ -192,6 +196,10 @@ impl HotFields { me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, + me_health_interval_ms_unhealthy: cfg.general.me_health_interval_ms_unhealthy, + me_health_interval_ms_healthy: cfg.general.me_health_interval_ms_healthy, + me_admission_poll_ms: cfg.general.me_admission_poll_ms, + me_warn_rate_limit_ms: cfg.general.me_warn_rate_limit_ms, users: cfg.access.users.clone(), user_ad_tags: cfg.access.user_ad_tags.clone(), user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(), @@ -335,6 +343,10 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { new.general.me_route_backpressure_high_timeout_ms; cfg.general.me_route_backpressure_high_watermark_pct = new.general.me_route_backpressure_high_watermark_pct; + cfg.general.me_health_interval_ms_unhealthy = new.general.me_health_interval_ms_unhealthy; + cfg.general.me_health_interval_ms_healthy = new.general.me_health_interval_ms_healthy; + cfg.general.me_admission_poll_ms = new.general.me_admission_poll_ms; + cfg.general.me_warn_rate_limit_ms = new.general.me_warn_rate_limit_ms; cfg.access.users = new.access.users.clone(); cfg.access.user_ad_tags = new.access.user_ad_tags.clone(); @@ -796,12 +808,21 @@ fn log_changes( != new_hot.me_route_backpressure_high_timeout_ms || old_hot.me_route_backpressure_high_watermark_pct != new_hot.me_route_backpressure_high_watermark_pct + || old_hot.me_health_interval_ms_unhealthy + != new_hot.me_health_interval_ms_unhealthy + || old_hot.me_health_interval_ms_healthy != new_hot.me_health_interval_ms_healthy + || old_hot.me_admission_poll_ms != new_hot.me_admission_poll_ms + || old_hot.me_warn_rate_limit_ms != new_hot.me_warn_rate_limit_ms { info!( - "config reload: me_route_backpressure: base={}ms high={}ms watermark={}%", + "config reload: me_route_backpressure: base={}ms high={}ms watermark={}%; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms", new_hot.me_route_backpressure_base_timeout_ms, new_hot.me_route_backpressure_high_timeout_ms, new_hot.me_route_backpressure_high_watermark_pct, + new_hot.me_health_interval_ms_unhealthy, + new_hot.me_health_interval_ms_healthy, + new_hot.me_admission_poll_ms, + new_hot.me_warn_rate_limit_ms, ); } diff --git a/src/config/load.rs b/src/config/load.rs index 25a9994..c013b1a 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -285,6 +285,48 @@ impl ProxyConfig { )); } + if config.general.me_writer_cmd_channel_capacity == 0 { + return Err(ProxyError::Config( + "general.me_writer_cmd_channel_capacity must be > 0".to_string(), + )); + } + + if config.general.me_route_channel_capacity == 0 { + return Err(ProxyError::Config( + "general.me_route_channel_capacity must be > 0".to_string(), + )); + } + + if config.general.me_c2me_channel_capacity == 0 { + return Err(ProxyError::Config( + "general.me_c2me_channel_capacity must be > 0".to_string(), + )); + } + + if config.general.me_health_interval_ms_unhealthy == 0 { + return Err(ProxyError::Config( + "general.me_health_interval_ms_unhealthy must be > 0".to_string(), + )); + } + + if config.general.me_health_interval_ms_healthy == 0 { + return Err(ProxyError::Config( + "general.me_health_interval_ms_healthy must be > 0".to_string(), + )); + } + + if config.general.me_admission_poll_ms == 0 { + return Err(ProxyError::Config( + "general.me_admission_poll_ms must be > 0".to_string(), + )); + } + + if config.general.me_warn_rate_limit_ms == 0 { + return Err(ProxyError::Config( + "general.me_warn_rate_limit_ms must be > 0".to_string(), + )); + } + if config.access.user_max_unique_ips_window_secs == 0 { return Err(ProxyError::Config( "access.user_max_unique_ips_window_secs must be > 0".to_string(), diff --git a/src/config/types.rs b/src/config/types.rs index 0c89df8..b2be9cf 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -420,6 +420,18 @@ pub struct GeneralConfig { #[serde(default = "default_rpc_proxy_req_every")] pub rpc_proxy_req_every: u64, + /// Capacity of per-ME writer command channel. + #[serde(default = "default_me_writer_cmd_channel_capacity")] + pub me_writer_cmd_channel_capacity: usize, + + /// Capacity of per-connection ME response route channel. + #[serde(default = "default_me_route_channel_capacity")] + pub me_route_channel_capacity: usize, + + /// Capacity of per-client command queue from client reader to ME sender task. + #[serde(default = "default_me_c2me_channel_capacity")] + pub me_c2me_channel_capacity: usize, + /// Max pending ciphertext buffer per client writer (bytes). /// Controls FakeTLS backpressure vs throughput. #[serde(default = "default_crypto_pending_buffer")] @@ -620,6 +632,22 @@ pub struct GeneralConfig { #[serde(default = "default_me_route_backpressure_high_watermark_pct")] pub me_route_backpressure_high_watermark_pct: u8, + /// Health monitor interval in milliseconds while writer coverage is degraded. + #[serde(default = "default_me_health_interval_ms_unhealthy")] + pub me_health_interval_ms_unhealthy: u64, + + /// Health monitor interval in milliseconds while writer coverage is stable. + #[serde(default = "default_me_health_interval_ms_healthy")] + pub me_health_interval_ms_healthy: u64, + + /// Poll interval in milliseconds for conditional-admission state checks. + #[serde(default = "default_me_admission_poll_ms")] + pub me_admission_poll_ms: u64, + + /// Cooldown for repetitive ME warning logs in milliseconds. + #[serde(default = "default_me_warn_rate_limit_ms")] + pub me_warn_rate_limit_ms: u64, + /// ME route behavior when no writer is immediately available. #[serde(default)] pub me_route_no_writer_mode: MeRouteNoWriterMode, @@ -796,6 +824,9 @@ impl Default for GeneralConfig { me_keepalive_jitter_secs: default_keepalive_jitter(), me_keepalive_payload_random: default_true(), rpc_proxy_req_every: default_rpc_proxy_req_every(), + me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(), + me_route_channel_capacity: default_me_route_channel_capacity(), + me_c2me_channel_capacity: default_me_c2me_channel_capacity(), me_warmup_stagger_enabled: default_true(), me_warmup_step_delay_ms: default_warmup_step_delay_ms(), me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(), @@ -837,6 +868,10 @@ impl Default for GeneralConfig { me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(), me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(), me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(), + me_health_interval_ms_unhealthy: default_me_health_interval_ms_unhealthy(), + me_health_interval_ms_healthy: default_me_health_interval_ms_healthy(), + me_admission_poll_ms: default_me_admission_poll_ms(), + me_warn_rate_limit_ms: default_me_warn_rate_limit_ms(), me_route_no_writer_mode: MeRouteNoWriterMode::default(), me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(), me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(), diff --git a/src/main.rs b/src/main.rs index 58c70cd..c1059c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1048,9 +1048,14 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_secret_atomic_snapshot, config.general.me_deterministic_writer_sort, config.general.me_socks_kdf_policy, + config.general.me_writer_cmd_channel_capacity, + config.general.me_route_channel_capacity, config.general.me_route_backpressure_base_timeout_ms, config.general.me_route_backpressure_high_timeout_ms, config.general.me_route_backpressure_high_watermark_pct, + config.general.me_health_interval_ms_unhealthy, + config.general.me_health_interval_ms_healthy, + config.general.me_warn_rate_limit_ms, config.general.me_route_no_writer_mode, config.general.me_route_no_writer_wait_ms, config.general.me_route_inline_recovery_attempts, @@ -1784,11 +1789,24 @@ async fn main() -> std::result::Result<(), Box> { let pool_for_gate = pool.clone(); let admission_tx_gate = admission_tx.clone(); + let mut config_rx_gate = config_rx.clone(); + let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1); tokio::spawn(async move { let mut gate_open = initial_open; let mut open_streak = if initial_open { 1u32 } else { 0u32 }; let mut close_streak = if initial_open { 0u32 } else { 1u32 }; loop { + tokio::select! { + changed = config_rx_gate.changed() => { + if changed.is_err() { + break; + } + let cfg = config_rx_gate.borrow_and_update().clone(); + admission_poll_ms = cfg.general.me_admission_poll_ms.max(1); + continue; + } + _ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {} + } let ready = pool_for_gate.admission_ready_conditional_cast().await; if ready { open_streak = open_streak.saturating_add(1); @@ -1813,7 +1831,6 @@ async fn main() -> std::result::Result<(), Box> { ); } } - tokio::time::sleep(Duration::from_millis(250)).await; } }); } else { diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index 707c8af..cae8273 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -27,7 +27,7 @@ enum C2MeCommand { const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60); const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync"; -const C2ME_CHANNEL_CAPACITY: usize = 1024; +const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128; const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64; const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32; static DESYNC_DEDUP: OnceLock>> = OnceLock::new(); @@ -271,7 +271,11 @@ where let frame_limit = config.general.max_client_frame; - let (c2me_tx, mut c2me_rx) = mpsc::channel::(C2ME_CHANNEL_CAPACITY); + let c2me_channel_capacity = config + .general + .me_c2me_channel_capacity + .max(C2ME_CHANNEL_CAPACITY_FALLBACK); + let (c2me_tx, mut c2me_rx) = mpsc::channel::(c2me_channel_capacity); let me_pool_c2me = me_pool.clone(); let effective_tag = effective_tag; let c2me_sender = tokio::spawn(async move { diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index cccf381..1bcda14 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -325,6 +325,9 @@ async fn run_update_cycle( cfg.general.me_adaptive_floor_max_warm_writers_per_core, cfg.general.me_adaptive_floor_max_active_writers_global, cfg.general.me_adaptive_floor_max_warm_writers_global, + cfg.general.me_health_interval_ms_unhealthy, + cfg.general.me_health_interval_ms_healthy, + cfg.general.me_warn_rate_limit_ms, ); let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1); @@ -546,6 +549,9 @@ pub async fn me_config_updater( cfg.general.me_adaptive_floor_max_warm_writers_per_core, cfg.general.me_adaptive_floor_max_active_writers_global, cfg.general.me_adaptive_floor_max_warm_writers_global, + cfg.general.me_health_interval_ms_unhealthy, + cfg.general.me_health_interval_ms_healthy, + cfg.general.me_warn_rate_limit_ms, ); let new_secs = cfg.general.effective_update_every_secs().max(1); if new_secs == update_every_secs { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index aefeead..b422dc6 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -13,7 +13,6 @@ use crate::network::IpFamily; use super::MePool; -const HEALTH_INTERVAL_SECS: u64 = 1; const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff #[allow(dead_code)] const MAX_CONCURRENT_PER_DC_DEFAULT: usize = 1; @@ -62,11 +61,18 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c let mut idle_refresh_next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut degraded_interval = true; loop { - tokio::time::sleep(Duration::from_secs(HEALTH_INTERVAL_SECS)).await; + let interval = if degraded_interval { + pool.health_interval_unhealthy() + } else { + pool.health_interval_healthy() + }; + tokio::time::sleep(interval).await; pool.prune_closed_writers().await; reap_draining_writers(&pool).await; - check_family( + let v4_degraded = check_family( IpFamily::V4, &pool, &rng, @@ -80,9 +86,10 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut idle_refresh_next_attempt, &mut adaptive_idle_since, &mut adaptive_recover_until, + &mut floor_warn_next_allowed, ) .await; - check_family( + let v6_degraded = check_family( IpFamily::V6, &pool, &rng, @@ -96,8 +103,10 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut idle_refresh_next_attempt, &mut adaptive_idle_since, &mut adaptive_recover_until, + &mut floor_warn_next_allowed, ) .await; + degraded_interval = v4_degraded || v6_degraded; } } @@ -137,15 +146,18 @@ async fn check_family( idle_refresh_next_attempt: &mut HashMap<(i32, IpFamily), Instant>, adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, -) { + floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>, +) -> bool { let enabled = match family { IpFamily::V4 => pool.decision.ipv4_me, IpFamily::V6 => pool.decision.ipv6_me, }; if !enabled { - return; + return false; } + let mut family_degraded = false; + let mut dc_endpoints = HashMap::>::new(); let map_guard = match family { IpFamily::V4 => pool.proxy_map_v4.read().await, @@ -234,6 +246,7 @@ async fn check_family( .sum::(); if endpoints.len() == 1 && pool.single_endpoint_outage_mode_enabled() && alive == 0 { + family_degraded = true; if single_endpoint_outage.insert(key) { pool.stats.increment_me_single_endpoint_outage_enter_total(); warn!( @@ -310,6 +323,7 @@ async fn check_family( continue; } let missing = required - alive; + family_degraded = true; let now = Instant::now(); if reconnect_budget == 0 { @@ -438,15 +452,23 @@ async fn check_family( + Duration::from_millis(rand::rng().random_range(0..=jitter.max(1))); next_attempt.insert(key, now + wait); if pool.is_runtime_ready() { - warn!( - dc = %dc, - ?family, - alive = now_alive, - required, - endpoint_count = endpoints.len(), - backoff_ms = next_ms, - "DC writer floor is below required level, scheduled reconnect" - ); + let warn_cooldown = pool.warn_rate_limit_duration(); + if should_emit_rate_limited_warn( + floor_warn_next_allowed, + key, + now, + warn_cooldown, + ) { + warn!( + dc = %dc, + ?family, + alive = now_alive, + required, + endpoint_count = endpoints.len(), + backoff_ms = next_ms, + "DC writer floor is below required level, scheduled reconnect" + ); + } } else { info!( dc = %dc, @@ -463,6 +485,8 @@ async fn check_family( *v = v.saturating_sub(1); } } + + family_degraded } fn health_reconnect_budget(pool: &Arc, dc_groups: usize) -> usize { @@ -474,6 +498,23 @@ fn health_reconnect_budget(pool: &Arc, dc_groups: usize) -> usize { .clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX) } +fn should_emit_rate_limited_warn( + next_allowed: &mut HashMap<(i32, IpFamily), Instant>, + key: (i32, IpFamily), + now: Instant, + cooldown: Duration, +) -> bool { + let Some(ready_at) = next_allowed.get(&key).copied() else { + next_allowed.insert(key, now + cooldown); + return true; + }; + if now >= ready_at { + next_allowed.insert(key, now + cooldown); + return true; + } + false +} + fn adaptive_floor_class_min( pool: &Arc, endpoint_count: usize, diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 5ec512a..0c9c30c 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -103,6 +103,7 @@ pub struct MePool { pub(super) me_keepalive_jitter: Duration, pub(super) me_keepalive_payload_random: bool, pub(super) rpc_proxy_req_every_secs: AtomicU64, + pub(super) writer_cmd_channel_capacity: usize, pub(super) me_warmup_stagger_enabled: bool, pub(super) me_warmup_step_delay: Duration, pub(super) me_warmup_step_jitter: Duration, @@ -181,8 +182,12 @@ pub struct MePool { pub(super) me_route_no_writer_wait: Duration, pub(super) me_route_inline_recovery_attempts: u32, pub(super) me_route_inline_recovery_wait: Duration, + pub(super) me_health_interval_ms_unhealthy: AtomicU64, + pub(super) me_health_interval_ms_healthy: AtomicU64, + pub(super) me_warn_rate_limit_ms: AtomicU64, pub(super) runtime_ready: AtomicBool, pool_size: usize, + pub(super) preferred_endpoints_by_dc: Arc>>>, } #[derive(Debug, Default)] @@ -270,16 +275,25 @@ impl MePool { me_secret_atomic_snapshot: bool, me_deterministic_writer_sort: bool, me_socks_kdf_policy: MeSocksKdfPolicy, + me_writer_cmd_channel_capacity: usize, + me_route_channel_capacity: usize, me_route_backpressure_base_timeout_ms: u64, me_route_backpressure_high_timeout_ms: u64, me_route_backpressure_high_watermark_pct: u8, + me_health_interval_ms_unhealthy: u64, + me_health_interval_ms_healthy: u64, + me_warn_rate_limit_ms: u64, me_route_no_writer_mode: MeRouteNoWriterMode, me_route_no_writer_wait_ms: u64, me_route_inline_recovery_attempts: u32, me_route_inline_recovery_wait_ms: u64, ) -> Arc { let endpoint_dc_map = Self::build_endpoint_dc_map_from_maps(&proxy_map_v4, &proxy_map_v6); - let registry = Arc::new(ConnRegistry::new()); + let preferred_endpoints_by_dc = + Self::build_preferred_endpoints_by_dc(&decision, &proxy_map_v4, &proxy_map_v6); + let registry = Arc::new(ConnRegistry::with_route_channel_capacity( + me_route_channel_capacity, + )); registry.update_route_backpressure_policy( me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms, @@ -326,6 +340,7 @@ impl MePool { me_keepalive_jitter: Duration::from_secs(me_keepalive_jitter_secs), me_keepalive_payload_random, rpc_proxy_req_every_secs: AtomicU64::new(rpc_proxy_req_every_secs), + writer_cmd_channel_capacity: me_writer_cmd_channel_capacity.max(1), me_warmup_stagger_enabled, me_warmup_step_delay: Duration::from_millis(me_warmup_step_delay_ms), me_warmup_step_jitter: Duration::from_millis(me_warmup_step_jitter_ms), @@ -440,7 +455,11 @@ impl MePool { me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), me_route_inline_recovery_attempts, me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms), + me_health_interval_ms_unhealthy: AtomicU64::new(me_health_interval_ms_unhealthy.max(1)), + me_health_interval_ms_healthy: AtomicU64::new(me_health_interval_ms_healthy.max(1)), + me_warn_rate_limit_ms: AtomicU64::new(me_warn_rate_limit_ms.max(1)), runtime_ready: AtomicBool::new(false), + preferred_endpoints_by_dc: Arc::new(RwLock::new(preferred_endpoints_by_dc)), }) } @@ -489,6 +508,9 @@ impl MePool { adaptive_floor_max_warm_writers_per_core: u16, adaptive_floor_max_active_writers_global: u32, adaptive_floor_max_warm_writers_global: u32, + me_health_interval_ms_unhealthy: u64, + me_health_interval_ms_healthy: u64, + me_warn_rate_limit_ms: u64, ) { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs @@ -564,6 +586,12 @@ impl MePool { .store(adaptive_floor_max_active_writers_global, Ordering::Relaxed); self.me_adaptive_floor_max_warm_writers_global .store(adaptive_floor_max_warm_writers_global, Ordering::Relaxed); + self.me_health_interval_ms_unhealthy + .store(me_health_interval_ms_unhealthy.max(1), Ordering::Relaxed); + self.me_health_interval_ms_healthy + .store(me_health_interval_ms_healthy.max(1), Ordering::Relaxed); + self.me_warn_rate_limit_ms + .store(me_warn_rate_limit_ms.max(1), Ordering::Relaxed); if previous_floor_mode != floor_mode { self.stats.increment_me_floor_mode_switch_total(); match (previous_floor_mode, floor_mode) { @@ -1042,6 +1070,62 @@ impl MePool { } } + fn build_preferred_endpoints_by_dc( + decision: &NetworkDecision, + map_v4: &HashMap>, + map_v6: &HashMap>, + ) -> HashMap> { + let mut out = HashMap::>::new(); + let mut dcs = HashSet::::new(); + dcs.extend(map_v4.keys().copied()); + dcs.extend(map_v6.keys().copied()); + + for dc in dcs { + let v4 = map_v4 + .get(&dc) + .map(|items| { + items + .iter() + .map(|(ip, port)| SocketAddr::new(*ip, *port)) + .collect::>() + }) + .unwrap_or_default(); + let v6 = map_v6 + .get(&dc) + .map(|items| { + items + .iter() + .map(|(ip, port)| SocketAddr::new(*ip, *port)) + .collect::>() + }) + .unwrap_or_default(); + + let mut selected = if decision.effective_multipath { + let mut both = Vec::::with_capacity(v4.len().saturating_add(v6.len())); + if decision.prefer_ipv6() { + both.extend(v6.iter().copied()); + both.extend(v4.iter().copied()); + } else { + both.extend(v4.iter().copied()); + both.extend(v6.iter().copied()); + } + both + } else if decision.prefer_ipv6() { + if !v6.is_empty() { v6 } else { v4 } + } else if !v4.is_empty() { + v4 + } else { + v6 + }; + + selected.sort_unstable(); + selected.dedup(); + out.insert(dc, selected); + } + + out + } + fn build_endpoint_dc_map_from_maps( map_v4: &HashMap>, map_v6: &HashMap>, @@ -1064,6 +1148,25 @@ impl MePool { let map_v4 = self.proxy_map_v4.read().await.clone(); let map_v6 = self.proxy_map_v6.read().await.clone(); let rebuilt = Self::build_endpoint_dc_map_from_maps(&map_v4, &map_v6); + let preferred = Self::build_preferred_endpoints_by_dc(&self.decision, &map_v4, &map_v6); *self.endpoint_dc_map.write().await = rebuilt; + *self.preferred_endpoints_by_dc.write().await = preferred; + } + + pub(super) async fn preferred_endpoints_for_dc(&self, dc: i32) -> Vec { + let guard = self.preferred_endpoints_by_dc.read().await; + guard.get(&dc).cloned().unwrap_or_default() + } + + pub(super) fn health_interval_unhealthy(&self) -> Duration { + Duration::from_millis(self.me_health_interval_ms_unhealthy.load(Ordering::Relaxed).max(1)) + } + + pub(super) fn health_interval_healthy(&self) -> Duration { + Duration::from_millis(self.me_health_interval_ms_healthy.load(Ordering::Relaxed).max(1)) + } + + pub(super) fn warn_rate_limit_duration(&self) -> Duration { + Duration::from_millis(self.me_warn_rate_limit_ms.load(Ordering::Relaxed).max(1)) } } diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 90f8d0a..036572a 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -132,7 +132,7 @@ impl MePool { let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0)); let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0)); let allow_drain_fallback = Arc::new(AtomicBool::new(false)); - let (tx, mut rx) = mpsc::channel::(4096); + let (tx, mut rx) = mpsc::channel::(self.writer_cmd_channel_capacity); let mut rpc_writer = RpcWriter { writer: hs.wr, key: hs.write_key, diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index 0ee81e0..f2682d5 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -9,7 +9,6 @@ use tokio::sync::mpsc::error::TrySendError; use super::codec::WriterCommand; use super::MeResponse; -const ROUTE_CHANNEL_CAPACITY: usize = 4096; const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25; const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120; const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80; @@ -78,6 +77,7 @@ impl RegistryInner { pub struct ConnRegistry { inner: RwLock, next_id: AtomicU64, + route_channel_capacity: usize, route_backpressure_base_timeout_ms: AtomicU64, route_backpressure_high_timeout_ms: AtomicU64, route_backpressure_high_watermark_pct: AtomicU8, @@ -91,11 +91,12 @@ impl ConnRegistry { .as_secs() } - pub fn new() -> Self { + pub fn with_route_channel_capacity(route_channel_capacity: usize) -> Self { let start = rand::random::() | 1; Self { inner: RwLock::new(RegistryInner::new()), next_id: AtomicU64::new(start), + route_channel_capacity: route_channel_capacity.max(1), route_backpressure_base_timeout_ms: AtomicU64::new( ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS, ), @@ -108,6 +109,11 @@ impl ConnRegistry { } } + #[cfg(test)] + pub fn new() -> Self { + Self::with_route_channel_capacity(4096) + } + pub fn update_route_backpressure_policy( &self, base_timeout_ms: u64, @@ -127,7 +133,7 @@ impl ConnRegistry { pub async fn register(&self) -> (u64, mpsc::Receiver) { let id = self.next_id.fetch_add(1, Ordering::Relaxed); - let (tx, rx) = mpsc::channel(ROUTE_CHANNEL_CAPACITY); + let (tx, rx) = mpsc::channel(self.route_channel_capacity); self.inner.write().await.map.insert(id, tx); (id, rx) } @@ -179,11 +185,11 @@ impl ConnRegistry { .route_backpressure_high_watermark_pct .load(Ordering::Relaxed) .clamp(1, 100); - let used = ROUTE_CHANNEL_CAPACITY.saturating_sub(tx.capacity()); - let used_pct = if ROUTE_CHANNEL_CAPACITY == 0 { + let used = self.route_channel_capacity.saturating_sub(tx.capacity()); + let used_pct = if self.route_channel_capacity == 0 { 100 } else { - (used.saturating_mul(100) / ROUTE_CHANNEL_CAPACITY) as u8 + (used.saturating_mul(100) / self.route_channel_capacity) as u8 }; let high_profile = used_pct >= high_watermark_pct; let timeout_ms = if high_profile { diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 07d39f6..ec199fd 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -480,31 +480,7 @@ impl MePool { } async fn endpoint_candidates_for_target_dc(&self, routed_dc: i32) -> Vec { - let mut preferred = Vec::::new(); - let mut seen = HashSet::::new(); - - for family in self.family_order() { - let map_guard = match family { - IpFamily::V4 => self.proxy_map_v4.read().await, - IpFamily::V6 => self.proxy_map_v6.read().await, - }; - let mut family_selected = Vec::::new(); - if let Some(addrs) = map_guard.get(&routed_dc) { - for (ip, port) in addrs { - family_selected.push(SocketAddr::new(*ip, *port)); - } - } - for addr in family_selected { - if seen.insert(addr) { - preferred.push(addr); - } - } - if !preferred.is_empty() && !self.decision.effective_multipath { - break; - } - } - - preferred + self.preferred_endpoints_for_dc(routed_dc).await } async fn maybe_trigger_hybrid_recovery( @@ -591,28 +567,7 @@ impl MePool { routed_dc: i32, include_warm: bool, ) -> Vec { - let mut preferred = HashSet::::new(); - - for family in self.family_order() { - let map_guard = match family { - IpFamily::V4 => self.proxy_map_v4.read().await, - IpFamily::V6 => self.proxy_map_v6.read().await, - }; - let mut family_selected = Vec::::new(); - if let Some(v) = map_guard.get(&routed_dc) { - family_selected.extend(v.iter().map(|(ip, port)| SocketAddr::new(*ip, *port))); - } - for endpoint in family_selected { - preferred.insert(endpoint); - } - - drop(map_guard); - - if !preferred.is_empty() && !self.decision.effective_multipath { - break; - } - } - + let preferred = self.preferred_endpoints_for_dc(routed_dc).await; if preferred.is_empty() { return Vec::new(); } @@ -622,7 +577,7 @@ impl MePool { if !self.writer_eligible_for_selection(w, include_warm) { continue; } - if w.writer_dc == routed_dc && preferred.contains(&w.addr) { + if w.writer_dc == routed_dc && preferred.iter().any(|endpoint| *endpoint == w.addr) { out.push(idx); } } From ca2eaa9eadee54e766d9c6ecbd0cc575e2f92aa1 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 19:37:40 +0300 Subject: [PATCH 136/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3311a82..a95eef7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.9" +version = "3.3.10" edition = "2024" [dependencies] From 373ae3281e3fe9e8e8da966b10419d9a7ac78ad5 Mon Sep 17 00:00:00 2001 From: Mirotin Artem Date: Sat, 7 Mar 2026 19:43:55 +0300 Subject: [PATCH 137/261] Update install.sh --- install.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 006d33c..7516748 100644 --- a/install.sh +++ b/install.sh @@ -29,11 +29,13 @@ detect_arch() { } detect_libc() { - if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -iq musl; then - printf 'musl\n' - else - printf 'gnu\n' + if command -v ldd >/dev/null 2>&1; then + if ldd --version 2>&1 | grep -iq musl; then + printf 'musl\n' + return + fi fi + printf 'gnu\n' } fetch_to_stdout() { From 7eb2b60855603d0a194ff22888053c8be202062d Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 19:49:45 +0300 Subject: [PATCH 138/261] Update install.sh --- install.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/install.sh b/install.sh index 7516748..9d96f0e 100644 --- a/install.sh +++ b/install.sh @@ -29,13 +29,10 @@ detect_arch() { } detect_libc() { - if command -v ldd >/dev/null 2>&1; then - if ldd --version 2>&1 | grep -iq musl; then - printf 'musl\n' - return - fi - fi - printf 'gnu\n' + case "$(ldd --version 2>&1 || true)" in + *musl*) printf 'musl\n' ;; + *) printf 'gnu\n' ;; + esac } fetch_to_stdout() { From c942c492ad4211948ed08296ba1bdea5782c9536 Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Sat, 7 Mar 2026 19:51:37 +0300 Subject: [PATCH 139/261] Apply suggestions from code review Co-authored-by: Alexey <247128645+axkurcom@users.noreply.github.com> --- telemt.service | 1 - 1 file changed, 1 deletion(-) diff --git a/telemt.service b/telemt.service index 55fd9fd..2ec36bf 100644 --- a/telemt.service +++ b/telemt.service @@ -9,7 +9,6 @@ WorkingDirectory=/bin ExecStart=/bin/telemt /etc/telemt.toml Restart=on-failure LimitNOFILE=65536 -SELinuxContext=system_u:system_r:unconfined_service_t:s0 [Install] WantedBy=multi-user.target From cfee7de66b3ab82339c858ac35c83456b1bf93ff Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 7 Mar 2026 19:55:28 +0300 Subject: [PATCH 140/261] Update telemt.service --- telemt.service | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/telemt.service b/telemt.service index 2ec36bf..f163057 100644 --- a/telemt.service +++ b/telemt.service @@ -5,10 +5,12 @@ Wants=network-online.target [Service] Type=simple -WorkingDirectory=/bin +WorkingDirectory=/etc/telemt ExecStart=/bin/telemt /etc/telemt.toml Restart=on-failure -LimitNOFILE=65536 +LimitNOFILE=262144 +TasksMax=8192 +MemoryAccounting=yes [Install] WantedBy=multi-user.target From cd5c60ce1e7a33cb4bca46f032a1c98b4e9f7c89 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:03:35 +0300 Subject: [PATCH 141/261] Update reader.rs --- src/transport/middle_proxy/reader.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index 61bd69c..32de774 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::io::ErrorKind; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::time::Instant; use bytes::{Bytes, BytesMut}; @@ -34,6 +34,7 @@ pub(crate) async fn reader_loop( stats: Arc, _writer_id: u64, degraded: Arc, + writer_rtt_ema_ms_x10: Arc, cancel: CancellationToken, ) -> Result<()> { let mut raw = enc_leftover; @@ -208,6 +209,8 @@ pub(crate) async fn reader_loop( } let degraded_now = entry.1 > entry.0 * 2.0; degraded.store(degraded_now, Ordering::Relaxed); + writer_rtt_ema_ms_x10 + .store((entry.1 * 10.0).clamp(0.0, u32::MAX as f64) as u32, Ordering::Relaxed); trace!(writer_id = wid, rtt_ms = rtt, ema_ms = entry.1, base_ms = entry.0, degraded = degraded_now, "ME RTT sample"); } } else { From c08160600e9fb6d1a54c9df8f737aee78b9a2bed Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:03:41 +0300 Subject: [PATCH 142/261] Update pool_writer.rs --- src/transport/middle_proxy/pool_writer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 036572a..43abf0c 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -1,6 +1,6 @@ use std::net::SocketAddr; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use std::io::ErrorKind; @@ -128,6 +128,7 @@ impl MePool { let contour = Arc::new(AtomicU8::new(contour.as_u8())); let cancel = CancellationToken::new(); let degraded = Arc::new(AtomicBool::new(false)); + let rtt_ema_ms_x10 = Arc::new(AtomicU32::new(0)); let draining = Arc::new(AtomicBool::new(false)); let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0)); let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0)); @@ -169,6 +170,7 @@ impl MePool { tx: tx.clone(), cancel: cancel.clone(), degraded: degraded.clone(), + rtt_ema_ms_x10: rtt_ema_ms_x10.clone(), draining: draining.clone(), draining_started_at_epoch_secs: draining_started_at_epoch_secs.clone(), drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(), @@ -222,6 +224,7 @@ impl MePool { stats_reader, writer_id, degraded.clone(), + rtt_ema_ms_x10.clone(), cancel_reader_token.clone(), ) .await; From 6b3697ee872e5ad7ddcc452c018f8074c1d4a2f3 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:04:27 +0300 Subject: [PATCH 143/261] ME Active-by-Endpoint --- src/transport/middle_proxy/pool_refill.rs | 29 +++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 544d048..fc916f4 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -113,10 +113,35 @@ impl MePool { contour: WriterContour, allow_coverage_override: bool, ) -> bool { - let candidates = self.connectable_endpoints(endpoints).await; + let mut candidates = self.connectable_endpoints(endpoints).await; if candidates.is_empty() { return false; } + if candidates.len() > 1 { + let mut active_by_endpoint = HashMap::::new(); + let ws = self.writers.read().await; + for writer in ws.iter() { + if writer.draining.load(Ordering::Relaxed) { + continue; + } + if writer.writer_dc != dc { + continue; + } + if !matches!( + super::pool::WriterContour::from_u8( + writer.contour.load(Ordering::Relaxed), + ), + super::pool::WriterContour::Active + ) { + continue; + } + if candidates.contains(&writer.addr) { + *active_by_endpoint.entry(writer.addr).or_insert(0) += 1; + } + } + drop(ws); + candidates.sort_by_key(|addr| (active_by_endpoint.get(addr).copied().unwrap_or(0), *addr)); + } let start = (self.rr.fetch_add(1, Ordering::Relaxed) as usize) % candidates.len(); for offset in 0..candidates.len() { let idx = (start + offset) % candidates.len(); From 9401c46727296d73fefdddf54243511ef33eba57 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:05:47 +0300 Subject: [PATCH 144/261] ME Writer Pick --- src/config/defaults.rs | 9 +- src/config/hot_reload.rs | 19 +- src/config/load.rs | 6 + src/config/types.rs | 36 +++ src/main.rs | 2 + src/transport/middle_proxy/config_updater.rs | 4 + src/transport/middle_proxy/pool.rs | 31 ++- src/transport/middle_proxy/pool_status.rs | 28 +++ src/transport/middle_proxy/send.rs | 226 ++++++++++++++----- 9 files changed, 300 insertions(+), 61 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 798d881..68dd71e 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -21,9 +21,10 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256; -const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 512; +const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 1024; const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 512; -const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 128; +const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 256; +const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3; const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000; const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000; const DEFAULT_ME_ADMISSION_POLL_MS: u64 = 1000; @@ -315,6 +316,10 @@ pub(crate) fn default_me_c2me_channel_capacity() -> usize { DEFAULT_ME_C2ME_CHANNEL_CAPACITY } +pub(crate) fn default_me_writer_pick_sample_size() -> u8 { + DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE +} + pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 { DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index e7029a3..34b2d76 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -29,7 +29,10 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher}; use tokio::sync::{mpsc, watch}; use tracing::{error, info, warn}; -use crate::config::{LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel}; +use crate::config::{ + LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel, + MeWriterPickMode, +}; use super::load::ProxyConfig; // ── Hot fields ──────────────────────────────────────────────────────────────── @@ -57,6 +60,8 @@ pub struct HotFields { pub me_bind_stale_ttl_secs: u64, pub me_secret_atomic_snapshot: bool, pub me_deterministic_writer_sort: bool, + pub me_writer_pick_mode: MeWriterPickMode, + pub me_writer_pick_sample_size: u8, pub me_single_endpoint_shadow_writers: u8, pub me_single_endpoint_outage_mode_enabled: bool, pub me_single_endpoint_outage_disable_quarantine: bool, @@ -130,6 +135,8 @@ impl HotFields { me_bind_stale_ttl_secs: cfg.general.me_bind_stale_ttl_secs, me_secret_atomic_snapshot: cfg.general.me_secret_atomic_snapshot, me_deterministic_writer_sort: cfg.general.me_deterministic_writer_sort, + me_writer_pick_mode: cfg.general.me_writer_pick_mode, + me_writer_pick_sample_size: cfg.general.me_writer_pick_sample_size, me_single_endpoint_shadow_writers: cfg.general.me_single_endpoint_shadow_writers, me_single_endpoint_outage_mode_enabled: cfg .general @@ -292,6 +299,8 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.general.me_bind_stale_ttl_secs = new.general.me_bind_stale_ttl_secs; cfg.general.me_secret_atomic_snapshot = new.general.me_secret_atomic_snapshot; cfg.general.me_deterministic_writer_sort = new.general.me_deterministic_writer_sort; + cfg.general.me_writer_pick_mode = new.general.me_writer_pick_mode; + cfg.general.me_writer_pick_sample_size = new.general.me_writer_pick_sample_size; cfg.general.me_single_endpoint_shadow_writers = new.general.me_single_endpoint_shadow_writers; cfg.general.me_single_endpoint_outage_mode_enabled = new.general.me_single_endpoint_outage_mode_enabled; @@ -683,11 +692,15 @@ fn log_changes( } if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot || old_hot.me_deterministic_writer_sort != new_hot.me_deterministic_writer_sort + || old_hot.me_writer_pick_mode != new_hot.me_writer_pick_mode + || old_hot.me_writer_pick_sample_size != new_hot.me_writer_pick_sample_size { info!( - "config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={}", + "config reload: me_runtime_flags: secret_atomic_snapshot={} deterministic_sort={} writer_pick_mode={:?} writer_pick_sample_size={}", new_hot.me_secret_atomic_snapshot, - new_hot.me_deterministic_writer_sort + new_hot.me_deterministic_writer_sort, + new_hot.me_writer_pick_mode, + new_hot.me_writer_pick_sample_size, ); } if old_hot.me_single_endpoint_shadow_writers != new_hot.me_single_endpoint_shadow_writers diff --git a/src/config/load.rs b/src/config/load.rs index c013b1a..623ec4d 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -519,6 +519,12 @@ impl ProxyConfig { )); } + if !(2..=4).contains(&config.general.me_writer_pick_sample_size) { + return Err(ProxyError::Config( + "general.me_writer_pick_sample_size must be within [2, 4]".to_string(), + )); + } + if config.general.me_route_inline_recovery_attempts == 0 { return Err(ProxyError::Config( "general.me_route_inline_recovery_attempts must be > 0".to_string(), diff --git a/src/config/types.rs b/src/config/types.rs index b2be9cf..588c82f 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -212,6 +212,32 @@ impl MeRouteNoWriterMode { } } +/// Middle-End writer selection mode for new client bindings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum MeWriterPickMode { + SortedRr, + #[default] + P2c, +} + +impl MeWriterPickMode { + pub fn as_u8(self) -> u8 { + match self { + MeWriterPickMode::SortedRr => 0, + MeWriterPickMode::P2c => 1, + } + } + + pub fn from_u8(raw: u8) -> Self { + match raw { + 0 => MeWriterPickMode::SortedRr, + 1 => MeWriterPickMode::P2c, + _ => MeWriterPickMode::P2c, + } + } +} + /// Per-user unique source IP limit mode. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] @@ -782,6 +808,14 @@ pub struct GeneralConfig { #[serde(default = "default_me_deterministic_writer_sort")] pub me_deterministic_writer_sort: bool, + /// Writer selection mode for ME route bind path. + #[serde(default)] + pub me_writer_pick_mode: MeWriterPickMode, + + /// Number of candidates sampled by writer picker in `p2c` mode. + #[serde(default = "default_me_writer_pick_sample_size")] + pub me_writer_pick_sample_size: u8, + /// Enable NTP drift check at startup. #[serde(default = "default_ntp_check")] pub ntp_check: bool, @@ -912,6 +946,8 @@ impl Default for GeneralConfig { me_reinit_trigger_channel: default_me_reinit_trigger_channel(), me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(), me_deterministic_writer_sort: default_me_deterministic_writer_sort(), + me_writer_pick_mode: MeWriterPickMode::default(), + me_writer_pick_sample_size: default_me_writer_pick_sample_size(), ntp_check: default_ntp_check(), ntp_servers: default_ntp_servers(), auto_degradation_enabled: default_true(), diff --git a/src/main.rs b/src/main.rs index c1059c3..d28fabe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1047,6 +1047,8 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_bind_stale_ttl_secs, config.general.me_secret_atomic_snapshot, config.general.me_deterministic_writer_sort, + config.general.me_writer_pick_mode, + config.general.me_writer_pick_sample_size, config.general.me_socks_kdf_policy, config.general.me_writer_cmd_channel_capacity, config.general.me_route_channel_capacity, diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 1bcda14..2c6a07a 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -306,6 +306,8 @@ async fn run_update_cycle( cfg.general.me_bind_stale_ttl_secs, cfg.general.me_secret_atomic_snapshot, cfg.general.me_deterministic_writer_sort, + cfg.general.me_writer_pick_mode, + cfg.general.me_writer_pick_sample_size, cfg.general.me_single_endpoint_shadow_writers, cfg.general.me_single_endpoint_outage_mode_enabled, cfg.general.me_single_endpoint_outage_disable_quarantine, @@ -530,6 +532,8 @@ pub async fn me_config_updater( cfg.general.me_bind_stale_ttl_secs, cfg.general.me_secret_atomic_snapshot, cfg.general.me_deterministic_writer_sort, + cfg.general.me_writer_pick_mode, + cfg.general.me_writer_pick_sample_size, cfg.general.me_single_endpoint_shadow_writers, cfg.general.me_single_endpoint_outage_mode_enabled, cfg.general.me_single_endpoint_outage_disable_quarantine, diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 0c9c30c..07ad67b 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -7,7 +7,9 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, Notify, RwLock, mpsc}; use tokio_util::sync::CancellationToken; -use crate::config::{MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy}; +use crate::config::{ + MeBindStaleMode, MeFloorMode, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode, +}; use crate::crypto::SecureRandom; use crate::network::IpFamily; use crate::network::probe::NetworkDecision; @@ -39,6 +41,7 @@ pub struct MeWriter { pub tx: mpsc::Sender, pub cancel: CancellationToken, pub degraded: Arc, + pub rtt_ema_ms_x10: Arc, pub draining: Arc, pub draining_started_at_epoch_secs: Arc, pub drain_deadline_epoch_secs: Arc, @@ -177,6 +180,8 @@ pub struct MePool { pub(super) me_bind_stale_ttl_secs: AtomicU64, pub(super) secret_atomic_snapshot: AtomicBool, pub(super) me_deterministic_writer_sort: AtomicBool, + pub(super) me_writer_pick_mode: AtomicU8, + pub(super) me_writer_pick_sample_size: AtomicU8, pub(super) me_socks_kdf_policy: AtomicU8, pub(super) me_route_no_writer_mode: AtomicU8, pub(super) me_route_no_writer_wait: Duration, @@ -274,6 +279,8 @@ impl MePool { me_bind_stale_ttl_secs: u64, me_secret_atomic_snapshot: bool, me_deterministic_writer_sort: bool, + me_writer_pick_mode: MeWriterPickMode, + me_writer_pick_sample_size: u8, me_socks_kdf_policy: MeSocksKdfPolicy, me_writer_cmd_channel_capacity: usize, me_route_channel_capacity: usize, @@ -450,6 +457,8 @@ impl MePool { me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs), secret_atomic_snapshot: AtomicBool::new(me_secret_atomic_snapshot), me_deterministic_writer_sort: AtomicBool::new(me_deterministic_writer_sort), + me_writer_pick_mode: AtomicU8::new(me_writer_pick_mode.as_u8()), + me_writer_pick_sample_size: AtomicU8::new(me_writer_pick_sample_size.clamp(2, 4)), me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()), me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()), me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), @@ -489,6 +498,8 @@ impl MePool { bind_stale_ttl_secs: u64, secret_atomic_snapshot: bool, deterministic_writer_sort: bool, + writer_pick_mode: MeWriterPickMode, + writer_pick_sample_size: u8, single_endpoint_shadow_writers: u8, single_endpoint_outage_mode_enabled: bool, single_endpoint_outage_disable_quarantine: bool, @@ -535,6 +546,14 @@ impl MePool { .store(secret_atomic_snapshot, Ordering::Relaxed); self.me_deterministic_writer_sort .store(deterministic_writer_sort, Ordering::Relaxed); + let previous_writer_pick_mode = self.writer_pick_mode(); + self.me_writer_pick_mode + .store(writer_pick_mode.as_u8(), Ordering::Relaxed); + self.me_writer_pick_sample_size + .store(writer_pick_sample_size.clamp(2, 4), Ordering::Relaxed); + if previous_writer_pick_mode != writer_pick_mode { + self.stats.increment_me_writer_pick_mode_switch_total(); + } self.me_single_endpoint_shadow_writers .store(single_endpoint_shadow_writers, Ordering::Relaxed); self.me_single_endpoint_outage_mode_enabled @@ -692,6 +711,16 @@ impl MePool { MeBindStaleMode::from_u8(self.me_bind_stale_mode.load(Ordering::Relaxed)) } + pub(super) fn writer_pick_mode(&self) -> MeWriterPickMode { + MeWriterPickMode::from_u8(self.me_writer_pick_mode.load(Ordering::Relaxed)) + } + + pub(super) fn writer_pick_sample_size(&self) -> usize { + self.me_writer_pick_sample_size + .load(Ordering::Relaxed) + .clamp(2, 4) as usize + } + pub(super) fn required_writers_for_dc(&self, endpoint_count: usize) -> usize { if endpoint_count == 0 { return 0; diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index cc1be5b..6673cf2 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -25,6 +25,7 @@ pub(crate) struct MeApiWriterStatusSnapshot { pub(crate) struct MeApiDcStatusSnapshot { pub dc: i16, pub endpoints: Vec, + pub endpoint_writers: Vec, pub available_endpoints: usize, pub available_pct: f64, pub required_writers: usize, @@ -38,6 +39,12 @@ pub(crate) struct MeApiDcStatusSnapshot { pub load: usize, } +#[derive(Clone, Debug)] +pub(crate) struct MeApiDcEndpointWriterSnapshot { + pub endpoint: SocketAddr, + pub active_writers: usize, +} + #[derive(Clone, Debug)] pub(crate) struct MeApiStatusSnapshot { pub generated_at_epoch_secs: u64, @@ -118,6 +125,8 @@ pub(crate) struct MeApiRuntimeSnapshot { pub me_single_endpoint_outage_backoff_max_ms: u64, pub me_single_endpoint_shadow_rotate_every_secs: u64, pub me_deterministic_writer_sort: bool, + pub me_writer_pick_mode: &'static str, + pub me_writer_pick_sample_size: u8, pub me_socks_kdf_policy: &'static str, pub quarantined_endpoints: Vec, pub network_path: Vec, @@ -338,6 +347,16 @@ impl MePool { dcs.push(MeApiDcStatusSnapshot { dc, + endpoint_writers: endpoints + .iter() + .map(|endpoint| MeApiDcEndpointWriterSnapshot { + endpoint: *endpoint, + active_writers: live_writers_by_dc_endpoint + .get(&(dc, *endpoint)) + .copied() + .unwrap_or(0), + }) + .collect(), endpoints: endpoints.into_iter().collect(), available_endpoints: dc_available_endpoints, available_pct: ratio_pct(dc_available_endpoints, endpoint_count), @@ -522,6 +541,8 @@ impl MePool { me_deterministic_writer_sort: self .me_deterministic_writer_sort .load(Ordering::Relaxed), + me_writer_pick_mode: writer_pick_mode_label(self.writer_pick_mode()), + me_writer_pick_sample_size: self.writer_pick_sample_size() as u8, me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()), quarantined_endpoints, network_path, @@ -570,6 +591,13 @@ fn bind_stale_mode_label(mode: MeBindStaleMode) -> &'static str { } } +fn writer_pick_mode_label(mode: crate::config::MeWriterPickMode) -> &'static str { + match mode { + crate::config::MeWriterPickMode::SortedRr => "sorted_rr", + crate::config::MeWriterPickMode::P2c => "p2c", + } +} + fn socks_kdf_policy_label(policy: MeSocksKdfPolicy) -> &'static str { match policy { MeSocksKdfPolicy::Strict => "strict", diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index ec199fd..79cfa54 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -9,7 +9,7 @@ use bytes::Bytes; use tokio::sync::mpsc::error::TrySendError; use tracing::{debug, warn}; -use crate::config::MeRouteNoWriterMode; +use crate::config::{MeRouteNoWriterMode, MeWriterPickMode}; use crate::error::{ProxyError, Result}; use crate::network::IpFamily; use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32}; @@ -24,6 +24,10 @@ use super::registry::ConnMeta; const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45; const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55; const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4; +const PICK_PENALTY_WARM: u64 = 200; +const PICK_PENALTY_DRAINING: u64 = 600; +const PICK_PENALTY_STALE: u64 = 300; +const PICK_PENALTY_DEGRADED: u64 = 250; impl MePool { /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default. @@ -181,6 +185,7 @@ impl MePool { .await; } if candidate_indices.is_empty() { + let pick_mode = self.writer_pick_mode(); match no_writer_mode { MeRouteNoWriterMode::AsyncRecoveryFailfast => { let deadline = *no_writer_deadline.get_or_insert_with(|| { @@ -196,6 +201,7 @@ impl MePool { if self.wait_for_candidate_until(routed_dc, deadline).await { continue; } + self.stats.increment_me_writer_pick_no_candidate_total(pick_mode); self.stats.increment_me_no_writer_failfast_total(); return Err(ProxyError::Proxy( "No ME writers available for target DC in failfast window".into(), @@ -209,10 +215,12 @@ impl MePool { if self.wait_for_candidate_until(routed_dc, deadline).await { continue; } + self.stats.increment_me_writer_pick_no_candidate_total(pick_mode); self.stats.increment_me_no_writer_failfast_total(); return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) { + self.stats.increment_me_writer_pick_no_candidate_total(pick_mode); self.stats.increment_me_no_writer_failfast_total(); return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } @@ -237,6 +245,7 @@ impl MePool { .await; } if candidate_indices.is_empty() { + self.stats.increment_me_writer_pick_no_candidate_total(pick_mode); return Err(ProxyError::Proxy("No ME writers available for target DC".into())); } } @@ -259,6 +268,8 @@ impl MePool { } } hybrid_wait_current = hybrid_wait_step; + let pick_mode = self.writer_pick_mode(); + let pick_sample_size = self.writer_pick_sample_size(); let writer_ids: Vec = candidate_indices .iter() .map(|idx| writers_snapshot[*idx].id) @@ -268,69 +279,84 @@ impl MePool { .writer_idle_since_for_writer_ids(&writer_ids) .await; let now_epoch_secs = Self::now_epoch_secs(); - - if self.me_deterministic_writer_sort.load(Ordering::Relaxed) { - candidate_indices.sort_by(|lhs, rhs| { - let left = &writers_snapshot[*lhs]; - let right = &writers_snapshot[*rhs]; - let left_key = ( - self.writer_contour_rank_for_selection(left), - (left.generation < self.current_generation()) as usize, - left.degraded.load(Ordering::Relaxed) as usize, - self.writer_idle_rank_for_selection( - left, - &writer_idle_since, - now_epoch_secs, - ), - Reverse(left.tx.capacity()), - left.addr, - left.id, - ); - let right_key = ( - self.writer_contour_rank_for_selection(right), - (right.generation < self.current_generation()) as usize, - right.degraded.load(Ordering::Relaxed) as usize, - self.writer_idle_rank_for_selection( - right, - &writer_idle_since, - now_epoch_secs, - ), - Reverse(right.tx.capacity()), - right.addr, - right.id, - ); - left_key.cmp(&right_key) - }); - } else { - candidate_indices.sort_by_key(|idx| { - let w = &writers_snapshot[*idx]; - let degraded = w.degraded.load(Ordering::Relaxed); - let stale = (w.generation < self.current_generation()) as usize; - ( - self.writer_contour_rank_for_selection(w), - stale, - degraded as usize, - self.writer_idle_rank_for_selection( - w, - &writer_idle_since, - now_epoch_secs, - ), - Reverse(w.tx.capacity()), - ) - }); - } - let start = self.rr.fetch_add(1, Ordering::Relaxed) as usize % candidate_indices.len(); + let ordered_candidate_indices = if pick_mode == MeWriterPickMode::P2c { + self.p2c_ordered_candidate_indices( + &candidate_indices, + &writers_snapshot, + &writer_idle_since, + now_epoch_secs, + start, + pick_sample_size, + ) + } else { + if self.me_deterministic_writer_sort.load(Ordering::Relaxed) { + candidate_indices.sort_by(|lhs, rhs| { + let left = &writers_snapshot[*lhs]; + let right = &writers_snapshot[*rhs]; + let left_key = ( + self.writer_contour_rank_for_selection(left), + (left.generation < self.current_generation()) as usize, + left.degraded.load(Ordering::Relaxed) as usize, + self.writer_idle_rank_for_selection( + left, + &writer_idle_since, + now_epoch_secs, + ), + Reverse(left.tx.capacity()), + left.addr, + left.id, + ); + let right_key = ( + self.writer_contour_rank_for_selection(right), + (right.generation < self.current_generation()) as usize, + right.degraded.load(Ordering::Relaxed) as usize, + self.writer_idle_rank_for_selection( + right, + &writer_idle_since, + now_epoch_secs, + ), + Reverse(right.tx.capacity()), + right.addr, + right.id, + ); + left_key.cmp(&right_key) + }); + } else { + candidate_indices.sort_by_key(|idx| { + let w = &writers_snapshot[*idx]; + let degraded = w.degraded.load(Ordering::Relaxed); + let stale = (w.generation < self.current_generation()) as usize; + ( + self.writer_contour_rank_for_selection(w), + stale, + degraded as usize, + self.writer_idle_rank_for_selection( + w, + &writer_idle_since, + now_epoch_secs, + ), + Reverse(w.tx.capacity()), + ) + }); + } + + let mut ordered = Vec::::with_capacity(candidate_indices.len()); + for offset in 0..candidate_indices.len() { + ordered.push(candidate_indices[(start + offset) % candidate_indices.len()]); + } + ordered + }; let mut fallback_blocking_idx: Option = None; - for offset in 0..candidate_indices.len() { - let idx = candidate_indices[(start + offset) % candidate_indices.len()]; + for idx in ordered_candidate_indices { let w = &writers_snapshot[idx]; if !self.writer_accepts_new_binding(w) { continue; } match w.tx.try_send(WriterCommand::Data(payload.clone())) { Ok(()) => { + self.stats.increment_me_writer_pick_success_try_total(pick_mode); self.registry .bind_writer(conn_id, w.id, w.tx.clone(), meta.clone()) .await; @@ -352,6 +378,7 @@ impl MePool { } } Err(TrySendError::Closed(_)) => { + self.stats.increment_me_writer_pick_closed_total(pick_mode); warn!(writer_id = w.id, "ME writer channel closed"); self.remove_writer_and_close_clients(w.id).await; continue; @@ -360,15 +387,20 @@ impl MePool { } let Some(blocking_idx) = fallback_blocking_idx else { + self.stats.increment_me_writer_pick_full_total(pick_mode); continue; }; let w = writers_snapshot[blocking_idx].clone(); if !self.writer_accepts_new_binding(&w) { + self.stats.increment_me_writer_pick_full_total(pick_mode); continue; } + self.stats.increment_me_writer_pick_blocking_fallback_total(); match w.tx.send(WriterCommand::Data(payload.clone())).await { Ok(()) => { + self.stats + .increment_me_writer_pick_success_fallback_total(pick_mode); self.registry .bind_writer(conn_id, w.id, w.tx.clone(), meta.clone()) .await; @@ -378,6 +410,7 @@ impl MePool { return Ok(()); } Err(_) => { + self.stats.increment_me_writer_pick_closed_total(pick_mode); warn!(writer_id = w.id, "ME writer channel closed (blocking)"); self.remove_writer_and_close_clients(w.id).await; } @@ -626,4 +659,87 @@ impl MePool { 0 } } + + fn writer_pick_score( + &self, + writer: &super::pool::MeWriter, + idle_since_by_writer: &HashMap, + now_epoch_secs: u64, + ) -> u64 { + let contour_penalty = match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) { + WriterContour::Active => 0, + WriterContour::Warm => PICK_PENALTY_WARM, + WriterContour::Draining => PICK_PENALTY_DRAINING, + }; + let stale_penalty = if writer.generation < self.current_generation() { + PICK_PENALTY_STALE + } else { + 0 + }; + let degraded_penalty = if writer.degraded.load(Ordering::Relaxed) { + PICK_PENALTY_DEGRADED + } else { + 0 + }; + let idle_penalty = + (self.writer_idle_rank_for_selection(writer, idle_since_by_writer, now_epoch_secs) as u64) + * 100; + let queue_cap = self.writer_cmd_channel_capacity.max(1) as u64; + let queue_remaining = writer.tx.capacity() as u64; + let queue_used = queue_cap.saturating_sub(queue_remaining.min(queue_cap)); + let queue_util_pct = queue_used.saturating_mul(100) / queue_cap; + let queue_penalty = queue_util_pct.saturating_mul(4); + let rtt_penalty = ((writer.rtt_ema_ms_x10.load(Ordering::Relaxed) as u64).saturating_add(5) / 10) + .min(400); + + contour_penalty + .saturating_add(stale_penalty) + .saturating_add(degraded_penalty) + .saturating_add(idle_penalty) + .saturating_add(queue_penalty) + .saturating_add(rtt_penalty) + } + + fn p2c_ordered_candidate_indices( + &self, + candidate_indices: &[usize], + writers_snapshot: &[super::pool::MeWriter], + idle_since_by_writer: &HashMap, + now_epoch_secs: u64, + start: usize, + sample_size: usize, + ) -> Vec { + let total = candidate_indices.len(); + if total == 0 { + return Vec::new(); + } + + let mut sampled = Vec::::with_capacity(sample_size.min(total)); + let mut seen = HashSet::::with_capacity(total); + for offset in 0..sample_size.min(total) { + let idx = candidate_indices[(start + offset) % total]; + if seen.insert(idx) { + sampled.push(idx); + } + } + + sampled.sort_by_key(|idx| { + let writer = &writers_snapshot[*idx]; + ( + self.writer_pick_score(writer, idle_since_by_writer, now_epoch_secs), + writer.addr, + writer.id, + ) + }); + + let mut ordered = Vec::::with_capacity(total); + ordered.extend(sampled.iter().copied()); + for offset in 0..total { + let idx = candidate_indices[(start + offset) % total]; + if seen.insert(idx) { + ordered.push(idx); + } + } + ordered + } } From fca0e3f619bdad5ac7f0a549118d1728bb58357a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:06:45 +0300 Subject: [PATCH 145/261] ME Writer Pick in Metrics+API --- src/api/model.rs | 9 +++ src/api/runtime_stats.rs | 18 +++-- src/api/runtime_zero.rs | 13 +++- src/metrics.rs | 129 ++++++++++++++++++++++++++++++++++ src/stats/mod.rs | 147 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 310 insertions(+), 6 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index fd678f6..0bc52de 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -266,6 +266,7 @@ pub(super) struct MeWritersData { pub(super) struct DcStatus { pub(super) dc: i16, pub(super) endpoints: Vec, + pub(super) endpoint_writers: Vec, pub(super) available_endpoints: usize, pub(super) available_pct: f64, pub(super) required_writers: usize, @@ -279,6 +280,12 @@ pub(super) struct DcStatus { pub(super) load: usize, } +#[derive(Serialize, Clone)] +pub(super) struct DcEndpointWriters { + pub(super) endpoint: String, + pub(super) active_writers: usize, +} + #[derive(Serialize, Clone)] pub(super) struct DcStatusData { pub(super) middle_proxy_enabled: bool, @@ -354,6 +361,8 @@ pub(super) struct MinimalMeRuntimeData { pub(super) me_single_endpoint_outage_backoff_max_ms: u64, pub(super) me_single_endpoint_shadow_rotate_every_secs: u64, pub(super) me_deterministic_writer_sort: bool, + pub(super) me_writer_pick_mode: &'static str, + pub(super) me_writer_pick_sample_size: u8, pub(super) me_socks_kdf_policy: &'static str, pub(super) quarantined_endpoints_total: usize, pub(super) quarantined_endpoints: Vec, diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index f90abe3..139a4c5 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -7,10 +7,10 @@ use crate::transport::UpstreamRouteKind; use super::ApiShared; use super::model::{ - DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary, MinimalAllData, - MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData, MinimalQuarantineData, - UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData, ZeroAllData, - ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData, + DcEndpointWriters, DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary, + MinimalAllData, MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData, + MinimalQuarantineData, UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData, + ZeroAllData, ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData, ZeroUpstreamData, }; @@ -346,6 +346,14 @@ async fn get_minimal_payload_cached( .into_iter() .map(|value| value.to_string()) .collect(), + endpoint_writers: entry + .endpoint_writers + .into_iter() + .map(|coverage| DcEndpointWriters { + endpoint: coverage.endpoint.to_string(), + active_writers: coverage.active_writers, + }) + .collect(), available_endpoints: entry.available_endpoints, available_pct: entry.available_pct, required_writers: entry.required_writers, @@ -422,6 +430,8 @@ async fn get_minimal_payload_cached( me_single_endpoint_shadow_rotate_every_secs: runtime .me_single_endpoint_shadow_rotate_every_secs, me_deterministic_writer_sort: runtime.me_deterministic_writer_sort, + me_writer_pick_mode: runtime.me_writer_pick_mode, + me_writer_pick_sample_size: runtime.me_writer_pick_sample_size, me_socks_kdf_policy: runtime.me_socks_kdf_policy, quarantined_endpoints_total: runtime.quarantined_endpoints.len(), quarantined_endpoints: runtime diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index 5020705..7d3d778 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -2,7 +2,7 @@ use std::sync::atomic::Ordering; use serde::Serialize; -use crate::config::{MeFloorMode, ProxyConfig, UserMaxUniqueIpsMode}; +use crate::config::{MeFloorMode, MeWriterPickMode, ProxyConfig, UserMaxUniqueIpsMode}; use super::ApiShared; use super::runtime_init::build_runtime_startup_summary; @@ -78,6 +78,8 @@ pub(super) struct EffectiveMiddleProxyLimits { pub(super) reconnect_backoff_base_ms: u64, pub(super) reconnect_backoff_cap_ms: u64, pub(super) reconnect_fast_retry_count: u32, + pub(super) writer_pick_mode: &'static str, + pub(super) writer_pick_sample_size: u8, pub(super) me2dc_fallback: bool, } @@ -237,6 +239,8 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD reconnect_backoff_base_ms: cfg.general.me_reconnect_backoff_base_ms, reconnect_backoff_cap_ms: cfg.general.me_reconnect_backoff_cap_ms, reconnect_fast_retry_count: cfg.general.me_reconnect_fast_retry_count, + writer_pick_mode: me_writer_pick_mode_label(cfg.general.me_writer_pick_mode), + writer_pick_sample_size: cfg.general.me_writer_pick_sample_size, me2dc_fallback: cfg.general.me2dc_fallback, }, user_ip_policy: EffectiveUserIpPolicyLimits { @@ -274,3 +278,10 @@ fn me_floor_mode_label(mode: MeFloorMode) -> &'static str { MeFloorMode::Adaptive => "adaptive", } } + +fn me_writer_pick_mode_label(mode: MeWriterPickMode) -> &'static str { + match mode { + MeWriterPickMode::SortedRr => "sorted_rr", + MeWriterPickMode::P2c => "p2c", + } +} diff --git a/src/metrics.rs b/src/metrics.rs index b338df5..917c9b3 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -689,6 +689,135 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_writer_pick_total ME writer-pick outcomes by mode and result" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_pick_total counter"); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_try\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_sorted_rr_success_try_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"success_fallback\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_sorted_rr_success_fallback_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"full\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_sorted_rr_full_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"closed\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_sorted_rr_closed_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"sorted_rr\",result=\"no_candidate\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_sorted_rr_no_candidate_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_try\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_p2c_success_try_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"p2c\",result=\"success_fallback\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_p2c_success_fallback_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"p2c\",result=\"full\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_p2c_full_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"p2c\",result=\"closed\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_p2c_closed_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_total{{mode=\"p2c\",result=\"no_candidate\"}} {}", + if me_allows_normal { + stats.get_me_writer_pick_p2c_no_candidate_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_writer_pick_blocking_fallback_total ME writer-pick blocking fallback attempts" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_writer_pick_blocking_fallback_total counter" + ); + let _ = writeln!( + out, + "telemt_me_writer_pick_blocking_fallback_total {}", + if me_allows_normal { + stats.get_me_writer_pick_blocking_fallback_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_writer_pick_mode_switch_total Writer-pick mode switches via runtime updates" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_pick_mode_switch_total counter"); + let _ = writeln!( + out, + "telemt_me_writer_pick_mode_switch_total {}", + if me_allows_normal { + stats.get_me_writer_pick_mode_switch_total() + } else { + 0 + } + ); + let _ = writeln!( out, "# HELP telemt_me_socks_kdf_policy_total SOCKS KDF policy outcomes" diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 10d8882..25905b2 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -16,7 +16,7 @@ use std::collections::hash_map::DefaultHasher; use std::collections::VecDeque; use tracing::debug; -use crate::config::MeTelemetryLevel; +use crate::config::{MeTelemetryLevel, MeWriterPickMode}; use self::telemetry::TelemetryPolicy; // ============= Stats ============= @@ -95,6 +95,18 @@ pub struct Stats { me_route_drop_queue_full: AtomicU64, me_route_drop_queue_full_base: AtomicU64, me_route_drop_queue_full_high: AtomicU64, + me_writer_pick_sorted_rr_success_try_total: AtomicU64, + me_writer_pick_sorted_rr_success_fallback_total: AtomicU64, + me_writer_pick_sorted_rr_full_total: AtomicU64, + me_writer_pick_sorted_rr_closed_total: AtomicU64, + me_writer_pick_sorted_rr_no_candidate_total: AtomicU64, + me_writer_pick_p2c_success_try_total: AtomicU64, + me_writer_pick_p2c_success_fallback_total: AtomicU64, + me_writer_pick_p2c_full_total: AtomicU64, + me_writer_pick_p2c_closed_total: AtomicU64, + me_writer_pick_p2c_no_candidate_total: AtomicU64, + me_writer_pick_blocking_fallback_total: AtomicU64, + me_writer_pick_mode_switch_total: AtomicU64, me_socks_kdf_strict_reject: AtomicU64, me_socks_kdf_compat_fallback: AtomicU64, secure_padding_invalid: AtomicU64, @@ -497,6 +509,93 @@ impl Stats { self.me_route_drop_queue_full_high.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_writer_pick_success_try_total(&self, mode: MeWriterPickMode) { + if !self.telemetry_me_allows_normal() { + return; + } + match mode { + MeWriterPickMode::SortedRr => { + self.me_writer_pick_sorted_rr_success_try_total + .fetch_add(1, Ordering::Relaxed); + } + MeWriterPickMode::P2c => { + self.me_writer_pick_p2c_success_try_total + .fetch_add(1, Ordering::Relaxed); + } + } + } + pub fn increment_me_writer_pick_success_fallback_total(&self, mode: MeWriterPickMode) { + if !self.telemetry_me_allows_normal() { + return; + } + match mode { + MeWriterPickMode::SortedRr => { + self.me_writer_pick_sorted_rr_success_fallback_total + .fetch_add(1, Ordering::Relaxed); + } + MeWriterPickMode::P2c => { + self.me_writer_pick_p2c_success_fallback_total + .fetch_add(1, Ordering::Relaxed); + } + } + } + pub fn increment_me_writer_pick_full_total(&self, mode: MeWriterPickMode) { + if !self.telemetry_me_allows_normal() { + return; + } + match mode { + MeWriterPickMode::SortedRr => { + self.me_writer_pick_sorted_rr_full_total + .fetch_add(1, Ordering::Relaxed); + } + MeWriterPickMode::P2c => { + self.me_writer_pick_p2c_full_total + .fetch_add(1, Ordering::Relaxed); + } + } + } + pub fn increment_me_writer_pick_closed_total(&self, mode: MeWriterPickMode) { + if !self.telemetry_me_allows_normal() { + return; + } + match mode { + MeWriterPickMode::SortedRr => { + self.me_writer_pick_sorted_rr_closed_total + .fetch_add(1, Ordering::Relaxed); + } + MeWriterPickMode::P2c => { + self.me_writer_pick_p2c_closed_total + .fetch_add(1, Ordering::Relaxed); + } + } + } + pub fn increment_me_writer_pick_no_candidate_total(&self, mode: MeWriterPickMode) { + if !self.telemetry_me_allows_normal() { + return; + } + match mode { + MeWriterPickMode::SortedRr => { + self.me_writer_pick_sorted_rr_no_candidate_total + .fetch_add(1, Ordering::Relaxed); + } + MeWriterPickMode::P2c => { + self.me_writer_pick_p2c_no_candidate_total + .fetch_add(1, Ordering::Relaxed); + } + } + } + pub fn increment_me_writer_pick_blocking_fallback_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_pick_blocking_fallback_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_pick_mode_switch_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_pick_mode_switch_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_socks_kdf_strict_reject(&self) { if self.telemetry_me_allows_normal() { self.me_socks_kdf_strict_reject.fetch_add(1, Ordering::Relaxed); @@ -1001,6 +1100,52 @@ impl Stats { pub fn get_me_route_drop_queue_full_high(&self) -> u64 { self.me_route_drop_queue_full_high.load(Ordering::Relaxed) } + pub fn get_me_writer_pick_sorted_rr_success_try_total(&self) -> u64 { + self.me_writer_pick_sorted_rr_success_try_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_sorted_rr_success_fallback_total(&self) -> u64 { + self.me_writer_pick_sorted_rr_success_fallback_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_sorted_rr_full_total(&self) -> u64 { + self.me_writer_pick_sorted_rr_full_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_sorted_rr_closed_total(&self) -> u64 { + self.me_writer_pick_sorted_rr_closed_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_sorted_rr_no_candidate_total(&self) -> u64 { + self.me_writer_pick_sorted_rr_no_candidate_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_p2c_success_try_total(&self) -> u64 { + self.me_writer_pick_p2c_success_try_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_p2c_success_fallback_total(&self) -> u64 { + self.me_writer_pick_p2c_success_fallback_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_p2c_full_total(&self) -> u64 { + self.me_writer_pick_p2c_full_total.load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_p2c_closed_total(&self) -> u64 { + self.me_writer_pick_p2c_closed_total.load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_p2c_no_candidate_total(&self) -> u64 { + self.me_writer_pick_p2c_no_candidate_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_blocking_fallback_total(&self) -> u64 { + self.me_writer_pick_blocking_fallback_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_pick_mode_switch_total(&self) -> u64 { + self.me_writer_pick_mode_switch_total + .load(Ordering::Relaxed) + } pub fn get_me_socks_kdf_strict_reject(&self) -> u64 { self.me_socks_kdf_strict_reject.load(Ordering::Relaxed) } From d641137537cdb0797d2b2b544caa3c0c41d1dabe Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:09:33 +0300 Subject: [PATCH 146/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a95eef7..c834e1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.10" +version = "3.3.11" edition = "2024" [dependencies] From c28b82a6181204618b4c354323f8e8dc34b6600e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:45:01 +0300 Subject: [PATCH 147/261] Update defaults.rs --- src/config/defaults.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 68dd71e..e885cbe 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -21,9 +21,9 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256; const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256; -const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 1024; -const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 512; -const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 256; +const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 4096; +const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 768; +const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024; const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3; const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000; const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000; From 76b28aea748e49476e0f8d09f5c7ed1225547626 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:45:46 +0300 Subject: [PATCH 148/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c834e1e..c3753db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.11" +version = "3.3.12" edition = "2024" [dependencies] From 633af93b199e6054f022f14ef74f7fdf693d225f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 04:51:46 +0300 Subject: [PATCH 149/261] DC to Client fine tuning --- src/config/defaults.rs | 35 +++ src/config/hot_reload.rs | 43 +++- src/config/load.rs | 36 +++ src/config/types.rs | 37 +++ src/main.rs | 2 + src/proxy/direct_relay.rs | 2 + src/proxy/middle_relay.rs | 287 ++++++++++++++++++---- src/proxy/relay.rs | 20 +- src/transport/middle_proxy/pool.rs | 6 + src/transport/middle_proxy/pool_writer.rs | 2 + src/transport/middle_proxy/reader.rs | 28 ++- src/transport/middle_proxy/registry.rs | 51 ++++ 12 files changed, 477 insertions(+), 72 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index e885cbe..cb95637 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -24,6 +24,13 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256; const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 4096; const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 768; const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024; +const DEFAULT_ME_READER_ROUTE_DATA_WAIT_MS: u64 = 2; +const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_FRAMES: usize = 32; +const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_BYTES: usize = 128 * 1024; +const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_DELAY_US: u64 = 1500; +const DEFAULT_ME_D2C_ACK_FLUSH_IMMEDIATE: bool = false; +const DEFAULT_DIRECT_RELAY_COPY_BUF_C2S_BYTES: usize = 64 * 1024; +const DEFAULT_DIRECT_RELAY_COPY_BUF_S2C_BYTES: usize = 256 * 1024; const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3; const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000; const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000; @@ -316,6 +323,34 @@ pub(crate) fn default_me_c2me_channel_capacity() -> usize { DEFAULT_ME_C2ME_CHANNEL_CAPACITY } +pub(crate) fn default_me_reader_route_data_wait_ms() -> u64 { + DEFAULT_ME_READER_ROUTE_DATA_WAIT_MS +} + +pub(crate) fn default_me_d2c_flush_batch_max_frames() -> usize { + DEFAULT_ME_D2C_FLUSH_BATCH_MAX_FRAMES +} + +pub(crate) fn default_me_d2c_flush_batch_max_bytes() -> usize { + DEFAULT_ME_D2C_FLUSH_BATCH_MAX_BYTES +} + +pub(crate) fn default_me_d2c_flush_batch_max_delay_us() -> u64 { + DEFAULT_ME_D2C_FLUSH_BATCH_MAX_DELAY_US +} + +pub(crate) fn default_me_d2c_ack_flush_immediate() -> bool { + DEFAULT_ME_D2C_ACK_FLUSH_IMMEDIATE +} + +pub(crate) fn default_direct_relay_copy_buf_c2s_bytes() -> usize { + DEFAULT_DIRECT_RELAY_COPY_BUF_C2S_BYTES +} + +pub(crate) fn default_direct_relay_copy_buf_s2c_bytes() -> usize { + DEFAULT_DIRECT_RELAY_COPY_BUF_S2C_BYTES +} + pub(crate) fn default_me_writer_pick_sample_size() -> u8 { DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 34b2d76..632ca8c 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -96,6 +96,13 @@ pub struct HotFields { pub me_route_backpressure_base_timeout_ms: u64, pub me_route_backpressure_high_timeout_ms: u64, pub me_route_backpressure_high_watermark_pct: u8, + pub me_reader_route_data_wait_ms: u64, + pub me_d2c_flush_batch_max_frames: usize, + pub me_d2c_flush_batch_max_bytes: usize, + pub me_d2c_flush_batch_max_delay_us: u64, + pub me_d2c_ack_flush_immediate: bool, + pub direct_relay_copy_buf_c2s_bytes: usize, + pub direct_relay_copy_buf_s2c_bytes: usize, pub me_health_interval_ms_unhealthy: u64, pub me_health_interval_ms_healthy: u64, pub me_admission_poll_ms: u64, @@ -203,6 +210,13 @@ impl HotFields { me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms, me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms, me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct, + me_reader_route_data_wait_ms: cfg.general.me_reader_route_data_wait_ms, + me_d2c_flush_batch_max_frames: cfg.general.me_d2c_flush_batch_max_frames, + me_d2c_flush_batch_max_bytes: cfg.general.me_d2c_flush_batch_max_bytes, + me_d2c_flush_batch_max_delay_us: cfg.general.me_d2c_flush_batch_max_delay_us, + me_d2c_ack_flush_immediate: cfg.general.me_d2c_ack_flush_immediate, + direct_relay_copy_buf_c2s_bytes: cfg.general.direct_relay_copy_buf_c2s_bytes, + direct_relay_copy_buf_s2c_bytes: cfg.general.direct_relay_copy_buf_s2c_bytes, me_health_interval_ms_unhealthy: cfg.general.me_health_interval_ms_unhealthy, me_health_interval_ms_healthy: cfg.general.me_health_interval_ms_healthy, me_admission_poll_ms: cfg.general.me_admission_poll_ms, @@ -352,6 +366,13 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { new.general.me_route_backpressure_high_timeout_ms; cfg.general.me_route_backpressure_high_watermark_pct = new.general.me_route_backpressure_high_watermark_pct; + cfg.general.me_reader_route_data_wait_ms = new.general.me_reader_route_data_wait_ms; + cfg.general.me_d2c_flush_batch_max_frames = new.general.me_d2c_flush_batch_max_frames; + cfg.general.me_d2c_flush_batch_max_bytes = new.general.me_d2c_flush_batch_max_bytes; + cfg.general.me_d2c_flush_batch_max_delay_us = new.general.me_d2c_flush_batch_max_delay_us; + cfg.general.me_d2c_ack_flush_immediate = new.general.me_d2c_ack_flush_immediate; + cfg.general.direct_relay_copy_buf_c2s_bytes = new.general.direct_relay_copy_buf_c2s_bytes; + cfg.general.direct_relay_copy_buf_s2c_bytes = new.general.direct_relay_copy_buf_s2c_bytes; cfg.general.me_health_interval_ms_unhealthy = new.general.me_health_interval_ms_unhealthy; cfg.general.me_health_interval_ms_healthy = new.general.me_health_interval_ms_healthy; cfg.general.me_admission_poll_ms = new.general.me_admission_poll_ms; @@ -821,6 +842,7 @@ fn log_changes( != new_hot.me_route_backpressure_high_timeout_ms || old_hot.me_route_backpressure_high_watermark_pct != new_hot.me_route_backpressure_high_watermark_pct + || old_hot.me_reader_route_data_wait_ms != new_hot.me_reader_route_data_wait_ms || old_hot.me_health_interval_ms_unhealthy != new_hot.me_health_interval_ms_unhealthy || old_hot.me_health_interval_ms_healthy != new_hot.me_health_interval_ms_healthy @@ -828,10 +850,11 @@ fn log_changes( || old_hot.me_warn_rate_limit_ms != new_hot.me_warn_rate_limit_ms { info!( - "config reload: me_route_backpressure: base={}ms high={}ms watermark={}%; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms", + "config reload: me_route_backpressure: base={}ms high={}ms watermark={}%; me_reader_route_data_wait_ms={}; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms", new_hot.me_route_backpressure_base_timeout_ms, new_hot.me_route_backpressure_high_timeout_ms, new_hot.me_route_backpressure_high_watermark_pct, + new_hot.me_reader_route_data_wait_ms, new_hot.me_health_interval_ms_unhealthy, new_hot.me_health_interval_ms_healthy, new_hot.me_admission_poll_ms, @@ -839,6 +862,24 @@ fn log_changes( ); } + if old_hot.me_d2c_flush_batch_max_frames != new_hot.me_d2c_flush_batch_max_frames + || old_hot.me_d2c_flush_batch_max_bytes != new_hot.me_d2c_flush_batch_max_bytes + || old_hot.me_d2c_flush_batch_max_delay_us != new_hot.me_d2c_flush_batch_max_delay_us + || old_hot.me_d2c_ack_flush_immediate != new_hot.me_d2c_ack_flush_immediate + || old_hot.direct_relay_copy_buf_c2s_bytes != new_hot.direct_relay_copy_buf_c2s_bytes + || old_hot.direct_relay_copy_buf_s2c_bytes != new_hot.direct_relay_copy_buf_s2c_bytes + { + info!( + "config reload: relay_tuning: me_d2c_frames={} me_d2c_bytes={} me_d2c_delay_us={} me_ack_flush_immediate={} direct_buf_c2s={} direct_buf_s2c={}", + new_hot.me_d2c_flush_batch_max_frames, + new_hot.me_d2c_flush_batch_max_bytes, + new_hot.me_d2c_flush_batch_max_delay_us, + new_hot.me_d2c_ack_flush_immediate, + new_hot.direct_relay_copy_buf_c2s_bytes, + new_hot.direct_relay_copy_buf_s2c_bytes, + ); + } + if old_hot.users != new_hot.users { let mut added: Vec<&String> = new_hot.users.keys() .filter(|u| !old_hot.users.contains_key(*u)) diff --git a/src/config/load.rs b/src/config/load.rs index 623ec4d..3f1cd5c 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -303,6 +303,42 @@ impl ProxyConfig { )); } + if config.general.me_reader_route_data_wait_ms > 20 { + return Err(ProxyError::Config( + "general.me_reader_route_data_wait_ms must be within [0, 20]".to_string(), + )); + } + + if !(1..=512).contains(&config.general.me_d2c_flush_batch_max_frames) { + return Err(ProxyError::Config( + "general.me_d2c_flush_batch_max_frames must be within [1, 512]".to_string(), + )); + } + + if !(4096..=2 * 1024 * 1024).contains(&config.general.me_d2c_flush_batch_max_bytes) { + return Err(ProxyError::Config( + "general.me_d2c_flush_batch_max_bytes must be within [4096, 2097152]".to_string(), + )); + } + + if config.general.me_d2c_flush_batch_max_delay_us > 5000 { + return Err(ProxyError::Config( + "general.me_d2c_flush_batch_max_delay_us must be within [0, 5000]".to_string(), + )); + } + + if !(4096..=1024 * 1024).contains(&config.general.direct_relay_copy_buf_c2s_bytes) { + return Err(ProxyError::Config( + "general.direct_relay_copy_buf_c2s_bytes must be within [4096, 1048576]".to_string(), + )); + } + + if !(8192..=2 * 1024 * 1024).contains(&config.general.direct_relay_copy_buf_s2c_bytes) { + return Err(ProxyError::Config( + "general.direct_relay_copy_buf_s2c_bytes must be within [8192, 2097152]".to_string(), + )); + } + if config.general.me_health_interval_ms_unhealthy == 0 { return Err(ProxyError::Config( "general.me_health_interval_ms_unhealthy must be > 0".to_string(), diff --git a/src/config/types.rs b/src/config/types.rs index 588c82f..eeb8cfa 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -458,6 +458,36 @@ pub struct GeneralConfig { #[serde(default = "default_me_c2me_channel_capacity")] pub me_c2me_channel_capacity: usize, + /// Bounded wait in milliseconds for routing ME DATA to per-connection queue. + /// `0` keeps legacy no-wait behavior. + #[serde(default = "default_me_reader_route_data_wait_ms")] + pub me_reader_route_data_wait_ms: u64, + + /// Maximum number of ME->Client responses coalesced before flush. + #[serde(default = "default_me_d2c_flush_batch_max_frames")] + pub me_d2c_flush_batch_max_frames: usize, + + /// Maximum total payload bytes coalesced before flush. + #[serde(default = "default_me_d2c_flush_batch_max_bytes")] + pub me_d2c_flush_batch_max_bytes: usize, + + /// Maximum wait in microseconds to coalesce additional ME->Client responses. + /// `0` disables timed coalescing. + #[serde(default = "default_me_d2c_flush_batch_max_delay_us")] + pub me_d2c_flush_batch_max_delay_us: u64, + + /// Flush client writer immediately after quick-ack write. + #[serde(default = "default_me_d2c_ack_flush_immediate")] + pub me_d2c_ack_flush_immediate: bool, + + /// Copy buffer size for client->DC direction in direct relay. + #[serde(default = "default_direct_relay_copy_buf_c2s_bytes")] + pub direct_relay_copy_buf_c2s_bytes: usize, + + /// Copy buffer size for DC->client direction in direct relay. + #[serde(default = "default_direct_relay_copy_buf_s2c_bytes")] + pub direct_relay_copy_buf_s2c_bytes: usize, + /// Max pending ciphertext buffer per client writer (bytes). /// Controls FakeTLS backpressure vs throughput. #[serde(default = "default_crypto_pending_buffer")] @@ -861,6 +891,13 @@ impl Default for GeneralConfig { me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(), me_route_channel_capacity: default_me_route_channel_capacity(), me_c2me_channel_capacity: default_me_c2me_channel_capacity(), + me_reader_route_data_wait_ms: default_me_reader_route_data_wait_ms(), + me_d2c_flush_batch_max_frames: default_me_d2c_flush_batch_max_frames(), + me_d2c_flush_batch_max_bytes: default_me_d2c_flush_batch_max_bytes(), + me_d2c_flush_batch_max_delay_us: default_me_d2c_flush_batch_max_delay_us(), + me_d2c_ack_flush_immediate: default_me_d2c_ack_flush_immediate(), + direct_relay_copy_buf_c2s_bytes: default_direct_relay_copy_buf_c2s_bytes(), + direct_relay_copy_buf_s2c_bytes: default_direct_relay_copy_buf_s2c_bytes(), me_warmup_stagger_enabled: default_true(), me_warmup_step_delay_ms: default_warmup_step_delay_ms(), me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(), diff --git a/src/main.rs b/src/main.rs index d28fabe..3b6a543 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1055,6 +1055,7 @@ async fn main() -> std::result::Result<(), Box> { config.general.me_route_backpressure_base_timeout_ms, config.general.me_route_backpressure_high_timeout_ms, config.general.me_route_backpressure_high_watermark_pct, + config.general.me_reader_route_data_wait_ms, config.general.me_health_interval_ms_unhealthy, config.general.me_health_interval_ms_healthy, config.general.me_warn_rate_limit_ms, @@ -1559,6 +1560,7 @@ async fn main() -> std::result::Result<(), Box> { cfg.general.me_route_backpressure_base_timeout_ms, cfg.general.me_route_backpressure_high_timeout_ms, cfg.general.me_route_backpressure_high_watermark_pct, + cfg.general.me_reader_route_data_wait_ms, ); } } diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index e39e446..d4b0f2e 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -64,6 +64,8 @@ where client_writer, tg_reader, tg_writer, + config.general.direct_relay_copy_buf_c2s_bytes, + config.general.direct_relay_copy_buf_s2c_bytes, user, Arc::clone(&stats), buffer_pool, diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index cae8273..0006914 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -30,6 +30,8 @@ const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync"; const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128; const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64; const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32; +const ME_D2C_FLUSH_BATCH_MAX_FRAMES_MIN: usize = 1; +const ME_D2C_FLUSH_BATCH_MAX_BYTES_MIN: usize = 4096; static DESYNC_DEDUP: OnceLock>> = OnceLock::new(); struct RelayForensicsState { @@ -44,6 +46,31 @@ struct RelayForensicsState { desync_all_full: bool, } +#[derive(Clone, Copy)] +struct MeD2cFlushPolicy { + max_frames: usize, + max_bytes: usize, + max_delay: Duration, + ack_flush_immediate: bool, +} + +impl MeD2cFlushPolicy { + fn from_config(config: &ProxyConfig) -> Self { + Self { + max_frames: config + .general + .me_d2c_flush_batch_max_frames + .max(ME_D2C_FLUSH_BATCH_MAX_FRAMES_MIN), + max_bytes: config + .general + .me_d2c_flush_batch_max_bytes + .max(ME_D2C_FLUSH_BATCH_MAX_BYTES_MIN), + max_delay: Duration::from_micros(config.general.me_d2c_flush_batch_max_delay_us), + ack_flush_immediate: config.general.me_d2c_ack_flush_immediate, + } + } +} + fn hash_value(value: &T) -> u64 { let mut hasher = DefaultHasher::new(); value.hash(&mut hasher); @@ -313,71 +340,152 @@ where let rng_clone = rng.clone(); let user_clone = user.clone(); let bytes_me2c_clone = bytes_me2c.clone(); + let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config); let me_writer = tokio::spawn(async move { let mut writer = crypto_writer; let mut frame_buf = Vec::with_capacity(16 * 1024); loop { tokio::select! { msg = me_rx_task.recv() => { - match msg { - Some(MeResponse::Data { flags, data }) => { - trace!(conn_id, bytes = data.len(), flags, "ME->C data"); - bytes_me2c_clone.fetch_add(data.len() as u64, Ordering::Relaxed); - stats_clone.add_user_octets_to(&user_clone, data.len() as u64); - write_client_payload( - &mut writer, - proto_tag, - flags, - &data, - rng_clone.as_ref(), - &mut frame_buf, - ) - .await?; + let Some(first) = msg else { + debug!(conn_id, "ME channel closed"); + return Err(ProxyError::Proxy("ME connection lost".into())); + }; - // Drain all immediately queued ME responses and flush once. - while let Ok(next) = me_rx_task.try_recv() { - match next { - MeResponse::Data { flags, data } => { - trace!(conn_id, bytes = data.len(), flags, "ME->C data (batched)"); - bytes_me2c_clone.fetch_add(data.len() as u64, Ordering::Relaxed); - stats_clone.add_user_octets_to(&user_clone, data.len() as u64); - write_client_payload( - &mut writer, - proto_tag, - flags, - &data, - rng_clone.as_ref(), - &mut frame_buf, - ).await?; + let mut batch_frames = 0usize; + let mut batch_bytes = 0usize; + let mut flush_immediately = false; + + match process_me_writer_response( + first, + &mut writer, + proto_tag, + rng_clone.as_ref(), + &mut frame_buf, + stats_clone.as_ref(), + &user_clone, + bytes_me2c_clone.as_ref(), + conn_id, + d2c_flush_policy.ack_flush_immediate, + false, + ).await? { + MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => { + batch_frames = batch_frames.saturating_add(frames); + batch_bytes = batch_bytes.saturating_add(bytes); + flush_immediately = immediate; + } + MeWriterResponseOutcome::Close => { + let _ = writer.flush().await; + return Ok(()); + } + } + + while !flush_immediately + && batch_frames < d2c_flush_policy.max_frames + && batch_bytes < d2c_flush_policy.max_bytes + { + let Ok(next) = me_rx_task.try_recv() else { + break; + }; + + match process_me_writer_response( + next, + &mut writer, + proto_tag, + rng_clone.as_ref(), + &mut frame_buf, + stats_clone.as_ref(), + &user_clone, + bytes_me2c_clone.as_ref(), + conn_id, + d2c_flush_policy.ack_flush_immediate, + true, + ).await? { + MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => { + batch_frames = batch_frames.saturating_add(frames); + batch_bytes = batch_bytes.saturating_add(bytes); + flush_immediately |= immediate; + } + MeWriterResponseOutcome::Close => { + let _ = writer.flush().await; + return Ok(()); + } + } + } + + if !flush_immediately + && !d2c_flush_policy.max_delay.is_zero() + && batch_frames < d2c_flush_policy.max_frames + && batch_bytes < d2c_flush_policy.max_bytes + { + match tokio::time::timeout(d2c_flush_policy.max_delay, me_rx_task.recv()).await { + Ok(Some(next)) => { + match process_me_writer_response( + next, + &mut writer, + proto_tag, + rng_clone.as_ref(), + &mut frame_buf, + stats_clone.as_ref(), + &user_clone, + bytes_me2c_clone.as_ref(), + conn_id, + d2c_flush_policy.ack_flush_immediate, + true, + ).await? { + MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => { + batch_frames = batch_frames.saturating_add(frames); + batch_bytes = batch_bytes.saturating_add(bytes); + flush_immediately |= immediate; } - MeResponse::Ack(confirm) => { - trace!(conn_id, confirm, "ME->C quickack (batched)"); - write_client_ack(&mut writer, proto_tag, confirm).await?; - } - MeResponse::Close => { - debug!(conn_id, "ME sent close (batched)"); + MeWriterResponseOutcome::Close => { let _ = writer.flush().await; return Ok(()); } } - } - writer.flush().await.map_err(ProxyError::Io)?; - } - Some(MeResponse::Ack(confirm)) => { - trace!(conn_id, confirm, "ME->C quickack"); - write_client_ack(&mut writer, proto_tag, confirm).await?; - } - Some(MeResponse::Close) => { - debug!(conn_id, "ME sent close"); - let _ = writer.flush().await; - return Ok(()); - } - None => { - debug!(conn_id, "ME channel closed"); - return Err(ProxyError::Proxy("ME connection lost".into())); + while !flush_immediately + && batch_frames < d2c_flush_policy.max_frames + && batch_bytes < d2c_flush_policy.max_bytes + { + let Ok(extra) = me_rx_task.try_recv() else { + break; + }; + + match process_me_writer_response( + extra, + &mut writer, + proto_tag, + rng_clone.as_ref(), + &mut frame_buf, + stats_clone.as_ref(), + &user_clone, + bytes_me2c_clone.as_ref(), + conn_id, + d2c_flush_policy.ack_flush_immediate, + true, + ).await? { + MeWriterResponseOutcome::Continue { frames, bytes, flush_immediately: immediate } => { + batch_frames = batch_frames.saturating_add(frames); + batch_bytes = batch_bytes.saturating_add(bytes); + flush_immediately |= immediate; + } + MeWriterResponseOutcome::Close => { + let _ = writer.flush().await; + return Ok(()); + } + } + } + } + Ok(None) => { + debug!(conn_id, "ME channel closed"); + return Err(ProxyError::Proxy("ME connection lost".into())); + } + Err(_) => {} } } + + writer.flush().await.map_err(ProxyError::Io)?; } _ = &mut stop_rx => { debug!(conn_id, "ME writer stop signal"); @@ -587,6 +695,81 @@ where } } +enum MeWriterResponseOutcome { + Continue { + frames: usize, + bytes: usize, + flush_immediately: bool, + }, + Close, +} + +async fn process_me_writer_response( + response: MeResponse, + client_writer: &mut CryptoWriter, + proto_tag: ProtoTag, + rng: &SecureRandom, + frame_buf: &mut Vec, + stats: &Stats, + user: &str, + bytes_me2c: &AtomicU64, + conn_id: u64, + ack_flush_immediate: bool, + batched: bool, +) -> Result +where + W: AsyncWrite + Unpin + Send + 'static, +{ + match response { + MeResponse::Data { flags, data } => { + if batched { + trace!(conn_id, bytes = data.len(), flags, "ME->C data (batched)"); + } else { + trace!(conn_id, bytes = data.len(), flags, "ME->C data"); + } + bytes_me2c.fetch_add(data.len() as u64, Ordering::Relaxed); + stats.add_user_octets_to(user, data.len() as u64); + write_client_payload( + client_writer, + proto_tag, + flags, + &data, + rng, + frame_buf, + ) + .await?; + + Ok(MeWriterResponseOutcome::Continue { + frames: 1, + bytes: data.len(), + flush_immediately: false, + }) + } + MeResponse::Ack(confirm) => { + if batched { + trace!(conn_id, confirm, "ME->C quickack (batched)"); + } else { + trace!(conn_id, confirm, "ME->C quickack"); + } + write_client_ack(client_writer, proto_tag, confirm).await?; + + Ok(MeWriterResponseOutcome::Continue { + frames: 1, + bytes: 4, + flush_immediately: ack_flush_immediate, + }) + } + MeResponse::Close => { + if batched { + debug!(conn_id, "ME sent close (batched)"); + } else { + debug!(conn_id, "ME sent close"); + } + Ok(MeWriterResponseOutcome::Close) + } + } +} + async fn write_client_payload( client_writer: &mut CryptoWriter, proto_tag: ProtoTag, @@ -696,9 +879,7 @@ where client_writer .write_all(&bytes) .await - .map_err(ProxyError::Io)?; - // ACK should remain low-latency. - client_writer.flush().await.map_err(ProxyError::Io) + .map_err(ProxyError::Io) } #[cfg(test)] diff --git a/src/proxy/relay.rs b/src/proxy/relay.rs index a155945..06ce0d8 100644 --- a/src/proxy/relay.rs +++ b/src/proxy/relay.rs @@ -57,7 +57,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::task::{Context, Poll}; use std::time::Duration; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf, copy_bidirectional}; +use tokio::io::{ + AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf, copy_bidirectional_with_sizes, +}; use tokio::time::Instant; use tracing::{debug, trace, warn}; use crate::error::Result; @@ -296,9 +298,8 @@ impl AsyncWrite for StatsIo { /// /// ## API compatibility /// -/// Signature is identical to the previous implementation. The `_buffer_pool` -/// parameter is retained for call-site compatibility — `copy_bidirectional` -/// manages its own internal buffers (8 KB per direction). +/// The `_buffer_pool` parameter is retained for call-site compatibility. +/// Effective relay copy buffers are configured by `c2s_buf_size` / `s2c_buf_size`. /// /// ## Guarantees preserved /// @@ -312,6 +313,8 @@ pub async fn relay_bidirectional( client_writer: CW, server_reader: SR, server_writer: SW, + c2s_buf_size: usize, + s2c_buf_size: usize, user: &str, stats: Arc, _buffer_pool: Arc, @@ -402,7 +405,12 @@ where // When the watchdog fires, select! drops the copy future, // releasing the &mut borrows on client and server. let copy_result = tokio::select! { - result = copy_bidirectional(&mut client, &mut server) => Some(result), + result = copy_bidirectional_with_sizes( + &mut client, + &mut server, + c2s_buf_size.max(1), + s2c_buf_size.max(1), + ) => Some(result), _ = watchdog => None, // Activity timeout — cancel relay }; @@ -463,4 +471,4 @@ where Ok(()) } } -} \ No newline at end of file +} diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 07ad67b..8d5b110 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -183,6 +183,7 @@ pub struct MePool { pub(super) me_writer_pick_mode: AtomicU8, pub(super) me_writer_pick_sample_size: AtomicU8, pub(super) me_socks_kdf_policy: AtomicU8, + pub(super) me_reader_route_data_wait_ms: Arc, pub(super) me_route_no_writer_mode: AtomicU8, pub(super) me_route_no_writer_wait: Duration, pub(super) me_route_inline_recovery_attempts: u32, @@ -287,6 +288,7 @@ impl MePool { me_route_backpressure_base_timeout_ms: u64, me_route_backpressure_high_timeout_ms: u64, me_route_backpressure_high_watermark_pct: u8, + me_reader_route_data_wait_ms: u64, me_health_interval_ms_unhealthy: u64, me_health_interval_ms_healthy: u64, me_warn_rate_limit_ms: u64, @@ -460,6 +462,7 @@ impl MePool { me_writer_pick_mode: AtomicU8::new(me_writer_pick_mode.as_u8()), me_writer_pick_sample_size: AtomicU8::new(me_writer_pick_sample_size.clamp(2, 4)), me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()), + me_reader_route_data_wait_ms: Arc::new(AtomicU64::new(me_reader_route_data_wait_ms)), me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()), me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), me_route_inline_recovery_attempts, @@ -650,9 +653,12 @@ impl MePool { route_backpressure_base_timeout_ms: u64, route_backpressure_high_timeout_ms: u64, route_backpressure_high_watermark_pct: u8, + reader_route_data_wait_ms: u64, ) { self.me_socks_kdf_policy .store(socks_kdf_policy.as_u8(), Ordering::Relaxed); + self.me_reader_route_data_wait_ms + .store(reader_route_data_wait_ms, Ordering::Relaxed); self.registry.update_route_backpressure_policy( route_backpressure_base_timeout_ms, route_backpressure_high_timeout_ms, diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 43abf0c..7e79f10 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -208,6 +208,7 @@ impl MePool { let keepalive_jitter_signal = self.me_keepalive_jitter; let cancel_reader_token = cancel.clone(); let cancel_ping_token = cancel_ping.clone(); + let reader_route_data_wait_ms = self.me_reader_route_data_wait_ms.clone(); tokio::spawn(async move { let res = reader_loop( @@ -225,6 +226,7 @@ impl MePool { writer_id, degraded.clone(), rtt_ema_ms_x10.clone(), + reader_route_data_wait_ms, cancel_reader_token.clone(), ) .await; diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index 32de774..785bc2c 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::io::ErrorKind; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::time::Instant; use bytes::{Bytes, BytesMut}; @@ -35,6 +35,7 @@ pub(crate) async fn reader_loop( _writer_id: u64, degraded: Arc, writer_rtt_ema_ms_x10: Arc, + reader_route_data_wait_ms: Arc, cancel: CancellationToken, ) -> Result<()> { let mut raw = enc_leftover; @@ -57,17 +58,14 @@ pub(crate) async fn reader_loop( let blocks = raw.len() / 16 * 16; if blocks > 0 { + let mut chunk = raw.split_to(blocks); let mut new_iv = [0u8; 16]; - new_iv.copy_from_slice(&raw[blocks - 16..blocks]); - - let mut chunk = vec![0u8; blocks]; - chunk.copy_from_slice(&raw[..blocks]); + new_iv.copy_from_slice(&chunk[blocks - 16..blocks]); AesCbc::new(dk, div) - .decrypt_in_place(&mut chunk) + .decrypt_in_place(&mut chunk[..]) .map_err(|e| ProxyError::Crypto(format!("{e}")))?; div = new_iv; dec.extend_from_slice(&chunk); - let _ = raw.split_to(blocks); } while dec.len() >= 12 { @@ -85,7 +83,7 @@ pub(crate) async fn reader_loop( break; } - let frame = dec.split_to(fl); + let frame = dec.split_to(fl).freeze(); let pe = fl - 4; let ec = u32::from_le_bytes(frame[pe..pe + 4].try_into().unwrap()); let actual_crc = rpc_crc(crc_mode, &frame[..pe]); @@ -111,21 +109,27 @@ pub(crate) async fn reader_loop( } expected_seq = expected_seq.wrapping_add(1); - let payload = &frame[8..pe]; + let payload = frame.slice(8..pe); if payload.len() < 4 { continue; } let pt = u32::from_le_bytes(payload[0..4].try_into().unwrap()); - let body = &payload[4..]; + let body = payload.slice(4..); if pt == RPC_PROXY_ANS_U32 && body.len() >= 12 { let flags = u32::from_le_bytes(body[0..4].try_into().unwrap()); let cid = u64::from_le_bytes(body[4..12].try_into().unwrap()); - let data = Bytes::copy_from_slice(&body[12..]); + let data = body.slice(12..); trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS"); - let routed = reg.route_nowait(cid, MeResponse::Data { flags, data }).await; + let data_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed); + let routed = if data_wait_ms == 0 { + reg.route_nowait(cid, MeResponse::Data { flags, data }).await + } else { + reg.route_with_timeout(cid, MeResponse::Data { flags, data }, data_wait_ms) + .await + }; if !matches!(routed, RouteResult::Routed) { match routed { RouteResult::NoConn => stats.increment_me_route_drop_no_conn(), diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index f2682d5..ee04969 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -231,6 +231,57 @@ impl ConnRegistry { } } + pub async fn route_with_timeout( + &self, + id: u64, + resp: MeResponse, + timeout_ms: u64, + ) -> RouteResult { + if timeout_ms == 0 { + return self.route_nowait(id, resp).await; + } + + let tx = { + let inner = self.inner.read().await; + inner.map.get(&id).cloned() + }; + + let Some(tx) = tx else { + return RouteResult::NoConn; + }; + + match tx.try_send(resp) { + Ok(()) => RouteResult::Routed, + Err(TrySendError::Closed(_)) => RouteResult::ChannelClosed, + Err(TrySendError::Full(resp)) => { + let high_watermark_pct = self + .route_backpressure_high_watermark_pct + .load(Ordering::Relaxed) + .clamp(1, 100); + let used = self.route_channel_capacity.saturating_sub(tx.capacity()); + let used_pct = if self.route_channel_capacity == 0 { + 100 + } else { + (used.saturating_mul(100) / self.route_channel_capacity) as u8 + }; + let high_profile = used_pct >= high_watermark_pct; + let timeout_dur = Duration::from_millis(timeout_ms.max(1)); + + match tokio::time::timeout(timeout_dur, tx.send(resp)).await { + Ok(Ok(())) => RouteResult::Routed, + Ok(Err(_)) => RouteResult::ChannelClosed, + Err(_) => { + if high_profile { + RouteResult::QueueFullHigh + } else { + RouteResult::QueueFullBase + } + } + } + } + } + } + pub async fn bind_writer( &self, conn_id: u64, From aa3fcfbbe1bef92a76e79fc1097374da12b0845c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 04:53:40 +0300 Subject: [PATCH 150/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c3753db..92e581c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.12" +version = "3.3.13" edition = "2024" [dependencies] From 5f7fb15dd8725cacd9e6dba561134231f96f94bd Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 06:20:56 +0300 Subject: [PATCH 151/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 92e581c..33dc9b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.13" +version = "3.3.14" edition = "2024" [dependencies] From 98f365be448433c58d903973f904a9f531dd60a8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 8 Mar 2026 06:22:20 +0300 Subject: [PATCH 152/261] Update README.md --- README.md | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6df79fb..b59e509 100644 --- a/README.md +++ b/README.md @@ -104,28 +104,19 @@ We welcome ideas, architectural feedback, and pull requests. - Extensive logging via `trace` and `debug` with `RUST_LOG` method # GOTO -- [Telemt - MTProxy on Rust + Tokio](#telemt---mtproxy-on-rust--tokio) - - [NEWS and EMERGENCY](#news-and-emergency) - - [✈️ Telemt 3 is released!](#️-telemt-3-is-released) - - [🇷🇺 RU](#-ru) - - [Релиз 3.3.5 LTS - 6 марта](#релиз-335-lts---6-марта) - - [🇬🇧 EN](#-en) - - [Release 3.3.5 LTS - March 6](#release-335-lts---march-6) -- [Features](#features) -- [GOTO](#goto) - - [Quick Start Guide](#quick-start-guide) - - [FAQ](#faq) - - [Recognizability for DPI and crawler](#recognizability-for-dpi-and-crawler) - - [Client WITH secret-key accesses the MTProxy resource:](#client-with-secret-key-accesses-the-mtproxy-resource) - - [Client WITHOUT secret-key gets transparent access to the specified resource:](#client-without-secret-key-gets-transparent-access-to-the-specified-resource) - - [Telegram Calls via MTProxy](#telegram-calls-via-mtproxy) - - [How does DPI see MTProxy TLS?](#how-does-dpi-see-mtproxy-tls) - - [Whitelist on IP](#whitelist-on-ip) - - [Too many open files](#too-many-open-files) - - [Build](#build) - - [Why Rust?](#why-rust) - - [Issues](#issues) - - [Roadmap](#roadmap) +- [Quick Start Guide](#quick-start-guide) +- [FAQ](#faq) + - [Recognizability for DPI and crawler](#recognizability-for-dpi-and-crawler) + - [Client WITH secret-key accesses the MTProxy resource:](#client-with-secret-key-accesses-the-mtproxy-resource) + - [Client WITHOUT secret-key gets transparent access to the specified resource:](#client-without-secret-key-gets-transparent-access-to-the-specified-resource) + - [Telegram Calls via MTProxy](#telegram-calls-via-mtproxy) + - [How does DPI see MTProxy TLS?](#how-does-dpi-see-mtproxy-tls) + - [Whitelist on IP](#whitelist-on-ip) + - [Too many open files](#too-many-open-files) +- [Build](#build) +- [Why Rust?](#why-rust) +- [Issues](#issues) +- [Roadmap](#roadmap) ## Quick Start Guide From ef2ed3daa0a432fc374caaaf65b36918f766042e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 00:53:47 +0300 Subject: [PATCH 153/261] ME/DC Reroute + ME Upper-limit tuning --- src/main.rs | 116 +++++++++++++++++++------ src/proxy/client.rs | 25 +++++- src/proxy/direct_relay.rs | 42 ++++++++- src/proxy/middle_relay.rs | 133 ++++++++++++++++++++--------- src/proxy/mod.rs | 1 + src/proxy/route_mode.rs | 117 +++++++++++++++++++++++++ src/transport/middle_proxy/pool.rs | 31 ++++++- 7 files changed, 392 insertions(+), 73 deletions(-) create mode 100644 src/proxy/route_mode.rs diff --git a/src/main.rs b/src/main.rs index 3b6a543..a706373 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,6 +37,7 @@ use crate::crypto::SecureRandom; use crate::ip_tracker::UserIpTracker; use crate::network::probe::{decide_network_capabilities, log_probe_result, run_probe}; use crate::proxy::ClientHandler; +use crate::proxy::route_mode::{ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteRuntimeController}; use crate::stats::beobachten::BeobachtenStore; use crate::stats::telemetry::TelemetryPolicy; use crate::stats::{ReplayChecker, Stats}; @@ -261,6 +262,10 @@ async fn wait_until_admission_open(admission_rx: &mut watch::Receiver) -> } } +fn is_expected_handshake_eof(err: &crate::error::ProxyError) -> bool { + err.to_string().contains("expected 64 bytes, got 0") +} + async fn load_startup_proxy_config_snapshot( url: &str, cache_path: Option<&str>, @@ -519,6 +524,12 @@ async fn main() -> std::result::Result<(), Box> { let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone())); let initial_admission_open = !config.general.use_middle_proxy; let (admission_tx, admission_rx) = watch::channel(initial_admission_open); + let initial_route_mode = if config.general.use_middle_proxy { + RelayRouteMode::Middle + } else { + RelayRouteMode::Direct + }; + let route_runtime = Arc::new(RouteRuntimeController::new(initial_route_mode)); let api_me_pool = Arc::new(RwLock::new(None::>)); startup_tracker .start_component(COMPONENT_API_BOOTSTRAP, Some("spawn API listener task".to_string())) @@ -1783,9 +1794,11 @@ async fn main() -> std::result::Result<(), Box> { if config.general.use_middle_proxy { if let Some(pool) = me_pool.as_ref() { - let initial_open = pool.admission_ready_conditional_cast().await; - admission_tx.send_replace(initial_open); - if initial_open { + let fallback_after = Duration::from_secs(6); + let initial_ready = pool.admission_ready_conditional_cast().await; + admission_tx.send_replace(initial_ready); + let _ = route_runtime.set_mode(RelayRouteMode::Middle); + if initial_ready { info!("Conditional-admission gate: open (ME pool ready)"); } else { warn!("Conditional-admission gate: closed (ME pool is not ready)"); @@ -1793,12 +1806,18 @@ async fn main() -> std::result::Result<(), Box> { let pool_for_gate = pool.clone(); let admission_tx_gate = admission_tx.clone(); + let route_runtime_gate = route_runtime.clone(); let mut config_rx_gate = config_rx.clone(); let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1); + let mut fallback_enabled = config.general.me2dc_fallback; tokio::spawn(async move { - let mut gate_open = initial_open; - let mut open_streak = if initial_open { 1u32 } else { 0u32 }; - let mut close_streak = if initial_open { 0u32 } else { 1u32 }; + let mut gate_open = initial_ready; + let mut route_mode = RelayRouteMode::Middle; + let mut not_ready_since = if initial_ready { + None + } else { + Some(Instant::now()) + }; loop { tokio::select! { changed = config_rx_gate.changed() => { @@ -1807,42 +1826,70 @@ async fn main() -> std::result::Result<(), Box> { } let cfg = config_rx_gate.borrow_and_update().clone(); admission_poll_ms = cfg.general.me_admission_poll_ms.max(1); + fallback_enabled = cfg.general.me2dc_fallback; continue; } _ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {} } let ready = pool_for_gate.admission_ready_conditional_cast().await; - if ready { - open_streak = open_streak.saturating_add(1); - close_streak = 0; - if !gate_open && open_streak >= 2 { - gate_open = true; - admission_tx_gate.send_replace(true); - info!( - open_streak, - "Conditional-admission gate opened (ME pool recovered)" - ); - } + let now = Instant::now(); + let (next_gate_open, next_route_mode, next_fallback_active) = if ready { + not_ready_since = None; + (true, RelayRouteMode::Middle, false) } else { - close_streak = close_streak.saturating_add(1); - open_streak = 0; - if gate_open && close_streak >= 2 { - gate_open = false; - admission_tx_gate.send_replace(false); - warn!( - close_streak, - "Conditional-admission gate closed (ME pool has uncovered DC groups)" - ); + let not_ready_started_at = *not_ready_since.get_or_insert(now); + let not_ready_for = now.saturating_duration_since(not_ready_started_at); + if fallback_enabled && not_ready_for > fallback_after { + (true, RelayRouteMode::Direct, true) + } else { + (false, RelayRouteMode::Middle, false) + } + }; + + if next_route_mode != route_mode { + route_mode = next_route_mode; + if let Some(snapshot) = route_runtime_gate.set_mode(route_mode) { + if matches!(route_mode, RelayRouteMode::Middle) { + info!( + target_mode = route_mode.as_str(), + cutover_generation = snapshot.generation, + "Middle-End routing restored for new sessions" + ); + } else { + warn!( + target_mode = route_mode.as_str(), + cutover_generation = snapshot.generation, + grace_secs = fallback_after.as_secs(), + "ME pool stayed not-ready beyond grace; routing new sessions via Direct-DC" + ); + } } } + + if next_gate_open != gate_open { + gate_open = next_gate_open; + admission_tx_gate.send_replace(gate_open); + if gate_open { + if next_fallback_active { + warn!("Conditional-admission gate opened in ME fallback mode"); + } else { + info!("Conditional-admission gate opened (ME pool ready)"); + } + } else { + warn!("Conditional-admission gate closed (ME pool is not ready)"); + } + } + } }); } else { admission_tx.send_replace(false); + let _ = route_runtime.set_mode(RelayRouteMode::Direct); warn!("Conditional-admission gate: closed (ME pool is unavailable)"); } } else { admission_tx.send_replace(true); + let _ = route_runtime.set_mode(RelayRouteMode::Direct); } let _admission_tx_hold = admission_tx; @@ -1886,6 +1933,7 @@ async fn main() -> std::result::Result<(), Box> { let buffer_pool = buffer_pool.clone(); let rng = rng.clone(); let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); let tls_cache = tls_cache.clone(); let ip_tracker = ip_tracker.clone(); let beobachten = beobachten.clone(); @@ -1918,6 +1966,7 @@ async fn main() -> std::result::Result<(), Box> { let buffer_pool = buffer_pool.clone(); let rng = rng.clone(); let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); let tls_cache = tls_cache.clone(); let ip_tracker = ip_tracker.clone(); let beobachten = beobachten.clone(); @@ -1928,7 +1977,7 @@ async fn main() -> std::result::Result<(), Box> { if let Err(e) = crate::proxy::client::handle_client_stream( stream, fake_peer, config, stats, upstream_manager, replay_checker, buffer_pool, rng, - me_pool, tls_cache, ip_tracker, beobachten, proxy_protocol_enabled, + me_pool, route_runtime, tls_cache, ip_tracker, beobachten, proxy_protocol_enabled, ).await { debug!(error = %e, "Unix socket connection error"); } @@ -2039,6 +2088,7 @@ async fn main() -> std::result::Result<(), Box> { let buffer_pool = buffer_pool.clone(); let rng = rng.clone(); let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); let tls_cache = tls_cache.clone(); let ip_tracker = ip_tracker.clone(); let beobachten = beobachten.clone(); @@ -2066,6 +2116,7 @@ async fn main() -> std::result::Result<(), Box> { let buffer_pool = buffer_pool.clone(); let rng = rng.clone(); let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); let tls_cache = tls_cache.clone(); let ip_tracker = ip_tracker.clone(); let beobachten = beobachten.clone(); @@ -2083,6 +2134,7 @@ async fn main() -> std::result::Result<(), Box> { buffer_pool, rng, me_pool, + route_runtime, tls_cache, ip_tracker, beobachten, @@ -2119,10 +2171,20 @@ async fn main() -> std::result::Result<(), Box> { &e, crate::error::ProxyError::Proxy(msg) if msg == "ME connection lost" ); + let route_switched = matches!( + &e, + crate::error::ProxyError::Proxy(msg) if msg == ROUTE_SWITCH_ERROR_MSG + ); match (peer_closed, me_closed) { (true, _) => debug!(peer = %peer_addr, error = %e, "Connection closed by client"), (_, true) => warn!(peer = %peer_addr, error = %e, "Connection closed: Middle-End dropped session"), + _ if route_switched => { + info!(peer = %peer_addr, error = %e, "Connection closed by controlled route cutover") + } + _ if is_expected_handshake_eof(&e) => { + info!(peer = %peer_addr, error = %e, "Connection closed during initial handshake") + } _ => warn!(peer = %peer_addr, error = %e, "Connection closed with error"), } } diff --git a/src/proxy/client.rs b/src/proxy/client.rs index ebfabcb..cbe59ce 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -39,6 +39,7 @@ use crate::proxy::direct_relay::handle_via_direct; use crate::proxy::handshake::{HandshakeSuccess, handle_mtproto_handshake, handle_tls_handshake}; use crate::proxy::masking::handle_bad_client; use crate::proxy::middle_relay::handle_via_middle_proxy; +use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController}; fn beobachten_ttl(config: &ProxyConfig) -> Duration { Duration::from_secs(config.general.beobachten_minutes.saturating_mul(60)) @@ -80,6 +81,7 @@ pub async fn handle_client_stream( buffer_pool: Arc, rng: Arc, me_pool: Option>, + route_runtime: Arc, tls_cache: Option>, ip_tracker: Arc, beobachten: Arc, @@ -214,6 +216,7 @@ where RunningClientHandler::handle_authenticated_static( crypto_reader, crypto_writer, success, upstream_manager, stats, config, buffer_pool, rng, me_pool, + route_runtime.clone(), local_addr, real_peer, ip_tracker.clone(), ), ))) @@ -274,6 +277,7 @@ where buffer_pool, rng, me_pool, + route_runtime.clone(), local_addr, real_peer, ip_tracker.clone(), @@ -324,6 +328,7 @@ pub struct RunningClientHandler { buffer_pool: Arc, rng: Arc, me_pool: Option>, + route_runtime: Arc, tls_cache: Option>, ip_tracker: Arc, beobachten: Arc, @@ -341,6 +346,7 @@ impl ClientHandler { buffer_pool: Arc, rng: Arc, me_pool: Option>, + route_runtime: Arc, tls_cache: Option>, ip_tracker: Arc, beobachten: Arc, @@ -356,6 +362,7 @@ impl ClientHandler { buffer_pool, rng, me_pool, + route_runtime, tls_cache, ip_tracker, beobachten, @@ -597,6 +604,7 @@ impl RunningClientHandler { buffer_pool, self.rng, self.me_pool, + self.route_runtime.clone(), local_addr, peer, self.ip_tracker, @@ -677,6 +685,7 @@ impl RunningClientHandler { buffer_pool, self.rng, self.me_pool, + self.route_runtime.clone(), local_addr, peer, self.ip_tracker, @@ -698,6 +707,7 @@ impl RunningClientHandler { buffer_pool: Arc, rng: Arc, me_pool: Option>, + route_runtime: Arc, local_addr: SocketAddr, peer_addr: SocketAddr, ip_tracker: Arc, @@ -713,7 +723,11 @@ impl RunningClientHandler { return Err(e); } - let relay_result = if config.general.use_middle_proxy { + let route_snapshot = route_runtime.snapshot(); + let session_id = rng.u64(); + let relay_result = if config.general.use_middle_proxy + && matches!(route_snapshot.mode, RelayRouteMode::Middle) + { if let Some(ref pool) = me_pool { handle_via_middle_proxy( client_reader, @@ -725,6 +739,9 @@ impl RunningClientHandler { buffer_pool, local_addr, rng, + route_runtime.subscribe(), + route_snapshot, + session_id, ) .await } else { @@ -738,6 +755,9 @@ impl RunningClientHandler { config, buffer_pool, rng, + route_runtime.subscribe(), + route_snapshot, + session_id, ) .await } @@ -752,6 +772,9 @@ impl RunningClientHandler { config, buffer_pool, rng, + route_runtime.subscribe(), + route_snapshot, + session_id, ) .await }; diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index d4b0f2e..7a7810a 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -5,14 +5,19 @@ use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpStream; +use tokio::sync::watch; use tracing::{debug, info, warn}; use crate::config::ProxyConfig; use crate::crypto::SecureRandom; -use crate::error::Result; +use crate::error::{ProxyError, Result}; use crate::protocol::constants::*; use crate::proxy::handshake::{HandshakeSuccess, encrypt_tg_nonce_with_ciphers, generate_tg_nonce}; use crate::proxy::relay::relay_bidirectional; +use crate::proxy::route_mode::{ + RelayRouteMode, RouteCutoverState, ROUTE_SWITCH_ERROR_MSG, affected_cutover_state, + cutover_stagger_delay, +}; use crate::stats::Stats; use crate::stream::{BufferPool, CryptoReader, CryptoWriter}; use crate::transport::UpstreamManager; @@ -26,6 +31,9 @@ pub(crate) async fn handle_via_direct( config: Arc, buffer_pool: Arc, rng: Arc, + mut route_rx: watch::Receiver, + route_snapshot: RouteCutoverState, + session_id: u64, ) -> Result<()> where R: AsyncRead + Unpin + Send + 'static, @@ -69,8 +77,36 @@ where user, Arc::clone(&stats), buffer_pool, - ) - .await; + ); + tokio::pin!(relay_result); + let relay_result = loop { + if let Some(cutover) = affected_cutover_state( + &route_rx, + RelayRouteMode::Direct, + route_snapshot.generation, + ) { + let delay = cutover_stagger_delay(session_id, cutover.generation); + warn!( + user = %user, + target_mode = cutover.mode.as_str(), + cutover_generation = cutover.generation, + delay_ms = delay.as_millis() as u64, + "Cutover affected direct session, closing client connection" + ); + tokio::time::sleep(delay).await; + break Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string())); + } + tokio::select! { + result = &mut relay_result => { + break result; + } + changed = route_rx.changed() => { + if changed.is_err() { + break relay_result.await; + } + } + } + }; stats.decrement_current_connections_direct(); stats.decrement_user_curr_connects(user); diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index 0006914..efaa8ba 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{mpsc, oneshot, watch}; use tracing::{debug, trace, warn}; use crate::config::ProxyConfig; @@ -16,6 +16,10 @@ use crate::crypto::SecureRandom; use crate::error::{ProxyError, Result}; use crate::protocol::constants::{*, secure_padding_len}; use crate::proxy::handshake::HandshakeSuccess; +use crate::proxy::route_mode::{ + RelayRouteMode, RouteCutoverState, ROUTE_SWITCH_ERROR_MSG, affected_cutover_state, + cutover_stagger_delay, +}; use crate::stats::Stats; use crate::stream::{BufferPool, CryptoReader, CryptoWriter}; use crate::transport::middle_proxy::{MePool, MeResponse, proto_flags_for_tag}; @@ -228,6 +232,9 @@ pub(crate) async fn handle_via_middle_proxy( _buffer_pool: Arc, local_addr: SocketAddr, rng: Arc, + mut route_rx: watch::Receiver, + route_snapshot: RouteCutoverState, + session_id: u64, ) -> Result<()> where R: AsyncRead + Unpin + Send + 'static, @@ -267,6 +274,27 @@ where stats.increment_user_curr_connects(&user); stats.increment_current_connections_me(); + if let Some(cutover) = affected_cutover_state( + &route_rx, + RelayRouteMode::Middle, + route_snapshot.generation, + ) { + let delay = cutover_stagger_delay(session_id, cutover.generation); + warn!( + conn_id, + target_mode = cutover.mode.as_str(), + cutover_generation = cutover.generation, + delay_ms = delay.as_millis() as u64, + "Cutover affected middle session before relay start, closing client connection" + ); + tokio::time::sleep(delay).await; + let _ = me_pool.send_close(conn_id).await; + me_pool.registry().unregister(conn_id).await; + stats.decrement_current_connections_me(); + stats.decrement_user_curr_connects(&user); + return Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string())); + } + // Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable) let user_tag: Option> = config .access @@ -498,46 +526,75 @@ where let mut main_result: Result<()> = Ok(()); let mut client_closed = false; let mut frame_counter: u64 = 0; + let mut route_watch_open = true; loop { - match read_client_payload( - &mut crypto_reader, - proto_tag, - frame_limit, - &forensics, - &mut frame_counter, - &stats, - ).await { - Ok(Some((payload, quickack))) => { - trace!(conn_id, bytes = payload.len(), "C->ME frame"); - forensics.bytes_c2me = forensics - .bytes_c2me - .saturating_add(payload.len() as u64); - stats.add_user_octets_from(&user, payload.len() as u64); - let mut flags = proto_flags; - if quickack { - flags |= RPC_FLAG_QUICKACK; - } - if payload.len() >= 8 && payload[..8].iter().all(|b| *b == 0) { - flags |= RPC_FLAG_NOT_ENCRYPTED; - } - // Keep client read loop lightweight: route heavy ME send path via a dedicated task. - if enqueue_c2me_command(&c2me_tx, C2MeCommand::Data { payload, flags }) - .await - .is_err() - { - main_result = Err(ProxyError::Proxy("ME sender channel closed".into())); - break; + if let Some(cutover) = affected_cutover_state( + &route_rx, + RelayRouteMode::Middle, + route_snapshot.generation, + ) { + let delay = cutover_stagger_delay(session_id, cutover.generation); + warn!( + conn_id, + target_mode = cutover.mode.as_str(), + cutover_generation = cutover.generation, + delay_ms = delay.as_millis() as u64, + "Cutover affected middle session, closing client connection" + ); + tokio::time::sleep(delay).await; + let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; + main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string())); + break; + } + + tokio::select! { + changed = route_rx.changed(), if route_watch_open => { + if changed.is_err() { + route_watch_open = false; } } - Ok(None) => { - debug!(conn_id, "Client EOF"); - client_closed = true; - let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; - break; - } - Err(e) => { - main_result = Err(e); - break; + payload_result = read_client_payload( + &mut crypto_reader, + proto_tag, + frame_limit, + &forensics, + &mut frame_counter, + &stats, + ) => { + match payload_result { + Ok(Some((payload, quickack))) => { + trace!(conn_id, bytes = payload.len(), "C->ME frame"); + forensics.bytes_c2me = forensics + .bytes_c2me + .saturating_add(payload.len() as u64); + stats.add_user_octets_from(&user, payload.len() as u64); + let mut flags = proto_flags; + if quickack { + flags |= RPC_FLAG_QUICKACK; + } + if payload.len() >= 8 && payload[..8].iter().all(|b| *b == 0) { + flags |= RPC_FLAG_NOT_ENCRYPTED; + } + // Keep client read loop lightweight: route heavy ME send path via a dedicated task. + if enqueue_c2me_command(&c2me_tx, C2MeCommand::Data { payload, flags }) + .await + .is_err() + { + main_result = Err(ProxyError::Proxy("ME sender channel closed".into())); + break; + } + } + Ok(None) => { + debug!(conn_id, "Client EOF"); + client_closed = true; + let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; + break; + } + Err(e) => { + main_result = Err(e); + break; + } + } } } } diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index bedae1a..1eed469 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -5,6 +5,7 @@ pub mod direct_relay; pub mod handshake; pub mod masking; pub mod middle_relay; +pub mod route_mode; pub mod relay; pub use client::ClientHandler; diff --git a/src/proxy/route_mode.rs b/src/proxy/route_mode.rs new file mode 100644 index 0000000..57830ca --- /dev/null +++ b/src/proxy/route_mode.rs @@ -0,0 +1,117 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; + +use tokio::sync::watch; + +pub(crate) const ROUTE_SWITCH_ERROR_MSG: &str = "Route mode switched by cutover"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum RelayRouteMode { + Direct = 0, + Middle = 1, +} + +impl RelayRouteMode { + pub(crate) fn as_u8(self) -> u8 { + self as u8 + } + + pub(crate) fn from_u8(value: u8) -> Self { + match value { + 1 => Self::Middle, + _ => Self::Direct, + } + } + + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Direct => "direct", + Self::Middle => "middle", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct RouteCutoverState { + pub mode: RelayRouteMode, + pub generation: u64, +} + +#[derive(Clone)] +pub(crate) struct RouteRuntimeController { + mode: Arc, + generation: Arc, + tx: watch::Sender, +} + +impl RouteRuntimeController { + pub(crate) fn new(initial_mode: RelayRouteMode) -> Self { + let initial = RouteCutoverState { + mode: initial_mode, + generation: 0, + }; + let (tx, _rx) = watch::channel(initial); + Self { + mode: Arc::new(AtomicU8::new(initial_mode.as_u8())), + generation: Arc::new(AtomicU64::new(0)), + tx, + } + } + + pub(crate) fn snapshot(&self) -> RouteCutoverState { + RouteCutoverState { + mode: RelayRouteMode::from_u8(self.mode.load(Ordering::Relaxed)), + generation: self.generation.load(Ordering::Relaxed), + } + } + + pub(crate) fn subscribe(&self) -> watch::Receiver { + self.tx.subscribe() + } + + pub(crate) fn set_mode(&self, mode: RelayRouteMode) -> Option { + let previous = self.mode.swap(mode.as_u8(), Ordering::Relaxed); + if previous == mode.as_u8() { + return None; + } + let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; + let next = RouteCutoverState { mode, generation }; + self.tx.send_replace(next); + Some(next) + } +} + +pub(crate) fn is_session_affected_by_cutover( + current: RouteCutoverState, + _session_mode: RelayRouteMode, + session_generation: u64, +) -> bool { + current.generation > session_generation +} + +pub(crate) fn affected_cutover_state( + rx: &watch::Receiver, + session_mode: RelayRouteMode, + session_generation: u64, +) -> Option { + let current = *rx.borrow(); + if is_session_affected_by_cutover(current, session_mode, session_generation) { + return Some(current); + } + None +} + +pub(crate) fn cutover_stagger_delay(session_id: u64, generation: u64) -> Duration { + let mut value = session_id + ^ generation.rotate_left(17) + ^ 0x9e37_79b9_7f4a_7c15; + value ^= value >> 30; + value = value.wrapping_mul(0xbf58_476d_1ce4_e5b9); + value ^= value >> 27; + value = value.wrapping_mul(0x94d0_49bb_1331_11eb); + value ^= value >> 31; + let ms = 1000 + (value % 1000); + Duration::from_millis(ms) +} diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 8d5b110..2d81e63 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -828,10 +828,29 @@ impl MePool { effective } + // Keeps per-contour (active/warm) writer budget bounded by CPU count. + // Baseline is 86 writers on the first core and +48 for each extra core. + fn adaptive_floor_cpu_budget_per_contour_cap(&self, cores: usize) -> usize { + const FIRST_CORE_WRITER_BUDGET: usize = 86; + const EXTRA_CORE_WRITER_BUDGET: usize = 48; + if cores == 0 { + return FIRST_CORE_WRITER_BUDGET; + } + FIRST_CORE_WRITER_BUDGET.saturating_add( + cores + .saturating_sub(1) + .saturating_mul(EXTRA_CORE_WRITER_BUDGET), + ) + } + pub(super) fn adaptive_floor_active_cap_configured_total(&self) -> usize { let cores = self.adaptive_floor_effective_cpu_cores(); - let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_active_writers_per_core()); - let configured = per_core_cap.min(self.adaptive_floor_max_active_writers_global()); + let per_contour_budget = self.adaptive_floor_cpu_budget_per_contour_cap(cores); + let configured = cores + .saturating_mul(self.adaptive_floor_max_active_writers_per_core()) + .min(self.adaptive_floor_max_active_writers_global()) + .min(per_contour_budget) + .max(1); self.me_adaptive_floor_active_cap_configured .store(configured as u64, Ordering::Relaxed); self.stats @@ -841,8 +860,12 @@ impl MePool { pub(super) fn adaptive_floor_warm_cap_configured_total(&self) -> usize { let cores = self.adaptive_floor_effective_cpu_cores(); - let per_core_cap = cores.saturating_mul(self.adaptive_floor_max_warm_writers_per_core()); - let configured = per_core_cap.min(self.adaptive_floor_max_warm_writers_global()); + let per_contour_budget = self.adaptive_floor_cpu_budget_per_contour_cap(cores); + let configured = cores + .saturating_mul(self.adaptive_floor_max_warm_writers_per_core()) + .min(self.adaptive_floor_max_warm_writers_global()) + .min(per_contour_budget) + .max(1); self.me_adaptive_floor_warm_cap_configured .store(configured as u64, Ordering::Relaxed); self.stats From d0f253b49bd8b726b2123f1e894c9e2ebb9bac8a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 01:55:07 +0300 Subject: [PATCH 154/261] PROXY Real IP in logs --- src/main.rs | 43 ++++++++++++++++++++++++++++++++++++++----- src/proxy/client.rs | 16 ++++++++++++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index a706373..b8b5d60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2121,6 +2121,8 @@ async fn main() -> std::result::Result<(), Box> { let ip_tracker = ip_tracker.clone(); let beobachten = beobachten.clone(); let proxy_protocol_enabled = listener_proxy_protocol; + let real_peer_report = Arc::new(std::sync::Mutex::new(None)); + let real_peer_report_for_handler = real_peer_report.clone(); tokio::spawn(async move { let _permit = permit; @@ -2139,10 +2141,15 @@ async fn main() -> std::result::Result<(), Box> { ip_tracker, beobachten, proxy_protocol_enabled, + real_peer_report_for_handler, ) .run() .await { + let real_peer = match real_peer_report.lock() { + Ok(guard) => *guard, + Err(_) => None, + }; let peer_closed = matches!( &e, crate::error::ProxyError::Io(ioe) @@ -2177,15 +2184,41 @@ async fn main() -> std::result::Result<(), Box> { ); match (peer_closed, me_closed) { - (true, _) => debug!(peer = %peer_addr, error = %e, "Connection closed by client"), - (_, true) => warn!(peer = %peer_addr, error = %e, "Connection closed: Middle-End dropped session"), + (true, _) => { + if let Some(real_peer) = real_peer { + debug!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by client"); + } else { + debug!(peer = %peer_addr, error = %e, "Connection closed by client"); + } + } + (_, true) => { + if let Some(real_peer) = real_peer { + warn!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed: Middle-End dropped session"); + } else { + warn!(peer = %peer_addr, error = %e, "Connection closed: Middle-End dropped session"); + } + } _ if route_switched => { - info!(peer = %peer_addr, error = %e, "Connection closed by controlled route cutover") + if let Some(real_peer) = real_peer { + info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by controlled route cutover"); + } else { + info!(peer = %peer_addr, error = %e, "Connection closed by controlled route cutover"); + } } _ if is_expected_handshake_eof(&e) => { - info!(peer = %peer_addr, error = %e, "Connection closed during initial handshake") + if let Some(real_peer) = real_peer { + info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed during initial handshake"); + } else { + info!(peer = %peer_addr, error = %e, "Connection closed during initial handshake"); + } + } + _ => { + if let Some(real_peer) = real_peer { + warn!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed with error"); + } else { + warn!(peer = %peer_addr, error = %e, "Connection closed with error"); + } } - _ => warn!(peer = %peer_addr, error = %e, "Connection closed with error"), } } }); diff --git a/src/proxy/client.rs b/src/proxy/client.rs index cbe59ce..4bc3751 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -321,6 +321,8 @@ pub struct ClientHandler; pub struct RunningClientHandler { stream: TcpStream, peer: SocketAddr, + real_peer_from_proxy: Option, + real_peer_report: Arc>>, config: Arc, stats: Arc, replay_checker: Arc, @@ -351,10 +353,14 @@ impl ClientHandler { ip_tracker: Arc, beobachten: Arc, proxy_protocol_enabled: bool, + real_peer_report: Arc>>, ) -> RunningClientHandler { + let normalized_peer = normalize_ip(peer); RunningClientHandler { stream, - peer, + peer: normalized_peer, + real_peer_from_proxy: None, + real_peer_report, config, stats, replay_checker, @@ -372,10 +378,8 @@ impl ClientHandler { } impl RunningClientHandler { - pub async fn run(mut self) -> Result<()> { + pub async fn run(self) -> Result<()> { self.stats.increment_connects_all(); - - self.peer = normalize_ip(self.peer); let peer = self.peer; let _ip_tracker = self.ip_tracker.clone(); debug!(peer = %peer, "New connection"); @@ -448,6 +452,10 @@ impl RunningClientHandler { "PROXY protocol header parsed" ); self.peer = normalize_ip(info.src_addr); + self.real_peer_from_proxy = Some(self.peer); + if let Ok(mut slot) = self.real_peer_report.lock() { + *slot = Some(self.peer); + } if let Some(dst) = info.dst_addr { local_addr = dst; } From 4c94f73546271671cef25c30e139b6f4ba68f579 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:05:46 +0300 Subject: [PATCH 155/261] Maestro - Refactored Main Format --- src/maestro/admission.rs | 116 ++ src/maestro/connectivity.rs | 220 ++++ src/maestro/helpers.rs | 320 +++++ src/maestro/listeners.rs | 465 +++++++ src/maestro/me_startup.rs | 425 +++++++ src/maestro/mod.rs | 546 ++++++++ src/maestro/runtime_tasks.rs | 317 +++++ src/maestro/shutdown.rs | 42 + src/maestro/tls_bootstrap.rs | 165 +++ src/main.rs | 2258 +--------------------------------- src/proxy/client.rs | 11 +- src/proxy/middle_relay.rs | 2 +- 12 files changed, 2627 insertions(+), 2260 deletions(-) create mode 100644 src/maestro/admission.rs create mode 100644 src/maestro/connectivity.rs create mode 100644 src/maestro/helpers.rs create mode 100644 src/maestro/listeners.rs create mode 100644 src/maestro/me_startup.rs create mode 100644 src/maestro/mod.rs create mode 100644 src/maestro/runtime_tasks.rs create mode 100644 src/maestro/shutdown.rs create mode 100644 src/maestro/tls_bootstrap.rs diff --git a/src/maestro/admission.rs b/src/maestro/admission.rs new file mode 100644 index 0000000..40a0c2d --- /dev/null +++ b/src/maestro/admission.rs @@ -0,0 +1,116 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::sync::watch; +use tracing::{info, warn}; + +use crate::config::ProxyConfig; +use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController}; +use crate::transport::middle_proxy::MePool; + +pub(crate) async fn configure_admission_gate( + config: &Arc, + me_pool: Option>, + route_runtime: Arc, + admission_tx: &watch::Sender, + config_rx: watch::Receiver>, +) { + if config.general.use_middle_proxy { + if let Some(pool) = me_pool.as_ref() { + let fallback_after = Duration::from_secs(6); + let initial_ready = pool.admission_ready_conditional_cast().await; + admission_tx.send_replace(initial_ready); + let _ = route_runtime.set_mode(RelayRouteMode::Middle); + if initial_ready { + info!("Conditional-admission gate: open (ME pool ready)"); + } else { + warn!("Conditional-admission gate: closed (ME pool is not ready)"); + } + + let pool_for_gate = pool.clone(); + let admission_tx_gate = admission_tx.clone(); + let route_runtime_gate = route_runtime.clone(); + let mut config_rx_gate = config_rx.clone(); + let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1); + let mut fallback_enabled = config.general.me2dc_fallback; + tokio::spawn(async move { + let mut gate_open = initial_ready; + let mut route_mode = RelayRouteMode::Middle; + let mut not_ready_since = if initial_ready { + None + } else { + Some(Instant::now()) + }; + loop { + tokio::select! { + changed = config_rx_gate.changed() => { + if changed.is_err() { + break; + } + let cfg = config_rx_gate.borrow_and_update().clone(); + admission_poll_ms = cfg.general.me_admission_poll_ms.max(1); + fallback_enabled = cfg.general.me2dc_fallback; + continue; + } + _ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {} + } + let ready = pool_for_gate.admission_ready_conditional_cast().await; + let now = Instant::now(); + let (next_gate_open, next_route_mode, next_fallback_active) = if ready { + not_ready_since = None; + (true, RelayRouteMode::Middle, false) + } else { + let not_ready_started_at = *not_ready_since.get_or_insert(now); + let not_ready_for = now.saturating_duration_since(not_ready_started_at); + if fallback_enabled && not_ready_for > fallback_after { + (true, RelayRouteMode::Direct, true) + } else { + (false, RelayRouteMode::Middle, false) + } + }; + + if next_route_mode != route_mode { + route_mode = next_route_mode; + if let Some(snapshot) = route_runtime_gate.set_mode(route_mode) { + if matches!(route_mode, RelayRouteMode::Middle) { + info!( + target_mode = route_mode.as_str(), + cutover_generation = snapshot.generation, + "Middle-End routing restored for new sessions" + ); + } else { + warn!( + target_mode = route_mode.as_str(), + cutover_generation = snapshot.generation, + grace_secs = fallback_after.as_secs(), + "ME pool stayed not-ready beyond grace; routing new sessions via Direct-DC" + ); + } + } + } + + if next_gate_open != gate_open { + gate_open = next_gate_open; + admission_tx_gate.send_replace(gate_open); + if gate_open { + if next_fallback_active { + warn!("Conditional-admission gate opened in ME fallback mode"); + } else { + info!("Conditional-admission gate opened (ME pool ready)"); + } + } else { + warn!("Conditional-admission gate closed (ME pool is not ready)"); + } + } + } + }); + } else { + admission_tx.send_replace(false); + let _ = route_runtime.set_mode(RelayRouteMode::Direct); + warn!("Conditional-admission gate: closed (ME pool is unavailable)"); + } + } else { + admission_tx.send_replace(true); + let _ = route_runtime.set_mode(RelayRouteMode::Direct); + } +} diff --git a/src/maestro/connectivity.rs b/src/maestro/connectivity.rs new file mode 100644 index 0000000..c843223 --- /dev/null +++ b/src/maestro/connectivity.rs @@ -0,0 +1,220 @@ +use std::sync::Arc; +use std::time::Instant; + +use tokio::sync::RwLock; +use tracing::info; + +use crate::config::ProxyConfig; +use crate::crypto::SecureRandom; +use crate::network::probe::NetworkDecision; +use crate::startup::{ + COMPONENT_DC_CONNECTIVITY_PING, COMPONENT_ME_CONNECTIVITY_PING, COMPONENT_RUNTIME_READY, + StartupTracker, +}; +use crate::transport::middle_proxy::{ + MePingFamily, MePingSample, MePool, format_me_route, format_sample_line, run_me_ping, +}; +use crate::transport::UpstreamManager; + +pub(crate) async fn run_startup_connectivity( + config: &Arc, + me_pool: &Option>, + rng: Arc, + startup_tracker: &Arc, + upstream_manager: Arc, + prefer_ipv6: bool, + decision: &NetworkDecision, + process_started_at: Instant, + api_me_pool: Arc>>>, +) { + if me_pool.is_some() { + startup_tracker + .start_component( + COMPONENT_ME_CONNECTIVITY_PING, + Some("run startup ME connectivity check".to_string()), + ) + .await; + } else { + startup_tracker + .skip_component( + COMPONENT_ME_CONNECTIVITY_PING, + Some("ME pool is not available".to_string()), + ) + .await; + } + if let Some(pool) = me_pool { + let me_results = run_me_ping(pool, &rng).await; + + let v4_ok = me_results.iter().any(|r| { + matches!(r.family, MePingFamily::V4) + && r.samples.iter().any(|s| s.error.is_none() && s.handshake_ms.is_some()) + }); + let v6_ok = me_results.iter().any(|r| { + matches!(r.family, MePingFamily::V6) + && r.samples.iter().any(|s| s.error.is_none() && s.handshake_ms.is_some()) + }); + + info!("================= Telegram ME Connectivity ================="); + if v4_ok && v6_ok { + info!(" IPv4 and IPv6 available"); + } else if v4_ok { + info!(" IPv4 only / IPv6 unavailable"); + } else if v6_ok { + info!(" IPv6 only / IPv4 unavailable"); + } else { + info!(" No ME connectivity"); + } + let me_route = + format_me_route(&config.upstreams, &me_results, prefer_ipv6, v4_ok, v6_ok).await; + info!(" via {}", me_route); + info!("============================================================"); + + use std::collections::BTreeMap; + let mut grouped: BTreeMap> = BTreeMap::new(); + for report in me_results { + for s in report.samples { + grouped.entry(s.dc).or_default().push(s); + } + } + + let family_order = if prefer_ipv6 { + vec![MePingFamily::V6, MePingFamily::V4] + } else { + vec![MePingFamily::V4, MePingFamily::V6] + }; + + for (dc, samples) in grouped { + for family in &family_order { + let fam_samples: Vec<&MePingSample> = samples + .iter() + .filter(|s| matches!(s.family, f if &f == family)) + .collect(); + if fam_samples.is_empty() { + continue; + } + + let fam_label = match family { + MePingFamily::V4 => "IPv4", + MePingFamily::V6 => "IPv6", + }; + info!(" DC{} [{}]", dc, fam_label); + for sample in fam_samples { + let line = format_sample_line(sample); + info!("{}", line); + } + } + } + info!("============================================================"); + startup_tracker + .complete_component( + COMPONENT_ME_CONNECTIVITY_PING, + Some("startup ME connectivity check completed".to_string()), + ) + .await; + } + + info!("================= Telegram DC Connectivity ================="); + startup_tracker + .start_component( + COMPONENT_DC_CONNECTIVITY_PING, + Some("run startup DC connectivity check".to_string()), + ) + .await; + + let ping_results = upstream_manager + .ping_all_dcs( + prefer_ipv6, + &config.dc_overrides, + decision.ipv4_dc, + decision.ipv6_dc, + ) + .await; + + for upstream_result in &ping_results { + let v6_works = upstream_result.v6_results.iter().any(|r| r.rtt_ms.is_some()); + let v4_works = upstream_result.v4_results.iter().any(|r| r.rtt_ms.is_some()); + + if upstream_result.both_available { + if prefer_ipv6 { + info!(" IPv6 in use / IPv4 is fallback"); + } else { + info!(" IPv4 in use / IPv6 is fallback"); + } + } else if v6_works && !v4_works { + info!(" IPv6 only / IPv4 unavailable"); + } else if v4_works && !v6_works { + info!(" IPv4 only / IPv6 unavailable"); + } else if !v6_works && !v4_works { + info!(" No DC connectivity"); + } + + info!(" via {}", upstream_result.upstream_name); + info!("============================================================"); + + if v6_works { + for dc in &upstream_result.v6_results { + let addr_str = format!("{}:{}", dc.dc_addr.ip(), dc.dc_addr.port()); + match &dc.rtt_ms { + Some(rtt) => { + info!(" DC{} [IPv6] {} - {:.0} ms", dc.dc_idx, addr_str, rtt); + } + None => { + let err = dc.error.as_deref().unwrap_or("fail"); + info!(" DC{} [IPv6] {} - FAIL ({})", dc.dc_idx, addr_str, err); + } + } + } + + info!("============================================================"); + } + + if v4_works { + for dc in &upstream_result.v4_results { + let addr_str = format!("{}:{}", dc.dc_addr.ip(), dc.dc_addr.port()); + match &dc.rtt_ms { + Some(rtt) => { + info!( + " DC{} [IPv4] {}\t\t\t\t{:.0} ms", + dc.dc_idx, addr_str, rtt + ); + } + None => { + let err = dc.error.as_deref().unwrap_or("fail"); + info!( + " DC{} [IPv4] {}:\t\t\t\tFAIL ({})", + dc.dc_idx, addr_str, err + ); + } + } + } + + info!("============================================================"); + } + } + startup_tracker + .complete_component( + COMPONENT_DC_CONNECTIVITY_PING, + Some("startup DC connectivity check completed".to_string()), + ) + .await; + + let initialized_secs = process_started_at.elapsed().as_secs(); + let second_suffix = if initialized_secs == 1 { "" } else { "s" }; + startup_tracker + .start_component( + COMPONENT_RUNTIME_READY, + Some("finalize startup runtime state".to_string()), + ) + .await; + info!("===================== Telegram Startup ====================="); + info!( + " DC/ME Initialized in {} second{}", + initialized_secs, second_suffix + ); + info!("============================================================"); + + if let Some(pool) = me_pool { + pool.set_runtime_ready(true); + } + *api_me_pool.write().await = me_pool.clone(); +} diff --git a/src/maestro/helpers.rs b/src/maestro/helpers.rs new file mode 100644 index 0000000..acaecdd --- /dev/null +++ b/src/maestro/helpers.rs @@ -0,0 +1,320 @@ +use std::time::Duration; + +use tokio::sync::watch; +use tracing::{debug, error, info, warn}; + +use crate::cli; +use crate::config::ProxyConfig; +use crate::transport::middle_proxy::{ + ProxyConfigData, fetch_proxy_config_with_raw, load_proxy_config_cache, save_proxy_config_cache, +}; + +pub(crate) fn parse_cli() -> (String, bool, Option) { + let mut config_path = "config.toml".to_string(); + let mut silent = false; + let mut log_level: Option = None; + + let args: Vec = std::env::args().skip(1).collect(); + + // Check for --init first (handled before tokio) + if let Some(init_opts) = cli::parse_init_args(&args) { + if let Err(e) = cli::run_init(init_opts) { + eprintln!("[telemt] Init failed: {}", e); + std::process::exit(1); + } + std::process::exit(0); + } + + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--silent" | "-s" => { + silent = true; + } + "--log-level" => { + i += 1; + if i < args.len() { + log_level = Some(args[i].clone()); + } + } + s if s.starts_with("--log-level=") => { + log_level = Some(s.trim_start_matches("--log-level=").to_string()); + } + "--help" | "-h" => { + eprintln!("Usage: telemt [config.toml] [OPTIONS]"); + eprintln!(); + eprintln!("Options:"); + eprintln!(" --silent, -s Suppress info logs"); + eprintln!(" --log-level debug|verbose|normal|silent"); + eprintln!(" --help, -h Show this help"); + eprintln!(); + eprintln!("Setup (fire-and-forget):"); + eprintln!( + " --init Generate config, install systemd service, start" + ); + eprintln!(" --port Listen port (default: 443)"); + eprintln!( + " --domain TLS domain for masking (default: www.google.com)" + ); + eprintln!( + " --secret 32-char hex secret (auto-generated if omitted)" + ); + eprintln!(" --user Username (default: user)"); + eprintln!(" --config-dir Config directory (default: /etc/telemt)"); + eprintln!(" --no-start Don't start the service after install"); + std::process::exit(0); + } + "--version" | "-V" => { + println!("telemt {}", env!("CARGO_PKG_VERSION")); + std::process::exit(0); + } + s if !s.starts_with('-') => { + config_path = s.to_string(); + } + other => { + eprintln!("Unknown option: {}", other); + } + } + i += 1; + } + + (config_path, silent, log_level) +} + +pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) { + info!(target: "telemt::links", "--- Proxy Links ({}) ---", host); + for user_name in config.general.links.show.resolve_users(&config.access.users) { + if let Some(secret) = config.access.users.get(user_name) { + info!(target: "telemt::links", "User: {}", user_name); + if config.general.modes.classic { + info!( + target: "telemt::links", + " Classic: tg://proxy?server={}&port={}&secret={}", + host, port, secret + ); + } + if config.general.modes.secure { + info!( + target: "telemt::links", + " DD: tg://proxy?server={}&port={}&secret=dd{}", + host, port, secret + ); + } + if config.general.modes.tls { + let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); + domains.push(config.censorship.tls_domain.clone()); + for d in &config.censorship.tls_domains { + if !domains.contains(d) { + domains.push(d.clone()); + } + } + + for domain in domains { + let domain_hex = hex::encode(&domain); + info!( + target: "telemt::links", + " EE-TLS: tg://proxy?server={}&port={}&secret=ee{}{}", + host, port, secret, domain_hex + ); + } + } + } else { + warn!(target: "telemt::links", "User '{}' in show_link not found", user_name); + } + } + info!(target: "telemt::links", "------------------------"); +} + +pub(crate) async fn write_beobachten_snapshot(path: &str, payload: &str) -> std::io::Result<()> { + if let Some(parent) = std::path::Path::new(path).parent() + && !parent.as_os_str().is_empty() + { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(path, payload).await +} + +pub(crate) fn unit_label(value: u64, singular: &'static str, plural: &'static str) -> &'static str { + if value == 1 { singular } else { plural } +} + +pub(crate) fn format_uptime(total_secs: u64) -> String { + const SECS_PER_MINUTE: u64 = 60; + const SECS_PER_HOUR: u64 = 60 * SECS_PER_MINUTE; + const SECS_PER_DAY: u64 = 24 * SECS_PER_HOUR; + const SECS_PER_MONTH: u64 = 30 * SECS_PER_DAY; + const SECS_PER_YEAR: u64 = 12 * SECS_PER_MONTH; + + let mut remaining = total_secs; + let years = remaining / SECS_PER_YEAR; + remaining %= SECS_PER_YEAR; + let months = remaining / SECS_PER_MONTH; + remaining %= SECS_PER_MONTH; + let days = remaining / SECS_PER_DAY; + remaining %= SECS_PER_DAY; + let hours = remaining / SECS_PER_HOUR; + remaining %= SECS_PER_HOUR; + let minutes = remaining / SECS_PER_MINUTE; + let seconds = remaining % SECS_PER_MINUTE; + + let mut parts = Vec::new(); + if total_secs > SECS_PER_YEAR { + parts.push(format!("{} {}", years, unit_label(years, "year", "years"))); + } + if total_secs > SECS_PER_MONTH { + parts.push(format!( + "{} {}", + months, + unit_label(months, "month", "months") + )); + } + if total_secs > SECS_PER_DAY { + parts.push(format!("{} {}", days, unit_label(days, "day", "days"))); + } + if total_secs > SECS_PER_HOUR { + parts.push(format!("{} {}", hours, unit_label(hours, "hour", "hours"))); + } + if total_secs > SECS_PER_MINUTE { + parts.push(format!( + "{} {}", + minutes, + unit_label(minutes, "minute", "minutes") + )); + } + parts.push(format!( + "{} {}", + seconds, + unit_label(seconds, "second", "seconds") + )); + + format!("{} / {} seconds", parts.join(", "), total_secs) +} + +pub(crate) async fn wait_until_admission_open(admission_rx: &mut watch::Receiver) -> bool { + loop { + if *admission_rx.borrow() { + return true; + } + if admission_rx.changed().await.is_err() { + return *admission_rx.borrow(); + } + } +} + +pub(crate) fn is_expected_handshake_eof(err: &crate::error::ProxyError) -> bool { + err.to_string().contains("expected 64 bytes, got 0") +} + +pub(crate) async fn load_startup_proxy_config_snapshot( + url: &str, + cache_path: Option<&str>, + me2dc_fallback: bool, + label: &'static str, +) -> Option { + loop { + match fetch_proxy_config_with_raw(url).await { + Ok((cfg, raw)) => { + if !cfg.map.is_empty() { + if let Some(path) = cache_path + && let Err(e) = save_proxy_config_cache(path, &raw).await + { + warn!(error = %e, path, snapshot = label, "Failed to store startup proxy-config cache"); + } + return Some(cfg); + } + + warn!(snapshot = label, url, "Startup proxy-config is empty; trying disk cache"); + if let Some(path) = cache_path { + match load_proxy_config_cache(path).await { + Ok(cached) if !cached.map.is_empty() => { + info!( + snapshot = label, + path, + proxy_for_lines = cached.proxy_for_lines, + "Loaded startup proxy-config from disk cache" + ); + return Some(cached); + } + Ok(_) => { + warn!( + snapshot = label, + path, + "Startup proxy-config cache is empty; ignoring cache file" + ); + } + Err(cache_err) => { + debug!( + snapshot = label, + path, + error = %cache_err, + "Startup proxy-config cache unavailable" + ); + } + } + } + + if me2dc_fallback { + error!( + snapshot = label, + "Startup proxy-config unavailable and no saved config found; falling back to direct mode" + ); + return None; + } + + warn!( + snapshot = label, + retry_in_secs = 2, + "Startup proxy-config unavailable and no saved config found; retrying because me2dc_fallback=false" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + } + Err(fetch_err) => { + if let Some(path) = cache_path { + match load_proxy_config_cache(path).await { + Ok(cached) if !cached.map.is_empty() => { + info!( + snapshot = label, + path, + proxy_for_lines = cached.proxy_for_lines, + "Loaded startup proxy-config from disk cache" + ); + return Some(cached); + } + Ok(_) => { + warn!( + snapshot = label, + path, + "Startup proxy-config cache is empty; ignoring cache file" + ); + } + Err(cache_err) => { + debug!( + snapshot = label, + path, + error = %cache_err, + "Startup proxy-config cache unavailable" + ); + } + } + } + + if me2dc_fallback { + error!( + snapshot = label, + error = %fetch_err, + "Startup proxy-config unavailable and no cached data; falling back to direct mode" + ); + return None; + } + + warn!( + snapshot = label, + error = %fetch_err, + retry_in_secs = 2, + "Startup proxy-config unavailable; retrying because me2dc_fallback=false" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } +} diff --git a/src/maestro/listeners.rs b/src/maestro/listeners.rs new file mode 100644 index 0000000..6296fd7 --- /dev/null +++ b/src/maestro/listeners.rs @@ -0,0 +1,465 @@ +use std::error::Error; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::Duration; + +use tokio::net::TcpListener; +#[cfg(unix)] +use tokio::net::UnixListener; +use tokio::sync::{Semaphore, watch}; +use tracing::{debug, error, info, warn}; + +use crate::config::ProxyConfig; +use crate::crypto::SecureRandom; +use crate::ip_tracker::UserIpTracker; +use crate::proxy::route_mode::{ROUTE_SWITCH_ERROR_MSG, RouteRuntimeController}; +use crate::proxy::ClientHandler; +use crate::startup::{COMPONENT_LISTENERS_BIND, StartupTracker}; +use crate::stats::beobachten::BeobachtenStore; +use crate::stats::{ReplayChecker, Stats}; +use crate::stream::BufferPool; +use crate::tls_front::TlsFrontCache; +use crate::transport::middle_proxy::MePool; +use crate::transport::{ + ListenOptions, UpstreamManager, create_listener, find_listener_processes, +}; + +use super::helpers::{is_expected_handshake_eof, print_proxy_links, wait_until_admission_open}; + +pub(crate) struct BoundListeners { + pub(crate) listeners: Vec<(TcpListener, bool)>, + pub(crate) has_unix_listener: bool, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn bind_listeners( + config: &Arc, + decision_ipv4_dc: bool, + decision_ipv6_dc: bool, + detected_ip_v4: Option, + detected_ip_v6: Option, + startup_tracker: &Arc, + config_rx: watch::Receiver>, + admission_rx: watch::Receiver, + stats: Arc, + upstream_manager: Arc, + replay_checker: Arc, + buffer_pool: Arc, + rng: Arc, + me_pool: Option>, + route_runtime: Arc, + tls_cache: Option>, + ip_tracker: Arc, + beobachten: Arc, + max_connections: Arc, +) -> Result> { + startup_tracker + .start_component( + COMPONENT_LISTENERS_BIND, + Some("bind TCP/Unix listeners".to_string()), + ) + .await; + let mut listeners = Vec::new(); + + for listener_conf in &config.server.listeners { + let addr = SocketAddr::new(listener_conf.ip, config.server.port); + if addr.is_ipv4() && !decision_ipv4_dc { + warn!(%addr, "Skipping IPv4 listener: IPv4 disabled by [network]"); + continue; + } + if addr.is_ipv6() && !decision_ipv6_dc { + warn!(%addr, "Skipping IPv6 listener: IPv6 disabled by [network]"); + continue; + } + let options = ListenOptions { + reuse_port: listener_conf.reuse_allow, + ipv6_only: listener_conf.ip.is_ipv6(), + ..Default::default() + }; + + match create_listener(addr, &options) { + Ok(socket) => { + let listener = TcpListener::from_std(socket.into())?; + info!("Listening on {}", addr); + let listener_proxy_protocol = + listener_conf.proxy_protocol.unwrap_or(config.server.proxy_protocol); + + let public_host = if let Some(ref announce) = listener_conf.announce { + announce.clone() + } else if listener_conf.ip.is_unspecified() { + if listener_conf.ip.is_ipv4() { + detected_ip_v4 + .map(|ip| ip.to_string()) + .unwrap_or_else(|| listener_conf.ip.to_string()) + } else { + detected_ip_v6 + .map(|ip| ip.to_string()) + .unwrap_or_else(|| listener_conf.ip.to_string()) + } + } else { + listener_conf.ip.to_string() + }; + + if config.general.links.public_host.is_none() && !config.general.links.show.is_empty() { + let link_port = config.general.links.public_port.unwrap_or(config.server.port); + print_proxy_links(&public_host, link_port, config); + } + + listeners.push((listener, listener_proxy_protocol)); + } + Err(e) => { + if e.kind() == std::io::ErrorKind::AddrInUse { + let owners = find_listener_processes(addr); + if owners.is_empty() { + error!( + %addr, + "Failed to bind: address already in use (owner process unresolved)" + ); + } else { + for owner in owners { + error!( + %addr, + pid = owner.pid, + process = %owner.process, + "Failed to bind: address already in use" + ); + } + } + + if !listener_conf.reuse_allow { + error!( + %addr, + "reuse_allow=false; set [[server.listeners]].reuse_allow=true to allow multi-instance listening" + ); + } + } else { + error!("Failed to bind to {}: {}", addr, e); + } + } + } + } + + if !config.general.links.show.is_empty() + && (config.general.links.public_host.is_some() || listeners.is_empty()) + { + let (host, port) = if let Some(ref h) = config.general.links.public_host { + ( + h.clone(), + config.general.links.public_port.unwrap_or(config.server.port), + ) + } else { + let ip = detected_ip_v4 + .or(detected_ip_v6) + .map(|ip| ip.to_string()); + if ip.is_none() { + warn!( + "show_link is configured but public IP could not be detected. Set public_host in config." + ); + } + ( + ip.unwrap_or_else(|| "UNKNOWN".to_string()), + config.general.links.public_port.unwrap_or(config.server.port), + ) + }; + + print_proxy_links(&host, port, config); + } + + let mut has_unix_listener = false; + #[cfg(unix)] + if let Some(ref unix_path) = config.server.listen_unix_sock { + let _ = tokio::fs::remove_file(unix_path).await; + + let unix_listener = UnixListener::bind(unix_path)?; + + if let Some(ref perm_str) = config.server.listen_unix_sock_perm { + match u32::from_str_radix(perm_str.trim_start_matches('0'), 8) { + Ok(mode) => { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(mode); + if let Err(e) = std::fs::set_permissions(unix_path, perms) { + error!("Failed to set unix socket permissions to {}: {}", perm_str, e); + } else { + info!("Listening on unix:{} (mode {})", unix_path, perm_str); + } + } + Err(e) => { + warn!("Invalid listen_unix_sock_perm '{}': {}. Ignoring.", perm_str, e); + info!("Listening on unix:{}", unix_path); + } + } + } else { + info!("Listening on unix:{}", unix_path); + } + + has_unix_listener = true; + + let mut config_rx_unix: watch::Receiver> = config_rx.clone(); + let mut admission_rx_unix = admission_rx.clone(); + let stats = stats.clone(); + let upstream_manager = upstream_manager.clone(); + let replay_checker = replay_checker.clone(); + let buffer_pool = buffer_pool.clone(); + let rng = rng.clone(); + let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); + let tls_cache = tls_cache.clone(); + let ip_tracker = ip_tracker.clone(); + let beobachten = beobachten.clone(); + let max_connections_unix = max_connections.clone(); + + tokio::spawn(async move { + let unix_conn_counter = Arc::new(std::sync::atomic::AtomicU64::new(1)); + + loop { + if !wait_until_admission_open(&mut admission_rx_unix).await { + warn!("Conditional-admission gate channel closed for unix listener"); + break; + } + match unix_listener.accept().await { + Ok((stream, _)) => { + let permit = match max_connections_unix.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + error!("Connection limiter is closed"); + break; + } + }; + let conn_id = + unix_conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let fake_peer = + SocketAddr::from(([127, 0, 0, 1], (conn_id % 65535) as u16)); + + let config = config_rx_unix.borrow_and_update().clone(); + let stats = stats.clone(); + let upstream_manager = upstream_manager.clone(); + let replay_checker = replay_checker.clone(); + let buffer_pool = buffer_pool.clone(); + let rng = rng.clone(); + let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); + let tls_cache = tls_cache.clone(); + let ip_tracker = ip_tracker.clone(); + let beobachten = beobachten.clone(); + let proxy_protocol_enabled = config.server.proxy_protocol; + + tokio::spawn(async move { + let _permit = permit; + if let Err(e) = crate::proxy::client::handle_client_stream( + stream, + fake_peer, + config, + stats, + upstream_manager, + replay_checker, + buffer_pool, + rng, + me_pool, + route_runtime, + tls_cache, + ip_tracker, + beobachten, + proxy_protocol_enabled, + ) + .await + { + debug!(error = %e, "Unix socket connection error"); + } + }); + } + Err(e) => { + error!("Unix socket accept error: {}", e); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + }); + } + + startup_tracker + .complete_component( + COMPONENT_LISTENERS_BIND, + Some(format!( + "listeners configured tcp={} unix={}", + listeners.len(), + has_unix_listener + )), + ) + .await; + + Ok(BoundListeners { + listeners, + has_unix_listener, + }) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn spawn_tcp_accept_loops( + listeners: Vec<(TcpListener, bool)>, + config_rx: watch::Receiver>, + admission_rx: watch::Receiver, + stats: Arc, + upstream_manager: Arc, + replay_checker: Arc, + buffer_pool: Arc, + rng: Arc, + me_pool: Option>, + route_runtime: Arc, + tls_cache: Option>, + ip_tracker: Arc, + beobachten: Arc, + max_connections: Arc, +) { + for (listener, listener_proxy_protocol) in listeners { + let mut config_rx: watch::Receiver> = config_rx.clone(); + let mut admission_rx_tcp = admission_rx.clone(); + let stats = stats.clone(); + let upstream_manager = upstream_manager.clone(); + let replay_checker = replay_checker.clone(); + let buffer_pool = buffer_pool.clone(); + let rng = rng.clone(); + let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); + let tls_cache = tls_cache.clone(); + let ip_tracker = ip_tracker.clone(); + let beobachten = beobachten.clone(); + let max_connections_tcp = max_connections.clone(); + + tokio::spawn(async move { + loop { + if !wait_until_admission_open(&mut admission_rx_tcp).await { + warn!("Conditional-admission gate channel closed for tcp listener"); + break; + } + match listener.accept().await { + Ok((stream, peer_addr)) => { + let permit = match max_connections_tcp.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + error!("Connection limiter is closed"); + break; + } + }; + let config = config_rx.borrow_and_update().clone(); + let stats = stats.clone(); + let upstream_manager = upstream_manager.clone(); + let replay_checker = replay_checker.clone(); + let buffer_pool = buffer_pool.clone(); + let rng = rng.clone(); + let me_pool = me_pool.clone(); + let route_runtime = route_runtime.clone(); + let tls_cache = tls_cache.clone(); + let ip_tracker = ip_tracker.clone(); + let beobachten = beobachten.clone(); + let proxy_protocol_enabled = listener_proxy_protocol; + let real_peer_report = Arc::new(std::sync::Mutex::new(None)); + let real_peer_report_for_handler = real_peer_report.clone(); + + tokio::spawn(async move { + let _permit = permit; + if let Err(e) = ClientHandler::new( + stream, + peer_addr, + config, + stats, + upstream_manager, + replay_checker, + buffer_pool, + rng, + me_pool, + route_runtime, + tls_cache, + ip_tracker, + beobachten, + proxy_protocol_enabled, + real_peer_report_for_handler, + ) + .run() + .await + { + let real_peer = match real_peer_report.lock() { + Ok(guard) => *guard, + Err(_) => None, + }; + let peer_closed = matches!( + &e, + crate::error::ProxyError::Io(ioe) + if matches!( + ioe.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::NotConnected + ) + ) || matches!( + &e, + crate::error::ProxyError::Stream( + crate::error::StreamError::Io(ioe) + ) + if matches!( + ioe.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::NotConnected + ) + ); + + let me_closed = matches!( + &e, + crate::error::ProxyError::Proxy(msg) if msg == "ME connection lost" + ); + let route_switched = matches!( + &e, + crate::error::ProxyError::Proxy(msg) if msg == ROUTE_SWITCH_ERROR_MSG + ); + + match (peer_closed, me_closed) { + (true, _) => { + if let Some(real_peer) = real_peer { + debug!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by client"); + } else { + debug!(peer = %peer_addr, error = %e, "Connection closed by client"); + } + } + (_, true) => { + if let Some(real_peer) = real_peer { + warn!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed: Middle-End dropped session"); + } else { + warn!(peer = %peer_addr, error = %e, "Connection closed: Middle-End dropped session"); + } + } + _ if route_switched => { + if let Some(real_peer) = real_peer { + info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by controlled route cutover"); + } else { + info!(peer = %peer_addr, error = %e, "Connection closed by controlled route cutover"); + } + } + _ if is_expected_handshake_eof(&e) => { + if let Some(real_peer) = real_peer { + info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed during initial handshake"); + } else { + info!(peer = %peer_addr, error = %e, "Connection closed during initial handshake"); + } + } + _ => { + if let Some(real_peer) = real_peer { + warn!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed with error"); + } else { + warn!(peer = %peer_addr, error = %e, "Connection closed with error"); + } + } + } + } + }); + } + Err(e) => { + error!("Accept error: {}", e); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + }); + } +} diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs new file mode 100644 index 0000000..9c674cd --- /dev/null +++ b/src/maestro/me_startup.rs @@ -0,0 +1,425 @@ +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::RwLock; +use tracing::{error, info, warn}; + +use crate::config::ProxyConfig; +use crate::crypto::SecureRandom; +use crate::network::probe::{NetworkDecision, NetworkProbe}; +use crate::startup::{ + COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, COMPONENT_ME_PROXY_CONFIG_V4, + COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, StartupMeStatus, StartupTracker, +}; +use crate::stats::Stats; +use crate::transport::middle_proxy::MePool; +use crate::transport::UpstreamManager; + +use super::helpers::load_startup_proxy_config_snapshot; + +pub(crate) async fn initialize_me_pool( + use_middle_proxy: bool, + config: &ProxyConfig, + decision: &NetworkDecision, + probe: &NetworkProbe, + startup_tracker: &Arc, + upstream_manager: Arc, + rng: Arc, + stats: Arc, + api_me_pool: Arc>>>, +) -> Option> { + if !use_middle_proxy { + return None; + } + + info!("=== Middle Proxy Mode ==="); + let me_nat_probe = config.general.middle_proxy_nat_probe && config.network.stun_use; + if config.general.middle_proxy_nat_probe && !config.network.stun_use { + info!("Middle-proxy STUN probing disabled by network.stun_use=false"); + } + + let me2dc_fallback = config.general.me2dc_fallback; + let me_init_retry_attempts = config.general.me_init_retry_attempts; + let me_init_warn_after_attempts: u32 = 3; + + // Global ad_tag (pool default). Used when user has no per-user tag in access.user_ad_tags. + let proxy_tag = config + .general + .ad_tag + .as_ref() + .map(|tag| hex::decode(tag).expect("general.ad_tag must be validated before startup")); + + // ============================================================= + // CRITICAL: Download Telegram proxy-secret (NOT user secret!) + // + // C MTProxy uses TWO separate secrets: + // -S flag = 16-byte user secret for client obfuscation + // --aes-pwd = 32-512 byte binary file for ME RPC auth + // + // proxy-secret is from: https://core.telegram.org/getProxySecret + // ============================================================= + let proxy_secret_path = config.general.proxy_secret_path.as_deref(); + let pool_size = config.general.middle_proxy_pool_size.max(1); + let proxy_secret = loop { + match crate::transport::middle_proxy::fetch_proxy_secret( + proxy_secret_path, + config.general.proxy_secret_len_max, + ) + .await + { + Ok(proxy_secret) => break Some(proxy_secret), + Err(e) => { + startup_tracker.set_me_last_error(Some(e.to_string())).await; + if me2dc_fallback { + error!( + error = %e, + "ME startup failed: proxy-secret is unavailable and no saved secret found; falling back to direct mode" + ); + break None; + } + + warn!( + error = %e, + retry_in_secs = 2, + "ME startup failed: proxy-secret is unavailable and no saved secret found; retrying because me2dc_fallback=false" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + }; + match proxy_secret { + Some(proxy_secret) => { + startup_tracker + .complete_component( + COMPONENT_ME_SECRET_FETCH, + Some("proxy-secret loaded".to_string()), + ) + .await; + info!( + secret_len = proxy_secret.len(), + key_sig = format_args!( + "0x{:08x}", + if proxy_secret.len() >= 4 { + u32::from_le_bytes([ + proxy_secret[0], + proxy_secret[1], + proxy_secret[2], + proxy_secret[3], + ]) + } else { + 0 + } + ), + "Proxy-secret loaded" + ); + + startup_tracker + .start_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("load startup proxy-config v4".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4) + .await; + let cfg_v4 = load_startup_proxy_config_snapshot( + "https://core.telegram.org/getProxyConfig", + config.general.proxy_config_v4_cache_path.as_deref(), + me2dc_fallback, + "getProxyConfig", + ) + .await; + if cfg_v4.is_some() { + startup_tracker + .complete_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("proxy-config v4 loaded".to_string()), + ) + .await; + } else { + startup_tracker + .fail_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("proxy-config v4 unavailable".to_string()), + ) + .await; + } + startup_tracker + .start_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("load startup proxy-config v6".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6) + .await; + let cfg_v6 = load_startup_proxy_config_snapshot( + "https://core.telegram.org/getProxyConfigV6", + config.general.proxy_config_v6_cache_path.as_deref(), + me2dc_fallback, + "getProxyConfigV6", + ) + .await; + if cfg_v6.is_some() { + startup_tracker + .complete_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("proxy-config v6 loaded".to_string()), + ) + .await; + } else { + startup_tracker + .fail_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("proxy-config v6 unavailable".to_string()), + ) + .await; + } + + if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) { + startup_tracker + .start_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("construct ME pool".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_CONSTRUCT) + .await; + let pool = MePool::new( + proxy_tag.clone(), + proxy_secret, + config.general.middle_proxy_nat_ip, + me_nat_probe, + None, + config.network.stun_servers.clone(), + config.general.stun_nat_probe_concurrency, + probe.detected_ipv6, + config.timeouts.me_one_retry, + config.timeouts.me_one_timeout_ms, + cfg_v4.map.clone(), + cfg_v6.map.clone(), + cfg_v4.default_dc.or(cfg_v6.default_dc), + decision.clone(), + Some(upstream_manager.clone()), + rng.clone(), + stats.clone(), + config.general.me_keepalive_enabled, + config.general.me_keepalive_interval_secs, + config.general.me_keepalive_jitter_secs, + config.general.me_keepalive_payload_random, + config.general.rpc_proxy_req_every, + config.general.me_warmup_stagger_enabled, + config.general.me_warmup_step_delay_ms, + config.general.me_warmup_step_jitter_ms, + config.general.me_reconnect_max_concurrent_per_dc, + config.general.me_reconnect_backoff_base_ms, + config.general.me_reconnect_backoff_cap_ms, + config.general.me_reconnect_fast_retry_count, + config.general.me_single_endpoint_shadow_writers, + config.general.me_single_endpoint_outage_mode_enabled, + config.general.me_single_endpoint_outage_disable_quarantine, + config.general.me_single_endpoint_outage_backoff_min_ms, + config.general.me_single_endpoint_outage_backoff_max_ms, + config.general.me_single_endpoint_shadow_rotate_every_secs, + config.general.me_floor_mode, + config.general.me_adaptive_floor_idle_secs, + config.general.me_adaptive_floor_min_writers_single_endpoint, + config.general.me_adaptive_floor_min_writers_multi_endpoint, + config.general.me_adaptive_floor_recover_grace_secs, + config.general.me_adaptive_floor_writers_per_core_total, + config.general.me_adaptive_floor_cpu_cores_override, + config.general.me_adaptive_floor_max_extra_writers_single_per_core, + config.general.me_adaptive_floor_max_extra_writers_multi_per_core, + config.general.me_adaptive_floor_max_active_writers_per_core, + config.general.me_adaptive_floor_max_warm_writers_per_core, + config.general.me_adaptive_floor_max_active_writers_global, + config.general.me_adaptive_floor_max_warm_writers_global, + config.general.hardswap, + config.general.me_pool_drain_ttl_secs, + config.general.effective_me_pool_force_close_secs(), + config.general.me_pool_min_fresh_ratio, + config.general.me_hardswap_warmup_delay_min_ms, + config.general.me_hardswap_warmup_delay_max_ms, + config.general.me_hardswap_warmup_extra_passes, + config.general.me_hardswap_warmup_pass_backoff_base_ms, + config.general.me_bind_stale_mode, + config.general.me_bind_stale_ttl_secs, + config.general.me_secret_atomic_snapshot, + config.general.me_deterministic_writer_sort, + config.general.me_writer_pick_mode, + config.general.me_writer_pick_sample_size, + config.general.me_socks_kdf_policy, + config.general.me_writer_cmd_channel_capacity, + config.general.me_route_channel_capacity, + config.general.me_route_backpressure_base_timeout_ms, + config.general.me_route_backpressure_high_timeout_ms, + config.general.me_route_backpressure_high_watermark_pct, + config.general.me_reader_route_data_wait_ms, + config.general.me_health_interval_ms_unhealthy, + config.general.me_health_interval_ms_healthy, + config.general.me_warn_rate_limit_ms, + config.general.me_route_no_writer_mode, + config.general.me_route_no_writer_wait_ms, + config.general.me_route_inline_recovery_attempts, + config.general.me_route_inline_recovery_wait_ms, + ); + startup_tracker + .complete_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("ME pool object created".to_string()), + ) + .await; + *api_me_pool.write().await = Some(pool.clone()); + startup_tracker + .start_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("initialize ME pool writers".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_INIT_STAGE1) + .await; + + let mut init_attempt: u32 = 0; + loop { + init_attempt = init_attempt.saturating_add(1); + startup_tracker.set_me_init_attempt(init_attempt).await; + match pool.init(pool_size, &rng).await { + Ok(()) => { + startup_tracker.set_me_last_error(None).await; + startup_tracker + .complete_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME pool initialized".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Ready, "ready") + .await; + info!( + attempt = init_attempt, + "Middle-End pool initialized successfully" + ); + + // Phase 4: Start health monitor + let pool_clone = pool.clone(); + let rng_clone = rng.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + pool_clone, rng_clone, min_conns, + ) + .await; + }); + + break Some(pool); + } + Err(e) => { + startup_tracker.set_me_last_error(Some(e.to_string())).await; + let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; + if retries_limited && init_attempt >= me_init_retry_attempts { + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME init retry budget exhausted".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; + error!( + error = %e, + attempt = init_attempt, + retry_limit = me_init_retry_attempts, + "ME pool init retries exhausted; falling back to direct mode" + ); + break None; + } + + let retry_limit = if !me2dc_fallback || me_init_retry_attempts == 0 { + String::from("unlimited") + } else { + me_init_retry_attempts.to_string() + }; + if init_attempt >= me_init_warn_after_attempts { + warn!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool is not ready yet; retrying startup initialization" + ); + } else { + info!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool startup warmup: retrying initialization" + ); + } + pool.reset_stun_state(); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } + } else { + startup_tracker + .skip_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("ME configs are incomplete".to_string()), + ) + .await; + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME configs are incomplete".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; + None + } + } + None => { + startup_tracker + .fail_component( + COMPONENT_ME_SECRET_FETCH, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("proxy-secret unavailable".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; + None + } + } +} diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs new file mode 100644 index 0000000..92b42e3 --- /dev/null +++ b/src/maestro/mod.rs @@ -0,0 +1,546 @@ +//! telemt — Telegram MTProto Proxy + +#![allow(unused_assignments)] + +// Runtime orchestration modules. +// - helpers: CLI and shared startup/runtime helper routines. +// - tls_bootstrap: TLS front cache bootstrap and refresh tasks. +// - me_startup: Middle-End secret/config fetch and pool initialization. +// - connectivity: startup ME/DC connectivity diagnostics. +// - runtime_tasks: hot-reload and background task orchestration. +// - admission: conditional-cast gate and route mode switching. +// - listeners: TCP/Unix listener bind and accept-loop orchestration. +// - shutdown: graceful shutdown sequence and uptime logging. +mod helpers; +mod admission; +mod connectivity; +mod listeners; +mod me_startup; +mod runtime_tasks; +mod shutdown; +mod tls_bootstrap; + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::sync::{RwLock, Semaphore, watch}; +use tracing::{error, info, warn}; +use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; + +use crate::api; +use crate::config::{LogLevel, ProxyConfig}; +use crate::crypto::SecureRandom; +use crate::ip_tracker::UserIpTracker; +use crate::network::probe::{decide_network_capabilities, log_probe_result, run_probe}; +use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController}; +use crate::stats::beobachten::BeobachtenStore; +use crate::stats::telemetry::TelemetryPolicy; +use crate::stats::{ReplayChecker, Stats}; +use crate::startup::{ + COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, + COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, + COMPONENT_ME_PROXY_CONFIG_V4, COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, + COMPONENT_NETWORK_PROBE, COMPONENT_TRACING_INIT, StartupMeStatus, StartupTracker, +}; +use crate::stream::BufferPool; +use crate::transport::middle_proxy::MePool; +use crate::transport::UpstreamManager; +use helpers::parse_cli; + +/// Runs the full telemt runtime startup pipeline and blocks until shutdown. +pub async fn run() -> std::result::Result<(), Box> { + let process_started_at = Instant::now(); + let process_started_at_epoch_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let startup_tracker = Arc::new(StartupTracker::new(process_started_at_epoch_secs)); + startup_tracker + .start_component(COMPONENT_CONFIG_LOAD, Some("load and validate config".to_string())) + .await; + let (config_path, cli_silent, cli_log_level) = parse_cli(); + + let mut config = match ProxyConfig::load(&config_path) { + Ok(c) => c, + Err(e) => { + if std::path::Path::new(&config_path).exists() { + eprintln!("[telemt] Error: {}", e); + std::process::exit(1); + } else { + let default = ProxyConfig::default(); + std::fs::write(&config_path, toml::to_string_pretty(&default).unwrap()).unwrap(); + eprintln!("[telemt] Created default config at {}", config_path); + default + } + } + }; + + if let Err(e) = config.validate() { + eprintln!("[telemt] Invalid config: {}", e); + std::process::exit(1); + } + + if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) { + eprintln!("[telemt] Invalid network.dns_overrides: {}", e); + std::process::exit(1); + } + startup_tracker + .complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string())) + .await; + + let has_rust_log = std::env::var("RUST_LOG").is_ok(); + let effective_log_level = if cli_silent { + LogLevel::Silent + } else if let Some(ref s) = cli_log_level { + LogLevel::from_str_loose(s) + } else { + config.general.log_level.clone() + }; + + let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info")); + startup_tracker + .start_component(COMPONENT_TRACING_INIT, Some("initialize tracing subscriber".to_string())) + .await; + + // Configure color output based on config + let fmt_layer = if config.general.disable_colors { + fmt::Layer::default().with_ansi(false) + } else { + fmt::Layer::default().with_ansi(true) + }; + + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .init(); + startup_tracker + .complete_component(COMPONENT_TRACING_INIT, Some("tracing initialized".to_string())) + .await; + + info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION")); + info!("Log level: {}", effective_log_level); + if config.general.disable_colors { + info!("Colors: disabled"); + } + info!( + "Modes: classic={} secure={} tls={}", + config.general.modes.classic, config.general.modes.secure, config.general.modes.tls + ); + if config.general.modes.classic { + warn!("Classic mode is vulnerable to DPI detection; enable only for legacy clients"); + } + info!("TLS domain: {}", config.censorship.tls_domain); + if let Some(ref sock) = config.censorship.mask_unix_sock { + info!("Mask: {} -> unix:{}", config.censorship.mask, sock); + if !std::path::Path::new(sock).exists() { + warn!( + "Unix socket '{}' does not exist yet. Masking will fail until it appears.", + sock + ); + } + } else { + info!( + "Mask: {} -> {}:{}", + config.censorship.mask, + config + .censorship + .mask_host + .as_deref() + .unwrap_or(&config.censorship.tls_domain), + config.censorship.mask_port + ); + } + + if config.censorship.tls_domain == "www.google.com" { + warn!("Using default tls_domain. Consider setting a custom domain."); + } + + let stats = Arc::new(Stats::new()); + stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry)); + + let upstream_manager = Arc::new(UpstreamManager::new( + config.upstreams.clone(), + config.general.upstream_connect_retry_attempts, + config.general.upstream_connect_retry_backoff_ms, + config.general.upstream_connect_budget_ms, + config.general.upstream_unhealthy_fail_threshold, + config.general.upstream_connect_failfast_hard_errors, + stats.clone(), + )); + let ip_tracker = Arc::new(UserIpTracker::new()); + ip_tracker.load_limits(&config.access.user_max_unique_ips).await; + ip_tracker + .set_limit_policy( + config.access.user_max_unique_ips_mode, + config.access.user_max_unique_ips_window_secs, + ) + .await; + if !config.access.user_max_unique_ips.is_empty() { + info!( + "IP limits configured for {} users", + config.access.user_max_unique_ips.len() + ); + } + if !config.network.dns_overrides.is_empty() { + info!( + "Runtime DNS overrides configured: {} entries", + config.network.dns_overrides.len() + ); + } + + let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone())); + let initial_admission_open = !config.general.use_middle_proxy; + let (admission_tx, admission_rx) = watch::channel(initial_admission_open); + let initial_route_mode = if config.general.use_middle_proxy { + RelayRouteMode::Middle + } else { + RelayRouteMode::Direct + }; + let route_runtime = Arc::new(RouteRuntimeController::new(initial_route_mode)); + let api_me_pool = Arc::new(RwLock::new(None::>)); + startup_tracker + .start_component(COMPONENT_API_BOOTSTRAP, Some("spawn API listener task".to_string())) + .await; + + if config.server.api.enabled { + let listen = match config.server.api.listen.parse::() { + Ok(listen) => listen, + Err(error) => { + warn!( + error = %error, + listen = %config.server.api.listen, + "Invalid server.api.listen; API is disabled" + ); + SocketAddr::from(([127, 0, 0, 1], 0)) + } + }; + if listen.port() != 0 { + let stats_api = stats.clone(); + let ip_tracker_api = ip_tracker.clone(); + let me_pool_api = api_me_pool.clone(); + let upstream_manager_api = upstream_manager.clone(); + let config_rx_api = api_config_rx.clone(); + let admission_rx_api = admission_rx.clone(); + let config_path_api = std::path::PathBuf::from(&config_path); + let startup_tracker_api = startup_tracker.clone(); + tokio::spawn(async move { + api::serve( + listen, + stats_api, + ip_tracker_api, + me_pool_api, + upstream_manager_api, + config_rx_api, + admission_rx_api, + config_path_api, + None, + None, + process_started_at_epoch_secs, + startup_tracker_api, + ) + .await; + }); + startup_tracker + .complete_component( + COMPONENT_API_BOOTSTRAP, + Some(format!("api task spawned on {}", listen)), + ) + .await; + } else { + startup_tracker + .skip_component( + COMPONENT_API_BOOTSTRAP, + Some("server.api.listen has zero port".to_string()), + ) + .await; + } + } else { + startup_tracker + .skip_component( + COMPONENT_API_BOOTSTRAP, + Some("server.api.enabled is false".to_string()), + ) + .await; + } + + let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); + tls_domains.push(config.censorship.tls_domain.clone()); + for d in &config.censorship.tls_domains { + if !tls_domains.contains(d) { + tls_domains.push(d.clone()); + } + } + + let tls_cache = tls_bootstrap::bootstrap_tls_front( + &config, + &tls_domains, + upstream_manager.clone(), + &startup_tracker, + ) + .await; + + startup_tracker + .start_component(COMPONENT_NETWORK_PROBE, Some("probe network capabilities".to_string())) + .await; + let probe = run_probe( + &config.network, + config.general.middle_proxy_nat_probe, + config.general.stun_nat_probe_concurrency, + ) + .await?; + let decision = decide_network_capabilities(&config.network, &probe); + log_probe_result(&probe, &decision); + startup_tracker + .complete_component( + COMPONENT_NETWORK_PROBE, + Some("network capabilities determined".to_string()), + ) + .await; + + let prefer_ipv6 = decision.prefer_ipv6(); + let mut use_middle_proxy = config.general.use_middle_proxy; + let beobachten = Arc::new(BeobachtenStore::new()); + let rng = Arc::new(SecureRandom::new()); + + // Connection concurrency limit + let max_connections = Arc::new(Semaphore::new(10_000)); + + let me2dc_fallback = config.general.me2dc_fallback; + let me_init_retry_attempts = config.general.me_init_retry_attempts; + if use_middle_proxy && !decision.ipv4_me && !decision.ipv6_me { + if me2dc_fallback { + warn!("No usable IP family for Middle Proxy detected; falling back to direct DC"); + use_middle_proxy = false; + } else { + warn!( + "No usable IP family for Middle Proxy detected; me2dc_fallback=false, ME init retries stay active" + ); + } + } + + if use_middle_proxy { + startup_tracker + .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_SECRET_FETCH) + .await; + startup_tracker + .start_component( + COMPONENT_ME_SECRET_FETCH, + Some("fetch proxy-secret from source/cache".to_string()), + ) + .await; + startup_tracker + .set_me_retry_limit(if !me2dc_fallback || me_init_retry_attempts == 0 { + "unlimited".to_string() + } else { + me_init_retry_attempts.to_string() + }) + .await; + } else { + startup_tracker + .set_me_status(StartupMeStatus::Skipped, "skipped") + .await; + startup_tracker + .skip_component( + COMPONENT_ME_SECRET_FETCH, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V4, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_PROXY_CONFIG_V6, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_POOL_CONSTRUCT, + Some("middle proxy mode disabled".to_string()), + ) + .await; + startup_tracker + .skip_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("middle proxy mode disabled".to_string()), + ) + .await; + } + + let me_pool: Option> = me_startup::initialize_me_pool( + use_middle_proxy, + &config, + &decision, + &probe, + &startup_tracker, + upstream_manager.clone(), + rng.clone(), + stats.clone(), + api_me_pool.clone(), + ) + .await; + + // If ME failed to initialize, force direct-only mode. + if me_pool.is_some() { + startup_tracker + .set_transport_mode("middle_proxy") + .await; + startup_tracker + .set_degraded(false) + .await; + info!("Transport: Middle-End Proxy - all DC-over-RPC"); + } else { + let _ = use_middle_proxy; + use_middle_proxy = false; + // Make runtime config reflect direct-only mode for handlers. + config.general.use_middle_proxy = false; + startup_tracker + .set_transport_mode("direct") + .await; + startup_tracker + .set_degraded(true) + .await; + if me2dc_fallback { + startup_tracker + .set_me_status(StartupMeStatus::Failed, "fallback_to_direct") + .await; + } else { + startup_tracker + .set_me_status(StartupMeStatus::Skipped, "skipped") + .await; + } + info!("Transport: Direct DC - TCP - standard DC-over-TCP"); + } + + // Freeze config after possible fallback decision + let config = Arc::new(config); + + let replay_checker = Arc::new(ReplayChecker::new( + config.access.replay_check_len, + Duration::from_secs(config.access.replay_window_secs), + )); + + let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096)); + + connectivity::run_startup_connectivity( + &config, + &me_pool, + rng.clone(), + &startup_tracker, + upstream_manager.clone(), + prefer_ipv6, + &decision, + process_started_at, + api_me_pool.clone(), + ) + .await; + + let runtime_watches = runtime_tasks::spawn_runtime_tasks( + &config, + &config_path, + &probe, + prefer_ipv6, + decision.ipv4_dc, + decision.ipv6_dc, + &startup_tracker, + stats.clone(), + upstream_manager.clone(), + replay_checker.clone(), + me_pool.clone(), + rng.clone(), + ip_tracker.clone(), + beobachten.clone(), + api_config_tx.clone(), + me_pool.clone(), + ) + .await; + let config_rx = runtime_watches.config_rx; + let log_level_rx = runtime_watches.log_level_rx; + let detected_ip_v4 = runtime_watches.detected_ip_v4; + let detected_ip_v6 = runtime_watches.detected_ip_v6; + + admission::configure_admission_gate( + &config, + me_pool.clone(), + route_runtime.clone(), + &admission_tx, + config_rx.clone(), + ) + .await; + let _admission_tx_hold = admission_tx; + + let bound = listeners::bind_listeners( + &config, + decision.ipv4_dc, + decision.ipv6_dc, + detected_ip_v4, + detected_ip_v6, + &startup_tracker, + config_rx.clone(), + admission_rx.clone(), + stats.clone(), + upstream_manager.clone(), + replay_checker.clone(), + buffer_pool.clone(), + rng.clone(), + me_pool.clone(), + route_runtime.clone(), + tls_cache.clone(), + ip_tracker.clone(), + beobachten.clone(), + max_connections.clone(), + ) + .await?; + let listeners = bound.listeners; + let has_unix_listener = bound.has_unix_listener; + + if listeners.is_empty() && !has_unix_listener { + error!("No listeners. Exiting."); + std::process::exit(1); + } + + runtime_tasks::apply_runtime_log_filter( + has_rust_log, + &effective_log_level, + filter_handle, + log_level_rx, + ) + .await; + + runtime_tasks::spawn_metrics_if_configured( + &config, + &startup_tracker, + stats.clone(), + beobachten.clone(), + ip_tracker.clone(), + config_rx.clone(), + ) + .await; + + runtime_tasks::mark_runtime_ready(&startup_tracker).await; + + listeners::spawn_tcp_accept_loops( + listeners, + config_rx.clone(), + admission_rx.clone(), + stats.clone(), + upstream_manager.clone(), + replay_checker.clone(), + buffer_pool.clone(), + rng.clone(), + me_pool.clone(), + route_runtime.clone(), + tls_cache.clone(), + ip_tracker.clone(), + beobachten.clone(), + max_connections.clone(), + ); + + shutdown::wait_for_shutdown(process_started_at, me_pool).await; + + Ok(()) +} diff --git a/src/maestro/runtime_tasks.rs b/src/maestro/runtime_tasks.rs new file mode 100644 index 0000000..c8aa534 --- /dev/null +++ b/src/maestro/runtime_tasks.rs @@ -0,0 +1,317 @@ +use std::net::IpAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use tokio::sync::{mpsc, watch}; +use tracing::{debug, warn}; +use tracing_subscriber::reload; +use tracing_subscriber::EnvFilter; + +use crate::config::{LogLevel, ProxyConfig}; +use crate::config::hot_reload::spawn_config_watcher; +use crate::crypto::SecureRandom; +use crate::ip_tracker::UserIpTracker; +use crate::metrics; +use crate::network::probe::NetworkProbe; +use crate::startup::{COMPONENT_CONFIG_WATCHER_START, COMPONENT_METRICS_START, COMPONENT_RUNTIME_READY, StartupTracker}; +use crate::stats::beobachten::BeobachtenStore; +use crate::stats::telemetry::TelemetryPolicy; +use crate::stats::{ReplayChecker, Stats}; +use crate::transport::middle_proxy::{MePool, MeReinitTrigger}; +use crate::transport::UpstreamManager; + +use super::helpers::write_beobachten_snapshot; + +pub(crate) struct RuntimeWatches { + pub(crate) config_rx: watch::Receiver>, + pub(crate) log_level_rx: watch::Receiver, + pub(crate) detected_ip_v4: Option, + pub(crate) detected_ip_v6: Option, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn spawn_runtime_tasks( + config: &Arc, + config_path: &str, + probe: &NetworkProbe, + prefer_ipv6: bool, + decision_ipv4_dc: bool, + decision_ipv6_dc: bool, + startup_tracker: &Arc, + stats: Arc, + upstream_manager: Arc, + replay_checker: Arc, + me_pool: Option>, + rng: Arc, + ip_tracker: Arc, + beobachten: Arc, + api_config_tx: watch::Sender>, + me_pool_for_policy: Option>, +) -> RuntimeWatches { + let um_clone = upstream_manager.clone(); + let dc_overrides_for_health = config.dc_overrides.clone(); + tokio::spawn(async move { + um_clone + .run_health_checks( + prefer_ipv6, + decision_ipv4_dc, + decision_ipv6_dc, + dc_overrides_for_health, + ) + .await; + }); + + let rc_clone = replay_checker.clone(); + tokio::spawn(async move { + rc_clone.run_periodic_cleanup().await; + }); + + let detected_ip_v4: Option = probe.detected_ipv4.map(IpAddr::V4); + let detected_ip_v6: Option = probe.detected_ipv6.map(IpAddr::V6); + debug!( + "Detected IPs: v4={:?} v6={:?}", + detected_ip_v4, detected_ip_v6 + ); + + startup_tracker + .start_component( + COMPONENT_CONFIG_WATCHER_START, + Some("spawn config hot-reload watcher".to_string()), + ) + .await; + let (config_rx, log_level_rx): ( + watch::Receiver>, + watch::Receiver, + ) = spawn_config_watcher( + PathBuf::from(config_path), + config.clone(), + detected_ip_v4, + detected_ip_v6, + ); + startup_tracker + .complete_component( + COMPONENT_CONFIG_WATCHER_START, + Some("config hot-reload watcher started".to_string()), + ) + .await; + let mut config_rx_api_bridge = config_rx.clone(); + let api_config_tx_bridge = api_config_tx.clone(); + tokio::spawn(async move { + loop { + if config_rx_api_bridge.changed().await.is_err() { + break; + } + let cfg = config_rx_api_bridge.borrow_and_update().clone(); + api_config_tx_bridge.send_replace(cfg); + } + }); + + let stats_policy = stats.clone(); + let mut config_rx_policy = config_rx.clone(); + tokio::spawn(async move { + loop { + if config_rx_policy.changed().await.is_err() { + break; + } + let cfg = config_rx_policy.borrow_and_update().clone(); + stats_policy.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry)); + if let Some(pool) = &me_pool_for_policy { + pool.update_runtime_transport_policy( + cfg.general.me_socks_kdf_policy, + cfg.general.me_route_backpressure_base_timeout_ms, + cfg.general.me_route_backpressure_high_timeout_ms, + cfg.general.me_route_backpressure_high_watermark_pct, + cfg.general.me_reader_route_data_wait_ms, + ); + } + } + }); + + let ip_tracker_policy = ip_tracker.clone(); + let mut config_rx_ip_limits = config_rx.clone(); + tokio::spawn(async move { + let mut prev_limits = config_rx_ip_limits.borrow().access.user_max_unique_ips.clone(); + let mut prev_mode = config_rx_ip_limits.borrow().access.user_max_unique_ips_mode; + let mut prev_window = config_rx_ip_limits + .borrow() + .access + .user_max_unique_ips_window_secs; + + loop { + if config_rx_ip_limits.changed().await.is_err() { + break; + } + let cfg = config_rx_ip_limits.borrow_and_update().clone(); + + if prev_limits != cfg.access.user_max_unique_ips { + ip_tracker_policy.load_limits(&cfg.access.user_max_unique_ips).await; + prev_limits = cfg.access.user_max_unique_ips.clone(); + } + + if prev_mode != cfg.access.user_max_unique_ips_mode + || prev_window != cfg.access.user_max_unique_ips_window_secs + { + ip_tracker_policy + .set_limit_policy( + cfg.access.user_max_unique_ips_mode, + cfg.access.user_max_unique_ips_window_secs, + ) + .await; + prev_mode = cfg.access.user_max_unique_ips_mode; + prev_window = cfg.access.user_max_unique_ips_window_secs; + } + } + }); + + let beobachten_writer = beobachten.clone(); + let config_rx_beobachten = config_rx.clone(); + tokio::spawn(async move { + loop { + let cfg = config_rx_beobachten.borrow().clone(); + let sleep_secs = cfg.general.beobachten_flush_secs.max(1); + + if cfg.general.beobachten { + let ttl = std::time::Duration::from_secs(cfg.general.beobachten_minutes.saturating_mul(60)); + let path = cfg.general.beobachten_file.clone(); + let snapshot = beobachten_writer.snapshot_text(ttl); + if let Err(e) = write_beobachten_snapshot(&path, &snapshot).await { + warn!(error = %e, path = %path, "Failed to flush beobachten snapshot"); + } + } + + tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await; + } + }); + + if let Some(pool) = me_pool { + let reinit_trigger_capacity = config.general.me_reinit_trigger_channel.max(1); + let (reinit_tx, reinit_rx) = mpsc::channel::(reinit_trigger_capacity); + + let pool_clone_sched = pool.clone(); + let rng_clone_sched = rng.clone(); + let config_rx_clone_sched = config_rx.clone(); + tokio::spawn(async move { + crate::transport::middle_proxy::me_reinit_scheduler( + pool_clone_sched, + rng_clone_sched, + config_rx_clone_sched, + reinit_rx, + ) + .await; + }); + + let pool_clone = pool.clone(); + let config_rx_clone = config_rx.clone(); + let reinit_tx_updater = reinit_tx.clone(); + tokio::spawn(async move { + crate::transport::middle_proxy::me_config_updater( + pool_clone, + config_rx_clone, + reinit_tx_updater, + ) + .await; + }); + + let config_rx_clone_rot = config_rx.clone(); + let reinit_tx_rotation = reinit_tx.clone(); + tokio::spawn(async move { + crate::transport::middle_proxy::me_rotation_task(config_rx_clone_rot, reinit_tx_rotation) + .await; + }); + } + + RuntimeWatches { + config_rx, + log_level_rx, + detected_ip_v4, + detected_ip_v6, + } +} + +pub(crate) async fn apply_runtime_log_filter( + has_rust_log: bool, + effective_log_level: &LogLevel, + filter_handle: reload::Handle, + mut log_level_rx: watch::Receiver, +) { + let runtime_filter = if has_rust_log { + EnvFilter::from_default_env() + } else if matches!(effective_log_level, LogLevel::Silent) { + EnvFilter::new("warn,telemt::links=info") + } else { + EnvFilter::new(effective_log_level.to_filter_str()) + }; + filter_handle + .reload(runtime_filter) + .expect("Failed to switch log filter"); + + tokio::spawn(async move { + loop { + if log_level_rx.changed().await.is_err() { + break; + } + let level = log_level_rx.borrow_and_update().clone(); + let new_filter = tracing_subscriber::EnvFilter::new(level.to_filter_str()); + if let Err(e) = filter_handle.reload(new_filter) { + tracing::error!("config reload: failed to update log filter: {}", e); + } + } + }); +} + +pub(crate) async fn spawn_metrics_if_configured( + config: &Arc, + startup_tracker: &Arc, + stats: Arc, + beobachten: Arc, + ip_tracker: Arc, + config_rx: watch::Receiver>, +) { + if let Some(port) = config.server.metrics_port { + startup_tracker + .start_component( + COMPONENT_METRICS_START, + Some(format!("spawn metrics endpoint on {}", port)), + ) + .await; + let stats = stats.clone(); + let beobachten = beobachten.clone(); + let config_rx_metrics = config_rx.clone(); + let ip_tracker_metrics = ip_tracker.clone(); + let whitelist = config.server.metrics_whitelist.clone(); + tokio::spawn(async move { + metrics::serve( + port, + stats, + beobachten, + ip_tracker_metrics, + config_rx_metrics, + whitelist, + ) + .await; + }); + startup_tracker + .complete_component( + COMPONENT_METRICS_START, + Some("metrics task spawned".to_string()), + ) + .await; + } else { + startup_tracker + .skip_component( + COMPONENT_METRICS_START, + Some("server.metrics_port is not configured".to_string()), + ) + .await; + } +} + +pub(crate) async fn mark_runtime_ready(startup_tracker: &Arc) { + startup_tracker + .complete_component( + COMPONENT_RUNTIME_READY, + Some("startup pipeline is fully initialized".to_string()), + ) + .await; + startup_tracker.mark_ready().await; +} diff --git a/src/maestro/shutdown.rs b/src/maestro/shutdown.rs new file mode 100644 index 0000000..b73df30 --- /dev/null +++ b/src/maestro/shutdown.rs @@ -0,0 +1,42 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::signal; +use tracing::{error, info, warn}; + +use crate::transport::middle_proxy::MePool; + +use super::helpers::{format_uptime, unit_label}; + +pub(crate) async fn wait_for_shutdown(process_started_at: Instant, me_pool: Option>) { + match signal::ctrl_c().await { + Ok(()) => { + let shutdown_started_at = Instant::now(); + info!("Shutting down..."); + let uptime_secs = process_started_at.elapsed().as_secs(); + info!("Uptime: {}", format_uptime(uptime_secs)); + if let Some(pool) = &me_pool { + match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all()) + .await + { + Ok(total) => { + info!( + close_conn_sent = total, + "ME shutdown: RPC_CLOSE_CONN broadcast completed" + ); + } + Err(_) => { + warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out"); + } + } + } + let shutdown_secs = shutdown_started_at.elapsed().as_secs(); + info!( + "Shutdown completed successfully in {} {}.", + shutdown_secs, + unit_label(shutdown_secs, "second", "seconds") + ); + } + Err(e) => error!("Signal error: {}", e), + } +} diff --git a/src/maestro/tls_bootstrap.rs b/src/maestro/tls_bootstrap.rs new file mode 100644 index 0000000..a0b0b5a --- /dev/null +++ b/src/maestro/tls_bootstrap.rs @@ -0,0 +1,165 @@ +use std::sync::Arc; +use std::time::Duration; + +use rand::Rng; +use tracing::warn; + +use crate::config::ProxyConfig; +use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker}; +use crate::tls_front::TlsFrontCache; +use crate::transport::UpstreamManager; + +pub(crate) async fn bootstrap_tls_front( + config: &ProxyConfig, + tls_domains: &[String], + upstream_manager: Arc, + startup_tracker: &Arc, +) -> Option> { + startup_tracker + .start_component( + COMPONENT_TLS_FRONT_BOOTSTRAP, + Some("initialize TLS front cache/bootstrap tasks".to_string()), + ) + .await; + + let tls_cache: Option> = if config.censorship.tls_emulation { + let cache = Arc::new(TlsFrontCache::new( + tls_domains, + config.censorship.fake_cert_len, + &config.censorship.tls_front_dir, + )); + cache.load_from_disk().await; + + let port = config.censorship.mask_port; + let proxy_protocol = config.censorship.mask_proxy_protocol; + let mask_host = config + .censorship + .mask_host + .clone() + .unwrap_or_else(|| config.censorship.tls_domain.clone()); + let mask_unix_sock = config.censorship.mask_unix_sock.clone(); + let fetch_timeout = Duration::from_secs(5); + + let cache_initial = cache.clone(); + let domains_initial = tls_domains.to_vec(); + let host_initial = mask_host.clone(); + let unix_sock_initial = mask_unix_sock.clone(); + let upstream_initial = upstream_manager.clone(); + tokio::spawn(async move { + let mut join = tokio::task::JoinSet::new(); + for domain in domains_initial { + let cache_domain = cache_initial.clone(); + let host_domain = host_initial.clone(); + let unix_sock_domain = unix_sock_initial.clone(); + let upstream_domain = upstream_initial.clone(); + join.spawn(async move { + match crate::tls_front::fetcher::fetch_real_tls( + &host_domain, + port, + &domain, + fetch_timeout, + Some(upstream_domain), + proxy_protocol, + unix_sock_domain.as_deref(), + ) + .await + { + Ok(res) => cache_domain.update_from_fetch(&domain, res).await, + Err(e) => { + warn!(domain = %domain, error = %e, "TLS emulation initial fetch failed") + } + } + }); + } + while let Some(res) = join.join_next().await { + if let Err(e) = res { + warn!(error = %e, "TLS emulation initial fetch task join failed"); + } + } + }); + + let cache_timeout = cache.clone(); + let domains_timeout = tls_domains.to_vec(); + let fake_cert_len = config.censorship.fake_cert_len; + tokio::spawn(async move { + tokio::time::sleep(fetch_timeout).await; + for domain in domains_timeout { + let cached = cache_timeout.get(&domain).await; + if cached.domain == "default" { + warn!( + domain = %domain, + timeout_secs = fetch_timeout.as_secs(), + fake_cert_len, + "TLS-front fetch not ready within timeout; using cache/default fake cert fallback" + ); + } + } + }); + + let cache_refresh = cache.clone(); + let domains_refresh = tls_domains.to_vec(); + let host_refresh = mask_host.clone(); + let unix_sock_refresh = mask_unix_sock.clone(); + let upstream_refresh = upstream_manager.clone(); + tokio::spawn(async move { + loop { + let base_secs = rand::rng().random_range(4 * 3600..=6 * 3600); + let jitter_secs = rand::rng().random_range(0..=7200); + tokio::time::sleep(Duration::from_secs(base_secs + jitter_secs)).await; + + let mut join = tokio::task::JoinSet::new(); + for domain in domains_refresh.clone() { + let cache_domain = cache_refresh.clone(); + let host_domain = host_refresh.clone(); + let unix_sock_domain = unix_sock_refresh.clone(); + let upstream_domain = upstream_refresh.clone(); + join.spawn(async move { + match crate::tls_front::fetcher::fetch_real_tls( + &host_domain, + port, + &domain, + fetch_timeout, + Some(upstream_domain), + proxy_protocol, + unix_sock_domain.as_deref(), + ) + .await + { + Ok(res) => cache_domain.update_from_fetch(&domain, res).await, + Err(e) => { + warn!(domain = %domain, error = %e, "TLS emulation refresh failed") + } + } + }); + } + + while let Some(res) = join.join_next().await { + if let Err(e) = res { + warn!(error = %e, "TLS emulation refresh task join failed"); + } + } + } + }); + + Some(cache) + } else { + startup_tracker + .skip_component( + COMPONENT_TLS_FRONT_BOOTSTRAP, + Some("censorship.tls_emulation is false".to_string()), + ) + .await; + None + }; + + if tls_cache.is_some() { + startup_tracker + .complete_component( + COMPONENT_TLS_FRONT_BOOTSTRAP, + Some("tls front cache is initialized".to_string()), + ) + .await; + } + + tls_cache +} diff --git a/src/main.rs b/src/main.rs index b8b5d60..73ada8c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,2270 +1,24 @@ //! telemt — Telegram MTProto Proxy -#![allow(unused_assignments)] - -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use rand::Rng; -use tokio::net::TcpListener; -use tokio::signal; -use tokio::sync::{RwLock, Semaphore, mpsc, watch}; -use tracing::{debug, error, info, warn}; -use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload}; -#[cfg(unix)] -use tokio::net::UnixListener; - -mod cli; mod api; +mod cli; mod config; mod crypto; mod error; mod ip_tracker; -mod network; +mod maestro; mod metrics; +mod network; mod protocol; mod proxy; +mod startup; mod stats; mod stream; -mod startup; -mod transport; mod tls_front; +mod transport; mod util; -use crate::config::{LogLevel, ProxyConfig}; -use crate::config::hot_reload::spawn_config_watcher; -use crate::crypto::SecureRandom; -use crate::ip_tracker::UserIpTracker; -use crate::network::probe::{decide_network_capabilities, log_probe_result, run_probe}; -use crate::proxy::ClientHandler; -use crate::proxy::route_mode::{ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteRuntimeController}; -use crate::stats::beobachten::BeobachtenStore; -use crate::stats::telemetry::TelemetryPolicy; -use crate::stats::{ReplayChecker, Stats}; -use crate::startup::{ - COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, COMPONENT_CONFIG_WATCHER_START, - COMPONENT_DC_CONNECTIVITY_PING, COMPONENT_LISTENERS_BIND, COMPONENT_ME_CONNECTIVITY_PING, - COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1, COMPONENT_ME_PROXY_CONFIG_V4, - COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH, COMPONENT_METRICS_START, - COMPONENT_NETWORK_PROBE, COMPONENT_RUNTIME_READY, COMPONENT_TLS_FRONT_BOOTSTRAP, - COMPONENT_TRACING_INIT, StartupMeStatus, StartupTracker, -}; -use crate::stream::BufferPool; -use crate::transport::middle_proxy::{ - MePool, ProxyConfigData, fetch_proxy_config_with_raw, format_me_route, format_sample_line, - load_proxy_config_cache, run_me_ping, save_proxy_config_cache, MePingFamily, MePingSample, - MeReinitTrigger, -}; -use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes}; -use crate::tls_front::TlsFrontCache; - -fn parse_cli() -> (String, bool, Option) { - let mut config_path = "config.toml".to_string(); - let mut silent = false; - let mut log_level: Option = None; - - let args: Vec = std::env::args().skip(1).collect(); - - // Check for --init first (handled before tokio) - if let Some(init_opts) = cli::parse_init_args(&args) { - if let Err(e) = cli::run_init(init_opts) { - eprintln!("[telemt] Init failed: {}", e); - std::process::exit(1); - } - std::process::exit(0); - } - - let mut i = 0; - while i < args.len() { - match args[i].as_str() { - "--silent" | "-s" => { - silent = true; - } - "--log-level" => { - i += 1; - if i < args.len() { - log_level = Some(args[i].clone()); - } - } - s if s.starts_with("--log-level=") => { - log_level = Some(s.trim_start_matches("--log-level=").to_string()); - } - "--help" | "-h" => { - eprintln!("Usage: telemt [config.toml] [OPTIONS]"); - eprintln!(); - eprintln!("Options:"); - eprintln!(" --silent, -s Suppress info logs"); - eprintln!(" --log-level debug|verbose|normal|silent"); - eprintln!(" --help, -h Show this help"); - eprintln!(); - eprintln!("Setup (fire-and-forget):"); - eprintln!( - " --init Generate config, install systemd service, start" - ); - eprintln!(" --port Listen port (default: 443)"); - eprintln!( - " --domain TLS domain for masking (default: www.google.com)" - ); - eprintln!( - " --secret 32-char hex secret (auto-generated if omitted)" - ); - eprintln!(" --user Username (default: user)"); - eprintln!(" --config-dir Config directory (default: /etc/telemt)"); - eprintln!(" --no-start Don't start the service after install"); - std::process::exit(0); - } - "--version" | "-V" => { - println!("telemt {}", env!("CARGO_PKG_VERSION")); - std::process::exit(0); - } - s if !s.starts_with('-') => { - config_path = s.to_string(); - } - other => { - eprintln!("Unknown option: {}", other); - } - } - i += 1; - } - - (config_path, silent, log_level) -} - -fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) { - info!(target: "telemt::links", "--- Proxy Links ({}) ---", host); - for user_name in config.general.links.show.resolve_users(&config.access.users) { - if let Some(secret) = config.access.users.get(user_name) { - info!(target: "telemt::links", "User: {}", user_name); - if config.general.modes.classic { - info!( - target: "telemt::links", - " Classic: tg://proxy?server={}&port={}&secret={}", - host, port, secret - ); - } - if config.general.modes.secure { - info!( - target: "telemt::links", - " DD: tg://proxy?server={}&port={}&secret=dd{}", - host, port, secret - ); - } - if config.general.modes.tls { - let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); - domains.push(config.censorship.tls_domain.clone()); - for d in &config.censorship.tls_domains { - if !domains.contains(d) { - domains.push(d.clone()); - } - } - - for domain in domains { - let domain_hex = hex::encode(&domain); - info!( - target: "telemt::links", - " EE-TLS: tg://proxy?server={}&port={}&secret=ee{}{}", - host, port, secret, domain_hex - ); - } - } - } else { - warn!(target: "telemt::links", "User '{}' in show_link not found", user_name); - } - } - info!(target: "telemt::links", "------------------------"); -} - -async fn write_beobachten_snapshot(path: &str, payload: &str) -> std::io::Result<()> { - if let Some(parent) = std::path::Path::new(path).parent() - && !parent.as_os_str().is_empty() - { - tokio::fs::create_dir_all(parent).await?; - } - tokio::fs::write(path, payload).await -} - -fn unit_label(value: u64, singular: &'static str, plural: &'static str) -> &'static str { - if value == 1 { singular } else { plural } -} - -fn format_uptime(total_secs: u64) -> String { - const SECS_PER_MINUTE: u64 = 60; - const SECS_PER_HOUR: u64 = 60 * SECS_PER_MINUTE; - const SECS_PER_DAY: u64 = 24 * SECS_PER_HOUR; - const SECS_PER_MONTH: u64 = 30 * SECS_PER_DAY; - const SECS_PER_YEAR: u64 = 12 * SECS_PER_MONTH; - - let mut remaining = total_secs; - let years = remaining / SECS_PER_YEAR; - remaining %= SECS_PER_YEAR; - let months = remaining / SECS_PER_MONTH; - remaining %= SECS_PER_MONTH; - let days = remaining / SECS_PER_DAY; - remaining %= SECS_PER_DAY; - let hours = remaining / SECS_PER_HOUR; - remaining %= SECS_PER_HOUR; - let minutes = remaining / SECS_PER_MINUTE; - let seconds = remaining % SECS_PER_MINUTE; - - let mut parts = Vec::new(); - if total_secs > SECS_PER_YEAR { - parts.push(format!( - "{} {}", - years, - unit_label(years, "year", "years") - )); - } - if total_secs > SECS_PER_MONTH { - parts.push(format!( - "{} {}", - months, - unit_label(months, "month", "months") - )); - } - if total_secs > SECS_PER_DAY { - parts.push(format!( - "{} {}", - days, - unit_label(days, "day", "days") - )); - } - if total_secs > SECS_PER_HOUR { - parts.push(format!( - "{} {}", - hours, - unit_label(hours, "hour", "hours") - )); - } - if total_secs > SECS_PER_MINUTE { - parts.push(format!( - "{} {}", - minutes, - unit_label(minutes, "minute", "minutes") - )); - } - parts.push(format!( - "{} {}", - seconds, - unit_label(seconds, "second", "seconds") - )); - - format!("{} / {} seconds", parts.join(", "), total_secs) -} - -async fn wait_until_admission_open(admission_rx: &mut watch::Receiver) -> bool { - loop { - if *admission_rx.borrow() { - return true; - } - if admission_rx.changed().await.is_err() { - return *admission_rx.borrow(); - } - } -} - -fn is_expected_handshake_eof(err: &crate::error::ProxyError) -> bool { - err.to_string().contains("expected 64 bytes, got 0") -} - -async fn load_startup_proxy_config_snapshot( - url: &str, - cache_path: Option<&str>, - me2dc_fallback: bool, - label: &'static str, -) -> Option { - loop { - match fetch_proxy_config_with_raw(url).await { - Ok((cfg, raw)) => { - if !cfg.map.is_empty() { - if let Some(path) = cache_path - && let Err(e) = save_proxy_config_cache(path, &raw).await - { - warn!(error = %e, path, snapshot = label, "Failed to store startup proxy-config cache"); - } - return Some(cfg); - } - - warn!(snapshot = label, url, "Startup proxy-config is empty; trying disk cache"); - if let Some(path) = cache_path { - match load_proxy_config_cache(path).await { - Ok(cached) if !cached.map.is_empty() => { - info!( - snapshot = label, - path, - proxy_for_lines = cached.proxy_for_lines, - "Loaded startup proxy-config from disk cache" - ); - return Some(cached); - } - Ok(_) => { - warn!( - snapshot = label, - path, - "Startup proxy-config cache is empty; ignoring cache file" - ); - } - Err(cache_err) => { - debug!( - snapshot = label, - path, - error = %cache_err, - "Startup proxy-config cache unavailable" - ); - } - } - } - - if me2dc_fallback { - error!( - snapshot = label, - "Startup proxy-config unavailable and no saved config found; falling back to direct mode" - ); - return None; - } - - warn!( - snapshot = label, - retry_in_secs = 2, - "Startup proxy-config unavailable and no saved config found; retrying because me2dc_fallback=false" - ); - tokio::time::sleep(Duration::from_secs(2)).await; - } - Err(fetch_err) => { - if let Some(path) = cache_path { - match load_proxy_config_cache(path).await { - Ok(cached) if !cached.map.is_empty() => { - info!( - snapshot = label, - path, - proxy_for_lines = cached.proxy_for_lines, - "Loaded startup proxy-config from disk cache" - ); - return Some(cached); - } - Ok(_) => { - warn!( - snapshot = label, - path, - "Startup proxy-config cache is empty; ignoring cache file" - ); - } - Err(cache_err) => { - debug!( - snapshot = label, - path, - error = %cache_err, - "Startup proxy-config cache unavailable" - ); - } - } - } - - if me2dc_fallback { - error!( - snapshot = label, - error = %fetch_err, - "Startup proxy-config unavailable and no cached data; falling back to direct mode" - ); - return None; - } - - warn!( - snapshot = label, - error = %fetch_err, - retry_in_secs = 2, - "Startup proxy-config unavailable; retrying because me2dc_fallback=false" - ); - tokio::time::sleep(Duration::from_secs(2)).await; - } - } - } -} - #[tokio::main] async fn main() -> std::result::Result<(), Box> { - let process_started_at = Instant::now(); - let process_started_at_epoch_secs = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - let startup_tracker = Arc::new(StartupTracker::new(process_started_at_epoch_secs)); - startup_tracker - .start_component(COMPONENT_CONFIG_LOAD, Some("load and validate config".to_string())) - .await; - let (config_path, cli_silent, cli_log_level) = parse_cli(); - - let mut config = match ProxyConfig::load(&config_path) { - Ok(c) => c, - Err(e) => { - if std::path::Path::new(&config_path).exists() { - eprintln!("[telemt] Error: {}", e); - std::process::exit(1); - } else { - let default = ProxyConfig::default(); - std::fs::write(&config_path, toml::to_string_pretty(&default).unwrap()).unwrap(); - eprintln!("[telemt] Created default config at {}", config_path); - default - } - } - }; - - if let Err(e) = config.validate() { - eprintln!("[telemt] Invalid config: {}", e); - std::process::exit(1); - } - - if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) { - eprintln!("[telemt] Invalid network.dns_overrides: {}", e); - std::process::exit(1); - } - startup_tracker - .complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string())) - .await; - - let has_rust_log = std::env::var("RUST_LOG").is_ok(); - let effective_log_level = if cli_silent { - LogLevel::Silent - } else if let Some(ref s) = cli_log_level { - LogLevel::from_str_loose(s) - } else { - config.general.log_level.clone() - }; - - let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info")); - startup_tracker - .start_component(COMPONENT_TRACING_INIT, Some("initialize tracing subscriber".to_string())) - .await; - - // Configure color output based on config - let fmt_layer = if config.general.disable_colors { - fmt::Layer::default().with_ansi(false) - } else { - fmt::Layer::default().with_ansi(true) - }; - - tracing_subscriber::registry() - .with(filter_layer) - .with(fmt_layer) - .init(); - startup_tracker - .complete_component(COMPONENT_TRACING_INIT, Some("tracing initialized".to_string())) - .await; - - info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION")); - info!("Log level: {}", effective_log_level); - if config.general.disable_colors { - info!("Colors: disabled"); - } - info!( - "Modes: classic={} secure={} tls={}", - config.general.modes.classic, config.general.modes.secure, config.general.modes.tls - ); - if config.general.modes.classic { - warn!("Classic mode is vulnerable to DPI detection; enable only for legacy clients"); - } - info!("TLS domain: {}", config.censorship.tls_domain); - if let Some(ref sock) = config.censorship.mask_unix_sock { - info!("Mask: {} -> unix:{}", config.censorship.mask, sock); - if !std::path::Path::new(sock).exists() { - warn!( - "Unix socket '{}' does not exist yet. Masking will fail until it appears.", - sock - ); - } - } else { - info!( - "Mask: {} -> {}:{}", - config.censorship.mask, - config - .censorship - .mask_host - .as_deref() - .unwrap_or(&config.censorship.tls_domain), - config.censorship.mask_port - ); - } - - if config.censorship.tls_domain == "www.google.com" { - warn!("Using default tls_domain. Consider setting a custom domain."); - } - - let stats = Arc::new(Stats::new()); - stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry)); - - let upstream_manager = Arc::new(UpstreamManager::new( - config.upstreams.clone(), - config.general.upstream_connect_retry_attempts, - config.general.upstream_connect_retry_backoff_ms, - config.general.upstream_connect_budget_ms, - config.general.upstream_unhealthy_fail_threshold, - config.general.upstream_connect_failfast_hard_errors, - stats.clone(), - )); - let ip_tracker = Arc::new(UserIpTracker::new()); - ip_tracker.load_limits(&config.access.user_max_unique_ips).await; - ip_tracker - .set_limit_policy( - config.access.user_max_unique_ips_mode, - config.access.user_max_unique_ips_window_secs, - ) - .await; - if !config.access.user_max_unique_ips.is_empty() { - info!( - "IP limits configured for {} users", - config.access.user_max_unique_ips.len() - ); - } - if !config.network.dns_overrides.is_empty() { - info!( - "Runtime DNS overrides configured: {} entries", - config.network.dns_overrides.len() - ); - } - - let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone())); - let initial_admission_open = !config.general.use_middle_proxy; - let (admission_tx, admission_rx) = watch::channel(initial_admission_open); - let initial_route_mode = if config.general.use_middle_proxy { - RelayRouteMode::Middle - } else { - RelayRouteMode::Direct - }; - let route_runtime = Arc::new(RouteRuntimeController::new(initial_route_mode)); - let api_me_pool = Arc::new(RwLock::new(None::>)); - startup_tracker - .start_component(COMPONENT_API_BOOTSTRAP, Some("spawn API listener task".to_string())) - .await; - - if config.server.api.enabled { - let listen = match config.server.api.listen.parse::() { - Ok(listen) => listen, - Err(error) => { - warn!( - error = %error, - listen = %config.server.api.listen, - "Invalid server.api.listen; API is disabled" - ); - SocketAddr::from(([127, 0, 0, 1], 0)) - } - }; - if listen.port() != 0 { - let stats_api = stats.clone(); - let ip_tracker_api = ip_tracker.clone(); - let me_pool_api = api_me_pool.clone(); - let upstream_manager_api = upstream_manager.clone(); - let config_rx_api = api_config_rx.clone(); - let admission_rx_api = admission_rx.clone(); - let config_path_api = std::path::PathBuf::from(&config_path); - let startup_tracker_api = startup_tracker.clone(); - tokio::spawn(async move { - api::serve( - listen, - stats_api, - ip_tracker_api, - me_pool_api, - upstream_manager_api, - config_rx_api, - admission_rx_api, - config_path_api, - None, - None, - process_started_at_epoch_secs, - startup_tracker_api, - ) - .await; - }); - startup_tracker - .complete_component( - COMPONENT_API_BOOTSTRAP, - Some(format!("api task spawned on {}", listen)), - ) - .await; - } else { - startup_tracker - .skip_component( - COMPONENT_API_BOOTSTRAP, - Some("server.api.listen has zero port".to_string()), - ) - .await; - } - } else { - startup_tracker - .skip_component( - COMPONENT_API_BOOTSTRAP, - Some("server.api.enabled is false".to_string()), - ) - .await; - } - - let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); - tls_domains.push(config.censorship.tls_domain.clone()); - for d in &config.censorship.tls_domains { - if !tls_domains.contains(d) { - tls_domains.push(d.clone()); - } - } - - // Start TLS front fetching in background immediately, in parallel with STUN probing. - startup_tracker - .start_component( - COMPONENT_TLS_FRONT_BOOTSTRAP, - Some("initialize TLS front cache/bootstrap tasks".to_string()), - ) - .await; - let tls_cache: Option> = if config.censorship.tls_emulation { - let cache = Arc::new(TlsFrontCache::new( - &tls_domains, - config.censorship.fake_cert_len, - &config.censorship.tls_front_dir, - )); - cache.load_from_disk().await; - - let port = config.censorship.mask_port; - let proxy_protocol = config.censorship.mask_proxy_protocol; - let mask_host = config - .censorship - .mask_host - .clone() - .unwrap_or_else(|| config.censorship.tls_domain.clone()); - let mask_unix_sock = config.censorship.mask_unix_sock.clone(); - let fetch_timeout = Duration::from_secs(5); - - let cache_initial = cache.clone(); - let domains_initial = tls_domains.clone(); - let host_initial = mask_host.clone(); - let unix_sock_initial = mask_unix_sock.clone(); - let upstream_initial = upstream_manager.clone(); - tokio::spawn(async move { - let mut join = tokio::task::JoinSet::new(); - for domain in domains_initial { - let cache_domain = cache_initial.clone(); - let host_domain = host_initial.clone(); - let unix_sock_domain = unix_sock_initial.clone(); - let upstream_domain = upstream_initial.clone(); - join.spawn(async move { - match crate::tls_front::fetcher::fetch_real_tls( - &host_domain, - port, - &domain, - fetch_timeout, - Some(upstream_domain), - proxy_protocol, - unix_sock_domain.as_deref(), - ) - .await - { - Ok(res) => cache_domain.update_from_fetch(&domain, res).await, - Err(e) => { - warn!(domain = %domain, error = %e, "TLS emulation initial fetch failed") - } - } - }); - } - while let Some(res) = join.join_next().await { - if let Err(e) = res { - warn!(error = %e, "TLS emulation initial fetch task join failed"); - } - } - }); - - let cache_timeout = cache.clone(); - let domains_timeout = tls_domains.clone(); - let fake_cert_len = config.censorship.fake_cert_len; - tokio::spawn(async move { - tokio::time::sleep(fetch_timeout).await; - for domain in domains_timeout { - let cached = cache_timeout.get(&domain).await; - if cached.domain == "default" { - warn!( - domain = %domain, - timeout_secs = fetch_timeout.as_secs(), - fake_cert_len, - "TLS-front fetch not ready within timeout; using cache/default fake cert fallback" - ); - } - } - }); - - // Periodic refresh with jitter. - let cache_refresh = cache.clone(); - let domains_refresh = tls_domains.clone(); - let host_refresh = mask_host.clone(); - let unix_sock_refresh = mask_unix_sock.clone(); - let upstream_refresh = upstream_manager.clone(); - tokio::spawn(async move { - loop { - let base_secs = rand::rng().random_range(4 * 3600..=6 * 3600); - let jitter_secs = rand::rng().random_range(0..=7200); - tokio::time::sleep(Duration::from_secs(base_secs + jitter_secs)).await; - - let mut join = tokio::task::JoinSet::new(); - for domain in domains_refresh.clone() { - let cache_domain = cache_refresh.clone(); - let host_domain = host_refresh.clone(); - let unix_sock_domain = unix_sock_refresh.clone(); - let upstream_domain = upstream_refresh.clone(); - join.spawn(async move { - match crate::tls_front::fetcher::fetch_real_tls( - &host_domain, - port, - &domain, - fetch_timeout, - Some(upstream_domain), - proxy_protocol, - unix_sock_domain.as_deref(), - ) - .await - { - Ok(res) => cache_domain.update_from_fetch(&domain, res).await, - Err(e) => warn!(domain = %domain, error = %e, "TLS emulation refresh failed"), - } - }); - } - - while let Some(res) = join.join_next().await { - if let Err(e) = res { - warn!(error = %e, "TLS emulation refresh task join failed"); - } - } - } - }); - - Some(cache) - } else { - startup_tracker - .skip_component( - COMPONENT_TLS_FRONT_BOOTSTRAP, - Some("censorship.tls_emulation is false".to_string()), - ) - .await; - None - }; - if tls_cache.is_some() { - startup_tracker - .complete_component( - COMPONENT_TLS_FRONT_BOOTSTRAP, - Some("tls front cache is initialized".to_string()), - ) - .await; - } - - startup_tracker - .start_component(COMPONENT_NETWORK_PROBE, Some("probe network capabilities".to_string())) - .await; - let probe = run_probe( - &config.network, - config.general.middle_proxy_nat_probe, - config.general.stun_nat_probe_concurrency, - ) - .await?; - let decision = decide_network_capabilities(&config.network, &probe); - log_probe_result(&probe, &decision); - startup_tracker - .complete_component( - COMPONENT_NETWORK_PROBE, - Some("network capabilities determined".to_string()), - ) - .await; - - let prefer_ipv6 = decision.prefer_ipv6(); - let mut use_middle_proxy = config.general.use_middle_proxy; - let beobachten = Arc::new(BeobachtenStore::new()); - let rng = Arc::new(SecureRandom::new()); - - // Connection concurrency limit - let max_connections = Arc::new(Semaphore::new(10_000)); - - let me2dc_fallback = config.general.me2dc_fallback; - let me_init_retry_attempts = config.general.me_init_retry_attempts; - let me_init_warn_after_attempts: u32 = 3; - if use_middle_proxy && !decision.ipv4_me && !decision.ipv6_me { - if me2dc_fallback { - warn!("No usable IP family for Middle Proxy detected; falling back to direct DC"); - use_middle_proxy = false; - } else { - warn!( - "No usable IP family for Middle Proxy detected; me2dc_fallback=false, ME init retries stay active" - ); - } - } - - if use_middle_proxy { - startup_tracker - .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_SECRET_FETCH) - .await; - startup_tracker - .start_component( - COMPONENT_ME_SECRET_FETCH, - Some("fetch proxy-secret from source/cache".to_string()), - ) - .await; - startup_tracker - .set_me_retry_limit(if !me2dc_fallback || me_init_retry_attempts == 0 { - "unlimited".to_string() - } else { - me_init_retry_attempts.to_string() - }) - .await; - } else { - startup_tracker - .set_me_status(StartupMeStatus::Skipped, "skipped") - .await; - startup_tracker - .skip_component( - COMPONENT_ME_SECRET_FETCH, - Some("middle proxy mode disabled".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_PROXY_CONFIG_V4, - Some("middle proxy mode disabled".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_PROXY_CONFIG_V6, - Some("middle proxy mode disabled".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_POOL_CONSTRUCT, - Some("middle proxy mode disabled".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("middle proxy mode disabled".to_string()), - ) - .await; - } - - // ===================================================================== - // Middle Proxy initialization (if enabled) - // ===================================================================== - let me_pool: Option> = if use_middle_proxy { - info!("=== Middle Proxy Mode ==="); - let me_nat_probe = config.general.middle_proxy_nat_probe && config.network.stun_use; - if config.general.middle_proxy_nat_probe && !config.network.stun_use { - info!("Middle-proxy STUN probing disabled by network.stun_use=false"); - } - - // Global ad_tag (pool default). Used when user has no per-user tag in access.user_ad_tags. - let proxy_tag = config - .general - .ad_tag - .as_ref() - .map(|tag| hex::decode(tag).expect("general.ad_tag must be validated before startup")); - - // ============================================================= - // CRITICAL: Download Telegram proxy-secret (NOT user secret!) - // - // C MTProxy uses TWO separate secrets: - // -S flag = 16-byte user secret for client obfuscation - // --aes-pwd = 32-512 byte binary file for ME RPC auth - // - // proxy-secret is from: https://core.telegram.org/getProxySecret - // ============================================================= - let proxy_secret_path = config.general.proxy_secret_path.as_deref(); - let pool_size = config.general.middle_proxy_pool_size.max(1); - let proxy_secret = loop { - match crate::transport::middle_proxy::fetch_proxy_secret( - proxy_secret_path, - config.general.proxy_secret_len_max, - ) - .await - { - Ok(proxy_secret) => break Some(proxy_secret), - Err(e) => { - startup_tracker - .set_me_last_error(Some(e.to_string())) - .await; - if me2dc_fallback { - error!( - error = %e, - "ME startup failed: proxy-secret is unavailable and no saved secret found; falling back to direct mode" - ); - break None; - } - - warn!( - error = %e, - retry_in_secs = 2, - "ME startup failed: proxy-secret is unavailable and no saved secret found; retrying because me2dc_fallback=false" - ); - tokio::time::sleep(Duration::from_secs(2)).await; - } - } - }; - match proxy_secret { - Some(proxy_secret) => { - startup_tracker - .complete_component( - COMPONENT_ME_SECRET_FETCH, - Some("proxy-secret loaded".to_string()), - ) - .await; - info!( - secret_len = proxy_secret.len(), - key_sig = format_args!( - "0x{:08x}", - if proxy_secret.len() >= 4 { - u32::from_le_bytes([ - proxy_secret[0], - proxy_secret[1], - proxy_secret[2], - proxy_secret[3], - ]) - } else { - 0 - } - ), - "Proxy-secret loaded" - ); - - startup_tracker - .start_component( - COMPONENT_ME_PROXY_CONFIG_V4, - Some("load startup proxy-config v4".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4) - .await; - let cfg_v4 = load_startup_proxy_config_snapshot( - "https://core.telegram.org/getProxyConfig", - config.general.proxy_config_v4_cache_path.as_deref(), - me2dc_fallback, - "getProxyConfig", - ) - .await; - if cfg_v4.is_some() { - startup_tracker - .complete_component( - COMPONENT_ME_PROXY_CONFIG_V4, - Some("proxy-config v4 loaded".to_string()), - ) - .await; - } else { - startup_tracker - .fail_component( - COMPONENT_ME_PROXY_CONFIG_V4, - Some("proxy-config v4 unavailable".to_string()), - ) - .await; - } - startup_tracker - .start_component( - COMPONENT_ME_PROXY_CONFIG_V6, - Some("load startup proxy-config v6".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6) - .await; - let cfg_v6 = load_startup_proxy_config_snapshot( - "https://core.telegram.org/getProxyConfigV6", - config.general.proxy_config_v6_cache_path.as_deref(), - me2dc_fallback, - "getProxyConfigV6", - ) - .await; - if cfg_v6.is_some() { - startup_tracker - .complete_component( - COMPONENT_ME_PROXY_CONFIG_V6, - Some("proxy-config v6 loaded".to_string()), - ) - .await; - } else { - startup_tracker - .fail_component( - COMPONENT_ME_PROXY_CONFIG_V6, - Some("proxy-config v6 unavailable".to_string()), - ) - .await; - } - - if let (Some(cfg_v4), Some(cfg_v6)) = (cfg_v4, cfg_v6) { - startup_tracker - .start_component( - COMPONENT_ME_POOL_CONSTRUCT, - Some("construct ME pool".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_CONSTRUCT) - .await; - let pool = MePool::new( - proxy_tag.clone(), - proxy_secret, - config.general.middle_proxy_nat_ip, - me_nat_probe, - None, - config.network.stun_servers.clone(), - config.general.stun_nat_probe_concurrency, - probe.detected_ipv6, - config.timeouts.me_one_retry, - config.timeouts.me_one_timeout_ms, - cfg_v4.map.clone(), - cfg_v6.map.clone(), - cfg_v4.default_dc.or(cfg_v6.default_dc), - decision.clone(), - Some(upstream_manager.clone()), - rng.clone(), - stats.clone(), - config.general.me_keepalive_enabled, - config.general.me_keepalive_interval_secs, - config.general.me_keepalive_jitter_secs, - config.general.me_keepalive_payload_random, - config.general.rpc_proxy_req_every, - config.general.me_warmup_stagger_enabled, - config.general.me_warmup_step_delay_ms, - config.general.me_warmup_step_jitter_ms, - config.general.me_reconnect_max_concurrent_per_dc, - config.general.me_reconnect_backoff_base_ms, - config.general.me_reconnect_backoff_cap_ms, - config.general.me_reconnect_fast_retry_count, - config.general.me_single_endpoint_shadow_writers, - config.general.me_single_endpoint_outage_mode_enabled, - config.general.me_single_endpoint_outage_disable_quarantine, - config.general.me_single_endpoint_outage_backoff_min_ms, - config.general.me_single_endpoint_outage_backoff_max_ms, - config.general.me_single_endpoint_shadow_rotate_every_secs, - config.general.me_floor_mode, - config.general.me_adaptive_floor_idle_secs, - config.general.me_adaptive_floor_min_writers_single_endpoint, - config.general.me_adaptive_floor_min_writers_multi_endpoint, - config.general.me_adaptive_floor_recover_grace_secs, - config.general.me_adaptive_floor_writers_per_core_total, - config.general.me_adaptive_floor_cpu_cores_override, - config.general.me_adaptive_floor_max_extra_writers_single_per_core, - config.general.me_adaptive_floor_max_extra_writers_multi_per_core, - config.general.me_adaptive_floor_max_active_writers_per_core, - config.general.me_adaptive_floor_max_warm_writers_per_core, - config.general.me_adaptive_floor_max_active_writers_global, - config.general.me_adaptive_floor_max_warm_writers_global, - config.general.hardswap, - config.general.me_pool_drain_ttl_secs, - config.general.effective_me_pool_force_close_secs(), - config.general.me_pool_min_fresh_ratio, - config.general.me_hardswap_warmup_delay_min_ms, - config.general.me_hardswap_warmup_delay_max_ms, - config.general.me_hardswap_warmup_extra_passes, - config.general.me_hardswap_warmup_pass_backoff_base_ms, - config.general.me_bind_stale_mode, - config.general.me_bind_stale_ttl_secs, - config.general.me_secret_atomic_snapshot, - config.general.me_deterministic_writer_sort, - config.general.me_writer_pick_mode, - config.general.me_writer_pick_sample_size, - config.general.me_socks_kdf_policy, - config.general.me_writer_cmd_channel_capacity, - config.general.me_route_channel_capacity, - config.general.me_route_backpressure_base_timeout_ms, - config.general.me_route_backpressure_high_timeout_ms, - config.general.me_route_backpressure_high_watermark_pct, - config.general.me_reader_route_data_wait_ms, - config.general.me_health_interval_ms_unhealthy, - config.general.me_health_interval_ms_healthy, - config.general.me_warn_rate_limit_ms, - config.general.me_route_no_writer_mode, - config.general.me_route_no_writer_wait_ms, - config.general.me_route_inline_recovery_attempts, - config.general.me_route_inline_recovery_wait_ms, - ); - startup_tracker - .complete_component( - COMPONENT_ME_POOL_CONSTRUCT, - Some("ME pool object created".to_string()), - ) - .await; - *api_me_pool.write().await = Some(pool.clone()); - startup_tracker - .start_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("initialize ME pool writers".to_string()), - ) - .await; - startup_tracker - .set_me_status( - StartupMeStatus::Initializing, - COMPONENT_ME_POOL_INIT_STAGE1, - ) - .await; - - let mut init_attempt: u32 = 0; - loop { - init_attempt = init_attempt.saturating_add(1); - startup_tracker.set_me_init_attempt(init_attempt).await; - match pool.init(pool_size, &rng).await { - Ok(()) => { - startup_tracker - .set_me_last_error(None) - .await; - startup_tracker - .complete_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("ME pool initialized".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Ready, "ready") - .await; - info!( - attempt = init_attempt, - "Middle-End pool initialized successfully" - ); - - // Phase 4: Start health monitor - let pool_clone = pool.clone(); - let rng_clone = rng.clone(); - let min_conns = pool_size; - tokio::spawn(async move { - crate::transport::middle_proxy::me_health_monitor( - pool_clone, rng_clone, min_conns, - ) - .await; - }); - - break Some(pool); - } - Err(e) => { - startup_tracker - .set_me_last_error(Some(e.to_string())) - .await; - let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; - if retries_limited && init_attempt >= me_init_retry_attempts { - startup_tracker - .fail_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("ME init retry budget exhausted".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Failed, "failed") - .await; - error!( - error = %e, - attempt = init_attempt, - retry_limit = me_init_retry_attempts, - "ME pool init retries exhausted; falling back to direct mode" - ); - break None; - } - - let retry_limit = if !me2dc_fallback || me_init_retry_attempts == 0 { - String::from("unlimited") - } else { - me_init_retry_attempts.to_string() - }; - if init_attempt >= me_init_warn_after_attempts { - warn!( - error = %e, - attempt = init_attempt, - retry_limit = retry_limit, - me2dc_fallback = me2dc_fallback, - retry_in_secs = 2, - "ME pool is not ready yet; retrying startup initialization" - ); - } else { - info!( - error = %e, - attempt = init_attempt, - retry_limit = retry_limit, - me2dc_fallback = me2dc_fallback, - retry_in_secs = 2, - "ME pool startup warmup: retrying initialization" - ); - } - pool.reset_stun_state(); - tokio::time::sleep(Duration::from_secs(2)).await; - } - } - } - } else { - startup_tracker - .skip_component( - COMPONENT_ME_POOL_CONSTRUCT, - Some("ME configs are incomplete".to_string()), - ) - .await; - startup_tracker - .fail_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("ME configs are incomplete".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Failed, "failed") - .await; - None - } - } - None => { - startup_tracker - .fail_component( - COMPONENT_ME_SECRET_FETCH, - Some("proxy-secret unavailable".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_PROXY_CONFIG_V4, - Some("proxy-secret unavailable".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_PROXY_CONFIG_V6, - Some("proxy-secret unavailable".to_string()), - ) - .await; - startup_tracker - .skip_component( - COMPONENT_ME_POOL_CONSTRUCT, - Some("proxy-secret unavailable".to_string()), - ) - .await; - startup_tracker - .fail_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("proxy-secret unavailable".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Failed, "failed") - .await; - None - } - } - } else { - None - }; - - // If ME failed to initialize, force direct-only mode. - if me_pool.is_some() { - startup_tracker - .set_transport_mode("middle_proxy") - .await; - startup_tracker - .set_degraded(false) - .await; - info!("Transport: Middle-End Proxy - all DC-over-RPC"); - } else { - let _ = use_middle_proxy; - use_middle_proxy = false; - // Make runtime config reflect direct-only mode for handlers. - config.general.use_middle_proxy = false; - startup_tracker - .set_transport_mode("direct") - .await; - startup_tracker - .set_degraded(true) - .await; - if me2dc_fallback { - startup_tracker - .set_me_status(StartupMeStatus::Failed, "fallback_to_direct") - .await; - } else { - startup_tracker - .set_me_status(StartupMeStatus::Skipped, "skipped") - .await; - } - info!("Transport: Direct DC - TCP - standard DC-over-TCP"); - } - - // Freeze config after possible fallback decision - let config = Arc::new(config); - - let replay_checker = Arc::new(ReplayChecker::new( - config.access.replay_check_len, - Duration::from_secs(config.access.replay_window_secs), - )); - - let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096)); - - // Middle-End ping before DC connectivity - if me_pool.is_some() { - startup_tracker - .start_component( - COMPONENT_ME_CONNECTIVITY_PING, - Some("run startup ME connectivity check".to_string()), - ) - .await; - } else { - startup_tracker - .skip_component( - COMPONENT_ME_CONNECTIVITY_PING, - Some("ME pool is not available".to_string()), - ) - .await; - } - if let Some(ref pool) = me_pool { - let me_results = run_me_ping(pool, &rng).await; - - let v4_ok = me_results.iter().any(|r| { - matches!(r.family, MePingFamily::V4) - && r.samples.iter().any(|s| s.error.is_none() && s.handshake_ms.is_some()) - }); - let v6_ok = me_results.iter().any(|r| { - matches!(r.family, MePingFamily::V6) - && r.samples.iter().any(|s| s.error.is_none() && s.handshake_ms.is_some()) - }); - - info!("================= Telegram ME Connectivity ================="); - if v4_ok && v6_ok { - info!(" IPv4 and IPv6 available"); - } else if v4_ok { - info!(" IPv4 only / IPv6 unavailable"); - } else if v6_ok { - info!(" IPv6 only / IPv4 unavailable"); - } else { - info!(" No ME connectivity"); - } - let me_route = format_me_route( - &config.upstreams, - &me_results, - prefer_ipv6, - v4_ok, - v6_ok, - ) - .await; - info!(" via {}", me_route); - info!("============================================================"); - - use std::collections::BTreeMap; - let mut grouped: BTreeMap> = BTreeMap::new(); - for report in me_results { - for s in report.samples { - grouped.entry(s.dc).or_default().push(s); - } - } - - let family_order = if prefer_ipv6 { - vec![MePingFamily::V6, MePingFamily::V4] - } else { - vec![MePingFamily::V4, MePingFamily::V6] - }; - - for (dc, samples) in grouped { - for family in &family_order { - let fam_samples: Vec<&MePingSample> = samples - .iter() - .filter(|s| matches!(s.family, f if &f == family)) - .collect(); - if fam_samples.is_empty() { - continue; - } - - let fam_label = match family { - MePingFamily::V4 => "IPv4", - MePingFamily::V6 => "IPv6", - }; - info!(" DC{} [{}]", dc, fam_label); - for sample in fam_samples { - let line = format_sample_line(sample); - info!("{}", line); - } - } - } - info!("============================================================"); - startup_tracker - .complete_component( - COMPONENT_ME_CONNECTIVITY_PING, - Some("startup ME connectivity check completed".to_string()), - ) - .await; - } - - info!("================= Telegram DC Connectivity ================="); - startup_tracker - .start_component( - COMPONENT_DC_CONNECTIVITY_PING, - Some("run startup DC connectivity check".to_string()), - ) - .await; - - let ping_results = upstream_manager - .ping_all_dcs( - prefer_ipv6, - &config.dc_overrides, - decision.ipv4_dc, - decision.ipv6_dc, - ) - .await; - - for upstream_result in &ping_results { - let v6_works = upstream_result - .v6_results - .iter() - .any(|r| r.rtt_ms.is_some()); - let v4_works = upstream_result - .v4_results - .iter() - .any(|r| r.rtt_ms.is_some()); - - if upstream_result.both_available { - if prefer_ipv6 { - info!(" IPv6 in use / IPv4 is fallback"); - } else { - info!(" IPv4 in use / IPv6 is fallback"); - } - } else if v6_works && !v4_works { - info!(" IPv6 only / IPv4 unavailable"); - } else if v4_works && !v6_works { - info!(" IPv4 only / IPv6 unavailable"); - } else if !v6_works && !v4_works { - info!(" No DC connectivity"); - } - - info!(" via {}", upstream_result.upstream_name); - info!("============================================================"); - - // Print IPv6 results first (only if IPv6 is available) - if v6_works { - for dc in &upstream_result.v6_results { - let addr_str = format!("{}:{}", dc.dc_addr.ip(), dc.dc_addr.port()); - match &dc.rtt_ms { - Some(rtt) => { - info!(" DC{} [IPv6] {} - {:.0} ms", dc.dc_idx, addr_str, rtt); - } - None => { - let err = dc.error.as_deref().unwrap_or("fail"); - info!(" DC{} [IPv6] {} - FAIL ({})", dc.dc_idx, addr_str, err); - } - } - } - - info!("============================================================"); - } - - // Print IPv4 results (only if IPv4 is available) - if v4_works { - for dc in &upstream_result.v4_results { - let addr_str = format!("{}:{}", dc.dc_addr.ip(), dc.dc_addr.port()); - match &dc.rtt_ms { - Some(rtt) => { - info!( - " DC{} [IPv4] {}\t\t\t\t{:.0} ms", - dc.dc_idx, addr_str, rtt - ); - } - None => { - let err = dc.error.as_deref().unwrap_or("fail"); - info!( - " DC{} [IPv4] {}:\t\t\t\tFAIL ({})", - dc.dc_idx, addr_str, err - ); - } - } - } - - info!("============================================================"); - } - } - startup_tracker - .complete_component( - COMPONENT_DC_CONNECTIVITY_PING, - Some("startup DC connectivity check completed".to_string()), - ) - .await; - - let initialized_secs = process_started_at.elapsed().as_secs(); - let second_suffix = if initialized_secs == 1 { "" } else { "s" }; - startup_tracker - .start_component( - COMPONENT_RUNTIME_READY, - Some("finalize startup runtime state".to_string()), - ) - .await; - info!("===================== Telegram Startup ====================="); - info!( - " DC/ME Initialized in {} second{}", - initialized_secs, second_suffix - ); - info!("============================================================"); - - if let Some(ref pool) = me_pool { - pool.set_runtime_ready(true); - } - *api_me_pool.write().await = me_pool.clone(); - - // Background tasks - let um_clone = upstream_manager.clone(); - let decision_clone = decision.clone(); - let dc_overrides_for_health = config.dc_overrides.clone(); - tokio::spawn(async move { - um_clone - .run_health_checks( - prefer_ipv6, - decision_clone.ipv4_dc, - decision_clone.ipv6_dc, - dc_overrides_for_health, - ) - .await; - }); - - let rc_clone = replay_checker.clone(); - tokio::spawn(async move { - rc_clone.run_periodic_cleanup().await; - }); - - let detected_ip_v4: Option = probe.detected_ipv4.map(std::net::IpAddr::V4); - let detected_ip_v6: Option = probe.detected_ipv6.map(std::net::IpAddr::V6); - debug!( - "Detected IPs: v4={:?} v6={:?}", - detected_ip_v4, detected_ip_v6 - ); - - // ── Hot-reload watcher ──────────────────────────────────────────────── - // Uses inotify to detect file changes instantly (SIGHUP also works). - // detected_ip_v4/v6 are passed so newly added users get correct TG links. - startup_tracker - .start_component( - COMPONENT_CONFIG_WATCHER_START, - Some("spawn config hot-reload watcher".to_string()), - ) - .await; - let (config_rx, mut log_level_rx): ( - tokio::sync::watch::Receiver>, - tokio::sync::watch::Receiver, - ) = spawn_config_watcher( - std::path::PathBuf::from(&config_path), - config.clone(), - detected_ip_v4, - detected_ip_v6, - ); - startup_tracker - .complete_component( - COMPONENT_CONFIG_WATCHER_START, - Some("config hot-reload watcher started".to_string()), - ) - .await; - let mut config_rx_api_bridge = config_rx.clone(); - let api_config_tx_bridge = api_config_tx.clone(); - tokio::spawn(async move { - loop { - if config_rx_api_bridge.changed().await.is_err() { - break; - } - let cfg = config_rx_api_bridge.borrow_and_update().clone(); - api_config_tx_bridge.send_replace(cfg); - } - }); - - let stats_policy = stats.clone(); - let mut config_rx_policy = config_rx.clone(); - let me_pool_policy = me_pool.clone(); - tokio::spawn(async move { - loop { - if config_rx_policy.changed().await.is_err() { - break; - } - let cfg = config_rx_policy.borrow_and_update().clone(); - stats_policy.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry)); - if let Some(pool) = &me_pool_policy { - pool.update_runtime_transport_policy( - cfg.general.me_socks_kdf_policy, - cfg.general.me_route_backpressure_base_timeout_ms, - cfg.general.me_route_backpressure_high_timeout_ms, - cfg.general.me_route_backpressure_high_watermark_pct, - cfg.general.me_reader_route_data_wait_ms, - ); - } - } - }); - - let ip_tracker_policy = ip_tracker.clone(); - let mut config_rx_ip_limits = config_rx.clone(); - tokio::spawn(async move { - let mut prev_limits = config_rx_ip_limits - .borrow() - .access - .user_max_unique_ips - .clone(); - let mut prev_mode = config_rx_ip_limits - .borrow() - .access - .user_max_unique_ips_mode; - let mut prev_window = config_rx_ip_limits - .borrow() - .access - .user_max_unique_ips_window_secs; - - loop { - if config_rx_ip_limits.changed().await.is_err() { - break; - } - let cfg = config_rx_ip_limits.borrow_and_update().clone(); - - if prev_limits != cfg.access.user_max_unique_ips { - ip_tracker_policy - .load_limits(&cfg.access.user_max_unique_ips) - .await; - prev_limits = cfg.access.user_max_unique_ips.clone(); - } - - if prev_mode != cfg.access.user_max_unique_ips_mode - || prev_window != cfg.access.user_max_unique_ips_window_secs - { - ip_tracker_policy - .set_limit_policy( - cfg.access.user_max_unique_ips_mode, - cfg.access.user_max_unique_ips_window_secs, - ) - .await; - prev_mode = cfg.access.user_max_unique_ips_mode; - prev_window = cfg.access.user_max_unique_ips_window_secs; - } - } - }); - - let beobachten_writer = beobachten.clone(); - let config_rx_beobachten = config_rx.clone(); - tokio::spawn(async move { - loop { - let cfg = config_rx_beobachten.borrow().clone(); - let sleep_secs = cfg.general.beobachten_flush_secs.max(1); - - if cfg.general.beobachten { - let ttl = Duration::from_secs(cfg.general.beobachten_minutes.saturating_mul(60)); - let path = cfg.general.beobachten_file.clone(); - let snapshot = beobachten_writer.snapshot_text(ttl); - if let Err(e) = write_beobachten_snapshot(&path, &snapshot).await { - warn!(error = %e, path = %path, "Failed to flush beobachten snapshot"); - } - } - - tokio::time::sleep(Duration::from_secs(sleep_secs)).await; - } - }); - - if let Some(ref pool) = me_pool { - let reinit_trigger_capacity = config - .general - .me_reinit_trigger_channel - .max(1); - let (reinit_tx, reinit_rx) = mpsc::channel::(reinit_trigger_capacity); - - let pool_clone_sched = pool.clone(); - let rng_clone_sched = rng.clone(); - let config_rx_clone_sched = config_rx.clone(); - tokio::spawn(async move { - crate::transport::middle_proxy::me_reinit_scheduler( - pool_clone_sched, - rng_clone_sched, - config_rx_clone_sched, - reinit_rx, - ) - .await; - }); - - let pool_clone = pool.clone(); - let config_rx_clone = config_rx.clone(); - let reinit_tx_updater = reinit_tx.clone(); - tokio::spawn(async move { - crate::transport::middle_proxy::me_config_updater( - pool_clone, - config_rx_clone, - reinit_tx_updater, - ) - .await; - }); - - let config_rx_clone_rot = config_rx.clone(); - let reinit_tx_rotation = reinit_tx.clone(); - tokio::spawn(async move { - crate::transport::middle_proxy::me_rotation_task( - config_rx_clone_rot, - reinit_tx_rotation, - ) - .await; - }); - } - - startup_tracker - .start_component( - COMPONENT_LISTENERS_BIND, - Some("bind TCP/Unix listeners".to_string()), - ) - .await; - let mut listeners = Vec::new(); - - for listener_conf in &config.server.listeners { - let addr = SocketAddr::new(listener_conf.ip, config.server.port); - if addr.is_ipv4() && !decision.ipv4_dc { - warn!(%addr, "Skipping IPv4 listener: IPv4 disabled by [network]"); - continue; - } - if addr.is_ipv6() && !decision.ipv6_dc { - warn!(%addr, "Skipping IPv6 listener: IPv6 disabled by [network]"); - continue; - } - let options = ListenOptions { - reuse_port: listener_conf.reuse_allow, - ipv6_only: listener_conf.ip.is_ipv6(), - ..Default::default() - }; - - match create_listener(addr, &options) { - Ok(socket) => { - let listener = TcpListener::from_std(socket.into())?; - info!("Listening on {}", addr); - let listener_proxy_protocol = - listener_conf.proxy_protocol.unwrap_or(config.server.proxy_protocol); - - // Resolve the public host for link generation - let public_host = if let Some(ref announce) = listener_conf.announce { - announce.clone() // Use announce (IP or hostname) if explicitly set - } else if listener_conf.ip.is_unspecified() { - // Auto-detect for unspecified addresses - if listener_conf.ip.is_ipv4() { - detected_ip_v4 - .map(|ip| ip.to_string()) - .unwrap_or_else(|| listener_conf.ip.to_string()) - } else { - detected_ip_v6 - .map(|ip| ip.to_string()) - .unwrap_or_else(|| listener_conf.ip.to_string()) - } - } else { - listener_conf.ip.to_string() - }; - - // Show per-listener proxy links only when public_host is not set - if config.general.links.public_host.is_none() && !config.general.links.show.is_empty() { - let link_port = config.general.links.public_port.unwrap_or(config.server.port); - print_proxy_links(&public_host, link_port, &config); - } - - listeners.push((listener, listener_proxy_protocol)); - } - Err(e) => { - if e.kind() == std::io::ErrorKind::AddrInUse { - let owners = find_listener_processes(addr); - if owners.is_empty() { - error!( - %addr, - "Failed to bind: address already in use (owner process unresolved)" - ); - } else { - for owner in owners { - error!( - %addr, - pid = owner.pid, - process = %owner.process, - "Failed to bind: address already in use" - ); - } - } - - if !listener_conf.reuse_allow { - error!( - %addr, - "reuse_allow=false; set [[server.listeners]].reuse_allow=true to allow multi-instance listening" - ); - } - } else { - error!("Failed to bind to {}: {}", addr, e); - } - } - } - } - - // Show proxy links once when public_host is set, OR when there are no TCP listeners - // (unix-only mode) — use detected IP as fallback - if !config.general.links.show.is_empty() && (config.general.links.public_host.is_some() || listeners.is_empty()) { - let (host, port) = if let Some(ref h) = config.general.links.public_host { - (h.clone(), config.general.links.public_port.unwrap_or(config.server.port)) - } else { - let ip = detected_ip_v4 - .or(detected_ip_v6) - .map(|ip| ip.to_string()); - if ip.is_none() { - warn!("show_link is configured but public IP could not be detected. Set public_host in config."); - } - (ip.unwrap_or_else(|| "UNKNOWN".to_string()), config.general.links.public_port.unwrap_or(config.server.port)) - }; - - print_proxy_links(&host, port, &config); - } - - if config.general.use_middle_proxy { - if let Some(pool) = me_pool.as_ref() { - let fallback_after = Duration::from_secs(6); - let initial_ready = pool.admission_ready_conditional_cast().await; - admission_tx.send_replace(initial_ready); - let _ = route_runtime.set_mode(RelayRouteMode::Middle); - if initial_ready { - info!("Conditional-admission gate: open (ME pool ready)"); - } else { - warn!("Conditional-admission gate: closed (ME pool is not ready)"); - } - - let pool_for_gate = pool.clone(); - let admission_tx_gate = admission_tx.clone(); - let route_runtime_gate = route_runtime.clone(); - let mut config_rx_gate = config_rx.clone(); - let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1); - let mut fallback_enabled = config.general.me2dc_fallback; - tokio::spawn(async move { - let mut gate_open = initial_ready; - let mut route_mode = RelayRouteMode::Middle; - let mut not_ready_since = if initial_ready { - None - } else { - Some(Instant::now()) - }; - loop { - tokio::select! { - changed = config_rx_gate.changed() => { - if changed.is_err() { - break; - } - let cfg = config_rx_gate.borrow_and_update().clone(); - admission_poll_ms = cfg.general.me_admission_poll_ms.max(1); - fallback_enabled = cfg.general.me2dc_fallback; - continue; - } - _ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {} - } - let ready = pool_for_gate.admission_ready_conditional_cast().await; - let now = Instant::now(); - let (next_gate_open, next_route_mode, next_fallback_active) = if ready { - not_ready_since = None; - (true, RelayRouteMode::Middle, false) - } else { - let not_ready_started_at = *not_ready_since.get_or_insert(now); - let not_ready_for = now.saturating_duration_since(not_ready_started_at); - if fallback_enabled && not_ready_for > fallback_after { - (true, RelayRouteMode::Direct, true) - } else { - (false, RelayRouteMode::Middle, false) - } - }; - - if next_route_mode != route_mode { - route_mode = next_route_mode; - if let Some(snapshot) = route_runtime_gate.set_mode(route_mode) { - if matches!(route_mode, RelayRouteMode::Middle) { - info!( - target_mode = route_mode.as_str(), - cutover_generation = snapshot.generation, - "Middle-End routing restored for new sessions" - ); - } else { - warn!( - target_mode = route_mode.as_str(), - cutover_generation = snapshot.generation, - grace_secs = fallback_after.as_secs(), - "ME pool stayed not-ready beyond grace; routing new sessions via Direct-DC" - ); - } - } - } - - if next_gate_open != gate_open { - gate_open = next_gate_open; - admission_tx_gate.send_replace(gate_open); - if gate_open { - if next_fallback_active { - warn!("Conditional-admission gate opened in ME fallback mode"); - } else { - info!("Conditional-admission gate opened (ME pool ready)"); - } - } else { - warn!("Conditional-admission gate closed (ME pool is not ready)"); - } - } - - } - }); - } else { - admission_tx.send_replace(false); - let _ = route_runtime.set_mode(RelayRouteMode::Direct); - warn!("Conditional-admission gate: closed (ME pool is unavailable)"); - } - } else { - admission_tx.send_replace(true); - let _ = route_runtime.set_mode(RelayRouteMode::Direct); - } - let _admission_tx_hold = admission_tx; - - // Unix socket setup (before listeners check so unix-only config works) - let mut has_unix_listener = false; - #[cfg(unix)] - if let Some(ref unix_path) = config.server.listen_unix_sock { - // Remove stale socket file if present (standard practice) - let _ = tokio::fs::remove_file(unix_path).await; - - let unix_listener = UnixListener::bind(unix_path)?; - - // Apply socket permissions if configured - if let Some(ref perm_str) = config.server.listen_unix_sock_perm { - match u32::from_str_radix(perm_str.trim_start_matches('0'), 8) { - Ok(mode) => { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(mode); - if let Err(e) = std::fs::set_permissions(unix_path, perms) { - error!("Failed to set unix socket permissions to {}: {}", perm_str, e); - } else { - info!("Listening on unix:{} (mode {})", unix_path, perm_str); - } - } - Err(e) => { - warn!("Invalid listen_unix_sock_perm '{}': {}. Ignoring.", perm_str, e); - info!("Listening on unix:{}", unix_path); - } - } - } else { - info!("Listening on unix:{}", unix_path); - } - - has_unix_listener = true; - - let mut config_rx_unix: tokio::sync::watch::Receiver> = config_rx.clone(); - let mut admission_rx_unix = admission_rx.clone(); - let stats = stats.clone(); - let upstream_manager = upstream_manager.clone(); - let replay_checker = replay_checker.clone(); - let buffer_pool = buffer_pool.clone(); - let rng = rng.clone(); - let me_pool = me_pool.clone(); - let route_runtime = route_runtime.clone(); - let tls_cache = tls_cache.clone(); - let ip_tracker = ip_tracker.clone(); - let beobachten = beobachten.clone(); - let max_connections_unix = max_connections.clone(); - - tokio::spawn(async move { - let unix_conn_counter = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)); - - loop { - if !wait_until_admission_open(&mut admission_rx_unix).await { - warn!("Conditional-admission gate channel closed for unix listener"); - break; - } - match unix_listener.accept().await { - Ok((stream, _)) => { - let permit = match max_connections_unix.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => { - error!("Connection limiter is closed"); - break; - } - }; - let conn_id = unix_conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let fake_peer = SocketAddr::from(([127, 0, 0, 1], (conn_id % 65535) as u16)); - - let config = config_rx_unix.borrow_and_update().clone(); - let stats = stats.clone(); - let upstream_manager = upstream_manager.clone(); - let replay_checker = replay_checker.clone(); - let buffer_pool = buffer_pool.clone(); - let rng = rng.clone(); - let me_pool = me_pool.clone(); - let route_runtime = route_runtime.clone(); - let tls_cache = tls_cache.clone(); - let ip_tracker = ip_tracker.clone(); - let beobachten = beobachten.clone(); - let proxy_protocol_enabled = config.server.proxy_protocol; - - tokio::spawn(async move { - let _permit = permit; - if let Err(e) = crate::proxy::client::handle_client_stream( - stream, fake_peer, config, stats, - upstream_manager, replay_checker, buffer_pool, rng, - me_pool, route_runtime, tls_cache, ip_tracker, beobachten, proxy_protocol_enabled, - ).await { - debug!(error = %e, "Unix socket connection error"); - } - }); - } - Err(e) => { - error!("Unix socket accept error: {}", e); - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - } - }); - } - startup_tracker - .complete_component( - COMPONENT_LISTENERS_BIND, - Some(format!( - "listeners configured tcp={} unix={}", - listeners.len(), - has_unix_listener - )), - ) - .await; - - if listeners.is_empty() && !has_unix_listener { - error!("No listeners. Exiting."); - std::process::exit(1); - } - - // Switch to user-configured log level after startup - let runtime_filter = if has_rust_log { - EnvFilter::from_default_env() - } else if matches!(effective_log_level, LogLevel::Silent) { - EnvFilter::new("warn,telemt::links=info") - } else { - EnvFilter::new(effective_log_level.to_filter_str()) - }; - filter_handle - .reload(runtime_filter) - .expect("Failed to switch log filter"); - - // Apply log_level changes from hot-reload to the tracing filter. - tokio::spawn(async move { - loop { - if log_level_rx.changed().await.is_err() { - break; - } - let level = log_level_rx.borrow_and_update().clone(); - let new_filter = tracing_subscriber::EnvFilter::new(level.to_filter_str()); - if let Err(e) = filter_handle.reload(new_filter) { - tracing::error!("config reload: failed to update log filter: {}", e); - } - } - }); - - if let Some(port) = config.server.metrics_port { - startup_tracker - .start_component( - COMPONENT_METRICS_START, - Some(format!("spawn metrics endpoint on {}", port)), - ) - .await; - let stats = stats.clone(); - let beobachten = beobachten.clone(); - let config_rx_metrics = config_rx.clone(); - let ip_tracker_metrics = ip_tracker.clone(); - let whitelist = config.server.metrics_whitelist.clone(); - tokio::spawn(async move { - metrics::serve( - port, - stats, - beobachten, - ip_tracker_metrics, - config_rx_metrics, - whitelist, - ) - .await; - }); - startup_tracker - .complete_component( - COMPONENT_METRICS_START, - Some("metrics task spawned".to_string()), - ) - .await; - } else { - startup_tracker - .skip_component( - COMPONENT_METRICS_START, - Some("server.metrics_port is not configured".to_string()), - ) - .await; - } - - startup_tracker - .complete_component( - COMPONENT_RUNTIME_READY, - Some("startup pipeline is fully initialized".to_string()), - ) - .await; - startup_tracker.mark_ready().await; - - for (listener, listener_proxy_protocol) in listeners { - let mut config_rx: tokio::sync::watch::Receiver> = config_rx.clone(); - let mut admission_rx_tcp = admission_rx.clone(); - let stats = stats.clone(); - let upstream_manager = upstream_manager.clone(); - let replay_checker = replay_checker.clone(); - let buffer_pool = buffer_pool.clone(); - let rng = rng.clone(); - let me_pool = me_pool.clone(); - let route_runtime = route_runtime.clone(); - let tls_cache = tls_cache.clone(); - let ip_tracker = ip_tracker.clone(); - let beobachten = beobachten.clone(); - let max_connections_tcp = max_connections.clone(); - - tokio::spawn(async move { - loop { - if !wait_until_admission_open(&mut admission_rx_tcp).await { - warn!("Conditional-admission gate channel closed for tcp listener"); - break; - } - match listener.accept().await { - Ok((stream, peer_addr)) => { - let permit = match max_connections_tcp.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => { - error!("Connection limiter is closed"); - break; - } - }; - let config = config_rx.borrow_and_update().clone(); - let stats = stats.clone(); - let upstream_manager = upstream_manager.clone(); - let replay_checker = replay_checker.clone(); - let buffer_pool = buffer_pool.clone(); - let rng = rng.clone(); - let me_pool = me_pool.clone(); - let route_runtime = route_runtime.clone(); - let tls_cache = tls_cache.clone(); - let ip_tracker = ip_tracker.clone(); - let beobachten = beobachten.clone(); - let proxy_protocol_enabled = listener_proxy_protocol; - let real_peer_report = Arc::new(std::sync::Mutex::new(None)); - let real_peer_report_for_handler = real_peer_report.clone(); - - tokio::spawn(async move { - let _permit = permit; - if let Err(e) = ClientHandler::new( - stream, - peer_addr, - config, - stats, - upstream_manager, - replay_checker, - buffer_pool, - rng, - me_pool, - route_runtime, - tls_cache, - ip_tracker, - beobachten, - proxy_protocol_enabled, - real_peer_report_for_handler, - ) - .run() - .await - { - let real_peer = match real_peer_report.lock() { - Ok(guard) => *guard, - Err(_) => None, - }; - let peer_closed = matches!( - &e, - crate::error::ProxyError::Io(ioe) - if matches!( - ioe.kind(), - std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::NotConnected - ) - ) || matches!( - &e, - crate::error::ProxyError::Stream( - crate::error::StreamError::Io(ioe) - ) - if matches!( - ioe.kind(), - std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::NotConnected - ) - ); - - let me_closed = matches!( - &e, - crate::error::ProxyError::Proxy(msg) if msg == "ME connection lost" - ); - let route_switched = matches!( - &e, - crate::error::ProxyError::Proxy(msg) if msg == ROUTE_SWITCH_ERROR_MSG - ); - - match (peer_closed, me_closed) { - (true, _) => { - if let Some(real_peer) = real_peer { - debug!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by client"); - } else { - debug!(peer = %peer_addr, error = %e, "Connection closed by client"); - } - } - (_, true) => { - if let Some(real_peer) = real_peer { - warn!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed: Middle-End dropped session"); - } else { - warn!(peer = %peer_addr, error = %e, "Connection closed: Middle-End dropped session"); - } - } - _ if route_switched => { - if let Some(real_peer) = real_peer { - info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by controlled route cutover"); - } else { - info!(peer = %peer_addr, error = %e, "Connection closed by controlled route cutover"); - } - } - _ if is_expected_handshake_eof(&e) => { - if let Some(real_peer) = real_peer { - info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed during initial handshake"); - } else { - info!(peer = %peer_addr, error = %e, "Connection closed during initial handshake"); - } - } - _ => { - if let Some(real_peer) = real_peer { - warn!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed with error"); - } else { - warn!(peer = %peer_addr, error = %e, "Connection closed with error"); - } - } - } - } - }); - } - Err(e) => { - error!("Accept error: {}", e); - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - } - }); - } - - match signal::ctrl_c().await { - Ok(()) => { - let shutdown_started_at = Instant::now(); - info!("Shutting down..."); - let uptime_secs = process_started_at.elapsed().as_secs(); - info!("Uptime: {}", format_uptime(uptime_secs)); - if let Some(pool) = &me_pool { - match tokio::time::timeout( - Duration::from_secs(2), - pool.shutdown_send_close_conn_all(), - ) - .await - { - Ok(total) => { - info!( - close_conn_sent = total, - "ME shutdown: RPC_CLOSE_CONN broadcast completed" - ); - } - Err(_) => { - warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out"); - } - } - } - let shutdown_secs = shutdown_started_at.elapsed().as_secs(); - info!( - "Shutdown completed successfully in {} {}.", - shutdown_secs, - unit_label(shutdown_secs, "second", "seconds") - ); - } - Err(e) => error!("Signal error: {}", e), - } - - Ok(()) + maestro::run().await } diff --git a/src/proxy/client.rs b/src/proxy/client.rs index 4bc3751..99e6837 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -806,12 +806,8 @@ impl RunningClientHandler { }); } - let mut ip_reserved = false; - // IP limit check - match ip_tracker.check_and_add(user, peer_addr.ip()).await { - Ok(()) => { - ip_reserved = true; - } + let ip_reserved = match ip_tracker.check_and_add(user, peer_addr.ip()).await { + Ok(()) => true, Err(reason) => { warn!( user = %user, @@ -823,7 +819,8 @@ impl RunningClientHandler { user: user.to_string(), }); } - } + }; + // IP limit check if let Some(limit) = config.access.user_max_tcp_conns.get(user) && stats.get_user_curr_connects(user) >= *limit as u64 diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index efaa8ba..aaae1b3 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -382,7 +382,7 @@ where let mut batch_frames = 0usize; let mut batch_bytes = 0usize; - let mut flush_immediately = false; + let mut flush_immediately; match process_me_writer_response( first, From 97926b05e811c5871d2785e6f1655f491f763d97 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:44:27 +0300 Subject: [PATCH 156/261] Update admission.rs --- src/maestro/admission.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/maestro/admission.rs b/src/maestro/admission.rs index 40a0c2d..bacd4d4 100644 --- a/src/maestro/admission.rs +++ b/src/maestro/admission.rs @@ -22,9 +22,9 @@ pub(crate) async fn configure_admission_gate( admission_tx.send_replace(initial_ready); let _ = route_runtime.set_mode(RelayRouteMode::Middle); if initial_ready { - info!("Conditional-admission gate: open (ME pool ready)"); + info!("Conditional-admission gate: open / ME pool READY"); } else { - warn!("Conditional-admission gate: closed (ME pool is not ready)"); + warn!("Conditional-admission gate: closed / ME pool is NOT ready)"); } let pool_for_gate = pool.clone(); @@ -96,10 +96,10 @@ pub(crate) async fn configure_admission_gate( if next_fallback_active { warn!("Conditional-admission gate opened in ME fallback mode"); } else { - info!("Conditional-admission gate opened (ME pool ready)"); + info!("Conditional-admission gate opened / ME pool READY"); } } else { - warn!("Conditional-admission gate closed (ME pool is not ready)"); + warn!("Conditional-admission gate closed / ME pool is NOT ready"); } } } @@ -107,7 +107,7 @@ pub(crate) async fn configure_admission_gate( } else { admission_tx.send_replace(false); let _ = route_runtime.set_mode(RelayRouteMode::Direct); - warn!("Conditional-admission gate: closed (ME pool is unavailable)"); + warn!("Conditional-admission gate: closed / ME pool is UNAVAILABLE"); } } else { admission_tx.send_replace(true); From 3a997fcf7157d0453a9d988f70b57408b8e524c0 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:23:25 +0300 Subject: [PATCH 157/261] Update API.md --- docs/API.md | 403 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 392 insertions(+), 11 deletions(-) diff --git a/docs/API.md b/docs/API.md index 2f98b62..c70b625 100644 --- a/docs/API.md +++ b/docs/API.md @@ -85,6 +85,7 @@ Notes: | `GET` | `/v1/health` | none | `200` | `HealthData` | | `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` | | `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` | +| `GET` | `/v1/runtime/initialization` | none | `200` | `RuntimeInitializationData` | | `GET` | `/v1/limits/effective` | none | `200` | `EffectiveLimitsData` | | `GET` | `/v1/security/posture` | none | `200` | `SecurityPostureData` | | `GET` | `/v1/security/whitelist` | none | `200` | `SecurityWhitelistData` | @@ -147,6 +148,12 @@ Notes: - `PATCH` updates only provided fields and does not support explicit clearing of optional fields. - `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed. +## Query Parameters + +| Endpoint | Query | Behavior | +| --- | --- | --- | +| `GET /v1/runtime/events/recent` | `limit=` | Optional. Invalid/missing value falls back to default `50`. Effective value is clamped to `[1, 1000]` and additionally bounded by ring-buffer capacity. | + ## Request Contracts ### `CreateUserRequest` @@ -219,6 +226,45 @@ Note: the request contract is defined, but the corresponding route currently ret | `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. | | `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. | | `use_middle_proxy` | `bool` | Current transport mode preference. | +| `startup_status` | `string` | Startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). | +| `startup_stage` | `string` | Current startup stage identifier. | +| `startup_progress_pct` | `f64` | Startup progress percentage (`0..100`). | + +### `RuntimeInitializationData` +| Field | Type | Description | +| --- | --- | --- | +| `status` | `string` | Startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). | +| `degraded` | `bool` | Whether runtime is currently in degraded mode. | +| `current_stage` | `string` | Current startup stage identifier. | +| `progress_pct` | `f64` | Overall startup progress percentage (`0..100`). | +| `started_at_epoch_secs` | `u64` | Process start timestamp (Unix seconds). | +| `ready_at_epoch_secs` | `u64?` | Timestamp when startup reached ready state; absent until ready. | +| `total_elapsed_ms` | `u64` | Elapsed startup duration in milliseconds. | +| `transport_mode` | `string` | Startup transport mode (`middle_proxy` or `direct`). | +| `me` | `RuntimeInitializationMeData` | ME startup substate snapshot. | +| `components` | `RuntimeInitializationComponentData[]` | Per-component startup timeline and status. | + +#### `RuntimeInitializationMeData` +| Field | Type | Description | +| --- | --- | --- | +| `status` | `string` | ME startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). | +| `current_stage` | `string` | Current ME startup stage identifier. | +| `progress_pct` | `f64` | ME startup progress percentage (`0..100`). | +| `init_attempt` | `u32` | Current ME init attempt counter. | +| `retry_limit` | `string` | Retry limit (`"unlimited"` or numeric string). | +| `last_error` | `string?` | Last ME initialization error text when present. | + +#### `RuntimeInitializationComponentData` +| Field | Type | Description | +| --- | --- | --- | +| `id` | `string` | Startup component identifier. | +| `title` | `string` | Human-readable component title. | +| `status` | `string` | Component status (`pending`, `running`, `ready`, `failed`, `skipped`). | +| `started_at_epoch_ms` | `u64?` | Component start timestamp in Unix milliseconds. | +| `finished_at_epoch_ms` | `u64?` | Component finish timestamp in Unix milliseconds. | +| `duration_ms` | `u64?` | Component duration in milliseconds. | +| `attempts` | `u32` | Attempt counter for this component. | +| `details` | `string?` | Optional short status details text. | ### `EffectiveLimitsData` | Field | Type | Description | @@ -256,11 +302,22 @@ Note: the request contract is defined, but the corresponding route currently ret | `floor_mode` | `string` | Effective floor mode (`static` or `adaptive`). | | `adaptive_floor_idle_secs` | `u64` | Adaptive floor idle threshold. | | `adaptive_floor_min_writers_single_endpoint` | `u8` | Adaptive floor minimum for single-endpoint DCs. | +| `adaptive_floor_min_writers_multi_endpoint` | `u8` | Adaptive floor minimum for multi-endpoint DCs. | | `adaptive_floor_recover_grace_secs` | `u64` | Adaptive floor recovery grace period. | +| `adaptive_floor_writers_per_core_total` | `u16` | Target total writers-per-core budget in adaptive mode. | +| `adaptive_floor_cpu_cores_override` | `u16` | Manual CPU core override (`0` means auto-detect). | +| `adaptive_floor_max_extra_writers_single_per_core` | `u16` | Extra per-core adaptive headroom for single-endpoint DCs. | +| `adaptive_floor_max_extra_writers_multi_per_core` | `u16` | Extra per-core adaptive headroom for multi-endpoint DCs. | +| `adaptive_floor_max_active_writers_per_core` | `u16` | Active writer cap per CPU core. | +| `adaptive_floor_max_warm_writers_per_core` | `u16` | Warm writer cap per CPU core. | +| `adaptive_floor_max_active_writers_global` | `u32` | Global active writer cap. | +| `adaptive_floor_max_warm_writers_global` | `u32` | Global warm writer cap. | | `reconnect_max_concurrent_per_dc` | `u32` | Max concurrent reconnects per DC. | | `reconnect_backoff_base_ms` | `u64` | Reconnect base backoff. | | `reconnect_backoff_cap_ms` | `u64` | Reconnect backoff cap. | | `reconnect_fast_retry_count` | `u32` | Number of fast retries before standard backoff strategy. | +| `writer_pick_mode` | `string` | Writer picker mode (`sorted_rr`, `p2c`). | +| `writer_pick_sample_size` | `u8` | Candidate sample size for `p2c` picker mode. | | `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. | #### `EffectiveUserIpPolicyLimits` @@ -290,16 +347,292 @@ Note: the request contract is defined, but the corresponding route currently ret | `entries_total` | `usize` | Number of whitelist CIDR entries. | | `entries` | `string[]` | Whitelist CIDR entries as strings. | -### Runtime Min Endpoints -- `/v1/runtime/me_pool_state`: generations, hardswap state, writer contour/health counts, refill inflight snapshot. -- `/v1/runtime/me_quality`: ME error/drift/reconnect counters and per-DC RTT coverage snapshot. -- `/v1/runtime/upstream_quality`: upstream runtime policy, connect counters, health summary and per-upstream DC latency/IP preference. -- `/v1/runtime/nat_stun`: NAT/STUN runtime flags, server lists, reflection cache state and backoff remaining. +### `RuntimeMePoolStateData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Runtime payload availability. | +| `reason` | `string?` | `source_unavailable` when ME pool snapshot is unavailable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeMePoolStatePayload?` | Null when unavailable. | -### Runtime Edge Endpoints -- `/v1/runtime/connections/summary`: cached connection totals (`total/me/direct`), active users and top-N users by connections/traffic. -- `/v1/runtime/events/recent?limit=N`: bounded control-plane ring-buffer events (`limit` clamped to `[1, 1000]`). -- If `server.api.runtime_edge_enabled=false`, runtime edge endpoints return `enabled=false` with `reason=feature_disabled`. +#### `RuntimeMePoolStatePayload` +| Field | Type | Description | +| --- | --- | --- | +| `generations` | `RuntimeMePoolStateGenerationData` | Active/warm/pending/draining generation snapshot. | +| `hardswap` | `RuntimeMePoolStateHardswapData` | Hardswap state flags. | +| `writers` | `RuntimeMePoolStateWriterData` | Writer total/contour/health counters. | +| `refill` | `RuntimeMePoolStateRefillData` | In-flight refill counters by DC/family. | + +#### `RuntimeMePoolStateGenerationData` +| Field | Type | Description | +| --- | --- | --- | +| `active_generation` | `u64` | Active pool generation id. | +| `warm_generation` | `u64` | Warm pool generation id. | +| `pending_hardswap_generation` | `u64` | Pending hardswap generation id (`0` when none). | +| `pending_hardswap_age_secs` | `u64?` | Age of pending hardswap generation in seconds. | +| `draining_generations` | `u64[]` | Distinct generation ids currently draining. | + +#### `RuntimeMePoolStateHardswapData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Hardswap feature toggle. | +| `pending` | `bool` | `true` when pending generation is non-zero. | + +#### `RuntimeMePoolStateWriterData` +| Field | Type | Description | +| --- | --- | --- | +| `total` | `usize` | Total writer rows in snapshot. | +| `alive_non_draining` | `usize` | Alive writers excluding draining ones. | +| `draining` | `usize` | Writers marked draining. | +| `degraded` | `usize` | Non-draining degraded writers. | +| `contour` | `RuntimeMePoolStateWriterContourData` | Counts by contour state. | +| `health` | `RuntimeMePoolStateWriterHealthData` | Counts by health bucket. | + +#### `RuntimeMePoolStateWriterContourData` +| Field | Type | Description | +| --- | --- | --- | +| `warm` | `usize` | Writers in warm contour. | +| `active` | `usize` | Writers in active contour. | +| `draining` | `usize` | Writers in draining contour. | + +#### `RuntimeMePoolStateWriterHealthData` +| Field | Type | Description | +| --- | --- | --- | +| `healthy` | `usize` | Non-draining non-degraded writers. | +| `degraded` | `usize` | Non-draining degraded writers. | +| `draining` | `usize` | Draining writers. | + +#### `RuntimeMePoolStateRefillData` +| Field | Type | Description | +| --- | --- | --- | +| `inflight_endpoints_total` | `usize` | Total in-flight endpoint refill operations. | +| `inflight_dc_total` | `usize` | Number of distinct DC+family keys with refill in flight. | +| `by_dc` | `RuntimeMePoolStateRefillDcData[]` | Per-DC refill rows. | + +#### `RuntimeMePoolStateRefillDcData` +| Field | Type | Description | +| --- | --- | --- | +| `dc` | `i16` | Telegram DC id. | +| `family` | `string` | Address family label (`V4`, `V6`). | +| `inflight` | `usize` | In-flight refill operations for this row. | + +### `RuntimeMeQualityData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Runtime payload availability. | +| `reason` | `string?` | `source_unavailable` when ME pool snapshot is unavailable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeMeQualityPayload?` | Null when unavailable. | + +#### `RuntimeMeQualityPayload` +| Field | Type | Description | +| --- | --- | --- | +| `counters` | `RuntimeMeQualityCountersData` | Key ME lifecycle/error counters. | +| `route_drops` | `RuntimeMeQualityRouteDropData` | Route drop counters by reason. | +| `dc_rtt` | `RuntimeMeQualityDcRttData[]` | Per-DC RTT and writer coverage rows. | + +#### `RuntimeMeQualityCountersData` +| Field | Type | Description | +| --- | --- | --- | +| `idle_close_by_peer_total` | `u64` | Peer-initiated idle closes. | +| `reader_eof_total` | `u64` | Reader EOF events. | +| `kdf_drift_total` | `u64` | KDF drift detections. | +| `kdf_port_only_drift_total` | `u64` | KDF port-only drift detections. | +| `reconnect_attempt_total` | `u64` | Reconnect attempts. | +| `reconnect_success_total` | `u64` | Successful reconnects. | + +#### `RuntimeMeQualityRouteDropData` +| Field | Type | Description | +| --- | --- | --- | +| `no_conn_total` | `u64` | Route drops with no connection mapping. | +| `channel_closed_total` | `u64` | Route drops because destination channel is closed. | +| `queue_full_total` | `u64` | Route drops due queue backpressure (aggregate). | +| `queue_full_base_total` | `u64` | Route drops in base-queue path. | +| `queue_full_high_total` | `u64` | Route drops in high-priority queue path. | + +#### `RuntimeMeQualityDcRttData` +| Field | Type | Description | +| --- | --- | --- | +| `dc` | `i16` | Telegram DC id. | +| `rtt_ema_ms` | `f64?` | RTT EMA for this DC. | +| `alive_writers` | `usize` | Alive writers currently mapped to this DC. | +| `required_writers` | `usize` | Target writer floor for this DC. | +| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. | + +### `RuntimeUpstreamQualityData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Runtime payload availability. | +| `reason` | `string?` | `source_unavailable` when upstream runtime snapshot is unavailable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `policy` | `RuntimeUpstreamQualityPolicyData` | Effective upstream policy values. | +| `counters` | `RuntimeUpstreamQualityCountersData` | Upstream connect counters. | +| `summary` | `RuntimeUpstreamQualitySummaryData?` | Aggregate runtime health summary. | +| `upstreams` | `RuntimeUpstreamQualityUpstreamData[]?` | Per-upstream runtime rows. | + +#### `RuntimeUpstreamQualityPolicyData` +| Field | Type | Description | +| --- | --- | --- | +| `connect_retry_attempts` | `u32` | Upstream connect retry attempts. | +| `connect_retry_backoff_ms` | `u64` | Upstream retry backoff delay. | +| `connect_budget_ms` | `u64` | Total connect wall-clock budget. | +| `unhealthy_fail_threshold` | `u32` | Consecutive fail threshold for unhealthy marking. | +| `connect_failfast_hard_errors` | `bool` | Whether hard errors skip retries. | + +#### `RuntimeUpstreamQualityCountersData` +| Field | Type | Description | +| --- | --- | --- | +| `connect_attempt_total` | `u64` | Total connect attempts. | +| `connect_success_total` | `u64` | Successful connects. | +| `connect_fail_total` | `u64` | Failed connects. | +| `connect_failfast_hard_error_total` | `u64` | Fail-fast hard errors. | + +#### `RuntimeUpstreamQualitySummaryData` +| Field | Type | Description | +| --- | --- | --- | +| `configured_total` | `usize` | Total configured upstream entries. | +| `healthy_total` | `usize` | Upstreams currently healthy. | +| `unhealthy_total` | `usize` | Upstreams currently unhealthy. | +| `direct_total` | `usize` | Direct-route upstream entries. | +| `socks4_total` | `usize` | SOCKS4 upstream entries. | +| `socks5_total` | `usize` | SOCKS5 upstream entries. | + +#### `RuntimeUpstreamQualityUpstreamData` +| Field | Type | Description | +| --- | --- | --- | +| `upstream_id` | `usize` | Runtime upstream index. | +| `route_kind` | `string` | `direct`, `socks4`, `socks5`. | +| `address` | `string` | Upstream address (`direct` literal for direct route kind). | +| `weight` | `u16` | Selection weight. | +| `scopes` | `string` | Configured scope selector. | +| `healthy` | `bool` | Current health flag. | +| `fails` | `u32` | Consecutive fail counter. | +| `last_check_age_secs` | `u64` | Seconds since last health update. | +| `effective_latency_ms` | `f64?` | Effective latency score used by selector. | +| `dc` | `RuntimeUpstreamQualityDcData[]` | Per-DC runtime rows. | + +#### `RuntimeUpstreamQualityDcData` +| Field | Type | Description | +| --- | --- | --- | +| `dc` | `i16` | Telegram DC id. | +| `latency_ema_ms` | `f64?` | Per-DC latency EMA. | +| `ip_preference` | `string` | `unknown`, `prefer_v4`, `prefer_v6`, `both_work`, `unavailable`. | + +### `RuntimeNatStunData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Runtime payload availability. | +| `reason` | `string?` | `source_unavailable` when shared STUN state is unavailable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeNatStunPayload?` | Null when unavailable. | + +#### `RuntimeNatStunPayload` +| Field | Type | Description | +| --- | --- | --- | +| `flags` | `RuntimeNatStunFlagsData` | NAT probe runtime flags. | +| `servers` | `RuntimeNatStunServersData` | Configured/live STUN server lists. | +| `reflection` | `RuntimeNatStunReflectionBlockData` | Reflection cache data for v4/v6. | +| `stun_backoff_remaining_ms` | `u64?` | Remaining retry backoff (milliseconds). | + +#### `RuntimeNatStunFlagsData` +| Field | Type | Description | +| --- | --- | --- | +| `nat_probe_enabled` | `bool` | Current NAT probe enable state. | +| `nat_probe_disabled_runtime` | `bool` | Runtime disable flag due failures/conditions. | +| `nat_probe_attempts` | `u8` | Configured NAT probe attempt count. | + +#### `RuntimeNatStunServersData` +| Field | Type | Description | +| --- | --- | --- | +| `configured` | `string[]` | Configured STUN server entries. | +| `live` | `string[]` | Runtime live STUN server entries. | +| `live_total` | `usize` | Number of live STUN entries. | + +#### `RuntimeNatStunReflectionBlockData` +| Field | Type | Description | +| --- | --- | --- | +| `v4` | `RuntimeNatStunReflectionData?` | IPv4 reflection data. | +| `v6` | `RuntimeNatStunReflectionData?` | IPv6 reflection data. | + +#### `RuntimeNatStunReflectionData` +| Field | Type | Description | +| --- | --- | --- | +| `addr` | `string` | Reflected public endpoint (`ip:port`). | +| `age_secs` | `u64` | Reflection value age in seconds. | + +### `RuntimeEdgeConnectionsSummaryData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Endpoint availability under `runtime_edge_enabled`. | +| `reason` | `string?` | `feature_disabled` or `source_unavailable`. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeEdgeConnectionsSummaryPayload?` | Null when unavailable. | + +#### `RuntimeEdgeConnectionsSummaryPayload` +| Field | Type | Description | +| --- | --- | --- | +| `cache` | `RuntimeEdgeConnectionCacheData` | Runtime edge cache metadata. | +| `totals` | `RuntimeEdgeConnectionTotalsData` | Connection totals block. | +| `top` | `RuntimeEdgeConnectionTopData` | Top-N leaderboard blocks. | +| `telemetry` | `RuntimeEdgeConnectionTelemetryData` | Telemetry-policy flags for counters. | + +#### `RuntimeEdgeConnectionCacheData` +| Field | Type | Description | +| --- | --- | --- | +| `ttl_ms` | `u64` | Configured cache TTL in milliseconds. | +| `served_from_cache` | `bool` | `true` when payload is served from cache. | +| `stale_cache_used` | `bool` | `true` when stale cache is used because recompute is busy. | + +#### `RuntimeEdgeConnectionTotalsData` +| Field | Type | Description | +| --- | --- | --- | +| `current_connections` | `u64` | Current global live connections. | +| `current_connections_me` | `u64` | Current live connections routed through ME. | +| `current_connections_direct` | `u64` | Current live connections routed through direct path. | +| `active_users` | `usize` | Users with `current_connections > 0`. | + +#### `RuntimeEdgeConnectionTopData` +| Field | Type | Description | +| --- | --- | --- | +| `limit` | `usize` | Effective Top-N row count. | +| `by_connections` | `RuntimeEdgeConnectionUserData[]` | Users sorted by current connections. | +| `by_throughput` | `RuntimeEdgeConnectionUserData[]` | Users sorted by cumulative octets. | + +#### `RuntimeEdgeConnectionUserData` +| Field | Type | Description | +| --- | --- | --- | +| `username` | `string` | Username. | +| `current_connections` | `u64` | Current live connections for user. | +| `total_octets` | `u64` | Cumulative (`client->proxy + proxy->client`) octets. | + +#### `RuntimeEdgeConnectionTelemetryData` +| Field | Type | Description | +| --- | --- | --- | +| `user_enabled` | `bool` | Per-user telemetry enable flag. | +| `throughput_is_cumulative` | `bool` | Always `true` in current implementation. | + +### `RuntimeEdgeEventsData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Endpoint availability under `runtime_edge_enabled`. | +| `reason` | `string?` | `feature_disabled` when endpoint is disabled. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeEdgeEventsPayload?` | Null when unavailable. | + +#### `RuntimeEdgeEventsPayload` +| Field | Type | Description | +| --- | --- | --- | +| `capacity` | `usize` | Effective ring-buffer capacity. | +| `dropped_total` | `u64` | Count of dropped oldest events due capacity pressure. | +| `events` | `ApiEventRecord[]` | Recent events in chronological order. | + +#### `ApiEventRecord` +| Field | Type | Description | +| --- | --- | --- | +| `seq` | `u64` | Monotonic sequence number. | +| `ts_epoch_secs` | `u64` | Event timestamp (Unix seconds). | +| `event_type` | `string` | Event kind identifier. | +| `context` | `string` | Context text (truncated to implementation-defined max length). | ### `ZeroAllData` | Field | Type | Description | @@ -485,7 +818,27 @@ Note: the request contract is defined, but the corresponding route currently ret | `floor_mode` | `string` | Writer floor mode. | | `adaptive_floor_idle_secs` | `u64` | Idle threshold for adaptive floor. | | `adaptive_floor_min_writers_single_endpoint` | `u8` | Minimum writers for single-endpoint DC in adaptive mode. | +| `adaptive_floor_min_writers_multi_endpoint` | `u8` | Minimum writers for multi-endpoint DC in adaptive mode. | | `adaptive_floor_recover_grace_secs` | `u64` | Grace period for floor recovery. | +| `adaptive_floor_writers_per_core_total` | `u16` | Target total writers-per-core budget in adaptive mode. | +| `adaptive_floor_cpu_cores_override` | `u16` | CPU core override (`0` means auto-detect). | +| `adaptive_floor_max_extra_writers_single_per_core` | `u16` | Extra single-endpoint writers budget per core. | +| `adaptive_floor_max_extra_writers_multi_per_core` | `u16` | Extra multi-endpoint writers budget per core. | +| `adaptive_floor_max_active_writers_per_core` | `u16` | Active writer cap per core. | +| `adaptive_floor_max_warm_writers_per_core` | `u16` | Warm writer cap per core. | +| `adaptive_floor_max_active_writers_global` | `u32` | Global active writer cap. | +| `adaptive_floor_max_warm_writers_global` | `u32` | Global warm writer cap. | +| `adaptive_floor_cpu_cores_detected` | `u32` | Runtime-detected CPU cores. | +| `adaptive_floor_cpu_cores_effective` | `u32` | Effective core count used for adaptive caps. | +| `adaptive_floor_global_cap_raw` | `u64` | Raw global cap before clamping. | +| `adaptive_floor_global_cap_effective` | `u64` | Effective global cap after clamping. | +| `adaptive_floor_target_writers_total` | `u64` | Current adaptive total writer target. | +| `adaptive_floor_active_cap_configured` | `u64` | Configured global active cap. | +| `adaptive_floor_active_cap_effective` | `u64` | Effective global active cap. | +| `adaptive_floor_warm_cap_configured` | `u64` | Configured global warm cap. | +| `adaptive_floor_warm_cap_effective` | `u64` | Effective global warm cap. | +| `adaptive_floor_active_writers_current` | `u64` | Current active writers count. | +| `adaptive_floor_warm_writers_current` | `u64` | Current warm writers count. | | `me_keepalive_enabled` | `bool` | ME keepalive toggle. | | `me_keepalive_interval_secs` | `u64` | Keepalive period. | | `me_keepalive_jitter_secs` | `u64` | Keepalive jitter. | @@ -507,6 +860,8 @@ Note: the request contract is defined, but the corresponding route currently ret | `me_single_endpoint_outage_backoff_max_ms` | `u64` | Outage mode max reconnect backoff. | | `me_single_endpoint_shadow_rotate_every_secs` | `u64` | Shadow rotation interval. | | `me_deterministic_writer_sort` | `bool` | Deterministic writer ordering toggle. | +| `me_writer_pick_mode` | `string` | Writer picker mode (`sorted_rr`, `p2c`). | +| `me_writer_pick_sample_size` | `u8` | Candidate sample size for `p2c` picker mode. | | `me_socks_kdf_policy` | `string` | Current SOCKS KDF policy mode. | | `quarantined_endpoints_total` | `usize` | Total quarantined endpoints. | | `quarantined_endpoints` | `MinimalQuarantineData[]` | Quarantine details. | @@ -572,14 +927,25 @@ Note: the request contract is defined, but the corresponding route currently ret | --- | --- | --- | | `dc` | `i16` | Telegram DC id. | | `endpoints` | `string[]` | Endpoints in this DC (`ip:port`). | +| `endpoint_writers` | `DcEndpointWriters[]` | Active writer counts grouped by endpoint. | | `available_endpoints` | `usize` | Endpoints currently available in this DC. | | `available_pct` | `f64` | `available_endpoints / endpoints_total * 100`. | | `required_writers` | `usize` | Required writer count for this DC. | +| `floor_min` | `usize` | Floor lower bound for this DC. | +| `floor_target` | `usize` | Floor target writer count for this DC. | +| `floor_max` | `usize` | Floor upper bound for this DC. | +| `floor_capped` | `bool` | `true` when computed floor target was capped by active limits. | | `alive_writers` | `usize` | Alive writers in this DC. | | `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. | | `rtt_ms` | `f64?` | Aggregated RTT for DC. | | `load` | `usize` | Active client sessions bound to this DC. | +#### `DcEndpointWriters` +| Field | Type | Description | +| --- | --- | --- | +| `endpoint` | `string` | Endpoint (`ip:port`). | +| `active_writers` | `usize` | Active writers currently mapped to endpoint. | + ### `UserInfo` | Field | Type | Description | | --- | --- | --- | @@ -591,6 +957,9 @@ Note: the request contract is defined, but the corresponding route currently ret | `max_unique_ips` | `usize?` | Optional unique IP limit. | | `current_connections` | `u64` | Current live connections. | | `active_unique_ips` | `usize` | Current active unique source IPs. | +| `active_unique_ips_list` | `ip[]` | Current active unique source IP list. | +| `recent_unique_ips` | `usize` | Unique source IP count inside the configured recent window. | +| `recent_unique_ips_list` | `ip[]` | Recent-window unique source IP list. | | `total_octets` | `u64` | Total traffic octets for this user. | | `links` | `UserLinks` | Active connection links derived from current config. | @@ -603,10 +972,10 @@ Note: the request contract is defined, but the corresponding route currently ret Link generation uses active config and enabled modes: - `[general.links].public_host/public_port` have priority. -- If `public_host` is not set, startup-detected public IPs are used (`IPv4`, `IPv6`, or both when available). +- If `public_host` is not set, startup-detected public IPs are used when they are present in API runtime context. - Fallback host sources: listener `announce`, `announce_ip`, explicit listener `ip`. - Legacy fallback: `listen_addr_ipv4` and `listen_addr_ipv6` when routable. -- Startup-detected IPs are fixed for process lifetime and refreshed on restart. +- Startup-detected IP values are process-static after API task bootstrap. - User rows are sorted by `username` in ascending lexical order. ### `CreateUserResponse` @@ -643,6 +1012,17 @@ All mutating endpoints: - ME endpoints: ME pool is absent (for example direct-only mode or failed ME initialization). - Upstreams endpoint: non-blocking upstream snapshot lock is unavailable at request time. +Additional runtime endpoint behavior: + +| Endpoint | Disabled by feature flag | `source_unavailable` condition | Normal mode | +| --- | --- | --- | --- | +| `/v1/runtime/me_pool_state` | No | ME pool snapshot unavailable | `enabled=true`, full payload | +| `/v1/runtime/me_quality` | No | ME pool snapshot unavailable | `enabled=true`, full payload | +| `/v1/runtime/upstream_quality` | No | Upstream runtime snapshot unavailable | `enabled=true`, full payload | +| `/v1/runtime/nat_stun` | No | STUN shared state unavailable | `enabled=true`, full payload | +| `/v1/runtime/connections/summary` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Recompute lock contention with no cache entry => `enabled=true`, `reason=source_unavailable` | `enabled=true`, full payload | +| `/v1/runtime/events/recent` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload | + ## Serialization Rules - Success responses always include `revision`. @@ -650,6 +1030,7 @@ All mutating endpoints: - Optional fields with `skip_serializing_if` are omitted when absent. - Nullable payload fields may still be `null` where contract uses `?` (for example `UserInfo` option fields). - For `/v1/stats/upstreams`, authentication details of SOCKS upstreams are intentionally omitted. +- `ip[]` fields are serialized as JSON string arrays (for example `"1.2.3.4"`, `"2001:db8::1"`). ## Operational Notes From 5ab3170f691e96b609f8564f501f83e8639f6151 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:43:46 +0300 Subject: [PATCH 158/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 33dc9b7..2f7ea3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.14" +version = "3.3.15" edition = "2024" [dependencies] From c920dc63810c33b1e639bb6219a97a206fbb3201 Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Mon, 9 Mar 2026 19:38:55 +0300 Subject: [PATCH 159/261] Fix config path and update service creation steps Updated paths and instructions in the quick start guide for Telemt configuration and service setup. --- docs/QUICK_START_GUIDE.en.md | 42 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/docs/QUICK_START_GUIDE.en.md b/docs/QUICK_START_GUIDE.en.md index 52ce66a..e140cf3 100644 --- a/docs/QUICK_START_GUIDE.en.md +++ b/docs/QUICK_START_GUIDE.en.md @@ -48,11 +48,16 @@ Save the obtained result somewhere. You will need it later! --- -**1. Place your config to /etc/telemt.toml** +**1. Place your config to /etc/telemt/telemt.toml** + +Create config directory: +```bash +mkdir /etc/telemt +``` Open nano ```bash -nano /etc/telemt.toml +nano /etc/telemt/telemt.toml ``` paste your config @@ -90,7 +95,14 @@ then Ctrl+S -> Ctrl+X to save --- -**2. Create service on /etc/systemd/system/telemt.service** +**2. Create telemt user** + +```bash +useradd -d /opt/telemt -m -r -U telemt +chown -R telemt:telemt /etc/telemt +``` + +**3. Create service on /etc/systemd/system/telemt.service** Open nano ```bash @@ -101,28 +113,38 @@ paste this Systemd Module ```bash [Unit] Description=Telemt -After=network.target +After=network-online.target +Wants=network-online.target [Service] Type=simple -WorkingDirectory=/bin -ExecStart=/bin/telemt /etc/telemt.toml +User=telemt +Group=telemt +WorkingDirectory=/opt/telemt +ExecStart=/bin/telemt /etc/telemt/telemt.toml Restart=on-failure LimitNOFILE=65536 +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=true [Install] WantedBy=multi-user.target ``` then Ctrl+S -> Ctrl+X to save +reload systemd units +```bash +systemctl daemon-reload +``` -**3.** To start it, enter the command `systemctl start telemt` +**4.** To start it, enter the command `systemctl start telemt` -**4.** To get status information, enter `systemctl status telemt` +**5.** To get status information, enter `systemctl status telemt` -**5.** For automatic startup at system boot, enter `systemctl enable telemt` +**6.** For automatic startup at system boot, enter `systemctl enable telemt` -**6.** To get the link(s), enter +**7.** To get the link(s), enter ```bash curl -s http://127.0.0.1:9091/v1/users | jq ``` From a09b597fabbac67201ff535d61686df7caf4ee94 Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Mon, 9 Mar 2026 19:39:55 +0300 Subject: [PATCH 160/261] Fix the ru translation also --- docs/QUICK_START_GUIDE.ru.md | 42 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index f893170..0ed5d31 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -48,11 +48,16 @@ python3 -c 'import os; print(os.urandom(16).hex())' --- -**1. Поместите свою конфигурацию в файл /etc/telemt.toml** +**1. Поместите свою конфигурацию в файл /etc/telemt/telemt.toml** + +Создаём директорию для конфига: +```bash +mkdir /etc/telemt +``` Открываем nano ```bash -nano /etc/telemt.toml +nano /etc/telemt/telemt.toml ``` Вставьте свою конфигурацию @@ -90,7 +95,14 @@ hello = "00000000000000000000000000000000" --- -**2. Создайте службу в /etc/systemd/system/telemt.service** +**2. Создайте пользователя для telemt** + +```bash +useradd -d /opt/telemt -m -r -U telemt +chown -R telemt:telemt /etc/telemt +``` + +**3. Создайте службу в /etc/systemd/system/telemt.service** Открываем nano ```bash @@ -101,28 +113,38 @@ nano /etc/systemd/system/telemt.service ```bash [Unit] Description=Telemt -After=network.target +After=network-online.target +Wants=network-online.target [Service] Type=simple -WorkingDirectory=/bin -ExecStart=/bin/telemt /etc/telemt.toml +User=telemt +Group=telemt +WorkingDirectory=/opt/telemt +ExecStart=/bin/telemt /etc/telemt/telemt.toml Restart=on-failure LimitNOFILE=65536 +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=true [Install] WantedBy=multi-user.target ``` Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить +перезагрузите конфигурацию systemd +```bash +systemctl daemon-reload +``` -**3.** Для запуска введите команду `systemctl start telemt` +**4.** Для запуска введите команду `systemctl start telemt` -**4.** Для получения информации о статусе введите `systemctl status telemt` +**5.** Для получения информации о статусе введите `systemctl status telemt` -**5.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt` +**6.** Для автоматического запуска при запуске системы в введите `systemctl enable telemt` -**6.** Для получения ссылки/ссылок введите +**7.** Для получения ссылки/ссылок введите ```bash curl -s http://127.0.0.1:9091/v1/users | jq ``` From b950987229203c885c57d29a4d5b640b3bef38ae Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 20:35:31 +0300 Subject: [PATCH 161/261] ME Selftest --- src/api/mod.rs | 7 + src/api/runtime_selftest.rs | 228 +++++++++++++++++++ src/network/probe.rs | 8 + src/transport/middle_proxy/config_updater.rs | 2 + src/transport/middle_proxy/handshake.rs | 29 +++ src/transport/middle_proxy/mod.rs | 2 + src/transport/middle_proxy/secret.rs | 2 + src/transport/middle_proxy/selftest.rs | 194 ++++++++++++++++ 8 files changed, 472 insertions(+) create mode 100644 src/api/runtime_selftest.rs create mode 100644 src/transport/middle_proxy/selftest.rs diff --git a/src/api/mod.rs b/src/api/mod.rs index 63fafad..0a51231 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -28,6 +28,7 @@ mod model; mod runtime_edge; mod runtime_init; mod runtime_min; +mod runtime_selftest; mod runtime_stats; mod runtime_watch; mod runtime_zero; @@ -48,6 +49,7 @@ use runtime_min::{ build_runtime_me_pool_state_data, build_runtime_me_quality_data, build_runtime_nat_stun_data, build_runtime_upstream_quality_data, build_security_whitelist_data, }; +use runtime_selftest::build_runtime_me_selftest_data; use runtime_stats::{ MinimalCacheEntry, build_dcs_data, build_me_writers_data, build_minimal_all_data, build_upstreams_data, build_zero_all_data, @@ -333,6 +335,11 @@ async fn handle( let data = build_runtime_nat_stun_data(shared.as_ref()).await; Ok(success_response(StatusCode::OK, data, revision)) } + ("GET", "/v1/runtime/me-selftest") => { + let revision = current_revision(&shared.config_path).await?; + let data = build_runtime_me_selftest_data(shared.as_ref()).await; + Ok(success_response(StatusCode::OK, data, revision)) + } ("GET", "/v1/runtime/connections/summary") => { let revision = current_revision(&shared.config_path).await?; let data = build_runtime_connections_summary_data(shared.as_ref(), cfg.as_ref()).await; diff --git a/src/api/runtime_selftest.rs b/src/api/runtime_selftest.rs new file mode 100644 index 0000000..da591b2 --- /dev/null +++ b/src/api/runtime_selftest.rs @@ -0,0 +1,228 @@ +use std::net::IpAddr; +use std::sync::{Mutex, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Serialize; + +use crate::network::probe::{detect_interface_ipv4, detect_interface_ipv6, is_bogon}; +use crate::transport::middle_proxy::{bnd_snapshot, timeskew_snapshot}; + +use super::ApiShared; + +const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable"; +const KDF_EWMA_TAU_SECS: f64 = 600.0; +const KDF_EWMA_THRESHOLD_ERRORS_PER_MIN: f64 = 0.30; +const TIMESKEW_THRESHOLD_SECS: u64 = 60; + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestKdfData { + pub(super) state: &'static str, + pub(super) ewma_errors_per_min: f64, + pub(super) threshold_errors_per_min: f64, + pub(super) errors_total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestTimeskewData { + pub(super) state: &'static str, + pub(super) max_skew_secs_15m: Option, + pub(super) samples_15m: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_skew_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_source: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_seen_age_secs: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestIpFamilyData { + pub(super) addr: String, + pub(super) state: &'static str, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestIpData { + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) v4: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) v6: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestPidData { + pub(super) pid: u32, + pub(super) state: &'static str, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestBndData { + pub(super) addr_state: &'static str, + pub(super) port_state: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_addr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) last_seen_age_secs: Option, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestPayload { + pub(super) kdf: RuntimeMeSelftestKdfData, + pub(super) timeskew: RuntimeMeSelftestTimeskewData, + pub(super) ip: RuntimeMeSelftestIpData, + pub(super) pid: RuntimeMeSelftestPidData, + pub(super) bnd: RuntimeMeSelftestBndData, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestData { + pub(super) enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reason: Option<&'static str>, + pub(super) generated_at_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) data: Option, +} + +#[derive(Default)] +struct KdfEwmaState { + initialized: bool, + last_epoch_secs: u64, + last_total_errors: u64, + ewma_errors_per_min: f64, +} + +static KDF_EWMA_STATE: OnceLock> = OnceLock::new(); + +fn kdf_ewma_state() -> &'static Mutex { + KDF_EWMA_STATE.get_or_init(|| Mutex::new(KdfEwmaState::default())) +} + +pub(super) async fn build_runtime_me_selftest_data(shared: &ApiShared) -> RuntimeMeSelftestData { + let now_epoch_secs = now_epoch_secs(); + if shared.me_pool.read().await.is_none() { + return RuntimeMeSelftestData { + enabled: false, + reason: Some(SOURCE_UNAVAILABLE_REASON), + generated_at_epoch_secs: now_epoch_secs, + data: None, + }; + } + + let kdf_errors_total = shared + .stats + .get_me_kdf_drift_total() + .saturating_add(shared.stats.get_me_socks_kdf_strict_reject()); + let kdf_ewma = update_kdf_ewma(now_epoch_secs, kdf_errors_total); + let kdf_state = if kdf_ewma >= KDF_EWMA_THRESHOLD_ERRORS_PER_MIN { + "error" + } else { + "ok" + }; + + let skew = timeskew_snapshot(); + let timeskew_state = if skew.max_skew_secs_15m.unwrap_or(0) > TIMESKEW_THRESHOLD_SECS { + "error" + } else { + "ok" + }; + + let ip_v4 = detect_interface_ipv4().map(|ip| RuntimeMeSelftestIpFamilyData { + addr: ip.to_string(), + state: classify_ip(IpAddr::V4(ip)), + }); + let ip_v6 = detect_interface_ipv6().map(|ip| RuntimeMeSelftestIpFamilyData { + addr: ip.to_string(), + state: classify_ip(IpAddr::V6(ip)), + }); + + let pid = std::process::id(); + let pid_state = if pid == 1 { "one" } else { "non-one" }; + + let bnd = bnd_snapshot(); + + RuntimeMeSelftestData { + enabled: true, + reason: None, + generated_at_epoch_secs: now_epoch_secs, + data: Some(RuntimeMeSelftestPayload { + kdf: RuntimeMeSelftestKdfData { + state: kdf_state, + ewma_errors_per_min: round3(kdf_ewma), + threshold_errors_per_min: KDF_EWMA_THRESHOLD_ERRORS_PER_MIN, + errors_total: kdf_errors_total, + }, + timeskew: RuntimeMeSelftestTimeskewData { + state: timeskew_state, + max_skew_secs_15m: skew.max_skew_secs_15m, + samples_15m: skew.samples_15m, + last_skew_secs: skew.last_skew_secs, + last_source: skew.last_source, + last_seen_age_secs: skew.last_seen_age_secs, + }, + ip: RuntimeMeSelftestIpData { + v4: ip_v4, + v6: ip_v6, + }, + pid: RuntimeMeSelftestPidData { + pid, + state: pid_state, + }, + bnd: RuntimeMeSelftestBndData { + addr_state: bnd.addr_status, + port_state: bnd.port_status, + last_addr: bnd.last_addr.map(|value| value.to_string()), + last_seen_age_secs: bnd.last_seen_age_secs, + }, + }), + } +} + +fn update_kdf_ewma(now_epoch_secs: u64, total_errors: u64) -> f64 { + let Ok(mut guard) = kdf_ewma_state().lock() else { + return 0.0; + }; + + if !guard.initialized { + guard.initialized = true; + guard.last_epoch_secs = now_epoch_secs; + guard.last_total_errors = total_errors; + guard.ewma_errors_per_min = 0.0; + return guard.ewma_errors_per_min; + } + + let dt_secs = now_epoch_secs.saturating_sub(guard.last_epoch_secs); + if dt_secs == 0 { + return guard.ewma_errors_per_min; + } + + let delta_errors = total_errors.saturating_sub(guard.last_total_errors); + let instant_rate_per_min = (delta_errors as f64) * 60.0 / (dt_secs as f64); + let alpha = 1.0 - f64::exp(-(dt_secs as f64) / KDF_EWMA_TAU_SECS); + guard.ewma_errors_per_min = guard.ewma_errors_per_min + + alpha * (instant_rate_per_min - guard.ewma_errors_per_min); + guard.last_epoch_secs = now_epoch_secs; + guard.last_total_errors = total_errors; + guard.ewma_errors_per_min +} + +fn classify_ip(ip: IpAddr) -> &'static str { + if ip.is_loopback() { + return "loopback"; + } + if is_bogon(ip) { + return "bogon"; + } + "good" +} + +fn round3(value: f64) -> f64 { + (value * 1000.0).round() / 1000.0 +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} diff --git a/src/network/probe.rs b/src/network/probe.rs index 2ceeb2c..63e23a1 100644 --- a/src/network/probe.rs +++ b/src/network/probe.rs @@ -280,6 +280,14 @@ fn detect_local_ip_v6() -> Option { } } +pub fn detect_interface_ipv4() -> Option { + detect_local_ip_v4() +} + +pub fn detect_interface_ipv6() -> Option { + detect_local_ip_v6() +} + pub fn is_bogon(ip: IpAddr) -> bool { match ip { IpAddr::V4(v4) => is_bogon_v4(v4), diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 2c6a07a..194da5b 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -15,6 +15,7 @@ use crate::error::Result; use super::MePool; use super::rotation::{MeReinitTrigger, enqueue_reinit_trigger}; use super::secret::download_proxy_secret_with_max_len; +use super::selftest::record_timeskew_sample; use std::time::SystemTime; async fn retry_fetch(url: &str) -> Option { @@ -109,6 +110,7 @@ pub async fn fetch_proxy_config_with_raw(url: &str) -> Result<(ProxyConfigData, }) { let skew_secs = skew.as_secs(); + record_timeskew_sample("proxy_config_date_header", skew_secs); if skew_secs > 60 { warn!(skew_secs, "Time skew >60s detected from fetch_proxy_config Date header"); } else if skew_secs > 30 { diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index f556b99..1c1b172 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -33,6 +33,7 @@ use super::codec::{ cbc_decrypt_inplace, cbc_encrypt_padded, parse_handshake_flags, parse_nonce_payload, read_rpc_frame_plaintext, rpc_crc, }; +use super::selftest::{BndAddrStatus, BndPortStatus, record_bnd_status}; use super::wire::{extract_ip_material, IpMaterial}; use super::MePool; @@ -131,6 +132,14 @@ impl MePool { ) } + fn bnd_port_status(bound: Option) -> BndPortStatus { + match bound { + Some(addr) if addr.port() == 0 => BndPortStatus::Zero, + Some(_) => BndPortStatus::Ok, + None => BndPortStatus::Error, + } + } + /// TCP connect with timeout + return RTT in milliseconds. pub(crate) async fn connect_tcp( &self, @@ -239,7 +248,27 @@ impl MePool { IpFamily::V6 }; let is_socks_route = Self::is_socks_route(upstream_egress); + let raw_socks_bound_addr = if is_socks_route { + upstream_egress.and_then(|info| info.socks_bound_addr) + } else { + None + }; let socks_bound_addr = Self::select_socks_bound_addr(family, upstream_egress); + let bnd_addr_status = if !is_socks_route { + BndAddrStatus::Error + } else if raw_socks_bound_addr.is_some() && socks_bound_addr.is_none() { + BndAddrStatus::Bogon + } else if socks_bound_addr.is_some() { + BndAddrStatus::Ok + } else { + BndAddrStatus::Error + }; + let bnd_port_status = if is_socks_route { + Self::bnd_port_status(raw_socks_bound_addr) + } else { + BndPortStatus::Error + }; + record_bnd_status(bnd_addr_status, bnd_port_status, raw_socks_bound_addr); let reflected = if let Some(bound) = socks_bound_addr { Some(bound) } else if is_socks_route { diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 87c012b..ecc963d 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -18,6 +18,7 @@ mod registry; mod rotation; mod send; mod secret; +mod selftest; mod wire; mod pool_status; @@ -37,6 +38,7 @@ pub use config_updater::{ me_config_updater, save_proxy_config_cache, }; pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task}; +pub(crate) use selftest::{bnd_snapshot, timeskew_snapshot}; pub use wire::proto_flags_for_tag; #[derive(Debug)] diff --git a/src/transport/middle_proxy/secret.rs b/src/transport/middle_proxy/secret.rs index 4991d32..b2cfcb6 100644 --- a/src/transport/middle_proxy/secret.rs +++ b/src/transport/middle_proxy/secret.rs @@ -3,6 +3,7 @@ use std::time::SystemTime; use httpdate; use crate::error::{ProxyError, Result}; +use super::selftest::record_timeskew_sample; pub const PROXY_SECRET_MIN_LEN: usize = 32; @@ -98,6 +99,7 @@ pub async fn download_proxy_secret_with_max_len(max_len: usize) -> Result 60 { warn!(skew_secs, "Time skew >60s detected from proxy-secret Date header"); } else if skew_secs > 30 { diff --git a/src/transport/middle_proxy/selftest.rs b/src/transport/middle_proxy/selftest.rs new file mode 100644 index 0000000..c1653ec --- /dev/null +++ b/src/transport/middle_proxy/selftest.rs @@ -0,0 +1,194 @@ +use std::collections::VecDeque; +use std::net::SocketAddr; +use std::sync::{Mutex, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BndAddrStatus { + Ok, + Bogon, + Error, +} + +impl BndAddrStatus { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Ok => "ok", + Self::Bogon => "bogon", + Self::Error => "error", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BndPortStatus { + Ok, + Zero, + Error, +} + +impl BndPortStatus { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Ok => "ok", + Self::Zero => "zero", + Self::Error => "error", + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct MeBndSnapshot { + pub addr_status: &'static str, + pub port_status: &'static str, + pub last_addr: Option, + pub last_seen_age_secs: Option, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct MeTimeskewSnapshot { + pub max_skew_secs_15m: Option, + pub samples_15m: usize, + pub last_skew_secs: Option, + pub last_source: Option<&'static str>, + pub last_seen_age_secs: Option, +} + +#[derive(Clone, Copy, Debug)] +struct MeTimeskewSample { + ts_epoch_secs: u64, + skew_secs: u64, + source: &'static str, +} + +#[derive(Debug)] +struct MeSelftestState { + bnd_addr_status: BndAddrStatus, + bnd_port_status: BndPortStatus, + bnd_last_addr: Option, + bnd_last_seen_epoch_secs: Option, + timeskew_samples: VecDeque, +} + +impl Default for MeSelftestState { + fn default() -> Self { + Self { + bnd_addr_status: BndAddrStatus::Error, + bnd_port_status: BndPortStatus::Error, + bnd_last_addr: None, + bnd_last_seen_epoch_secs: None, + timeskew_samples: VecDeque::new(), + } + } +} + +const MAX_TIMESKEW_SAMPLES: usize = 512; +const TIMESKEW_WINDOW_SECS: u64 = 15 * 60; + +static ME_SELFTEST_STATE: OnceLock> = OnceLock::new(); + +fn state() -> &'static Mutex { + ME_SELFTEST_STATE.get_or_init(|| Mutex::new(MeSelftestState::default())) +} + +pub(crate) fn record_bnd_status( + addr_status: BndAddrStatus, + port_status: BndPortStatus, + last_addr: Option, +) { + let now_epoch_secs = now_epoch_secs(); + let Ok(mut guard) = state().lock() else { + return; + }; + guard.bnd_addr_status = addr_status; + guard.bnd_port_status = port_status; + guard.bnd_last_addr = last_addr; + guard.bnd_last_seen_epoch_secs = Some(now_epoch_secs); +} + +pub(crate) fn bnd_snapshot() -> MeBndSnapshot { + let now_epoch_secs = now_epoch_secs(); + let Ok(guard) = state().lock() else { + return MeBndSnapshot { + addr_status: BndAddrStatus::Error.as_str(), + port_status: BndPortStatus::Error.as_str(), + last_addr: None, + last_seen_age_secs: None, + }; + }; + MeBndSnapshot { + addr_status: guard.bnd_addr_status.as_str(), + port_status: guard.bnd_port_status.as_str(), + last_addr: guard.bnd_last_addr, + last_seen_age_secs: guard + .bnd_last_seen_epoch_secs + .map(|value| now_epoch_secs.saturating_sub(value)), + } +} + +pub(crate) fn record_timeskew_sample(source: &'static str, skew_secs: u64) { + let now_epoch_secs = now_epoch_secs(); + let Ok(mut guard) = state().lock() else { + return; + }; + guard.timeskew_samples.push_back(MeTimeskewSample { + ts_epoch_secs: now_epoch_secs, + skew_secs, + source, + }); + while guard.timeskew_samples.len() > MAX_TIMESKEW_SAMPLES { + guard.timeskew_samples.pop_front(); + } + let cutoff = now_epoch_secs.saturating_sub(TIMESKEW_WINDOW_SECS * 2); + while guard + .timeskew_samples + .front() + .is_some_and(|sample| sample.ts_epoch_secs < cutoff) + { + guard.timeskew_samples.pop_front(); + } +} + +pub(crate) fn timeskew_snapshot() -> MeTimeskewSnapshot { + let now_epoch_secs = now_epoch_secs(); + let Ok(guard) = state().lock() else { + return MeTimeskewSnapshot::default(); + }; + + let mut max_skew_secs_15m = None; + let mut samples_15m = 0usize; + let window_start = now_epoch_secs.saturating_sub(TIMESKEW_WINDOW_SECS); + for sample in &guard.timeskew_samples { + if sample.ts_epoch_secs < window_start { + continue; + } + samples_15m = samples_15m.saturating_add(1); + max_skew_secs_15m = Some(max_skew_secs_15m.unwrap_or(0).max(sample.skew_secs)); + } + + let (last_skew_secs, last_source, last_seen_age_secs) = + if let Some(last) = guard.timeskew_samples.back() { + ( + Some(last.skew_secs), + Some(last.source), + Some(now_epoch_secs.saturating_sub(last.ts_epoch_secs)), + ) + } else { + (None, None, None) + }; + + MeTimeskewSnapshot { + max_skew_secs_15m, + samples_15m, + last_skew_secs, + last_source, + last_seen_age_secs, + } +} + +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} From 100ef0fa28dd9be4aef296335b006f22a1bf7205 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 9 Mar 2026 23:37:29 +0300 Subject: [PATCH 162/261] Correct IP:port/public-host:public-port in API --- src/api/mod.rs | 23 ++++++++------- src/api/users.rs | 72 +++++++++++++++++++++++++++------------------- src/maestro/mod.rs | 11 +++++-- 3 files changed, 63 insertions(+), 43 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 0a51231..ff9d2f9 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -75,8 +75,7 @@ pub(super) struct ApiShared { pub(super) me_pool: Arc>>>, pub(super) upstream_manager: Arc, pub(super) config_path: PathBuf, - pub(super) startup_detected_ip_v4: Option, - pub(super) startup_detected_ip_v6: Option, + pub(super) detected_ips_rx: watch::Receiver<(Option, Option)>, pub(super) mutation_lock: Arc>, pub(super) minimal_cache: Arc>>, pub(super) runtime_edge_connections_cache: Arc>>, @@ -91,6 +90,10 @@ impl ApiShared { fn next_request_id(&self) -> u64 { self.request_id.fetch_add(1, Ordering::Relaxed) } + + fn detected_link_ips(&self) -> (Option, Option) { + *self.detected_ips_rx.borrow() + } } pub async fn serve( @@ -102,8 +105,7 @@ pub async fn serve( config_rx: watch::Receiver>, admission_rx: watch::Receiver, config_path: PathBuf, - startup_detected_ip_v4: Option, - startup_detected_ip_v6: Option, + detected_ips_rx: watch::Receiver<(Option, Option)>, process_started_at_epoch_secs: u64, startup_tracker: Arc, ) { @@ -134,8 +136,7 @@ pub async fn serve( me_pool, upstream_manager, config_path, - startup_detected_ip_v4, - startup_detected_ip_v6, + detected_ips_rx, mutation_lock: Arc::new(Mutex::new(())), minimal_cache: Arc::new(Mutex::new(None)), runtime_edge_connections_cache: Arc::new(Mutex::new(None)), @@ -356,12 +357,13 @@ async fn handle( } ("GET", "/v1/stats/users") | ("GET", "/v1/users") => { let revision = current_revision(&shared.config_path).await?; + let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); let users = users_from_config( &cfg, &shared.stats, &shared.ip_tracker, - shared.startup_detected_ip_v4, - shared.startup_detected_ip_v6, + detected_ip_v4, + detected_ip_v6, ) .await; Ok(success_response(StatusCode::OK, users, revision)) @@ -399,12 +401,13 @@ async fn handle( { if method == Method::GET { let revision = current_revision(&shared.config_path).await?; + let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); let users = users_from_config( &cfg, &shared.stats, &shared.ip_tracker, - shared.startup_detected_ip_v4, - shared.startup_detected_ip_v6, + detected_ip_v4, + detected_ip_v6, ) .await; if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) diff --git a/src/api/users.rs b/src/api/users.rs index d156896..7265044 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -90,13 +90,14 @@ pub(super) async fn create_user( if let Some(limit) = updated_limit { shared.ip_tracker.set_user_limit(&body.username, limit).await; } + let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); let users = users_from_config( &cfg, &shared.stats, &shared.ip_tracker, - shared.startup_detected_ip_v4, - shared.startup_detected_ip_v6, + detected_ip_v4, + detected_ip_v6, ) .await; let user = users @@ -118,8 +119,8 @@ pub(super) async fn create_user( links: build_user_links( &cfg, &secret, - shared.startup_detected_ip_v4, - shared.startup_detected_ip_v6, + detected_ip_v4, + detected_ip_v6, ), }); @@ -185,12 +186,13 @@ pub(super) async fn patch_user( if let Some(limit) = updated_limit { shared.ip_tracker.set_user_limit(user, limit).await; } + let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); let users = users_from_config( &cfg, &shared.stats, &shared.ip_tracker, - shared.startup_detected_ip_v4, - shared.startup_detected_ip_v6, + detected_ip_v4, + detected_ip_v6, ) .await; let user_info = users @@ -232,12 +234,13 @@ pub(super) async fn rotate_secret( let revision = save_config_to_disk(&shared.config_path, &cfg).await?; drop(_guard); + let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); let users = users_from_config( &cfg, &shared.stats, &shared.ip_tracker, - shared.startup_detected_ip_v4, - shared.startup_detected_ip_v6, + detected_ip_v4, + detected_ip_v6, ) .await; let user_info = users @@ -418,17 +421,6 @@ fn resolve_link_hosts( return vec![host.to_string()]; } - let mut startup_hosts = Vec::new(); - if let Some(ip) = startup_detected_ip_v4 { - push_unique_host(&mut startup_hosts, &ip.to_string()); - } - if let Some(ip) = startup_detected_ip_v6 { - push_unique_host(&mut startup_hosts, &ip.to_string()); - } - if !startup_hosts.is_empty() { - return startup_hosts; - } - let mut hosts = Vec::new(); for listener in &cfg.server.listeners { if let Some(host) = listener @@ -443,24 +435,44 @@ fn resolve_link_hosts( if let Some(ip) = listener.announce_ip { if !ip.is_unspecified() { push_unique_host(&mut hosts, &ip.to_string()); + continue; + } + } + if listener.ip.is_unspecified() { + let detected_ip = if listener.ip.is_ipv4() { + startup_detected_ip_v4 + } else { + startup_detected_ip_v6 + }; + if let Some(ip) = detected_ip { + push_unique_host(&mut hosts, &ip.to_string()); + } else { + push_unique_host(&mut hosts, &listener.ip.to_string()); } continue; } - if !listener.ip.is_unspecified() { - push_unique_host(&mut hosts, &listener.ip.to_string()); - } + push_unique_host(&mut hosts, &listener.ip.to_string()); } - if hosts.is_empty() { - if let Some(host) = cfg.server.listen_addr_ipv4.as_deref() { - push_host_from_legacy_listen(&mut hosts, host); - } - if let Some(host) = cfg.server.listen_addr_ipv6.as_deref() { - push_host_from_legacy_listen(&mut hosts, host); - } + if !hosts.is_empty() { + return hosts; } - hosts + if let Some(ip) = startup_detected_ip_v4.or(startup_detected_ip_v6) { + return vec![ip.to_string()]; + } + + if let Some(host) = cfg.server.listen_addr_ipv4.as_deref() { + push_host_from_legacy_listen(&mut hosts, host); + } + if let Some(host) = cfg.server.listen_addr_ipv6.as_deref() { + push_host_from_legacy_listen(&mut hosts, host); + } + if !hosts.is_empty() { + return hosts; + } + + vec!["UNKNOWN".to_string()] } fn push_host_from_legacy_listen(hosts: &mut Vec, raw: &str) { diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index 92b42e3..fcb4d3f 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -20,7 +20,7 @@ mod runtime_tasks; mod shutdown; mod tls_bootstrap; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::{RwLock, Semaphore, watch}; @@ -189,6 +189,7 @@ pub async fn run() -> std::result::Result<(), Box> { } let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone())); + let (detected_ips_tx, detected_ips_rx) = watch::channel((None::, None::)); let initial_admission_open = !config.general.use_middle_proxy; let (admission_tx, admission_rx) = watch::channel(initial_admission_open); let initial_route_mode = if config.general.use_middle_proxy { @@ -223,6 +224,7 @@ pub async fn run() -> std::result::Result<(), Box> { let admission_rx_api = admission_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); let startup_tracker_api = startup_tracker.clone(); + let detected_ips_rx_api = detected_ips_rx.clone(); tokio::spawn(async move { api::serve( listen, @@ -233,8 +235,7 @@ pub async fn run() -> std::result::Result<(), Box> { config_rx_api, admission_rx_api, config_path_api, - None, - None, + detected_ips_rx_api, process_started_at_epoch_secs, startup_tracker_api, ) @@ -288,6 +289,10 @@ pub async fn run() -> std::result::Result<(), Box> { config.general.stun_nat_probe_concurrency, ) .await?; + detected_ips_tx.send_replace(( + probe.detected_ipv4.map(IpAddr::V4), + probe.detected_ipv6.map(IpAddr::V6), + )); let decision = decide_network_capabilities(&config.network, &probe); log_probe_result(&probe, &decision); startup_tracker From 161af515587010322b5ebdc98203122ed871e2bc Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:02:39 +0300 Subject: [PATCH 163/261] User Management in API --- src/api/config_store.rs | 162 ++++++++++++++++++++++++++ src/api/users.rs | 38 ++++++- src/maestro/admission.rs | 16 ++- src/maestro/me_startup.rs | 232 ++++++++++++++++++++++++++------------ src/metrics.rs | 93 +++++++++++++-- 5 files changed, 457 insertions(+), 84 deletions(-) diff --git a/src/api/config_store.rs b/src/api/config_store.rs index e7fbbca..f0da554 100644 --- a/src/api/config_store.rs +++ b/src/api/config_store.rs @@ -1,13 +1,39 @@ +use std::collections::BTreeMap; use std::io::Write; use std::path::{Path, PathBuf}; +use chrono::{DateTime, Utc}; use hyper::header::IF_MATCH; +use serde::Serialize; use sha2::{Digest, Sha256}; use crate::config::ProxyConfig; use super::model::ApiFailure; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum AccessSection { + Users, + UserAdTags, + UserMaxTcpConns, + UserExpirations, + UserDataQuota, + UserMaxUniqueIps, +} + +impl AccessSection { + fn table_name(self) -> &'static str { + match self { + Self::Users => "access.users", + Self::UserAdTags => "access.user_ad_tags", + Self::UserMaxTcpConns => "access.user_max_tcp_conns", + Self::UserExpirations => "access.user_expirations", + Self::UserDataQuota => "access.user_data_quota", + Self::UserMaxUniqueIps => "access.user_max_unique_ips", + } + } +} + pub(super) fn parse_if_match(headers: &hyper::HeaderMap) -> Option { headers .get(IF_MATCH) @@ -66,6 +92,142 @@ pub(super) async fn save_config_to_disk( Ok(compute_revision(&serialized)) } +pub(super) async fn save_access_sections_to_disk( + config_path: &Path, + cfg: &ProxyConfig, + sections: &[AccessSection], +) -> Result { + let mut content = tokio::fs::read_to_string(config_path) + .await + .map_err(|e| ApiFailure::internal(format!("failed to read config: {}", e)))?; + + let mut applied = Vec::new(); + for section in sections { + if applied.contains(section) { + continue; + } + let rendered = render_access_section(cfg, *section)?; + content = upsert_toml_table(&content, section.table_name(), &rendered); + applied.push(*section); + } + + write_atomic(config_path.to_path_buf(), content.clone()).await?; + Ok(compute_revision(&content)) +} + +fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result { + let body = match section { + AccessSection::Users => { + let rows: BTreeMap = cfg + .access + .users + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + serialize_table_body(&rows)? + } + AccessSection::UserAdTags => { + let rows: BTreeMap = cfg + .access + .user_ad_tags + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + serialize_table_body(&rows)? + } + AccessSection::UserMaxTcpConns => { + let rows: BTreeMap = cfg + .access + .user_max_tcp_conns + .iter() + .map(|(key, value)| (key.clone(), *value)) + .collect(); + serialize_table_body(&rows)? + } + AccessSection::UserExpirations => { + let rows: BTreeMap> = cfg + .access + .user_expirations + .iter() + .map(|(key, value)| (key.clone(), *value)) + .collect(); + serialize_table_body(&rows)? + } + AccessSection::UserDataQuota => { + let rows: BTreeMap = cfg + .access + .user_data_quota + .iter() + .map(|(key, value)| (key.clone(), *value)) + .collect(); + serialize_table_body(&rows)? + } + AccessSection::UserMaxUniqueIps => { + let rows: BTreeMap = cfg + .access + .user_max_unique_ips + .iter() + .map(|(key, value)| (key.clone(), *value)) + .collect(); + serialize_table_body(&rows)? + } + }; + + let mut out = format!("[{}]\n", section.table_name()); + if !body.is_empty() { + out.push_str(&body); + } + if !out.ends_with('\n') { + out.push('\n'); + } + Ok(out) +} + +fn serialize_table_body(value: &T) -> Result { + toml::to_string(value) + .map_err(|e| ApiFailure::internal(format!("failed to serialize access section: {}", e))) +} + +fn upsert_toml_table(source: &str, table_name: &str, replacement: &str) -> String { + if let Some((start, end)) = find_toml_table_bounds(source, table_name) { + let mut out = String::with_capacity(source.len() + replacement.len()); + out.push_str(&source[..start]); + out.push_str(replacement); + out.push_str(&source[end..]); + return out; + } + + let mut out = source.to_string(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + if !out.is_empty() { + out.push('\n'); + } + out.push_str(replacement); + out +} + +fn find_toml_table_bounds(source: &str, table_name: &str) -> Option<(usize, usize)> { + let target = format!("[{}]", table_name); + let mut offset = 0usize; + let mut start = None; + + for line in source.split_inclusive('\n') { + let trimmed = line.trim(); + if let Some(start_offset) = start { + if trimmed.starts_with('[') { + return Some((start_offset, offset)); + } + } else if trimmed == target { + start = Some(offset); + } + offset = offset.saturating_add(line.len()); + } + + start.map(|start_offset| (start_offset, source.len())) +} + async fn write_atomic(path: PathBuf, contents: String) -> Result<(), ApiFailure> { tokio::task::spawn_blocking(move || write_atomic_sync(&path, &contents)) .await diff --git a/src/api/users.rs b/src/api/users.rs index 7265044..d98790d 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -8,7 +8,8 @@ use crate::stats::Stats; use super::ApiShared; use super::config_store::{ - ensure_expected_revision, load_config_from_disk, save_config_to_disk, + AccessSection, ensure_expected_revision, load_config_from_disk, save_access_sections_to_disk, + save_config_to_disk, }; use super::model::{ ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest, @@ -21,6 +22,12 @@ pub(super) async fn create_user( expected_revision: Option, shared: &ApiShared, ) -> Result<(CreateUserResponse, String), ApiFailure> { + let touches_user_ad_tags = body.user_ad_tag.is_some(); + let touches_user_max_tcp_conns = body.max_tcp_conns.is_some(); + let touches_user_expirations = body.expiration_rfc3339.is_some(); + let touches_user_data_quota = body.data_quota_bytes.is_some(); + let touches_user_max_unique_ips = body.max_unique_ips.is_some(); + if !is_valid_username(&body.username) { return Err(ApiFailure::bad_request( "username must match [A-Za-z0-9_.-] and be 1..64 chars", @@ -84,7 +91,24 @@ pub(super) async fn create_user( cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; - let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + let mut touched_sections = vec![AccessSection::Users]; + if touches_user_ad_tags { + touched_sections.push(AccessSection::UserAdTags); + } + if touches_user_max_tcp_conns { + touched_sections.push(AccessSection::UserMaxTcpConns); + } + if touches_user_expirations { + touched_sections.push(AccessSection::UserExpirations); + } + if touches_user_data_quota { + touched_sections.push(AccessSection::UserDataQuota); + } + if touches_user_max_unique_ips { + touched_sections.push(AccessSection::UserMaxUniqueIps); + } + + let revision = save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?; drop(_guard); if let Some(limit) = updated_limit { @@ -231,7 +255,15 @@ pub(super) async fn rotate_secret( cfg.access.users.insert(user.to_string(), secret.clone()); cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; - let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + let touched_sections = [ + AccessSection::Users, + AccessSection::UserAdTags, + AccessSection::UserMaxTcpConns, + AccessSection::UserExpirations, + AccessSection::UserDataQuota, + AccessSection::UserMaxUniqueIps, + ]; + let revision = save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?; drop(_guard); let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips(); diff --git a/src/maestro/admission.rs b/src/maestro/admission.rs index bacd4d4..69a9c9f 100644 --- a/src/maestro/admission.rs +++ b/src/maestro/admission.rs @@ -8,6 +8,9 @@ use crate::config::ProxyConfig; use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController}; use crate::transport::middle_proxy::MePool; +const STARTUP_FALLBACK_AFTER: Duration = Duration::from_secs(80); +const RUNTIME_FALLBACK_AFTER: Duration = Duration::from_secs(6); + pub(crate) async fn configure_admission_gate( config: &Arc, me_pool: Option>, @@ -17,7 +20,6 @@ pub(crate) async fn configure_admission_gate( ) { if config.general.use_middle_proxy { if let Some(pool) = me_pool.as_ref() { - let fallback_after = Duration::from_secs(6); let initial_ready = pool.admission_ready_conditional_cast().await; admission_tx.send_replace(initial_ready); let _ = route_runtime.set_mode(RelayRouteMode::Middle); @@ -36,6 +38,7 @@ pub(crate) async fn configure_admission_gate( tokio::spawn(async move { let mut gate_open = initial_ready; let mut route_mode = RelayRouteMode::Middle; + let mut ready_observed = initial_ready; let mut not_ready_since = if initial_ready { None } else { @@ -57,11 +60,17 @@ pub(crate) async fn configure_admission_gate( let ready = pool_for_gate.admission_ready_conditional_cast().await; let now = Instant::now(); let (next_gate_open, next_route_mode, next_fallback_active) = if ready { + ready_observed = true; not_ready_since = None; (true, RelayRouteMode::Middle, false) } else { let not_ready_started_at = *not_ready_since.get_or_insert(now); let not_ready_for = now.saturating_duration_since(not_ready_started_at); + let fallback_after = if ready_observed { + RUNTIME_FALLBACK_AFTER + } else { + STARTUP_FALLBACK_AFTER + }; if fallback_enabled && not_ready_for > fallback_after { (true, RelayRouteMode::Direct, true) } else { @@ -79,6 +88,11 @@ pub(crate) async fn configure_admission_gate( "Middle-End routing restored for new sessions" ); } else { + let fallback_after = if ready_observed { + RUNTIME_FALLBACK_AFTER + } else { + STARTUP_FALLBACK_AFTER + }; warn!( target_mode = route_mode.as_str(), cutover_generation = snapshot.generation, diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs index 9c674cd..72fdd40 100644 --- a/src/maestro/me_startup.rs +++ b/src/maestro/me_startup.rs @@ -281,88 +281,178 @@ pub(crate) async fn initialize_me_pool( .set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_POOL_INIT_STAGE1) .await; - let mut init_attempt: u32 = 0; - loop { - init_attempt = init_attempt.saturating_add(1); - startup_tracker.set_me_init_attempt(init_attempt).await; - match pool.init(pool_size, &rng).await { - Ok(()) => { - startup_tracker.set_me_last_error(None).await; - startup_tracker - .complete_component( - COMPONENT_ME_POOL_INIT_STAGE1, - Some("ME pool initialized".to_string()), - ) - .await; - startup_tracker - .set_me_status(StartupMeStatus::Ready, "ready") - .await; - info!( - attempt = init_attempt, - "Middle-End pool initialized successfully" - ); + if me2dc_fallback { + let pool_bg = pool.clone(); + let rng_bg = rng.clone(); + let startup_tracker_bg = startup_tracker.clone(); + let retry_limit = if me_init_retry_attempts == 0 { + String::from("unlimited") + } else { + me_init_retry_attempts.to_string() + }; + std::thread::spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + error!(error = %error, "Failed to build background runtime for ME initialization"); + return; + } + }; + runtime.block_on(async move { + let mut init_attempt: u32 = 0; + loop { + init_attempt = init_attempt.saturating_add(1); + startup_tracker_bg.set_me_init_attempt(init_attempt).await; + match pool_bg.init(pool_size, &rng_bg).await { + Ok(()) => { + startup_tracker_bg.set_me_last_error(None).await; + startup_tracker_bg + .complete_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME pool initialized".to_string()), + ) + .await; + startup_tracker_bg + .set_me_status(StartupMeStatus::Ready, "ready") + .await; + info!( + attempt = init_attempt, + "Middle-End pool initialized successfully" + ); - // Phase 4: Start health monitor - let pool_clone = pool.clone(); - let rng_clone = rng.clone(); - let min_conns = pool_size; - tokio::spawn(async move { - crate::transport::middle_proxy::me_health_monitor( - pool_clone, rng_clone, min_conns, - ) - .await; - }); - - break Some(pool); - } - Err(e) => { - startup_tracker.set_me_last_error(Some(e.to_string())).await; - let retries_limited = me2dc_fallback && me_init_retry_attempts > 0; - if retries_limited && init_attempt >= me_init_retry_attempts { + let pool_health = pool_bg.clone(); + let rng_health = rng_bg.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + pool_health, + rng_health, + min_conns, + ) + .await; + }); + break; + } + Err(e) => { + startup_tracker_bg.set_me_last_error(Some(e.to_string())).await; + if init_attempt >= me_init_warn_after_attempts { + warn!( + error = %e, + attempt = init_attempt, + retry_limit = %retry_limit, + retry_in_secs = 2, + "ME pool is not ready yet; retrying background initialization" + ); + } else { + info!( + error = %e, + attempt = init_attempt, + retry_limit = %retry_limit, + retry_in_secs = 2, + "ME pool startup warmup: retrying background initialization" + ); + } + pool_bg.reset_stun_state(); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } + }); + }); + startup_tracker + .set_me_status(StartupMeStatus::Initializing, "background_init") + .await; + info!( + startup_grace_secs = 80, + "ME pool initialization continues in background; startup continues with conditional Direct fallback" + ); + Some(pool) + } else { + let mut init_attempt: u32 = 0; + loop { + init_attempt = init_attempt.saturating_add(1); + startup_tracker.set_me_init_attempt(init_attempt).await; + match pool.init(pool_size, &rng).await { + Ok(()) => { + startup_tracker.set_me_last_error(None).await; startup_tracker - .fail_component( + .complete_component( COMPONENT_ME_POOL_INIT_STAGE1, - Some("ME init retry budget exhausted".to_string()), + Some("ME pool initialized".to_string()), ) .await; startup_tracker - .set_me_status(StartupMeStatus::Failed, "failed") + .set_me_status(StartupMeStatus::Ready, "ready") .await; - error!( - error = %e, - attempt = init_attempt, - retry_limit = me_init_retry_attempts, - "ME pool init retries exhausted; falling back to direct mode" - ); - break None; - } - - let retry_limit = if !me2dc_fallback || me_init_retry_attempts == 0 { - String::from("unlimited") - } else { - me_init_retry_attempts.to_string() - }; - if init_attempt >= me_init_warn_after_attempts { - warn!( - error = %e, - attempt = init_attempt, - retry_limit = retry_limit, - me2dc_fallback = me2dc_fallback, - retry_in_secs = 2, - "ME pool is not ready yet; retrying startup initialization" - ); - } else { info!( - error = %e, attempt = init_attempt, - retry_limit = retry_limit, - me2dc_fallback = me2dc_fallback, - retry_in_secs = 2, - "ME pool startup warmup: retrying initialization" + "Middle-End pool initialized successfully" ); + + let pool_clone = pool.clone(); + let rng_clone = rng.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + pool_clone, rng_clone, min_conns, + ) + .await; + }); + + break Some(pool); + } + Err(e) => { + startup_tracker.set_me_last_error(Some(e.to_string())).await; + let retries_limited = me_init_retry_attempts > 0; + if retries_limited && init_attempt >= me_init_retry_attempts { + startup_tracker + .fail_component( + COMPONENT_ME_POOL_INIT_STAGE1, + Some("ME init retry budget exhausted".to_string()), + ) + .await; + startup_tracker + .set_me_status(StartupMeStatus::Failed, "failed") + .await; + error!( + error = %e, + attempt = init_attempt, + retry_limit = me_init_retry_attempts, + "ME pool init retries exhausted; startup cannot continue in middle-proxy mode" + ); + break None; + } + + let retry_limit = if me_init_retry_attempts == 0 { + String::from("unlimited") + } else { + me_init_retry_attempts.to_string() + }; + if init_attempt >= me_init_warn_after_attempts { + warn!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool is not ready yet; retrying startup initialization" + ); + } else { + info!( + error = %e, + attempt = init_attempt, + retry_limit = retry_limit, + me2dc_fallback = me2dc_fallback, + retry_in_secs = 2, + "ME pool startup warmup: retrying initialization" + ); + } + pool.reset_stun_state(); + tokio::time::sleep(Duration::from_secs(2)).await; } - pool.reset_stun_state(); - tokio::time::sleep(Duration::from_secs(2)).await; } } } diff --git a/src/metrics.rs b/src/metrics.rs index 917c9b3..c24dc54 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -17,6 +17,7 @@ use crate::config::ProxyConfig; use crate::ip_tracker::UserIpTracker; use crate::stats::beobachten::BeobachtenStore; use crate::stats::Stats; +use crate::transport::{ListenOptions, create_listener}; pub async fn serve( port: u16, @@ -26,16 +27,90 @@ pub async fn serve( config_rx: tokio::sync::watch::Receiver>, whitelist: Vec, ) { - let addr = SocketAddr::from(([0, 0, 0, 0], port)); - let listener = match TcpListener::bind(addr).await { - Ok(l) => l, - Err(e) => { - warn!(error = %e, "Failed to bind metrics on {}", addr); - return; - } - }; - info!("Metrics endpoint: http://{}/metrics and /beobachten", addr); + let whitelist = Arc::new(whitelist); + let mut listener_v4 = None; + let mut listener_v6 = None; + let addr_v4 = SocketAddr::from(([0, 0, 0, 0], port)); + match bind_metrics_listener(addr_v4, false) { + Ok(listener) => { + info!("Metrics endpoint: http://{}/metrics and /beobachten", addr_v4); + listener_v4 = Some(listener); + } + Err(e) => { + warn!(error = %e, "Failed to bind metrics on {}", addr_v4); + } + } + + let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], port)); + match bind_metrics_listener(addr_v6, true) { + Ok(listener) => { + info!("Metrics endpoint: http://[::]:{}/metrics and /beobachten", port); + listener_v6 = Some(listener); + } + Err(e) => { + warn!(error = %e, "Failed to bind metrics on {}", addr_v6); + } + } + + match (listener_v4, listener_v6) { + (None, None) => { + warn!("Metrics listener is unavailable on both IPv4 and IPv6"); + } + (Some(listener), None) | (None, Some(listener)) => { + serve_listener( + listener, stats, beobachten, ip_tracker, config_rx, whitelist, + ) + .await; + } + (Some(listener4), Some(listener6)) => { + let stats_v6 = stats.clone(); + let beobachten_v6 = beobachten.clone(); + let ip_tracker_v6 = ip_tracker.clone(); + let config_rx_v6 = config_rx.clone(); + let whitelist_v6 = whitelist.clone(); + tokio::spawn(async move { + serve_listener( + listener6, + stats_v6, + beobachten_v6, + ip_tracker_v6, + config_rx_v6, + whitelist_v6, + ) + .await; + }); + serve_listener( + listener4, + stats, + beobachten, + ip_tracker, + config_rx, + whitelist, + ) + .await; + } + } +} + +fn bind_metrics_listener(addr: SocketAddr, ipv6_only: bool) -> std::io::Result { + let options = ListenOptions { + reuse_port: false, + ipv6_only, + ..Default::default() + }; + let socket = create_listener(addr, &options)?; + TcpListener::from_std(socket.into()) +} + +async fn serve_listener( + listener: TcpListener, + stats: Arc, + beobachten: Arc, + ip_tracker: Arc, + config_rx: tokio::sync::watch::Receiver>, + whitelist: Arc>, +) { loop { let (stream, peer) = match listener.accept().await { Ok(v) => v, From fabb3c45f11f398bb5eb7087d8c34f47ec493d32 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:04:22 +0300 Subject: [PATCH 164/261] Runtime Selftest in API Docs --- docs/API.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/docs/API.md b/docs/API.md index c70b625..c7b9d88 100644 --- a/docs/API.md +++ b/docs/API.md @@ -99,6 +99,7 @@ Notes: | `GET` | `/v1/runtime/me_quality` | none | `200` | `RuntimeMeQualityData` | | `GET` | `/v1/runtime/upstream_quality` | none | `200` | `RuntimeUpstreamQualityData` | | `GET` | `/v1/runtime/nat_stun` | none | `200` | `RuntimeNatStunData` | +| `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` | | `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` | | `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` | | `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` | @@ -560,6 +561,67 @@ Note: the request contract is defined, but the corresponding route currently ret | `addr` | `string` | Reflected public endpoint (`ip:port`). | | `age_secs` | `u64` | Reflection value age in seconds. | +### `RuntimeMeSelftestData` +| Field | Type | Description | +| --- | --- | --- | +| `enabled` | `bool` | Runtime payload availability. | +| `reason` | `string?` | `source_unavailable` when ME pool is unavailable. | +| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. | +| `data` | `RuntimeMeSelftestPayload?` | Null when unavailable. | + +#### `RuntimeMeSelftestPayload` +| Field | Type | Description | +| --- | --- | --- | +| `kdf` | `RuntimeMeSelftestKdfData` | KDF EWMA health state. | +| `timeskew` | `RuntimeMeSelftestTimeskewData` | Date-header skew health state. | +| `ip` | `RuntimeMeSelftestIpData` | Interface IP family classification. | +| `pid` | `RuntimeMeSelftestPidData` | Process PID marker (`one|non-one`). | +| `bnd` | `RuntimeMeSelftestBndData` | SOCKS BND.ADDR/BND.PORT health state. | + +#### `RuntimeMeSelftestKdfData` +| Field | Type | Description | +| --- | --- | --- | +| `state` | `string` | `ok` or `error` based on EWMA threshold. | +| `ewma_errors_per_min` | `f64` | EWMA KDF error rate per minute. | +| `threshold_errors_per_min` | `f64` | Threshold used for `error` decision. | +| `errors_total` | `u64` | Total source errors (`kdf_drift + socks_kdf_strict_reject`). | + +#### `RuntimeMeSelftestTimeskewData` +| Field | Type | Description | +| --- | --- | --- | +| `state` | `string` | `ok` or `error` (`max_skew_secs_15m > 60` => `error`). | +| `max_skew_secs_15m` | `u64?` | Maximum observed skew in the last 15 minutes. | +| `samples_15m` | `usize` | Number of skew samples in the last 15 minutes. | +| `last_skew_secs` | `u64?` | Latest observed skew value. | +| `last_source` | `string?` | Latest skew source marker. | +| `last_seen_age_secs` | `u64?` | Age of the latest skew sample. | + +#### `RuntimeMeSelftestIpData` +| Field | Type | Description | +| --- | --- | --- | +| `v4` | `RuntimeMeSelftestIpFamilyData?` | IPv4 interface probe result; absent when unknown. | +| `v6` | `RuntimeMeSelftestIpFamilyData?` | IPv6 interface probe result; absent when unknown. | + +#### `RuntimeMeSelftestIpFamilyData` +| Field | Type | Description | +| --- | --- | --- | +| `addr` | `string` | Detected interface IP. | +| `state` | `string` | `good`, `bogon`, or `loopback`. | + +#### `RuntimeMeSelftestPidData` +| Field | Type | Description | +| --- | --- | --- | +| `pid` | `u32` | Current process PID. | +| `state` | `string` | `one` when PID=1, otherwise `non-one`. | + +#### `RuntimeMeSelftestBndData` +| Field | Type | Description | +| --- | --- | --- | +| `addr_state` | `string` | `ok`, `bogon`, or `error`. | +| `port_state` | `string` | `ok`, `zero`, or `error`. | +| `last_addr` | `string?` | Latest observed SOCKS BND address. | +| `last_seen_age_secs` | `u64?` | Age of latest BND sample. | + ### `RuntimeEdgeConnectionsSummaryData` | Field | Type | Description | | --- | --- | --- | @@ -972,10 +1034,11 @@ Note: the request contract is defined, but the corresponding route currently ret Link generation uses active config and enabled modes: - `[general.links].public_host/public_port` have priority. -- If `public_host` is not set, startup-detected public IPs are used when they are present in API runtime context. -- Fallback host sources: listener `announce`, `announce_ip`, explicit listener `ip`. -- Legacy fallback: `listen_addr_ipv4` and `listen_addr_ipv6` when routable. -- Startup-detected IP values are process-static after API task bootstrap. +- If `public_host` is not set, hosts are resolved from `server.listeners` in order: + `announce` -> `announce_ip` -> listener bind `ip`. +- For wildcard listener IPs (`0.0.0.0` / `::`), startup-detected external IP of the same family is used when available. +- If no host can be resolved from listeners, fallback is startup-detected `IPv4 -> IPv6`. +- Final compatibility fallback uses `listen_addr_ipv4`/`listen_addr_ipv6` when routable, otherwise `"UNKNOWN"`. - User rows are sorted by `username` in ascending lexical order. ### `CreateUserResponse` @@ -1020,6 +1083,7 @@ Additional runtime endpoint behavior: | `/v1/runtime/me_quality` | No | ME pool snapshot unavailable | `enabled=true`, full payload | | `/v1/runtime/upstream_quality` | No | Upstream runtime snapshot unavailable | `enabled=true`, full payload | | `/v1/runtime/nat_stun` | No | STUN shared state unavailable | `enabled=true`, full payload | +| `/v1/runtime/me-selftest` | No | ME pool unavailable => `enabled=false`, `reason=source_unavailable` | `enabled=true`, full payload | | `/v1/runtime/connections/summary` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Recompute lock contention with no cache entry => `enabled=true`, `reason=source_unavailable` | `enabled=true`, full payload | | `/v1/runtime/events/recent` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload | From 796279343eed7b2519796496d3bfce81d15c8de1 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:04:38 +0300 Subject: [PATCH 165/261] API User Deletion fixes --- src/api/users.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/api/users.rs b/src/api/users.rs index d98790d..848bb45 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -205,7 +205,15 @@ pub(super) async fn patch_user( cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; - let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + let touched_sections = [ + AccessSection::Users, + AccessSection::UserAdTags, + AccessSection::UserMaxTcpConns, + AccessSection::UserExpirations, + AccessSection::UserDataQuota, + AccessSection::UserMaxUniqueIps, + ]; + let revision = save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?; drop(_guard); if let Some(limit) = updated_limit { shared.ip_tracker.set_user_limit(user, limit).await; From d2db9b8cf93a8f4d586f840b9b030c24156aa562 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:05:38 +0300 Subject: [PATCH 166/261] Update API.md --- docs/API.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/API.md b/docs/API.md index c7b9d88..9296aff 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1033,10 +1033,13 @@ Note: the request contract is defined, but the corresponding route currently ret | `tls` | `string[]` | Active `tg://proxy` links for EE-TLS mode (for each host+TLS domain). | Link generation uses active config and enabled modes: -- `[general.links].public_host/public_port` have priority. +- Link port is `general.links.public_port` when configured; otherwise `server.port`. +- If `general.links.public_host` is non-empty, it is used as the single link host override. - If `public_host` is not set, hosts are resolved from `server.listeners` in order: `announce` -> `announce_ip` -> listener bind `ip`. - For wildcard listener IPs (`0.0.0.0` / `::`), startup-detected external IP of the same family is used when available. +- Listener-derived hosts are de-duplicated while preserving first-seen order. +- If multiple hosts are resolved, API returns links for all resolved hosts in every enabled mode. - If no host can be resolved from listeners, fallback is startup-detected `IPv4 -> IPv6`. - Final compatibility fallback uses `listen_addr_ipv4`/`listen_addr_ipv6` when routable, otherwise `"UNKNOWN"`. - User rows are sorted by `username` in ascending lexical order. @@ -1051,10 +1054,10 @@ Link generation uses active config and enabled modes: | Endpoint | Notes | | --- | --- | -| `POST /v1/users` | Creates user and validates resulting config before atomic save. | -| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. | +| `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). | +| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. Current implementation persists full config document on success. | | `POST /v1/users/{username}/rotate-secret` | Currently returns `404` in runtime route matcher; request schema is reserved for intended behavior. | -| `DELETE /v1/users/{username}` | Deletes user and related optional settings. Last user deletion is blocked. | +| `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. | All mutating endpoints: - Respect `read_only` mode. @@ -1062,6 +1065,10 @@ All mutating endpoints: - Return new `revision` after successful write. - Use process-local mutation lock + atomic write (`tmp + rename`) for config persistence. +Delete path cleanup guarantees: +- Config cleanup removes only the requested username keys. +- Runtime unique-IP cleanup removes only this user's limiter and tracked IP state. + ## Runtime State Matrix | Endpoint | `minimal_runtime_enabled=false` | `minimal_runtime_enabled=true` + source unavailable | `minimal_runtime_enabled=true` + source available | @@ -1087,6 +1094,16 @@ Additional runtime endpoint behavior: | `/v1/runtime/connections/summary` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Recompute lock contention with no cache entry => `enabled=true`, `reason=source_unavailable` | `enabled=true`, full payload | | `/v1/runtime/events/recent` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload | +## ME Fallback Behavior Exposed Via API + +When `general.use_middle_proxy=true` and `general.me2dc_fallback=true`: +- Startup does not block on full ME pool readiness; initialization can continue in background. +- Runtime initialization payload can expose ME stage `background_init` until pool becomes ready. +- Admission/routing decision uses two readiness grace windows for "ME not ready" periods: + `80s` before first-ever readiness is observed (startup grace), + `6s` after readiness has been observed at least once (runtime failover timeout). +- While in fallback window breach, new sessions are routed via Direct-DC; when ME becomes ready, routing returns to Middle mode for new sessions. + ## Serialization Rules - Success responses always include `revision`. @@ -1110,7 +1127,7 @@ Additional runtime endpoint behavior: | Runtime apply path | Successful writes are picked up by existing config watcher/hot-reload path. | | Exposure | Built-in TLS/mTLS is not provided. Use loopback bind + reverse proxy if needed. | | Pagination | User list currently has no pagination/filtering. | -| Serialization side effect | Config comments/manual formatting are not preserved on write. | +| Serialization side effect | Updated TOML table bodies are re-serialized on write. Endpoints that persist full config can still rewrite broader formatting/comments. | ## Known Limitations (Current Release) From 1d8de09a3236eae47483a0a1bfe6e39313ebacf9 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:06:43 +0300 Subject: [PATCH 167/261] Update users.rs --- src/api/users.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/api/users.rs b/src/api/users.rs index 848bb45..6265aea 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -330,7 +330,15 @@ pub(super) async fn delete_user( cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; - let revision = save_config_to_disk(&shared.config_path, &cfg).await?; + let touched_sections = [ + AccessSection::Users, + AccessSection::UserAdTags, + AccessSection::UserMaxTcpConns, + AccessSection::UserExpirations, + AccessSection::UserDataQuota, + AccessSection::UserMaxUniqueIps, + ]; + let revision = save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?; drop(_guard); shared.ip_tracker.remove_user_limit(user).await; shared.ip_tracker.clear_user_ips(user).await; From b315e8413601e2d36317e2ad778b4148f1e7fdba Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:09:11 +0300 Subject: [PATCH 168/261] Update users.rs --- src/api/users.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/api/users.rs b/src/api/users.rs index 6265aea..da360c7 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -205,15 +205,7 @@ pub(super) async fn patch_user( cfg.validate() .map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?; - let touched_sections = [ - AccessSection::Users, - AccessSection::UserAdTags, - AccessSection::UserMaxTcpConns, - AccessSection::UserExpirations, - AccessSection::UserDataQuota, - AccessSection::UserMaxUniqueIps, - ]; - let revision = save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?; + let revision = save_config_to_disk(&shared.config_path, &cfg).await?; drop(_guard); if let Some(limit) = updated_limit { shared.ip_tracker.set_user_limit(user, limit).await; From a383f3f1a3c7ed8ca6aa86cadb2aa2f9366d52ca Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:27:36 +0300 Subject: [PATCH 169/261] API Defaults --- src/config/defaults.rs | 6 +++--- src/config/types.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index cb95637..76479cf 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -119,11 +119,11 @@ pub(crate) fn default_metrics_whitelist() -> Vec { } pub(crate) fn default_api_listen() -> String { - "127.0.0.1:9091".to_string() + "0.0.0.0:9091".to_string() } pub(crate) fn default_api_whitelist() -> Vec { - default_metrics_whitelist() + vec!["127.0.0.0/8".parse().unwrap()] } pub(crate) fn default_api_request_body_limit_bytes() -> usize { @@ -131,7 +131,7 @@ pub(crate) fn default_api_request_body_limit_bytes() -> usize { } pub(crate) fn default_api_minimal_runtime_enabled() -> bool { - false + true } pub(crate) fn default_api_minimal_runtime_cache_ttl_ms() -> u64 { diff --git a/src/config/types.rs b/src/config/types.rs index eeb8cfa..24626a4 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1044,7 +1044,7 @@ impl Default for LinksConfig { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct ApiConfig { /// Enable or disable REST API. - #[serde(default)] + #[serde(default = "default_true")] pub enabled: bool, /// Listen address for API in `IP:PORT` format. @@ -1096,7 +1096,7 @@ pub struct ApiConfig { impl Default for ApiConfig { fn default() -> Self { Self { - enabled: false, + enabled: default_true(), listen: default_api_listen(), whitelist: default_api_whitelist(), auth_header: String::new(), From 959d3850155a670aed43ae20c3b57ab349e05065 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:59:25 +0300 Subject: [PATCH 170/261] ReRoute state in API --- src/api/mod.rs | 4 ++++ src/api/runtime_zero.rs | 18 ++++++++++++++++++ src/maestro/mod.rs | 2 ++ src/proxy/route_mode.rs | 27 ++++++++++++++++++++++++++- 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index ff9d2f9..6ee72a9 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -16,6 +16,7 @@ use tracing::{debug, info, warn}; use crate::config::ProxyConfig; use crate::ip_tracker::UserIpTracker; +use crate::proxy::route_mode::RouteRuntimeController; use crate::startup::StartupTracker; use crate::stats::Stats; use crate::transport::middle_proxy::MePool; @@ -84,6 +85,7 @@ pub(super) struct ApiShared { pub(super) request_id: Arc, pub(super) runtime_state: Arc, pub(super) startup_tracker: Arc, + pub(super) route_runtime: Arc, } impl ApiShared { @@ -101,6 +103,7 @@ pub async fn serve( stats: Arc, ip_tracker: Arc, me_pool: Arc>>>, + route_runtime: Arc, upstream_manager: Arc, config_rx: watch::Receiver>, admission_rx: watch::Receiver, @@ -147,6 +150,7 @@ pub async fn serve( request_id: Arc::new(AtomicU64::new(1)), runtime_state: runtime_state.clone(), startup_tracker, + route_runtime, }); spawn_runtime_watchers( diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index 7d3d778..93e3931 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -3,6 +3,7 @@ use std::sync::atomic::Ordering; use serde::Serialize; use crate::config::{MeFloorMode, MeWriterPickMode, ProxyConfig, UserMaxUniqueIpsMode}; +use crate::proxy::route_mode::RelayRouteMode; use super::ApiShared; use super::runtime_init::build_runtime_startup_summary; @@ -35,6 +36,10 @@ pub(super) struct RuntimeGatesData { pub(super) me_runtime_ready: bool, pub(super) me2dc_fallback_enabled: bool, pub(super) use_middle_proxy: bool, + pub(super) route_mode: &'static str, + pub(super) reroute_active: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) reroute_to_direct_at_epoch_secs: Option, pub(super) startup_status: &'static str, pub(super) startup_stage: String, pub(super) startup_progress_pct: f64, @@ -157,6 +162,16 @@ pub(super) async fn build_runtime_gates_data( cfg: &ProxyConfig, ) -> RuntimeGatesData { let startup_summary = build_runtime_startup_summary(shared).await; + let route_state = shared.route_runtime.snapshot(); + let route_mode = route_state.mode.as_str(); + let reroute_active = cfg.general.use_middle_proxy + && cfg.general.me2dc_fallback + && matches!(route_state.mode, RelayRouteMode::Direct); + let reroute_to_direct_at_epoch_secs = if reroute_active { + shared.route_runtime.direct_since_epoch_secs() + } else { + None + }; let me_runtime_ready = if !cfg.general.use_middle_proxy { true } else { @@ -175,6 +190,9 @@ pub(super) async fn build_runtime_gates_data( me_runtime_ready, me2dc_fallback_enabled: cfg.general.me2dc_fallback, use_middle_proxy: cfg.general.use_middle_proxy, + route_mode, + reroute_active, + reroute_to_direct_at_epoch_secs, startup_status: startup_summary.status, startup_stage: startup_summary.stage, startup_progress_pct: startup_summary.progress_pct, diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index fcb4d3f..996070e 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -220,6 +220,7 @@ pub async fn run() -> std::result::Result<(), Box> { let ip_tracker_api = ip_tracker.clone(); let me_pool_api = api_me_pool.clone(); let upstream_manager_api = upstream_manager.clone(); + let route_runtime_api = route_runtime.clone(); let config_rx_api = api_config_rx.clone(); let admission_rx_api = admission_rx.clone(); let config_path_api = std::path::PathBuf::from(&config_path); @@ -231,6 +232,7 @@ pub async fn run() -> std::result::Result<(), Box> { stats_api, ip_tracker_api, me_pool_api, + route_runtime_api, upstream_manager_api, config_rx_api, admission_rx_api, diff --git a/src/proxy/route_mode.rs b/src/proxy/route_mode.rs index 57830ca..306c536 100644 --- a/src/proxy/route_mode.rs +++ b/src/proxy/route_mode.rs @@ -1,6 +1,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::watch; @@ -43,6 +43,7 @@ pub(crate) struct RouteCutoverState { pub(crate) struct RouteRuntimeController { mode: Arc, generation: Arc, + direct_since_epoch_secs: Arc, tx: watch::Sender, } @@ -53,9 +54,15 @@ impl RouteRuntimeController { generation: 0, }; let (tx, _rx) = watch::channel(initial); + let direct_since_epoch_secs = if matches!(initial_mode, RelayRouteMode::Direct) { + now_epoch_secs() + } else { + 0 + }; Self { mode: Arc::new(AtomicU8::new(initial_mode.as_u8())), generation: Arc::new(AtomicU64::new(0)), + direct_since_epoch_secs: Arc::new(AtomicU64::new(direct_since_epoch_secs)), tx, } } @@ -71,11 +78,22 @@ impl RouteRuntimeController { self.tx.subscribe() } + pub(crate) fn direct_since_epoch_secs(&self) -> Option { + let value = self.direct_since_epoch_secs.load(Ordering::Relaxed); + (value > 0).then_some(value) + } + pub(crate) fn set_mode(&self, mode: RelayRouteMode) -> Option { let previous = self.mode.swap(mode.as_u8(), Ordering::Relaxed); if previous == mode.as_u8() { return None; } + if matches!(mode, RelayRouteMode::Direct) { + self.direct_since_epoch_secs + .store(now_epoch_secs(), Ordering::Relaxed); + } else { + self.direct_since_epoch_secs.store(0, Ordering::Relaxed); + } let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; let next = RouteCutoverState { mode, generation }; self.tx.send_replace(next); @@ -83,6 +101,13 @@ impl RouteRuntimeController { } } +fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|value| value.as_secs()) + .unwrap_or(0) +} + pub(crate) fn is_session_affected_by_cutover( current: RouteCutoverState, _session_mode: RelayRouteMode, From 8cd719da3f044459ff1e0edbe8e98c06aabe05d9 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 01:16:21 +0300 Subject: [PATCH 171/261] Bnd-block in API fixes --- src/api/mod.rs | 2 +- src/api/runtime_selftest.rs | 35 ++++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 6ee72a9..0e2edd4 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -342,7 +342,7 @@ async fn handle( } ("GET", "/v1/runtime/me-selftest") => { let revision = current_revision(&shared.config_path).await?; - let data = build_runtime_me_selftest_data(shared.as_ref()).await; + let data = build_runtime_me_selftest_data(shared.as_ref(), cfg.as_ref()).await; Ok(success_response(StatusCode::OK, data, revision)) } ("GET", "/v1/runtime/connections/summary") => { diff --git a/src/api/runtime_selftest.rs b/src/api/runtime_selftest.rs index da591b2..02bfb04 100644 --- a/src/api/runtime_selftest.rs +++ b/src/api/runtime_selftest.rs @@ -4,6 +4,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::Serialize; +use crate::config::{ProxyConfig, UpstreamType}; use crate::network::probe::{detect_interface_ipv4, detect_interface_ipv6, is_bogon}; use crate::transport::middle_proxy::{bnd_snapshot, timeskew_snapshot}; @@ -71,7 +72,7 @@ pub(super) struct RuntimeMeSelftestPayload { pub(super) timeskew: RuntimeMeSelftestTimeskewData, pub(super) ip: RuntimeMeSelftestIpData, pub(super) pid: RuntimeMeSelftestPidData, - pub(super) bnd: RuntimeMeSelftestBndData, + pub(super) bnd: Option, } #[derive(Serialize)] @@ -98,7 +99,10 @@ fn kdf_ewma_state() -> &'static Mutex { KDF_EWMA_STATE.get_or_init(|| Mutex::new(KdfEwmaState::default())) } -pub(super) async fn build_runtime_me_selftest_data(shared: &ApiShared) -> RuntimeMeSelftestData { +pub(super) async fn build_runtime_me_selftest_data( + shared: &ApiShared, + cfg: &ProxyConfig, +) -> RuntimeMeSelftestData { let now_epoch_secs = now_epoch_secs(); if shared.me_pool.read().await.is_none() { return RuntimeMeSelftestData { @@ -139,7 +143,25 @@ pub(super) async fn build_runtime_me_selftest_data(shared: &ApiShared) -> Runtim let pid = std::process::id(); let pid_state = if pid == 1 { "one" } else { "non-one" }; - let bnd = bnd_snapshot(); + let has_socks_upstreams = cfg.upstreams.iter().any(|upstream| { + upstream.enabled + && matches!( + upstream.upstream_type, + UpstreamType::Socks4 { .. } | UpstreamType::Socks5 { .. } + ) + }); + + let bnd = if has_socks_upstreams { + let snapshot = bnd_snapshot(); + Some(RuntimeMeSelftestBndData { + addr_state: snapshot.addr_status, + port_state: snapshot.port_status, + last_addr: snapshot.last_addr.map(|value| value.to_string()), + last_seen_age_secs: snapshot.last_seen_age_secs, + }) + } else { + None + }; RuntimeMeSelftestData { enabled: true, @@ -168,12 +190,7 @@ pub(super) async fn build_runtime_me_selftest_data(shared: &ApiShared) -> Runtim pid, state: pid_state, }, - bnd: RuntimeMeSelftestBndData { - addr_state: bnd.addr_status, - port_state: bnd.port_status, - last_addr: bnd.last_addr.map(|value| value.to_string()), - last_seen_age_secs: bnd.last_seen_age_secs, - }, + bnd, }), } } From be24b4730047df2cccdc974a657907449091996b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 10 Mar 2026 01:25:28 +0300 Subject: [PATCH 172/261] Per-upstream Runtime Selftest --- src/api/runtime_selftest.rs | 56 +++++++++++++++++++- src/transport/middle_proxy/handshake.rs | 14 ++++- src/transport/middle_proxy/mod.rs | 4 +- src/transport/middle_proxy/selftest.rs | 70 ++++++++++++++++++++++++- src/transport/upstream.rs | 16 ++++-- 5 files changed, 151 insertions(+), 9 deletions(-) diff --git a/src/api/runtime_selftest.rs b/src/api/runtime_selftest.rs index 02bfb04..0dce3dc 100644 --- a/src/api/runtime_selftest.rs +++ b/src/api/runtime_selftest.rs @@ -1,4 +1,5 @@ use std::net::IpAddr; +use std::collections::HashMap; use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -6,7 +7,8 @@ use serde::Serialize; use crate::config::{ProxyConfig, UpstreamType}; use crate::network::probe::{detect_interface_ipv4, detect_interface_ipv6, is_bogon}; -use crate::transport::middle_proxy::{bnd_snapshot, timeskew_snapshot}; +use crate::transport::middle_proxy::{bnd_snapshot, timeskew_snapshot, upstream_bnd_snapshots}; +use crate::transport::UpstreamRouteKind; use super::ApiShared; @@ -66,6 +68,17 @@ pub(super) struct RuntimeMeSelftestBndData { pub(super) last_seen_age_secs: Option, } +#[derive(Serialize)] +pub(super) struct RuntimeMeSelftestUpstreamData { + pub(super) upstream_id: usize, + pub(super) route_kind: &'static str, + pub(super) address: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) bnd: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) ip: Option, +} + #[derive(Serialize)] pub(super) struct RuntimeMeSelftestPayload { pub(super) kdf: RuntimeMeSelftestKdfData, @@ -73,6 +86,8 @@ pub(super) struct RuntimeMeSelftestPayload { pub(super) ip: RuntimeMeSelftestIpData, pub(super) pid: RuntimeMeSelftestPidData, pub(super) bnd: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) upstreams: Option>, } #[derive(Serialize)] @@ -162,6 +177,7 @@ pub(super) async fn build_runtime_me_selftest_data( } else { None }; + let upstreams = build_upstream_selftest_data(shared); RuntimeMeSelftestData { enabled: true, @@ -191,10 +207,40 @@ pub(super) async fn build_runtime_me_selftest_data( state: pid_state, }, bnd, + upstreams, }), } } +fn build_upstream_selftest_data(shared: &ApiShared) -> Option> { + let snapshot = shared.upstream_manager.try_api_snapshot()?; + if snapshot.summary.configured_total <= 1 { + return None; + } + + let mut upstream_bnd_by_id: HashMap = upstream_bnd_snapshots() + .into_iter() + .map(|entry| (entry.upstream_id, entry)) + .collect(); + let mut rows = Vec::with_capacity(snapshot.upstreams.len()); + for upstream in snapshot.upstreams { + let upstream_bnd = upstream_bnd_by_id.remove(&upstream.upstream_id); + rows.push(RuntimeMeSelftestUpstreamData { + upstream_id: upstream.upstream_id, + route_kind: map_route_kind(upstream.route_kind), + address: upstream.address, + bnd: upstream_bnd.as_ref().map(|entry| RuntimeMeSelftestBndData { + addr_state: entry.addr_status, + port_state: entry.port_status, + last_addr: entry.last_addr.map(|value| value.to_string()), + last_seen_age_secs: entry.last_seen_age_secs, + }), + ip: upstream_bnd.and_then(|entry| entry.last_ip.map(|value| value.to_string())), + }); + } + Some(rows) +} + fn update_kdf_ewma(now_epoch_secs: u64, total_errors: u64) -> f64 { let Ok(mut guard) = kdf_ewma_state().lock() else { return 0.0; @@ -233,6 +279,14 @@ fn classify_ip(ip: IpAddr) -> &'static str { "good" } +fn map_route_kind(value: UpstreamRouteKind) -> &'static str { + match value { + UpstreamRouteKind::Direct => "direct", + UpstreamRouteKind::Socks4 => "socks4", + UpstreamRouteKind::Socks5 => "socks5", + } +} + fn round3(value: f64) -> f64 { (value * 1000.0).round() / 1000.0 } diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 1c1b172..245a331 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -33,7 +33,7 @@ use super::codec::{ cbc_decrypt_inplace, cbc_encrypt_padded, parse_handshake_flags, parse_nonce_payload, read_rpc_frame_plaintext, rpc_crc, }; -use super::selftest::{BndAddrStatus, BndPortStatus, record_bnd_status}; +use super::selftest::{BndAddrStatus, BndPortStatus, record_bnd_status, record_upstream_bnd_status}; use super::wire::{extract_ip_material, IpMaterial}; use super::MePool; @@ -299,6 +299,18 @@ impl MePool { let local_addr_nat = self.translate_our_addr_with_reflection(local_addr, reflected); let peer_addr_nat = SocketAddr::new(self.translate_ip_for_nat(peer_addr.ip()), peer_addr.port()); + if let Some(upstream_info) = upstream_egress { + let client_ip_for_kdf = socks_bound_addr + .map(|value| value.ip()) + .unwrap_or(local_addr_nat.ip()); + record_upstream_bnd_status( + upstream_info.upstream_id, + bnd_addr_status, + bnd_port_status, + raw_socks_bound_addr, + Some(client_ip_for_kdf), + ); + } let (mut rd, mut wr) = tokio::io::split(stream); let my_nonce: [u8; 16] = rng.bytes(16).try_into().unwrap(); diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index ecc963d..92e222d 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -38,7 +38,9 @@ pub use config_updater::{ me_config_updater, save_proxy_config_cache, }; pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task}; -pub(crate) use selftest::{bnd_snapshot, timeskew_snapshot}; +pub(crate) use selftest::{ + bnd_snapshot, timeskew_snapshot, upstream_bnd_snapshots, +}; pub use wire::proto_flags_for_tag; #[derive(Debug)] diff --git a/src/transport/middle_proxy/selftest.rs b/src/transport/middle_proxy/selftest.rs index c1653ec..86a93e3 100644 --- a/src/transport/middle_proxy/selftest.rs +++ b/src/transport/middle_proxy/selftest.rs @@ -1,5 +1,5 @@ -use std::collections::VecDeque; -use std::net::SocketAddr; +use std::collections::{HashMap, VecDeque}; +use std::net::{IpAddr, SocketAddr}; use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -45,6 +45,16 @@ pub(crate) struct MeBndSnapshot { pub last_seen_age_secs: Option, } +#[derive(Clone, Debug)] +pub(crate) struct MeUpstreamBndSnapshot { + pub upstream_id: usize, + pub addr_status: &'static str, + pub port_status: &'static str, + pub last_addr: Option, + pub last_ip: Option, + pub last_seen_age_secs: Option, +} + #[derive(Clone, Debug, Default)] pub(crate) struct MeTimeskewSnapshot { pub max_skew_secs_15m: Option, @@ -67,9 +77,19 @@ struct MeSelftestState { bnd_port_status: BndPortStatus, bnd_last_addr: Option, bnd_last_seen_epoch_secs: Option, + upstream_bnd: HashMap, timeskew_samples: VecDeque, } +#[derive(Clone, Copy, Debug)] +struct UpstreamBndState { + addr_status: BndAddrStatus, + port_status: BndPortStatus, + last_addr: Option, + last_ip: Option, + last_seen_epoch_secs: Option, +} + impl Default for MeSelftestState { fn default() -> Self { Self { @@ -77,6 +97,7 @@ impl Default for MeSelftestState { bnd_port_status: BndPortStatus::Error, bnd_last_addr: None, bnd_last_seen_epoch_secs: None, + upstream_bnd: HashMap::new(), timeskew_samples: VecDeque::new(), } } @@ -126,6 +147,51 @@ pub(crate) fn bnd_snapshot() -> MeBndSnapshot { } } +pub(crate) fn record_upstream_bnd_status( + upstream_id: usize, + addr_status: BndAddrStatus, + port_status: BndPortStatus, + last_addr: Option, + last_ip: Option, +) { + let now_epoch_secs = now_epoch_secs(); + let Ok(mut guard) = state().lock() else { + return; + }; + guard.upstream_bnd.insert( + upstream_id, + UpstreamBndState { + addr_status, + port_status, + last_addr, + last_ip, + last_seen_epoch_secs: Some(now_epoch_secs), + }, + ); +} + +pub(crate) fn upstream_bnd_snapshots() -> Vec { + let now_epoch_secs = now_epoch_secs(); + let Ok(guard) = state().lock() else { + return Vec::new(); + }; + let mut out = Vec::with_capacity(guard.upstream_bnd.len()); + for (upstream_id, entry) in &guard.upstream_bnd { + out.push(MeUpstreamBndSnapshot { + upstream_id: *upstream_id, + addr_status: entry.addr_status.as_str(), + port_status: entry.port_status.as_str(), + last_addr: entry.last_addr, + last_ip: entry.last_ip, + last_seen_age_secs: entry + .last_seen_epoch_secs + .map(|value| now_epoch_secs.saturating_sub(value)), + }); + } + out.sort_by_key(|entry| entry.upstream_id); + out +} + pub(crate) fn record_timeskew_sample(source: &'static str, skew_secs: u64) { let now_epoch_secs = now_epoch_secs(); let Ok(mut guard) = state().lock() else { diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index 2424f9c..1355934 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -213,6 +213,7 @@ pub struct UpstreamApiPolicySnapshot { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct UpstreamEgressInfo { + pub upstream_id: usize, pub route_kind: UpstreamRouteKind, pub local_addr: Option, pub direct_bind_ip: Option, @@ -672,7 +673,7 @@ impl UpstreamManager { self.stats.increment_upstream_connect_attempt_total(); let start = Instant::now(); match self - .connect_via_upstream(&upstream, target, bind_rr.clone(), attempt_timeout) + .connect_via_upstream(idx, &upstream, target, bind_rr.clone(), attempt_timeout) .await { Ok((stream, egress)) => { @@ -779,6 +780,7 @@ impl UpstreamManager { async fn connect_via_upstream( &self, + upstream_id: usize, config: &UpstreamConfig, target: SocketAddr, bind_rr: Option>, @@ -828,6 +830,7 @@ impl UpstreamManager { Ok(( stream, UpstreamEgressInfo { + upstream_id, route_kind: UpstreamRouteKind::Direct, local_addr, direct_bind_ip: bind_ip, @@ -906,6 +909,7 @@ impl UpstreamManager { Ok(( stream, UpstreamEgressInfo { + upstream_id, route_kind: UpstreamRouteKind::Socks4, local_addr, direct_bind_ip: None, @@ -986,6 +990,7 @@ impl UpstreamManager { Ok(( stream, UpstreamEgressInfo { + upstream_id, route_kind: UpstreamRouteKind::Socks5, local_addr, direct_bind_ip: None, @@ -1048,7 +1053,7 @@ impl UpstreamManager { let result = tokio::time::timeout( Duration::from_secs(DC_PING_TIMEOUT_SECS), - self.ping_single_dc(upstream_config, Some(bind_rr.clone()), addr_v6) + self.ping_single_dc(*upstream_idx, upstream_config, Some(bind_rr.clone()), addr_v6) ).await; let ping_result = match result { @@ -1099,7 +1104,7 @@ impl UpstreamManager { let result = tokio::time::timeout( Duration::from_secs(DC_PING_TIMEOUT_SECS), - self.ping_single_dc(upstream_config, Some(bind_rr.clone()), addr_v4) + self.ping_single_dc(*upstream_idx, upstream_config, Some(bind_rr.clone()), addr_v4) ).await; let ping_result = match result { @@ -1162,7 +1167,7 @@ impl UpstreamManager { } let result = tokio::time::timeout( Duration::from_secs(DC_PING_TIMEOUT_SECS), - self.ping_single_dc(upstream_config, Some(bind_rr.clone()), addr) + self.ping_single_dc(*upstream_idx, upstream_config, Some(bind_rr.clone()), addr) ).await; let ping_result = match result { @@ -1233,6 +1238,7 @@ impl UpstreamManager { async fn ping_single_dc( &self, + upstream_id: usize, config: &UpstreamConfig, bind_rr: Option>, target: SocketAddr, @@ -1240,6 +1246,7 @@ impl UpstreamManager { let start = Instant::now(); let _ = self .connect_via_upstream( + upstream_id, config, target, bind_rr, @@ -1418,6 +1425,7 @@ impl UpstreamManager { let result = tokio::time::timeout( Duration::from_secs(HEALTH_CHECK_CONNECT_TIMEOUT_SECS), self.connect_via_upstream( + i, &config, endpoint, Some(bind_rr.clone()), From 8b5cbb7b4b3cdc901b9c565b2c1aba32aa29795c Mon Sep 17 00:00:00 2001 From: David Osipov Date: Wed, 11 Mar 2026 20:49:51 +0400 Subject: [PATCH 173/261] Add Rust coding conventions and self-explanatory commenting guidelines; update dependencies and version in Cargo files; enhance OpenBSD support in installation and documentation; improve TCP socket configuration and testing --- .../instructions/rust_rules.instructions.md | 135 +++++++++++++++ ...xplanatory-code-commenting.instructions.md | 162 ++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 3 + README.md | 11 ++ contrib/openbsd/telemt.rcd | 16 ++ docs/OPENBSD.en.md | 132 ++++++++++++++ install.sh | 22 +++ src/transport/middle_proxy/handshake.rs | 87 +++++++++- src/transport/socket.rs | 113 ++++++++++++ 10 files changed, 678 insertions(+), 5 deletions(-) create mode 100644 .github/instructions/rust_rules.instructions.md create mode 100644 .github/instructions/self-explanatory-code-commenting.instructions.md create mode 100644 contrib/openbsd/telemt.rcd create mode 100644 docs/OPENBSD.en.md diff --git a/.github/instructions/rust_rules.instructions.md b/.github/instructions/rust_rules.instructions.md new file mode 100644 index 0000000..75ac0e4 --- /dev/null +++ b/.github/instructions/rust_rules.instructions.md @@ -0,0 +1,135 @@ +--- +description: 'Rust programming language coding conventions and best practices' +applyTo: '**/*.rs' +--- + +# Rust Coding Conventions and Best Practices + +Follow idiomatic Rust practices and community standards when writing Rust code. + +These instructions are based on [The Rust Book](https://doc.rust-lang.org/book/), [Rust API Guidelines](https://rust-lang.github.io/api-guidelines/), [RFC 430 naming conventions](https://github.com/rust-lang/rfcs/blob/master/text/0430-finalizing-naming-conventions.md), and the broader Rust community at [users.rust-lang.org](https://users.rust-lang.org). + +## General Instructions + +- Always prioritize readability, safety, and maintainability. +- Use strong typing and leverage Rust's ownership system for memory safety. +- Break down complex functions into smaller, more manageable functions. +- For algorithm-related code, include explanations of the approach used. +- Write code with good maintainability practices, including comments on why certain design decisions were made. +- Handle errors gracefully using `Result` and provide meaningful error messages. +- For external dependencies, mention their usage and purpose in documentation. +- Use consistent naming conventions following [RFC 430](https://github.com/rust-lang/rfcs/blob/master/text/0430-finalizing-naming-conventions.md). +- Write idiomatic, safe, and efficient Rust code that follows the borrow checker's rules. +- Ensure code compiles without warnings. + +## Patterns to Follow + +- Use modules (`mod`) and public interfaces (`pub`) to encapsulate logic. +- Handle errors properly using `?`, `match`, or `if let`. +- Use `serde` for serialization and `thiserror` or `anyhow` for custom errors. +- Implement traits to abstract services or external dependencies. +- Structure async code using `async/await` and `tokio` or `async-std`. +- Prefer enums over flags and states for type safety. +- Use builders for complex object creation. +- Split binary and library code (`main.rs` vs `lib.rs`) for testability and reuse. +- Use `rayon` for data parallelism and CPU-bound tasks. +- Use iterators instead of index-based loops as they're often faster and safer. +- Use `&str` instead of `String` for function parameters when you don't need ownership. +- Prefer borrowing and zero-copy operations to avoid unnecessary allocations. + +### Ownership, Borrowing, and Lifetimes + +- Prefer borrowing (`&T`) over cloning unless ownership transfer is necessary. +- Use `&mut T` when you need to modify borrowed data. +- Explicitly annotate lifetimes when the compiler cannot infer them. +- Use `Rc` for single-threaded reference counting and `Arc` for thread-safe reference counting. +- Use `RefCell` for interior mutability in single-threaded contexts and `Mutex` or `RwLock` for multi-threaded contexts. + +## Patterns to Avoid + +- Don't use `unwrap()` or `expect()` unless absolutely necessary—prefer proper error handling. +- Avoid panics in library code—return `Result` instead. +- Don't rely on global mutable state—use dependency injection or thread-safe containers. +- Avoid deeply nested logic—refactor with functions or combinators. +- Don't ignore warnings—treat them as errors during CI. +- Avoid `unsafe` unless required and fully documented. +- Don't overuse `clone()`, use borrowing instead of cloning unless ownership transfer is needed. +- Avoid premature `collect()`, keep iterators lazy until you actually need the collection. +- Avoid unnecessary allocations—prefer borrowing and zero-copy operations. + +## Code Style and Formatting + +- Follow the Rust Style Guide and use `rustfmt` for automatic formatting. +- Keep lines under 100 characters when possible. +- Place function and struct documentation immediately before the item using `///`. +- Use `cargo clippy` to catch common mistakes and enforce best practices. + +## Error Handling + +- Use `Result` for recoverable errors and `panic!` only for unrecoverable errors. +- Prefer `?` operator over `unwrap()` or `expect()` for error propagation. +- Create custom error types using `thiserror` or implement `std::error::Error`. +- Use `Option` for values that may or may not exist. +- Provide meaningful error messages and context. +- Error types should be meaningful and well-behaved (implement standard traits). +- Validate function arguments and return appropriate errors for invalid input. + +## API Design Guidelines + +### Common Traits Implementation +Eagerly implement common traits where appropriate: +- `Copy`, `Clone`, `Eq`, `PartialEq`, `Ord`, `PartialOrd`, `Hash`, `Debug`, `Display`, `Default` +- Use standard conversion traits: `From`, `AsRef`, `AsMut` +- Collections should implement `FromIterator` and `Extend` +- Note: `Send` and `Sync` are auto-implemented by the compiler when safe; avoid manual implementation unless using `unsafe` code + +### Type Safety and Predictability +- Use newtypes to provide static distinctions +- Arguments should convey meaning through types; prefer specific types over generic `bool` parameters +- Use `Option` appropriately for truly optional values +- Functions with a clear receiver should be methods +- Only smart pointers should implement `Deref` and `DerefMut` + +### Future Proofing +- Use sealed traits to protect against downstream implementations +- Structs should have private fields +- Functions should validate their arguments +- All public types must implement `Debug` + +## Testing and Documentation + +- Write comprehensive unit tests using `#[cfg(test)]` modules and `#[test]` annotations. +- Use test modules alongside the code they test (`mod tests { ... }`). +- Write integration tests in `tests/` directory with descriptive filenames. +- Write clear and concise comments for each function, struct, enum, and complex logic. +- Ensure functions have descriptive names and include comprehensive documentation. +- Document all public APIs with rustdoc (`///` comments) following the [API Guidelines](https://rust-lang.github.io/api-guidelines/). +- Use `#[doc(hidden)]` to hide implementation details from public documentation. +- Document error conditions, panic scenarios, and safety considerations. +- Examples should use `?` operator, not `unwrap()` or deprecated `try!` macro. + +## Project Organization + +- Use semantic versioning in `Cargo.toml`. +- Include comprehensive metadata: `description`, `license`, `repository`, `keywords`, `categories`. +- Use feature flags for optional functionality. +- Organize code into modules using `mod.rs` or named files. +- Keep `main.rs` or `lib.rs` minimal - move logic to modules. + +## Quality Checklist + +Before publishing or reviewing Rust code, ensure: + +### Core Requirements +- [ ] **Naming**: Follows RFC 430 naming conventions +- [ ] **Traits**: Implements `Debug`, `Clone`, `PartialEq` where appropriate +- [ ] **Error Handling**: Uses `Result` and provides meaningful error types +- [ ] **Documentation**: All public items have rustdoc comments with examples +- [ ] **Testing**: Comprehensive test coverage including edge cases + +### Safety and Quality +- [ ] **Safety**: No unnecessary `unsafe` code, proper error handling +- [ ] **Performance**: Efficient use of iterators, minimal allocations +- [ ] **API Design**: Functions are predictable, flexible, and type-safe +- [ ] **Future Proofing**: Private fields in structs, sealed traits where appropriate +- [ ] **Tooling**: Code passes `cargo fmt`, `cargo clippy`, and `cargo test` diff --git a/.github/instructions/self-explanatory-code-commenting.instructions.md b/.github/instructions/self-explanatory-code-commenting.instructions.md new file mode 100644 index 0000000..03a559f --- /dev/null +++ b/.github/instructions/self-explanatory-code-commenting.instructions.md @@ -0,0 +1,162 @@ +--- +description: 'Guidelines for GitHub Copilot to write comments to achieve self-explanatory code with less comments. Examples are in JavaScript but it should work on any language that has comments.' +applyTo: '**' +--- + +# Self-explanatory Code Commenting Instructions + +## Core Principle +**Write code that speaks for itself. Comment only when necessary to explain WHY, not WHAT.** +We do not need comments most of the time. + +## Commenting Guidelines + +### ❌ AVOID These Comment Types + +**Obvious Comments** +```javascript +// Bad: States the obvious +let counter = 0; // Initialize counter to zero +counter++; // Increment counter by one +``` + +**Redundant Comments** +```javascript +// Bad: Comment repeats the code +function getUserName() { + return user.name; // Return the user's name +} +``` + +**Outdated Comments** +```javascript +// Bad: Comment doesn't match the code +// Calculate tax at 5% rate +const tax = price * 0.08; // Actually 8% +``` + +### ✅ WRITE These Comment Types + +**Complex Business Logic** +```javascript +// Good: Explains WHY this specific calculation +// Apply progressive tax brackets: 10% up to 10k, 20% above +const tax = calculateProgressiveTax(income, [0.10, 0.20], [10000]); +``` + +**Non-obvious Algorithms** +```javascript +// Good: Explains the algorithm choice +// Using Floyd-Warshall for all-pairs shortest paths +// because we need distances between all nodes +for (let k = 0; k < vertices; k++) { + for (let i = 0; i < vertices; i++) { + for (let j = 0; j < vertices; j++) { + // ... implementation + } + } +} +``` + +**Regex Patterns** +```javascript +// Good: Explains what the regex matches +// Match email format: username@domain.extension +const emailPattern = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/; +``` + +**API Constraints or Gotchas** +```javascript +// Good: Explains external constraint +// GitHub API rate limit: 5000 requests/hour for authenticated users +await rateLimiter.wait(); +const response = await fetch(githubApiUrl); +``` + +## Decision Framework + +Before writing a comment, ask: +1. **Is the code self-explanatory?** → No comment needed +2. **Would a better variable/function name eliminate the need?** → Refactor instead +3. **Does this explain WHY, not WHAT?** → Good comment +4. **Will this help future maintainers?** → Good comment + +## Special Cases for Comments + +### Public APIs +```javascript +/** + * Calculate compound interest using the standard formula. + * + * @param {number} principal - Initial amount invested + * @param {number} rate - Annual interest rate (as decimal, e.g., 0.05 for 5%) + * @param {number} time - Time period in years + * @param {number} compoundFrequency - How many times per year interest compounds (default: 1) + * @returns {number} Final amount after compound interest + */ +function calculateCompoundInterest(principal, rate, time, compoundFrequency = 1) { + // ... implementation +} +``` + +### Configuration and Constants +```javascript +// Good: Explains the source or reasoning +const MAX_RETRIES = 3; // Based on network reliability studies +const API_TIMEOUT = 5000; // AWS Lambda timeout is 15s, leaving buffer +``` + +### Annotations +```javascript +// TODO: Replace with proper user authentication after security review +// FIXME: Memory leak in production - investigate connection pooling +// HACK: Workaround for bug in library v2.1.0 - remove after upgrade +// NOTE: This implementation assumes UTC timezone for all calculations +// WARNING: This function modifies the original array instead of creating a copy +// PERF: Consider caching this result if called frequently in hot path +// SECURITY: Validate input to prevent SQL injection before using in query +// BUG: Edge case failure when array is empty - needs investigation +// REFACTOR: Extract this logic into separate utility function for reusability +// DEPRECATED: Use newApiFunction() instead - this will be removed in v3.0 +``` + +## Anti-Patterns to Avoid + +### Dead Code Comments +```javascript +// Bad: Don't comment out code +// const oldFunction = () => { ... }; +const newFunction = () => { ... }; +``` + +### Changelog Comments +```javascript +// Bad: Don't maintain history in comments +// Modified by John on 2023-01-15 +// Fixed bug reported by Sarah on 2023-02-03 +function processData() { + // ... implementation +} +``` + +### Divider Comments +```javascript +// Bad: Don't use decorative comments +//===================================== +// UTILITY FUNCTIONS +//===================================== +``` + +## Quality Checklist + +Before committing, ensure your comments: +- [ ] Explain WHY, not WHAT +- [ ] Are grammatically correct and clear +- [ ] Will remain accurate as code evolves +- [ ] Add genuine value to code understanding +- [ ] Are placed appropriately (above the code they describe) +- [ ] Use proper spelling and professional language + +## Summary + +Remember: **The best comment is the one you don't need to write because the code is self-documenting.** \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e29b473..06ea5c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2087,7 +2087,7 @@ dependencies = [ [[package]] name = "telemt" -version = "3.1.3" +version = "3.3.15" dependencies = [ "aes", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 2f7ea3c..afdf5b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,3 +73,6 @@ futures = "0.3" [[bench]] name = "crypto_bench" harness = false + +[profile.release] +lto = "thin" diff --git a/README.md b/README.md index b59e509..8807393 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,11 @@ git clone https://github.com/telemt/telemt cd telemt # Starting Release Build cargo build --release + +# Low-RAM devices (1 GB, e.g. NanoPi Neo3 / Raspberry Pi Zero 2): +# release profile uses lto = "thin" to reduce peak linker memory. +# If your custom toolchain overrides profiles, avoid enabling fat LTO. + # Move to /bin mv ./target/release/telemt /bin # Make executable @@ -272,6 +277,12 @@ chmod +x /bin/telemt telemt config.toml ``` +### OpenBSD +- Build and service setup guide: [OpenBSD Guide (EN)](docs/OPENBSD.en.md) +- Example rc.d script: [contrib/openbsd/telemt.rcd](contrib/openbsd/telemt.rcd) +- Status: OpenBSD sandbox hardening with `pledge(2)` and `unveil(2)` is not implemented yet. + + ## Why Rust? - Long-running reliability and idempotent behavior - Rust's deterministic resource management - RAII diff --git a/contrib/openbsd/telemt.rcd b/contrib/openbsd/telemt.rcd new file mode 100644 index 0000000..c3dece1 --- /dev/null +++ b/contrib/openbsd/telemt.rcd @@ -0,0 +1,16 @@ +#!/bin/ksh +# /etc/rc.d/telemt +# +# rc.d(8) script for Telemt MTProxy daemon. +# Tokio runtime does not daemonize itself, so rc_bg=YES is used. + +daemon="/usr/local/bin/telemt" +daemon_user="_telemt" +daemon_flags="/etc/telemt/config.toml" + +. /etc/rc.d/rc.subr + +rc_bg=YES +rc_reload=NO + +rc_cmd $1 diff --git a/docs/OPENBSD.en.md b/docs/OPENBSD.en.md new file mode 100644 index 0000000..943e599 --- /dev/null +++ b/docs/OPENBSD.en.md @@ -0,0 +1,132 @@ +# Telemt on OpenBSD (Build, Run, and rc.d) + +This guide covers a practical OpenBSD deployment flow for Telemt: +- build from source, +- install binary and config, +- run as an rc.d daemon, +- verify basic runtime behavior. + +## 1. Prerequisites + +Install required packages: + +```sh +doas pkg_add rust git +``` + +Notes: +- Telemt release installer (`install.sh`) is Linux-only. +- On OpenBSD, use source build with `cargo`. + +## 2. Build from source + +```sh +git clone https://github.com/telemt/telemt +cd telemt +cargo build --release +./target/release/telemt --version +``` + +For low-RAM systems, this repository already uses `lto = "thin"` in release profile. + +## 3. Install binary and config + +```sh +doas install -d -m 0755 /usr/local/bin +doas install -m 0755 ./target/release/telemt /usr/local/bin/telemt + +doas install -d -m 0750 /etc/telemt +doas install -m 0640 ./config.toml /etc/telemt/config.toml +``` + +## 4. Create runtime user + +```sh +doas useradd -L daemon -s /sbin/nologin -d /var/empty _telemt +``` + +If `_telemt` already exists, continue. + +## 5. Install rc.d service + +Install the provided script: + +```sh +doas install -m 0555 ./contrib/openbsd/telemt.rcd /etc/rc.d/telemt +``` + +Enable and start: + +```sh +doas rcctl enable telemt +# Optional: send daemon output to syslog +#doas rcctl set telemt logger daemon.info + +doas rcctl start telemt +``` + +Service controls: + +```sh +doas rcctl check telemt +doas rcctl restart telemt +doas rcctl stop telemt +``` + +## 6. Resource limits (recommended) + +OpenBSD rc.d can apply limits via login class. Add class `telemt` and assign it to `_telemt`. + +Example class entry: + +```text +telemt:\ + :openfiles-cur=8192:openfiles-max=16384:\ + :datasize-cur=768M:datasize-max=1024M:\ + :coredumpsize=0:\ + :tc=daemon: +``` + +These values are conservative defaults for small and medium deployments. +Increase `openfiles-*` only if logs show descriptor exhaustion under load. + +Then rebuild database and assign class: + +```sh +doas cap_mkdb /etc/login.conf +#doas usermod -L telemt _telemt +``` + +Uncomment `usermod` if you want this class bound to the Telemt user. + +## 7. Functional smoke test + +1. Validate service state: + +```sh +doas rcctl check telemt +``` + +2. Check listener is present (replace 443 if needed): + +```sh +netstat -n -f inet -p tcp | grep LISTEN | grep '\.443' +``` + +3. Verify process user: + +```sh +ps -o user,pid,command -ax | grep telemt | grep -v grep +``` + +4. If startup fails, debug in foreground: + +```sh +RUST_LOG=debug /usr/local/bin/telemt /etc/telemt/config.toml +``` + +## 8. OpenBSD-specific caveats + +- OpenBSD does not support per-socket keepalive retries/interval tuning in the same way as Linux. +- Telemt source already uses target-aware cfg gates for keepalive setup. +- Use rc.d/rcctl, not systemd. diff --git a/install.sh b/install.sh index 9d96f0e..2dd207b 100644 --- a/install.sh +++ b/install.sh @@ -19,6 +19,15 @@ need_cmd() { command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" } +detect_os() { + os="$(uname -s)" + case "$os" in + Linux) printf 'linux\n' ;; + OpenBSD) printf 'openbsd\n' ;; + *) printf '%s\n' "$os" ;; + esac +} + detect_arch() { arch="$(uname -m)" case "$arch" in @@ -68,6 +77,19 @@ need_cmd grep need_cmd install ARCH="$(detect_arch)" +OS="$(detect_os)" + +if [ "$OS" != "linux" ]; then + case "$OS" in + openbsd) + die "install.sh installs only Linux release artifacts. On OpenBSD, build from source (see docs/OPENBSD.en.md)." + ;; + *) + die "unsupported operating system for install.sh: $OS" + ;; + esac +fi + LIBC="$(detect_libc)" case "$VERSION" in diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 245a331..0d7626c 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -199,10 +199,26 @@ impl MePool { fn configure_keepalive(stream: &TcpStream) -> std::io::Result<()> { let sock = SockRef::from(stream); - let ka = TcpKeepalive::new() - .with_time(Duration::from_secs(30)) - .with_interval(Duration::from_secs(10)) - .with_retries(3); + let ka = TcpKeepalive::new().with_time(Duration::from_secs(30)); + + // Mirror socket2 v0.5.10 target gate for with_retries(), the stricter method. + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "ios", + target_os = "visionos", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "tvos", + target_os = "watchos", + target_os = "cygwin", + ))] + let ka = ka.with_interval(Duration::from_secs(10)).with_retries(3); + sock.set_tcp_keepalive(&ka)?; sock.set_keepalive(true)?; Ok(()) @@ -697,3 +713,66 @@ fn hex_dump(data: &[u8]) -> String { } out } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::ErrorKind; + use tokio::net::{TcpListener, TcpStream}; + + #[tokio::test] + async fn test_configure_keepalive_loopback() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(error) if error.kind() == ErrorKind::PermissionDenied => return, + Err(error) => panic!("bind failed: {error}"), + }; + + let addr = match listener.local_addr() { + Ok(addr) => addr, + Err(error) => panic!("local_addr failed: {error}"), + }; + + let stream = match TcpStream::connect(addr).await { + Ok(stream) => stream, + Err(error) if error.kind() == ErrorKind::PermissionDenied => return, + Err(error) => panic!("connect failed: {error}"), + }; + + if let Err(error) = MePool::configure_keepalive(&stream) { + if error.kind() == ErrorKind::PermissionDenied { + return; + } + panic!("configure_keepalive failed: {error}"); + } + } + + #[test] + #[cfg(target_os = "openbsd")] + fn test_openbsd_keepalive_cfg_path_compiles() { + let _ka = TcpKeepalive::new().with_time(Duration::from_secs(30)); + } + + #[test] + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "ios", + target_os = "visionos", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "tvos", + target_os = "watchos", + target_os = "cygwin", + ))] + fn test_retry_keepalive_cfg_path_compiles() { + let _ka = TcpKeepalive::new() + .with_time(Duration::from_secs(30)) + .with_interval(Duration::from_secs(10)) + .with_retries(3); + } +} diff --git a/src/transport/socket.rs b/src/transport/socket.rs index f1f8d5c..54eb143 100644 --- a/src/transport/socket.rs +++ b/src/transport/socket.rs @@ -1,6 +1,8 @@ //! TCP Socket Configuration +#[cfg(target_os = "linux")] use std::collections::HashSet; +#[cfg(target_os = "linux")] use std::fs; use std::io::Result; use std::net::{SocketAddr, IpAddr}; @@ -44,6 +46,7 @@ pub fn configure_tcp_socket( pub fn configure_client_socket( stream: &TcpStream, keepalive_secs: u64, + #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] ack_timeout_secs: u64, ) -> Result<()> { let socket = socket2::SockRef::from(stream); @@ -373,6 +376,7 @@ fn listening_inodes_for_port(addr: SocketAddr) -> HashSet { mod tests { use super::*; use std::io::ErrorKind; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; #[tokio::test] @@ -396,6 +400,115 @@ mod tests { panic!("configure_tcp_socket failed: {e}"); } } + + #[tokio::test] + async fn test_configure_client_socket() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(l) => l, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("bind failed: {e}"), + }; + let addr = match listener.local_addr() { + Ok(addr) => addr, + Err(e) => panic!("local_addr failed: {e}"), + }; + + let stream = match TcpStream::connect(addr).await { + Ok(s) => s, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("connect failed: {e}"), + }; + + if let Err(e) = configure_client_socket(&stream, 30, 30) { + if e.kind() == ErrorKind::PermissionDenied { + return; + } + panic!("configure_client_socket failed: {e}"); + } + } + + #[tokio::test] + async fn test_configure_client_socket_zero_ack_timeout() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(l) => l, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("bind failed: {e}"), + }; + let addr = match listener.local_addr() { + Ok(addr) => addr, + Err(e) => panic!("local_addr failed: {e}"), + }; + + let stream = match TcpStream::connect(addr).await { + Ok(s) => s, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("connect failed: {e}"), + }; + + if let Err(e) = configure_client_socket(&stream, 30, 0) { + if e.kind() == ErrorKind::PermissionDenied { + return; + } + panic!("configure_client_socket with zero ack timeout failed: {e}"); + } + } + + #[tokio::test] + async fn test_configure_client_socket_roundtrip_io() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(l) => l, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("bind failed: {e}"), + }; + let addr = match listener.local_addr() { + Ok(addr) => addr, + Err(e) => panic!("local_addr failed: {e}"), + }; + + let server_task = tokio::spawn(async move { + let (mut accepted, _) = match listener.accept().await { + Ok(v) => v, + Err(e) => panic!("accept failed: {e}"), + }; + let mut payload = [0u8; 4]; + if let Err(e) = accepted.read_exact(&mut payload).await { + panic!("server read_exact failed: {e}"); + } + if let Err(e) = accepted.write_all(b"pong").await { + panic!("server write_all failed: {e}"); + } + payload + }); + + let mut stream = match TcpStream::connect(addr).await { + Ok(s) => s, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("connect failed: {e}"), + }; + + if let Err(e) = configure_client_socket(&stream, 30, 30) { + if e.kind() == ErrorKind::PermissionDenied { + return; + } + panic!("configure_client_socket failed: {e}"); + } + + if let Err(e) = stream.write_all(b"ping").await { + panic!("client write_all failed: {e}"); + } + + let mut reply = [0u8; 4]; + if let Err(e) = stream.read_exact(&mut reply).await { + panic!("client read_exact failed: {e}"); + } + assert_eq!(&reply, b"pong"); + + let server_seen = match server_task.await { + Ok(value) => value, + Err(e) => panic!("server task join failed: {e}"), + }; + assert_eq!(&server_seen, b"ping"); + } #[test] fn test_normalize_ip() { From 40dc6a39c1d3b54fd000c943ac947d3037785737 Mon Sep 17 00:00:00 2001 From: David Osipov Date: Wed, 11 Mar 2026 21:10:58 +0400 Subject: [PATCH 174/261] fix(socket): validate ack_timeout_secs and check setsockopt rc --- src/transport/socket.rs | 45 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/transport/socket.rs b/src/transport/socket.rs index 54eb143..aa4dc01 100644 --- a/src/transport/socket.rs +++ b/src/transport/socket.rs @@ -68,17 +68,27 @@ pub fn configure_client_socket( // is implemented in relay_bidirectional instead #[cfg(target_os = "linux")] { + use std::io::{Error, ErrorKind}; use std::os::unix::io::AsRawFd; + let fd = stream.as_raw_fd(); - let timeout_ms = (ack_timeout_secs * 1000) as libc::c_int; - unsafe { + let timeout_ms_u64 = ack_timeout_secs + .checked_mul(1000) + .ok_or_else(|| Error::new(ErrorKind::InvalidInput, "ack_timeout_secs is too large"))?; + let timeout_ms = i32::try_from(timeout_ms_u64) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "ack_timeout_secs exceeds TCP_USER_TIMEOUT range"))?; + + let rc = unsafe { libc::setsockopt( fd, libc::IPPROTO_TCP, libc::TCP_USER_TIMEOUT, - &timeout_ms as *const _ as *const libc::c_void, + &timeout_ms as *const libc::c_int as *const libc::c_void, std::mem::size_of::() as libc::socklen_t, - ); + ) + }; + if rc != 0 { + return Err(Error::last_os_error()); } } @@ -509,6 +519,33 @@ mod tests { }; assert_eq!(&server_seen, b"ping"); } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn test_configure_client_socket_ack_timeout_overflow_rejected() { + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(l) => l, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("bind failed: {e}"), + }; + let addr = match listener.local_addr() { + Ok(addr) => addr, + Err(e) => panic!("local_addr failed: {e}"), + }; + + let stream = match TcpStream::connect(addr).await { + Ok(s) => s, + Err(e) if e.kind() == ErrorKind::PermissionDenied => return, + Err(e) => panic!("connect failed: {e}"), + }; + + let too_large_secs = (i32::MAX as u64 / 1000) + 1; + let err = match configure_client_socket(&stream, 30, too_large_secs) { + Ok(()) => panic!("expected overflow validation error"), + Err(e) => e, + }; + assert_eq!(err.kind(), ErrorKind::InvalidInput); + } #[test] fn test_normalize_ip() { From ae72e6f356daaca3858886ab50459f6700449e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9=20=D0=9C=D0=B0?= =?UTF-8?q?=D1=80=D0=BA=D0=BE=D0=B2?= <13werwolf13@mail.ru> Date: Thu, 12 Mar 2026 12:26:23 +0500 Subject: [PATCH 175/261] systemd contrib, add sysuser & tmpfiles configs, fix service --- contrib/systemd/system-user-telemt.conf | 3 +++ contrib/systemd/telemt.service | 29 +++++++++++++++++++++++++ contrib/systemd/tmpfiles-telemt.conf | 1 + telemt.service | 16 -------------- 4 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 contrib/systemd/system-user-telemt.conf create mode 100644 contrib/systemd/telemt.service create mode 100644 contrib/systemd/tmpfiles-telemt.conf delete mode 100644 telemt.service diff --git a/contrib/systemd/system-user-telemt.conf b/contrib/systemd/system-user-telemt.conf new file mode 100644 index 0000000..918956c --- /dev/null +++ b/contrib/systemd/system-user-telemt.conf @@ -0,0 +1,3 @@ +u telemt - "telemt user" /var/lib/telemt - +g telemt - - +m telemt telemt diff --git a/contrib/systemd/telemt.service b/contrib/systemd/telemt.service new file mode 100644 index 0000000..8e08efc --- /dev/null +++ b/contrib/systemd/telemt.service @@ -0,0 +1,29 @@ +[Unit] +Description=Telemt +Wants=network-online.target +After=multi-user.target network.target network-online.target + +[Service] +Type=simple +User=telemt +Group=telemt +WorkingDirectory=/var/lib/telemt +ExecStart=/bin/telemt /etc/telemt/telemt.toml +Restart=on-failure +RestartSec=10 +LimitNOFILE=65536 +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=true +PrivateTmp=yes +PrivateDevices=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +ProtectSystem=strict +ProtectHome=read-only +SystemCallFilter=~@mount +ReadWritePaths=/var/lib/telemt + +[Install] +WantedBy=multi-user.target diff --git a/contrib/systemd/tmpfiles-telemt.conf b/contrib/systemd/tmpfiles-telemt.conf new file mode 100644 index 0000000..7c6628b --- /dev/null +++ b/contrib/systemd/tmpfiles-telemt.conf @@ -0,0 +1 @@ +d /var/lib/telemt 700 telemt telemt diff --git a/telemt.service b/telemt.service deleted file mode 100644 index f163057..0000000 --- a/telemt.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Telemt -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -WorkingDirectory=/etc/telemt -ExecStart=/bin/telemt /etc/telemt.toml -Restart=on-failure -LimitNOFILE=262144 -TasksMax=8192 -MemoryAccounting=yes - -[Install] -WantedBy=multi-user.target From 5073248911d438161dd43425bf1d0f903a259ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9=20=D0=9C=D0=B0?= =?UTF-8?q?=D1=80=D0=BA=D0=BE=D0=B2?= <13werwolf13@mail.ru> Date: Thu, 12 Mar 2026 12:47:03 +0500 Subject: [PATCH 176/261] systemd contrib, add sysuser & tmpfiles configs, fix service --- contrib/systemd/telemt.service | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/contrib/systemd/telemt.service b/contrib/systemd/telemt.service index 8e08efc..e2bae67 100644 --- a/contrib/systemd/telemt.service +++ b/contrib/systemd/telemt.service @@ -8,22 +8,14 @@ Type=simple User=telemt Group=telemt WorkingDirectory=/var/lib/telemt -ExecStart=/bin/telemt /etc/telemt/telemt.toml +ExecStart=/usr/bin/telemt /etc/telemt/telemt.toml Restart=on-failure RestartSec=10 LimitNOFILE=65536 AmbientCapabilities=CAP_NET_BIND_SERVICE CapabilityBoundingSet=CAP_NET_BIND_SERVICE NoNewPrivileges=true -PrivateTmp=yes -PrivateDevices=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -ProtectSystem=strict -ProtectHome=read-only -SystemCallFilter=~@mount -ReadWritePaths=/var/lib/telemt + [Install] WantedBy=multi-user.target From d1372c5c1be0987141b9d9f8c27720ebd6e6993e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:56:59 +0300 Subject: [PATCH 177/261] Update README.md --- README.md | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 8807393..1990109 100644 --- a/README.md +++ b/README.md @@ -19,22 +19,9 @@ ### 🇷🇺 RU -#### Релиз 3.3.5 LTS - 6 марта +#### Релиз 3.3.16 -6 марта мы выпустили Telemt **3.3.5** - -Это [3.3.5 - первая LTS-версия telemt](https://github.com/telemt/telemt/releases/tag/3.3.5)! - -В ней используется: -- новый алгоритм ME NoWait для непревзойдённо быстрого восстановления пула -- Adaptive Floor, поддерживающий количество ME Writer на оптимальном уровне -- модель усовершенствованного доступа к KDF Fingerprint на RwLock -- строгая привязка Middle-End к DC-ID с предсказуемым алгоритмом деградации и самовосстановления - -Telemt Control API V1 в 3.3.5 включает: -- несколько режимов работы в зависимости от доступных ресурсов -- снапшот-модель для живых метрик без вмешательства в hot-path -- минималистичный набор запросов для управления пользователями +[3.3.16](https://github.com/telemt/telemt/releases/tag/3.3.16)! Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **API**, **статистики**, **UX** @@ -53,22 +40,9 @@ Telemt Control API V1 в 3.3.5 включает: ### 🇬🇧 EN -#### Release 3.3.5 LTS - March 6 +#### Release 3.3.16 -On March 6, we released Telemt **3.3.3** - -This is [3.3.5 - the first LTS release of telemt](https://github.com/telemt/telemt/releases/tag/3.3.5) - -It introduces: -- the new ME NoWait algorithm for exceptionally fast pool recovery -- Adaptive Floor, which maintains the number of ME Writers at an optimal level -- an improved KDF Fingerprint access model based on RwLock -- strict binding of Middle-End instances to DC-ID with a predictable degradation and self-recovery algorithm - -Telemt Control API V1 in version 3.3.5 includes: -- multiple operating modes depending on available resources -- a snapshot-based model for live metrics without interfering with the hot path -- a minimalistic request set for user management +[3.3.16](https://github.com/telemt/telemt/releases/tag/3.3.16) We are looking forward to your feedback and improvement proposals — especially regarding **API**, **statistics**, **UX** From a15f74a6f966c287343a2945a7f6bffc8682132f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 13 Mar 2026 16:52:24 +0300 Subject: [PATCH 178/261] Configured middle_proxy_nat_ip for ME Gate on strartup --- src/maestro/mod.rs | 6 ++- src/network/probe.rs | 64 ++++++++++++++++++++++++-- src/transport/middle_proxy/pool.rs | 4 +- src/transport/middle_proxy/pool_nat.rs | 8 +++- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index 996070e..6724188 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -295,7 +295,11 @@ pub async fn run() -> std::result::Result<(), Box> { probe.detected_ipv4.map(IpAddr::V4), probe.detected_ipv6.map(IpAddr::V6), )); - let decision = decide_network_capabilities(&config.network, &probe); + let decision = decide_network_capabilities( + &config.network, + &probe, + config.general.middle_proxy_nat_ip, + ); log_probe_result(&probe, &decision); startup_tracker .complete_component( diff --git a/src/network/probe.rs b/src/network/probe.rs index 63e23a1..cbd32d9 100644 --- a/src/network/probe.rs +++ b/src/network/probe.rs @@ -226,18 +226,24 @@ async fn probe_stun_servers_parallel( out } -pub fn decide_network_capabilities(config: &NetworkConfig, probe: &NetworkProbe) -> NetworkDecision { +pub fn decide_network_capabilities( + config: &NetworkConfig, + probe: &NetworkProbe, + middle_proxy_nat_ip: Option, +) -> NetworkDecision { let ipv4_dc = config.ipv4 && probe.detected_ipv4.is_some(); let ipv6_dc = config.ipv6.unwrap_or(probe.detected_ipv6.is_some()) && probe.detected_ipv6.is_some(); + let nat_ip_v4 = matches!(middle_proxy_nat_ip, Some(IpAddr::V4(_))); + let nat_ip_v6 = matches!(middle_proxy_nat_ip, Some(IpAddr::V6(_))); let ipv4_me = config.ipv4 && probe.detected_ipv4.is_some() - && (!probe.ipv4_is_bogon || probe.reflected_ipv4.is_some()); + && (!probe.ipv4_is_bogon || probe.reflected_ipv4.is_some() || nat_ip_v4); let ipv6_enabled = config.ipv6.unwrap_or(probe.detected_ipv6.is_some()); let ipv6_me = ipv6_enabled && probe.detected_ipv6.is_some() - && (!probe.ipv6_is_bogon || probe.reflected_ipv6.is_some()); + && (!probe.ipv6_is_bogon || probe.reflected_ipv6.is_some() || nat_ip_v6); let effective_prefer = match config.prefer { 6 if ipv6_me || ipv6_dc => 6, @@ -262,6 +268,58 @@ pub fn decide_network_capabilities(config: &NetworkConfig, probe: &NetworkProbe) } } +#[cfg(test)] +mod tests { + use super::*; + use crate::config::NetworkConfig; + + #[test] + fn manual_nat_ip_enables_ipv4_me_without_reflection() { + let config = NetworkConfig { + ipv4: true, + ..Default::default() + }; + let probe = NetworkProbe { + detected_ipv4: Some(Ipv4Addr::new(10, 0, 0, 10)), + ipv4_is_bogon: true, + ..Default::default() + }; + + let decision = decide_network_capabilities( + &config, + &probe, + Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))), + ); + + assert!(decision.ipv4_me); + } + + #[test] + fn manual_nat_ip_does_not_enable_other_family() { + let config = NetworkConfig { + ipv4: true, + ipv6: Some(true), + ..Default::default() + }; + let probe = NetworkProbe { + detected_ipv4: Some(Ipv4Addr::new(10, 0, 0, 10)), + detected_ipv6: Some(Ipv6Addr::LOCALHOST), + ipv4_is_bogon: true, + ipv6_is_bogon: true, + ..Default::default() + }; + + let decision = decide_network_capabilities( + &config, + &probe, + Some(IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4))), + ); + + assert!(decision.ipv4_me); + assert!(!decision.ipv6_me); + } +} + fn detect_local_ip_v4() -> Option { let socket = UdpSocket::bind("0.0.0.0:0").ok()?; socket.connect("8.8.8.8:80").ok()?; diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 2d81e63..08ef142 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -638,9 +638,9 @@ impl MePool { } } + /// Translate the local ME address into the address material sent to the proxy. pub fn translate_our_addr(&self, addr: SocketAddr) -> SocketAddr { - let ip = self.translate_ip_for_nat(addr.ip()); - SocketAddr::new(ip, addr.port()) + self.translate_our_addr_with_reflection(addr, None) } pub fn registry(&self) -> &Arc { diff --git a/src/transport/middle_proxy/pool_nat.rs b/src/transport/middle_proxy/pool_nat.rs index 07ae0b8..b3b7bca 100644 --- a/src/transport/middle_proxy/pool_nat.rs +++ b/src/transport/middle_proxy/pool_nat.rs @@ -159,7 +159,13 @@ impl MePool { addr: std::net::SocketAddr, reflected: Option, ) -> std::net::SocketAddr { - let ip = if let Some(r) = reflected { + let ip = if let Some(nat_ip) = self.nat_ip_cfg { + match (addr.ip(), nat_ip) { + (IpAddr::V4(_), IpAddr::V4(dst)) => IpAddr::V4(dst), + (IpAddr::V6(_), IpAddr::V6(dst)) => IpAddr::V6(dst), + _ => addr.ip(), + } + } else if let Some(r) = reflected { // Use reflected IP (not port) only when local address is non-public. if is_bogon(addr.ip()) || addr.ip().is_loopback() || addr.ip().is_unspecified() { r.ip() From 780fafa6041c65cc854469fc9f1474a46b86b482 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 02:20:51 +0300 Subject: [PATCH 179/261] Src-IP in ME Routing + more strict bind_addresses --- src/maestro/mod.rs | 1 + src/network/probe.rs | 141 ++++++++++++++++++++-- src/transport/middle_proxy/handshake.rs | 2 + src/transport/middle_proxy/pool.rs | 1 + src/transport/middle_proxy/pool_writer.rs | 1 + src/transport/middle_proxy/send.rs | 44 +++++-- src/transport/upstream.rs | 46 ++++++- 7 files changed, 206 insertions(+), 30 deletions(-) diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index 6724188..5f6c70a 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -287,6 +287,7 @@ pub async fn run() -> std::result::Result<(), Box> { .await; let probe = run_probe( &config.network, + &config.upstreams, config.general.middle_proxy_nat_probe, config.general.stun_nat_probe_concurrency, ) diff --git a/src/network/probe.rs b/src/network/probe.rs index cbd32d9..a9e369d 100644 --- a/src/network/probe.rs +++ b/src/network/probe.rs @@ -8,9 +8,10 @@ use tokio::task::JoinSet; use tokio::time::timeout; use tracing::{debug, info, warn}; -use crate::config::NetworkConfig; +use crate::config::{NetworkConfig, UpstreamConfig, UpstreamType}; use crate::error::Result; -use crate::network::stun::{stun_probe_dual, DualStunResult, IpFamily, StunProbeResult}; +use crate::network::stun::{stun_probe_family_with_bind, DualStunResult, IpFamily, StunProbeResult}; +use crate::transport::UpstreamManager; #[derive(Debug, Clone, Default)] pub struct NetworkProbe { @@ -57,19 +58,22 @@ const STUN_BATCH_TIMEOUT: Duration = Duration::from_secs(5); pub async fn run_probe( config: &NetworkConfig, + upstreams: &[UpstreamConfig], nat_probe: bool, stun_nat_probe_concurrency: usize, ) -> Result { let mut probe = NetworkProbe::default(); + let servers = collect_stun_servers(config); + let mut detected_ipv4 = detect_local_ip_v4(); + let mut detected_ipv6 = detect_local_ip_v6(); + let mut explicit_detected_ipv4 = false; + let mut explicit_detected_ipv6 = false; + let mut explicit_reflected_ipv4 = false; + let mut explicit_reflected_ipv6 = false; + let mut strict_bind_ipv4_requested = false; + let mut strict_bind_ipv6_requested = false; - probe.detected_ipv4 = detect_local_ip_v4(); - probe.detected_ipv6 = detect_local_ip_v6(); - - probe.ipv4_is_bogon = probe.detected_ipv4.map(is_bogon_v4).unwrap_or(false); - probe.ipv6_is_bogon = probe.detected_ipv6.map(is_bogon_v6).unwrap_or(false); - - let stun_res = if nat_probe && config.stun_use { - let servers = collect_stun_servers(config); + let global_stun_res = if nat_probe && config.stun_use { if servers.is_empty() { warn!("STUN probe is enabled but network.stun_servers is empty"); DualStunResult::default() @@ -77,6 +81,8 @@ pub async fn run_probe( probe_stun_servers_parallel( &servers, stun_nat_probe_concurrency.max(1), + None, + None, ) .await } @@ -86,8 +92,108 @@ pub async fn run_probe( } else { DualStunResult::default() }; - probe.reflected_ipv4 = stun_res.v4.map(|r| r.reflected_addr); - probe.reflected_ipv6 = stun_res.v6.map(|r| r.reflected_addr); + let mut reflected_ipv4 = global_stun_res.v4.map(|r| r.reflected_addr); + let mut reflected_ipv6 = global_stun_res.v6.map(|r| r.reflected_addr); + + for upstream in upstreams.iter().filter(|upstream| upstream.enabled) { + let UpstreamType::Direct { + interface, + bind_addresses, + } = &upstream.upstream_type else { + continue; + }; + if let Some(addrs) = bind_addresses.as_ref().filter(|v| !v.is_empty()) { + let mut saw_parsed_ip = false; + for value in addrs { + if let Ok(ip) = value.parse::() { + saw_parsed_ip = true; + if ip.is_ipv4() { + strict_bind_ipv4_requested = true; + } else { + strict_bind_ipv6_requested = true; + } + } + } + if !saw_parsed_ip { + strict_bind_ipv4_requested = true; + strict_bind_ipv6_requested = true; + } + } + + let bind_v4 = UpstreamManager::resolve_bind_address( + interface, + bind_addresses, + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), 443), + None, + true, + ); + let bind_v6 = UpstreamManager::resolve_bind_address( + interface, + bind_addresses, + SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1)), + 443, + ), + None, + true, + ); + + if let Some(IpAddr::V4(ip)) = bind_v4 + && !explicit_detected_ipv4 + { + detected_ipv4 = Some(ip); + explicit_detected_ipv4 = true; + } + if let Some(IpAddr::V6(ip)) = bind_v6 + && !explicit_detected_ipv6 + { + detected_ipv6 = Some(ip); + explicit_detected_ipv6 = true; + } + if bind_v4.is_none() && bind_v6.is_none() { + continue; + } + + if !(nat_probe && config.stun_use) || servers.is_empty() { + continue; + } + + let direct_stun_res = probe_stun_servers_parallel( + &servers, + stun_nat_probe_concurrency.max(1), + bind_v4, + bind_v6, + ) + .await; + if let Some(reflected) = direct_stun_res.v4.map(|r| r.reflected_addr) { + reflected_ipv4 = Some(reflected); + explicit_reflected_ipv4 = true; + } + if let Some(reflected) = direct_stun_res.v6.map(|r| r.reflected_addr) { + reflected_ipv6 = Some(reflected); + explicit_reflected_ipv6 = true; + } + } + + if strict_bind_ipv4_requested && !explicit_detected_ipv4 { + detected_ipv4 = None; + reflected_ipv4 = None; + } else if strict_bind_ipv4_requested && !explicit_reflected_ipv4 { + reflected_ipv4 = None; + } + if strict_bind_ipv6_requested && !explicit_detected_ipv6 { + detected_ipv6 = None; + reflected_ipv6 = None; + } else if strict_bind_ipv6_requested && !explicit_reflected_ipv6 { + reflected_ipv6 = None; + } + + probe.detected_ipv4 = detected_ipv4; + probe.detected_ipv6 = detected_ipv6; + probe.reflected_ipv4 = reflected_ipv4; + probe.reflected_ipv6 = reflected_ipv6; + probe.ipv4_is_bogon = probe.detected_ipv4.map(is_bogon_v4).unwrap_or(false); + probe.ipv6_is_bogon = probe.detected_ipv6.map(is_bogon_v6).unwrap_or(false); // If STUN is blocked but IPv4 is private, try HTTP public-IP fallback. if nat_probe @@ -162,6 +268,8 @@ fn collect_stun_servers(config: &NetworkConfig) -> Vec { async fn probe_stun_servers_parallel( servers: &[String], concurrency: usize, + bind_v4: Option, + bind_v6: Option, ) -> DualStunResult { let mut join_set = JoinSet::new(); let mut next_idx = 0usize; @@ -172,8 +280,15 @@ async fn probe_stun_servers_parallel( while next_idx < servers.len() && join_set.len() < concurrency { let stun_addr = servers[next_idx].clone(); next_idx += 1; + let bind_v4 = bind_v4; + let bind_v6 = bind_v6; join_set.spawn(async move { - let res = timeout(STUN_BATCH_TIMEOUT, stun_probe_dual(&stun_addr)).await; + let res = timeout(STUN_BATCH_TIMEOUT, async { + let v4 = stun_probe_family_with_bind(&stun_addr, IpFamily::V4, bind_v4).await?; + let v6 = stun_probe_family_with_bind(&stun_addr, IpFamily::V6, bind_v6).await?; + Ok::(DualStunResult { v4, v6 }) + }) + .await; (stun_addr, res) }); } diff --git a/src/transport/middle_proxy/handshake.rs b/src/transport/middle_proxy/handshake.rs index 0d7626c..39e34d7 100644 --- a/src/transport/middle_proxy/handshake.rs +++ b/src/transport/middle_proxy/handshake.rs @@ -59,6 +59,7 @@ impl KdfClientPortSource { pub(crate) struct HandshakeOutput { pub rd: ReadHalf, pub wr: WriteHalf, + pub source_ip: IpAddr, pub read_key: [u8; 32], pub read_iv: [u8; 16], pub write_key: [u8; 32], @@ -689,6 +690,7 @@ impl MePool { Ok(HandshakeOutput { rd, wr, + source_ip: local_addr_nat.ip(), read_key: rk, read_iv, write_key: wk, diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 08ef142..42cba81 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -34,6 +34,7 @@ pub(super) struct RefillEndpointKey { pub struct MeWriter { pub id: u64, pub addr: SocketAddr, + pub source_ip: IpAddr, pub writer_dc: i32, pub generation: u64, pub contour: Arc, diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 7e79f10..64fb700 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -163,6 +163,7 @@ impl MePool { let writer = MeWriter { id: writer_id, addr, + source_ip: hs.source_ip, writer_dc, generation, contour: contour.clone(), diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 79cfa54..f63662b 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -42,20 +42,30 @@ impl MePool { tag_override: Option<&[u8]>, ) -> Result<()> { let tag = tag_override.or(self.proxy_tag.as_deref()); - let payload = build_proxy_req_payload( - conn_id, - client_addr, - our_addr, - data, - tag, - proto_flags, - ); - let meta = ConnMeta { + let fallback_meta = ConnMeta { target_dc, client_addr, our_addr, proto_flags, }; + let build_routed_payload = |effective_our_addr: SocketAddr| { + ( + build_proxy_req_payload( + conn_id, + client_addr, + effective_our_addr, + data, + tag, + proto_flags, + ), + ConnMeta { + target_dc, + client_addr, + our_addr: effective_our_addr, + proto_flags, + }, + ) + }; let no_writer_mode = MeRouteNoWriterMode::from_u8(self.me_route_no_writer_mode.load(Ordering::Relaxed)); let (routed_dc, unknown_target_dc) = self @@ -70,8 +80,14 @@ impl MePool { let mut hybrid_wait_current = hybrid_wait_step; loop { + let current_meta = self + .registry + .get_meta(conn_id) + .await + .unwrap_or_else(|| fallback_meta.clone()); + let (current_payload, _) = build_routed_payload(current_meta.our_addr); if let Some(current) = self.registry.get_writer(conn_id).await { - match current.tx.try_send(WriterCommand::Data(payload.clone())) { + match current.tx.try_send(WriterCommand::Data(current_payload.clone())) { Ok(()) => return Ok(()), Err(TrySendError::Full(cmd)) => { if current.tx.send(cmd).await.is_ok() { @@ -354,11 +370,13 @@ impl MePool { if !self.writer_accepts_new_binding(w) { continue; } + let effective_our_addr = SocketAddr::new(w.source_ip, our_addr.port()); + let (payload, meta) = build_routed_payload(effective_our_addr); match w.tx.try_send(WriterCommand::Data(payload.clone())) { Ok(()) => { self.stats.increment_me_writer_pick_success_try_total(pick_mode); self.registry - .bind_writer(conn_id, w.id, w.tx.clone(), meta.clone()) + .bind_writer(conn_id, w.id, w.tx.clone(), meta) .await; if w.generation < self.current_generation() { self.stats.increment_pool_stale_pick_total(); @@ -397,12 +415,14 @@ impl MePool { continue; } self.stats.increment_me_writer_pick_blocking_fallback_total(); + let effective_our_addr = SocketAddr::new(w.source_ip, our_addr.port()); + let (payload, meta) = build_routed_payload(effective_our_addr); match w.tx.send(WriterCommand::Data(payload.clone())).await { Ok(()) => { self.stats .increment_me_writer_pick_success_fallback_total(pick_mode); self.registry - .bind_writer(conn_id, w.id, w.tx.clone(), meta.clone()) + .bind_writer(conn_id, w.id, w.tx.clone(), meta) .await; if w.generation < self.current_generation() { self.stats.increment_pool_stale_pick_total(); diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index 1355934..8360e1e 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -390,7 +390,7 @@ impl UpstreamManager { out } - fn resolve_bind_address( + pub(crate) fn resolve_bind_address( interface: &Option, bind_addresses: &Option>, target: SocketAddr, @@ -399,7 +399,7 @@ impl UpstreamManager { ) -> Option { let want_ipv6 = target.is_ipv6(); - if let Some(addrs) = bind_addresses { + if let Some(addrs) = bind_addresses.as_ref().filter(|v| !v.is_empty()) { let mut candidates: Vec = addrs .iter() .filter_map(|s| s.parse::().ok()) @@ -431,7 +431,7 @@ impl UpstreamManager { warn!( interface = %iface, target = %target, - "Configured interface has no addresses for target family; falling back to direct connect without bind" + "Configured interface has no addresses for target family" ); candidates.clear(); } @@ -454,10 +454,11 @@ impl UpstreamManager { warn!( interface = interface.as_deref().unwrap_or(""), target = %target, - "No valid bind_addresses left for interface; falling back to direct connect without bind" + "No valid bind_addresses left for interface" ); - return None; } + + return None; } if let Some(iface) = interface { @@ -795,6 +796,13 @@ impl UpstreamManager { bind_rr.as_deref(), true, ); + if bind_ip.is_none() + && bind_addresses.as_ref().is_some_and(|v| !v.is_empty()) + { + return Err(ProxyError::Config(format!( + "No valid bind_addresses for target family {target}" + ))); + } let socket = create_outgoing_socket_bound(target, bind_ip)?; if let Some(ip) = bind_ip { @@ -1642,4 +1650,32 @@ mod tests { }; assert!(!UpstreamManager::is_hard_connect_error(&error)); } + + #[test] + fn resolve_bind_address_prefers_explicit_bind_ip() { + let target = "203.0.113.10:443".parse::().unwrap(); + let bind = UpstreamManager::resolve_bind_address( + &Some("198.51.100.20".to_string()), + &Some(vec!["198.51.100.10".to_string()]), + target, + None, + true, + ); + + assert_eq!(bind, Some("198.51.100.10".parse::().unwrap())); + } + + #[test] + fn resolve_bind_address_does_not_fallback_to_interface_when_bind_addresses_present() { + let target = "203.0.113.10:443".parse::().unwrap(); + let bind = UpstreamManager::resolve_bind_address( + &Some("198.51.100.20".to_string()), + &Some(vec!["2001:db8::10".to_string()]), + target, + None, + true, + ); + + assert_eq!(bind, None); + } } From 89923dbaa2c77414b435a98c8757eca2942d3e26 Mon Sep 17 00:00:00 2001 From: farton1983 Date: Sat, 14 Mar 2026 11:07:12 +0300 Subject: [PATCH 180/261] Update QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index 0ed5d31..67dcbe2 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -151,7 +151,7 @@ curl -s http://127.0.0.1:9091/v1/users | jq > Одной ссылкой может пользоваться сколько угодно человек. > [!WARNING] -> Рабочую ссылку может выдать только команда из 6 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете! +> Рабочую ссылку может выдать только команда из 7 пункта. Не пытайтесь делать ее самостоятельно или копировать откуда-либо если вы не уверены в том, что делаете! --- From 50caeb180342b97876429ae82dde7daad09f2474 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 13:24:16 +0300 Subject: [PATCH 181/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index afdf5b9..dd3e5fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.15" +version = "3.3.17" edition = "2024" [dependencies] From 4506f38bfb53368d1a01e6b3c4932776ce10260f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:02:12 +0300 Subject: [PATCH 182/261] Update LICENSING.md --- LICENSING.md | 82 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/LICENSING.md b/LICENSING.md index 50d007b..6c0571b 100644 --- a/LICENSING.md +++ b/LICENSING.md @@ -1,17 +1,75 @@ # LICENSING ## Licenses for Versions -| Version | License | -|---------|---------------| -| 1.0 | NO LICNESE | -| 1.1 | NO LICENSE | -| 1.2 | NO LICENSE | -| 2.0 | NO LICENSE | -| 3.0 | TELEMT UL 1 | +| Version ≥ | Version ≤ | License | +|-----------|-----------|---------------| +| 1.0 | 2.0.0.1 | NO LICNESE | +| 3.0.0 | 3.4.0 | TELEMT PL 3 | ### License Types - **NO LICENSE** = ***ALL RIGHT RESERVED*** -- **TELEMT UL1** - work in progress license for source code of `telemt`, which encourages: - - fair use, - - contributions, - - distribution, - - but prohibits NOT mentioning the authors +- **TELEMT PL** - special Telemt Public License based on Apache License 2 principles + +## Telemt Public License 3 +```text +TELEMT Public License 3 - 2026 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions. + +#### 1 Attribution + +Redistributions of the Software, in source or binary form, must retain the +above copyright notice, this license text, and any existing attribution +notices. + +#### 2 Modification Notice + +If you modify the Software, you must clearly state that the Software has been +modified and include a brief description of the changes made. + +Modified versions must not be presented as the original Telemt project. + +#### 3 Trademark and Branding + +This license does not grant permission to use the name "Telemt", the Telemt +logo, or any Telemt trademarks or branding. + +Redistributed or modified versions of the Software may not use the Telemt +name in a way that suggests endorsement or official origin without explicit +permission from the Telemt project maintainers. + +#### 4 Binary Distribution Transparency + +If you distribute compiled binaries of the Software, you must make the +corresponding source code and build instructions available to recipients +of those binaries. + +This requirement exists to preserve transparency and allow verification +of distributed builds. + +#### 5 Patent Grant + +Each contributor grants you a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Software. + +If you initiate patent litigation claiming that the Software or a +contribution incorporated within the Software constitutes direct or +contributory patent infringement, then any patent licenses granted to you +under this license shall terminate as of the date such litigation is filed. + +#### 6 Disclaimer of Warranty + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE +``` From 4dc733d3e36c8e6f333069bcf0787a3df014c9be Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:03:29 +0300 Subject: [PATCH 183/261] Create LICENSE --- LICENSE | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d1e9634 --- /dev/null +++ b/LICENSE @@ -0,0 +1,61 @@ +### TELEMT Public License 3 - 2026 ### + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions. + +#### 1 Attribution + +Redistributions of the Software, in source or binary form, must retain the +above copyright notice, this license text, and any existing attribution +notices. + +#### 2 Modification Notice + +If you modify the Software, you must clearly state that the Software has been +modified and include a brief description of the changes made. + +Modified versions must not be presented as the original Telemt project. + +#### 3 Trademark and Branding + +This license does not grant permission to use the name "Telemt", the Telemt +logo, or any Telemt trademarks or branding. + +Redistributed or modified versions of the Software may not use the Telemt +name in a way that suggests endorsement or official origin without explicit +permission from the Telemt project maintainers. + +#### 4 Binary Distribution Transparency + +If you distribute compiled binaries of the Software, you must make the +corresponding source code and build instructions available to recipients +of those binaries. + +This requirement exists to preserve transparency and allow verification +of distributed builds. + +#### 5 Patent Grant + +Each contributor grants you a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Software. + +If you initiate patent litigation claiming that the Software or a +contribution incorporated within the Software constitutes direct or +contributory patent infringement, then any patent licenses granted to you +under this license shall terminate as of the date such litigation is filed. + +#### 6 Disclaimer of Warranty + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE From 543a87e16694f9346eb299a5455172fdf6cdcd10 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:08:08 +0300 Subject: [PATCH 184/261] Update LICENSE --- LICENSE | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index d1e9634..2f605e5 100644 --- a/LICENSE +++ b/LICENSE @@ -31,12 +31,11 @@ permission from the Telemt project maintainers. #### 4 Binary Distribution Transparency -If you distribute compiled binaries of the Software, you must make the -corresponding source code and build instructions available to recipients -of those binaries. +If you distribute compiled binaries of the Software, you are encouraged to +provide access to the corresponding source code and build instructions. -This requirement exists to preserve transparency and allow verification -of distributed builds. +This helps preserve transparency and allows recipients to verify the +integrity and reproducibility of distributed builds. #### 5 Patent Grant From 0c1a5c24d58ce11a738aafdd37a3ffa5b7a175b8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:27:45 +0300 Subject: [PATCH 185/261] Update LICENSE --- LICENSE | 60 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/LICENSE b/LICENSE index d1e9634..f1d399f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,55 +1,75 @@ -### TELEMT Public License 3 - 2026 ### +###### TELEMT Public License 3 ###### +##### Copyright (c) 2026 Telemt ##### Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights +in the Software WITHOUT restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions. #### 1 Attribution -Redistributions of the Software, in source or binary form, must retain the +Redistributions of the Software, in source or binary form, MUST RETAIN the above copyright notice, this license text, and any existing attribution notices. #### 2 Modification Notice -If you modify the Software, you must clearly state that the Software has been +If you modify the Software, you MUST clearly state that the Software has been modified and include a brief description of the changes made. -Modified versions must not be presented as the original Telemt project. +Modified versions MUST NOT be presented as the original Telemt. #### 3 Trademark and Branding -This license does not grant permission to use the name "Telemt", the Telemt -logo, or any Telemt trademarks or branding. +This license DOES NOT grant permission to use the name "Telemt", +the Telemt logo, or any Telemt trademarks or branding. -Redistributed or modified versions of the Software may not use the Telemt +Redistributed or modified versions of the Software MAY NOT use the Telemt name in a way that suggests endorsement or official origin without explicit -permission from the Telemt project maintainers. +permission from the Telemt maintainers. #### 4 Binary Distribution Transparency -If you distribute compiled binaries of the Software, you must make the -corresponding source code and build instructions available to recipients -of those binaries. +If you distribute compiled binaries of the Software, +you are ENCOURAGED to provide access to the corresponding +source code and build instructions where reasonably possible. -This requirement exists to preserve transparency and allow verification -of distributed builds. +This helps preserve transparency and allows recipients to verify the +integrity and reproducibility of distributed builds. -#### 5 Patent Grant +#### 5 Patent Grant and Defensive Termination Clause Each contributor grants you a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Software. -If you initiate patent litigation claiming that the Software or a -contribution incorporated within the Software constitutes direct or -contributory patent infringement, then any patent licenses granted to you -under this license shall terminate as of the date such litigation is filed. +This patent license applies only to those patent claims necessarily +infringed by the contributor’s contribution alone or by combination of +their contribution with the Software. -#### 6 Disclaimer of Warranty +If you initiate or participate in any patent litigation, including +cross-claims or counterclaims, alleging that the Software or any +contribution incorporated within the Software constitutes patent +infringement, then **all rights granted to you under this license shall +terminate immediately** as of the date such litigation is filed. + +Additionally, if you initiate legal action alleging that the +Software itself infringes your patent or other intellectual +property rights, then all rights granted to you under this +license SHALL TERMINATE automatically. + +#### 7 Network Use Attribution + +If the Software is used to provide a publicly accessible network +service, the operator of such service MUST RETAIN reasonable +attribution to the Telemt in documentation, service +descriptions, or user-visible materials. + +Such attribution MUST NOT imply endorsement by the Telemt. + +#### 8 Disclaimer of Warranty THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, From 1674ba36b20d95948d98e6f966d7a1538d7ec1b8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:31:57 +0300 Subject: [PATCH 186/261] Update LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index f1d399f..24d5c55 100644 --- a/LICENSE +++ b/LICENSE @@ -60,7 +60,7 @@ Software itself infringes your patent or other intellectual property rights, then all rights granted to you under this license SHALL TERMINATE automatically. -#### 7 Network Use Attribution +#### 6 Network Use Attribution If the Software is used to provide a publicly accessible network service, the operator of such service MUST RETAIN reasonable @@ -69,7 +69,7 @@ descriptions, or user-visible materials. Such attribution MUST NOT imply endorsement by the Telemt. -#### 8 Disclaimer of Warranty +#### 7 Disclaimer of Warranty THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, From 70cc6f22aa46cb3440638bbdd17ebafe97f45284 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:32:41 +0300 Subject: [PATCH 187/261] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 24d5c55..825e8b4 100644 --- a/LICENSE +++ b/LICENSE @@ -63,7 +63,7 @@ license SHALL TERMINATE automatically. #### 6 Network Use Attribution If the Software is used to provide a publicly accessible network -service, the operator of such service MUST RETAIN reasonable +service, the operator of such service MUST RETAIN visible attribution to the Telemt in documentation, service descriptions, or user-visible materials. From 8a579d9bdae4d63da4c060a00b892a50e52d1cb7 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:38:51 +0300 Subject: [PATCH 188/261] Update LICENSE --- LICENSE | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/LICENSE b/LICENSE index 825e8b4..0eece9a 100644 --- a/LICENSE +++ b/LICENSE @@ -8,6 +8,27 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions. +### Definitions + +For the purposes of this License: + +"Software" means the Telemt software, including source code, documentation, +and any associated files distributed under this License. + +"Contributor" means any person or entity that submits code, patches, +documentation, or other contributions to the Software that are accepted +into the Software by the maintainers. + +"Contribution" means any work of authorship intentionally submitted +to the Software for inclusion in the Software. + +"Modified Version" means any version of the Software that has been +changed, adapted, extended, or otherwise modified from the original +Software. + +"Maintainters" means the individuals or entities responsible for +the official Telemt project and its releases. + #### 1 Attribution Redistributions of the Software, in source or binary form, MUST RETAIN the @@ -30,6 +51,13 @@ Redistributed or modified versions of the Software MAY NOT use the Telemt name in a way that suggests endorsement or official origin without explicit permission from the Telemt maintainers. +Use of the name "Telemt" to describe a modified version of the Software +is permitted only if the modified version is clearly identified as a +modified or unofficial version. + +Any distribution that could reasonably confuse users into believing that +the software is an official Telemt release is prohibited. + #### 4 Binary Distribution Transparency If you distribute compiled binaries of the Software, @@ -60,16 +88,31 @@ Software itself infringes your patent or other intellectual property rights, then all rights granted to you under this license SHALL TERMINATE automatically. -#### 6 Network Use Attribution +#### 6 Contributions -If the Software is used to provide a publicly accessible network -service, the operator of such service MUST RETAIN visible -attribution to the Telemt in documentation, service -descriptions, or user-visible materials. +Unless you explicitly state otherwise, any Contribution intentionally +submitted for inclusion in the Software shall be licensed under the terms +of this License. -Such attribution MUST NOT imply endorsement by the Telemt. +By submitting a Contribution, you grant the Telemt maintainers and all +recipients of the Software the rights described in this License with +respect to that Contribution. -#### 7 Disclaimer of Warranty +#### 7 Network Use Attribution + +If the Software is used to provide a publicly accessible network service, +the operator of such service MUST provide attribution to Telemt in at least +one of the following locations: + +- service documentation +- service description +- an "About" or similar informational page +- other user-visible materials reasonably associated with the service + +Such attribution MUST NOT imply endorsement by the Telemt project or its +maintainers. + +#### 8 Disclaimer of Warranty and Severability Clause THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -79,3 +122,8 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE + +IF ANY OF PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, +SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT +OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS +SHALL REMAIN IN FULL FORCE AND EFFECT From d4ce304a37e2d9652c832ce58c1afedde1afcec2 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:40:10 +0300 Subject: [PATCH 189/261] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 0eece9a..e249ac3 100644 --- a/LICENSE +++ b/LICENSE @@ -26,7 +26,7 @@ to the Software for inclusion in the Software. changed, adapted, extended, or otherwise modified from the original Software. -"Maintainters" means the individuals or entities responsible for +"Maintainers" means the individuals or entities responsible for the official Telemt project and its releases. #### 1 Attribution From aa68ce531e87c558f387d5b0f8e37e76fe956601 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:42:36 +0300 Subject: [PATCH 190/261] Update LICENSE --- LICENSE | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index e249ac3..b972ca1 100644 --- a/LICENSE +++ b/LICENSE @@ -2,11 +2,12 @@ ##### Copyright (c) 2026 Telemt ##### Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software WITHOUT restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions. +of this Software and associated documentation files (the "Software"), +to use, reproduce, modify, prepare derivative works of, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, provided that all +copyright notices, license terms, and conditions set forth in this License +are preserved and complied with. ### Definitions @@ -123,7 +124,7 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE -IF ANY OF PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, +IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS SHALL REMAIN IN FULL FORCE AND EFFECT From 5cfb05b1f464a0e3bcb4ed10818f943533a0b28f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:47:21 +0300 Subject: [PATCH 191/261] Update LICENSING.md --- LICENSING.md | 117 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 24 deletions(-) diff --git a/LICENSING.md b/LICENSING.md index 6c0571b..9b9ac49 100644 --- a/LICENSING.md +++ b/LICENSING.md @@ -11,58 +11,122 @@ ## Telemt Public License 3 ```text -TELEMT Public License 3 - 2026 +###### TELEMT Public License 3 ###### +##### Copyright (c) 2026 Telemt ##### Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions. +of this Software and associated documentation files (the "Software"), +to use, reproduce, modify, prepare derivative works of, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, provided that all +copyright notices, license terms, and conditions set forth in this License +are preserved and complied with. + +### Definitions + +For the purposes of this License: + +"Software" means the Telemt software, including source code, documentation, +and any associated files distributed under this License. + +"Contributor" means any person or entity that submits code, patches, +documentation, or other contributions to the Software that are accepted +into the Software by the maintainers. + +"Contribution" means any work of authorship intentionally submitted +to the Software for inclusion in the Software. + +"Modified Version" means any version of the Software that has been +changed, adapted, extended, or otherwise modified from the original +Software. + +"Maintainers" means the individuals or entities responsible for +the official Telemt project and its releases. #### 1 Attribution -Redistributions of the Software, in source or binary form, must retain the +Redistributions of the Software, in source or binary form, MUST RETAIN the above copyright notice, this license text, and any existing attribution notices. #### 2 Modification Notice -If you modify the Software, you must clearly state that the Software has been +If you modify the Software, you MUST clearly state that the Software has been modified and include a brief description of the changes made. -Modified versions must not be presented as the original Telemt project. +Modified versions MUST NOT be presented as the original Telemt. #### 3 Trademark and Branding -This license does not grant permission to use the name "Telemt", the Telemt -logo, or any Telemt trademarks or branding. +This license DOES NOT grant permission to use the name "Telemt", +the Telemt logo, or any Telemt trademarks or branding. -Redistributed or modified versions of the Software may not use the Telemt +Redistributed or modified versions of the Software MAY NOT use the Telemt name in a way that suggests endorsement or official origin without explicit -permission from the Telemt project maintainers. +permission from the Telemt maintainers. + +Use of the name "Telemt" to describe a modified version of the Software +is permitted only if the modified version is clearly identified as a +modified or unofficial version. + +Any distribution that could reasonably confuse users into believing that +the software is an official Telemt release is prohibited. #### 4 Binary Distribution Transparency -If you distribute compiled binaries of the Software, you must make the -corresponding source code and build instructions available to recipients -of those binaries. +If you distribute compiled binaries of the Software, +you are ENCOURAGED to provide access to the corresponding +source code and build instructions where reasonably possible. -This requirement exists to preserve transparency and allow verification -of distributed builds. +This helps preserve transparency and allows recipients to verify the +integrity and reproducibility of distributed builds. -#### 5 Patent Grant +#### 5 Patent Grant and Defensive Termination Clause Each contributor grants you a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Software. -If you initiate patent litigation claiming that the Software or a -contribution incorporated within the Software constitutes direct or -contributory patent infringement, then any patent licenses granted to you -under this license shall terminate as of the date such litigation is filed. +This patent license applies only to those patent claims necessarily +infringed by the contributor’s contribution alone or by combination of +their contribution with the Software. -#### 6 Disclaimer of Warranty +If you initiate or participate in any patent litigation, including +cross-claims or counterclaims, alleging that the Software or any +contribution incorporated within the Software constitutes patent +infringement, then **all rights granted to you under this license shall +terminate immediately** as of the date such litigation is filed. + +Additionally, if you initiate legal action alleging that the +Software itself infringes your patent or other intellectual +property rights, then all rights granted to you under this +license SHALL TERMINATE automatically. + +#### 6 Contributions + +Unless you explicitly state otherwise, any Contribution intentionally +submitted for inclusion in the Software shall be licensed under the terms +of this License. + +By submitting a Contribution, you grant the Telemt maintainers and all +recipients of the Software the rights described in this License with +respect to that Contribution. + +#### 7 Network Use Attribution + +If the Software is used to provide a publicly accessible network service, +the operator of such service MUST provide attribution to Telemt in at least +one of the following locations: + +- service documentation +- service description +- an "About" or similar informational page +- other user-visible materials reasonably associated with the service + +Such attribution MUST NOT imply endorsement by the Telemt project or its +maintainers. + +#### 8 Disclaimer of Warranty and Severability Clause THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -72,4 +136,9 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE + +IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, +SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT +OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS +SHALL REMAIN IN FULL FORCE AND EFFECT ``` From 47b8f0f65663fe9aee00ee51b7a96fc55a1019de Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:55:48 +0300 Subject: [PATCH 192/261] Delete config.full.toml --- config.full.toml | 697 ----------------------------------------------- 1 file changed, 697 deletions(-) delete mode 100644 config.full.toml diff --git a/config.full.toml b/config.full.toml deleted file mode 100644 index 69e7d2b..0000000 --- a/config.full.toml +++ /dev/null @@ -1,697 +0,0 @@ -# ============================================================================== -# -# TELEMT — Advanced Rust-based Telegram MTProto Proxy -# Full Configuration Reference -# -# This file is both a working config and a complete documentation. -# Every parameter is explained. Read it top to bottom before deploying. -# -# Quick Start: -# 1. Set [server].port to your desired port (443 recommended) -# 2. Generate a secret: openssl rand -hex 16 -# 3. Put it in [access.users] under a name you choose -# 4. Set [censorship].tls_domain to a popular unblocked HTTPS site -# 5. Set your public IP in [general].middle_proxy_nat_ip -# and [general.links].public_host -# 6. Set announce IP in [[server.listeners]] -# 7. Run Telemt. It prints a tg:// link. Send it to your users. -# -# Modes of Operation: -# Direct Mode (use_middle_proxy = false) -# Connects straight to Telegram DCs via TCP. Simple, fast, low overhead. -# No ad_tag support. No CDN DC support (203, etc). -# -# Middle-Proxy Mode (use_middle_proxy = true) -# Connects to Telegram Middle-End servers via RPC protocol. -# Required for ad_tag monetization and CDN support. -# Requires proxy_secret_path and a valid public IP. -# -# ============================================================================== - - -# ============================================================================== -# LEGACY TOP-LEVEL FIELDS -# ============================================================================== - -# Deprecated. Use [general.links].show instead. -# Accepts "*" for all users, or an array like ["alice", "bob"]. -show_link = ["0"] - -# Fallback Datacenter index (1-5) when a client requests an unknown DC ID. -# DC 2 is Amsterdam (Europe), closest for most CIS users. -# default_dc = 2 - - -# ============================================================================== -# GENERAL SETTINGS -# ============================================================================== - -[general] - -# ------------------------------------------------------------------------------ -# Core Protocol -# ------------------------------------------------------------------------------ - -# Coalesce the MTProto handshake and first data payload into a single TCP packet. -# Significantly reduces connection latency. No reason to disable. -fast_mode = true - -# How the proxy connects to Telegram servers. -# false = Direct TCP to Telegram DCs (simple, low overhead) -# true = Middle-End RPC protocol (required for ad_tag and CDN DCs) -use_middle_proxy = true - -# 32-char hex Ad-Tag from @MTProxybot for sponsored channel injection. -# Only works when use_middle_proxy = true. -# Obtain yours: message @MTProxybot on Telegram, register your proxy. -# ad_tag = "00000000000000000000000000000000" - -# ------------------------------------------------------------------------------ -# Middle-End Authentication -# ------------------------------------------------------------------------------ - -# Path to the Telegram infrastructure AES key file. -# Auto-downloaded from https://core.telegram.org/getProxySecret on first run. -# This key authenticates your proxy with Middle-End servers. -proxy_secret_path = "proxy-secret" - -# ------------------------------------------------------------------------------ -# Public IP Configuration (Critical for Middle-Proxy Mode) -# ------------------------------------------------------------------------------ - -# Your server's PUBLIC IPv4 address. -# Middle-End servers need this for the cryptographic Key Derivation Function. -# If your server has a direct public IP, set it here. -# If behind NAT (AWS, Docker, etc.), this MUST be your external IP. -# If omitted, Telemt uses STUN to auto-detect (see middle_proxy_nat_probe). -# middle_proxy_nat_ip = "203.0.113.10" - -# Auto-detect public IP via STUN servers defined in [network]. -# Set to false if you hardcoded middle_proxy_nat_ip above. -# Set to true if you want automatic detection. -middle_proxy_nat_probe = true - -# ------------------------------------------------------------------------------ -# Middle-End Connection Pool -# ------------------------------------------------------------------------------ - -# Number of persistent multiplexed RPC connections to ME servers. -# All client traffic is routed through these "fat pipes". -# 8 handles thousands of concurrent users comfortably. -middle_proxy_pool_size = 8 - -# Legacy field. Connections kept initialized but idle as warm standby. -middle_proxy_warm_standby = 16 - -# ------------------------------------------------------------------------------ -# Middle-End Keepalive -# Telegram ME servers aggressively kill idle TCP connections. -# These settings send periodic RPC_PING frames to keep pipes alive. -# ------------------------------------------------------------------------------ - -me_keepalive_enabled = true - -# Base interval between pings in seconds. -me_keepalive_interval_secs = 25 - -# Random jitter added to interval to prevent all connections pinging simultaneously. -me_keepalive_jitter_secs = 5 - -# Randomize ping payload bytes to prevent DPI from fingerprinting ping patterns. -me_keepalive_payload_random = true - -# ------------------------------------------------------------------------------ -# Client-Side Limits -# ------------------------------------------------------------------------------ - -# Max buffered ciphertext per client (bytes) when upstream is slow. -# Acts as backpressure to prevent memory exhaustion. 256KB is safe. -crypto_pending_buffer = 262144 - -# Maximum single MTProto frame size from client. 16MB is protocol standard. -max_client_frame = 16777216 - -# ------------------------------------------------------------------------------ -# Crypto Desynchronization Logging -# Desync errors usually mean DPI/GFW is tampering with connections. -# ------------------------------------------------------------------------------ - -# true = full forensics (trace ID, IP hash, hex dumps) for EVERY desync event -# false = deduplicated logging, one entry per time window (prevents log spam) -# Set true if you are actively debugging DPI interference. -desync_all_full = true - -# ------------------------------------------------------------------------------ -# Beobachten — Built-in Honeypot / Active Probe Tracker -# Tracks IPs that fail handshakes or behave like TLS scanners. -# Output file can be fed into fail2ban or iptables for auto-blocking. -# ------------------------------------------------------------------------------ - -beobachten = true - -# How long (minutes) to remember a suspicious IP before expiring it. -beobachten_minutes = 30 - -# How often (seconds) to flush tracker state to disk. -beobachten_flush_secs = 15 - -# File path for the tracker output. -beobachten_file = "cache/beobachten.txt" - -# ------------------------------------------------------------------------------ -# Hardswap — Zero-Downtime ME Pool Rotation -# When Telegram updates ME server IPs, Hardswap creates a completely new pool, -# waits until it is fully ready, migrates traffic, then kills the old pool. -# Users experience zero interruption. -# ------------------------------------------------------------------------------ - -hardswap = true - -# ------------------------------------------------------------------------------ -# ME Pool Warmup Staggering -# When creating a new pool, connections are opened one by one with delays -# to avoid a burst of SYN packets that could trigger ISP flood protection. -# ------------------------------------------------------------------------------ - -me_warmup_stagger_enabled = true - -# Delay between each connection creation (milliseconds). -me_warmup_step_delay_ms = 500 - -# Random jitter added to the delay (milliseconds). -me_warmup_step_jitter_ms = 300 - -# ------------------------------------------------------------------------------ -# ME Reconnect Backoff -# If an ME server drops the connection, Telemt retries with this strategy. -# ------------------------------------------------------------------------------ - -# Max simultaneous reconnect attempts per DC. -me_reconnect_max_concurrent_per_dc = 8 - -# Exponential backoff base (milliseconds). -me_reconnect_backoff_base_ms = 500 - -# Backoff ceiling (milliseconds). Will never wait longer than this. -me_reconnect_backoff_cap_ms = 30000 - -# Number of instant retries before switching to exponential backoff. -me_reconnect_fast_retry_count = 12 - -# ------------------------------------------------------------------------------ -# NAT Mismatch Behavior -# If STUN-detected IP differs from local interface IP (you are behind NAT). -# false = abort ME mode (safe default) -# true = force ME mode anyway (use if you know your NAT setup is correct) -# ------------------------------------------------------------------------------ - -stun_iface_mismatch_ignore = false - -# ------------------------------------------------------------------------------ -# Logging -# ------------------------------------------------------------------------------ - -# File to log unknown DC requests (DC IDs outside standard 1-5). -unknown_dc_log_path = "unknown-dc.txt" - -# Verbosity: "debug" | "verbose" | "normal" | "silent" -log_level = "normal" - -# Disable ANSI color codes in log output (useful for file logging). -disable_colors = false - -# ------------------------------------------------------------------------------ -# FakeTLS Record Sizing -# Buffer small MTProto packets into larger TLS records to mimic real HTTPS. -# Real HTTPS servers send records close to MTU size (~1400 bytes). -# A stream of tiny TLS records is a strong DPI signal. -# Set to 0 to disable. Set to 1400 for realistic HTTPS emulation. -# ------------------------------------------------------------------------------ - -fast_mode_min_tls_record = 1400 - -# ------------------------------------------------------------------------------ -# Periodic Updates -# ------------------------------------------------------------------------------ - -# How often (seconds) to re-fetch ME server lists and proxy secrets -# from core.telegram.org. Keeps your proxy in sync with Telegram infrastructure. -update_every = 300 - -# How often (seconds) to force a Hardswap even if the ME map is unchanged. -# Shorter intervals mean shorter-lived TCP flows, harder for DPI to profile. -me_reinit_every_secs = 600 - -# ------------------------------------------------------------------------------ -# Hardswap Warmup Tuning -# Fine-grained control over how the new pool is warmed up before traffic switch. -# ------------------------------------------------------------------------------ - -me_hardswap_warmup_delay_min_ms = 1000 -me_hardswap_warmup_delay_max_ms = 2000 -me_hardswap_warmup_extra_passes = 3 -me_hardswap_warmup_pass_backoff_base_ms = 500 - -# ------------------------------------------------------------------------------ -# Config Update Debouncing -# Telegram sometimes pushes transient/broken configs. Debouncing requires -# N consecutive identical fetches before applying a change. -# ------------------------------------------------------------------------------ - -# ME server list must be identical for this many fetches before applying. -me_config_stable_snapshots = 2 - -# Minimum seconds between config applications. -me_config_apply_cooldown_secs = 300 - -# Proxy secret must be identical for this many fetches before applying. -proxy_secret_stable_snapshots = 2 - -# ------------------------------------------------------------------------------ -# Proxy Secret Rotation -# ------------------------------------------------------------------------------ - -# Apply newly downloaded secrets at runtime without restart. -proxy_secret_rotate_runtime = true - -# Maximum acceptable secret length (bytes). Rejects abnormally large secrets. -proxy_secret_len_max = 256 - -# ------------------------------------------------------------------------------ -# Hardswap Drain Settings -# Controls graceful shutdown of old ME connections during pool rotation. -# ------------------------------------------------------------------------------ - -# Seconds to keep old connections alive for in-flight data before force-closing. -me_pool_drain_ttl_secs = 90 - -# Minimum ratio of healthy connections in new pool before draining old pool. -# 0.8 = at least 80% of new pool must be ready. -me_pool_min_fresh_ratio = 0.8 - -# Maximum seconds to wait for drain to complete before force-killing. -me_reinit_drain_timeout_secs = 120 - -# ------------------------------------------------------------------------------ -# NTP Clock Check -# MTProto uses timestamps. Clock drift > 30 seconds breaks handshakes. -# Telemt checks on startup and warns if out of sync. -# ------------------------------------------------------------------------------ - -ntp_check = true -ntp_servers = ["pool.ntp.org"] - -# ------------------------------------------------------------------------------ -# Auto-Degradation -# If ME servers become completely unreachable (ISP blocking), -# automatically fall back to Direct Mode so users stay connected. -# ------------------------------------------------------------------------------ - -auto_degradation_enabled = true - -# Number of DC groups that must be unreachable before triggering fallback. -degradation_min_unavailable_dc_groups = 2 - - -# ============================================================================== -# ALLOWED CLIENT PROTOCOLS -# Only enable what you need. In censored regions, TLS-only is safest. -# ============================================================================== - -[general.modes] - -# Classic MTProto. Unobfuscated length prefixes. Trivially detected by DPI. -# No reason to enable unless you have ancient clients. -classic = false - -# Obfuscated MTProto with randomized padding. Better than classic, but -# still detectable by statistical analysis of packet sizes. -secure = false - -# FakeTLS (ee-secrets). Wraps MTProto in TLS 1.3 framing. -# To DPI, it looks like a normal HTTPS connection. -# This should be the ONLY enabled mode in censored environments. -tls = true - - -# ============================================================================== -# STARTUP LINK GENERATION -# Controls what tg:// invite links are printed to console on startup. -# ============================================================================== - -[general.links] - -# Which users to generate links for. -# "*" = all users, or an array like ["alice", "bob"]. -show = "*" - -# IP or domain to embed in the tg:// link. -# If omitted, Telemt uses STUN to auto-detect. -# Set this to your server's public IP or domain for reliable links. -# public_host = "proxy.example.com" - -# Port to embed in the tg:// link. -# If omitted, uses [server].port. -# public_port = 443 - - -# ============================================================================== -# NETWORK & IP RESOLUTION -# ============================================================================== - -[network] - -# Enable IPv4 for outbound connections to Telegram. -ipv4 = true - -# Enable IPv6 for outbound connections to Telegram. -ipv6 = false - -# Prefer IPv4 (4) or IPv6 (6) when both are available. -prefer = 4 - -# Experimental: use both IPv4 and IPv6 ME servers simultaneously. -# May improve reliability but doubles connection count. -multipath = false - -# STUN servers for external IP discovery. -# Used for Middle-Proxy KDF (if nat_probe=true) and link generation. -stun_servers = [ - "stun.l.google.com:5349", - "stun1.l.google.com:3478", - "stun.gmx.net:3478", - "stun.l.google.com:19302" -] - -# If UDP STUN is blocked, attempt TCP-based STUN as fallback. -stun_tcp_fallback = true - -# If all STUN fails, use HTTP APIs to discover public IP. -http_ip_detect_urls = [ - "https://ifconfig.me/ip", - "https://api.ipify.org" -] - -# Cache discovered public IP to this file to survive restarts. -cache_public_ip_path = "cache/public_ip.txt" - - -# ============================================================================== -# SERVER BINDING & METRICS -# ============================================================================== - -[server] - -# TCP port to listen on. -# 443 is recommended (looks like normal HTTPS traffic). -port = 443 - -# IPv4 bind address. "0.0.0.0" = all interfaces. -listen_addr_ipv4 = "0.0.0.0" - -# IPv6 bind address. "::" = all interfaces. -listen_addr_ipv6 = "::" - -# Unix socket listener (for reverse proxy setups with Nginx/HAProxy). -# listen_unix_sock = "/var/run/telemt.sock" -# listen_unix_sock_perm = "0660" - -# Enable PROXY protocol header parsing. -# Set true ONLY if Telemt is behind HAProxy/Nginx that injects PROXY headers. -# If enabled without a proxy in front, clients will fail to connect. -proxy_protocol = false - -# Prometheus metrics HTTP endpoint port. -# Uncomment to enable. Access at http://your-server:9090/metrics -# metrics_port = 9090 - -# IP ranges allowed to access the metrics endpoint. -metrics_whitelist = [ - "127.0.0.1/32", - "::1/128" -] - -# ------------------------------------------------------------------------------ -# Listener Overrides -# Define explicit listeners with specific bind IPs and announce IPs. -# The announce IP is what gets embedded in tg:// links and sent to ME servers. -# You MUST set announce to your server's public IP for ME mode to work. -# ------------------------------------------------------------------------------ - -# [[server.listeners]] -# ip = "0.0.0.0" -# announce = "203.0.113.10" -# reuse_allow = false - - -# ============================================================================== -# TIMEOUTS (seconds unless noted) -# ============================================================================== - -[timeouts] - -# Maximum time for client to complete FakeTLS + MTProto handshake. -client_handshake = 15 - -# Maximum time to establish TCP connection to upstream Telegram DC. -tg_connect = 10 - -# TCP keepalive interval for client connections. -client_keepalive = 60 - -# Maximum client inactivity before dropping the connection. -client_ack = 300 - -# Instant retry count for a single ME endpoint before giving up on it. -me_one_retry = 3 - -# Timeout (milliseconds) for a single ME endpoint connection attempt. -me_one_timeout_ms = 1500 - - -# ============================================================================== -# ANTI-CENSORSHIP / FAKETLS / MASKING -# This is where Telemt becomes invisible to Deep Packet Inspection. -# ============================================================================== - -[censorship] - -# ------------------------------------------------------------------------------ -# TLS Domain Fronting -# The SNI (Server Name Indication) your proxy presents to connecting clients. -# Must be a popular, unblocked HTTPS website in your target country. -# DPI sees traffic to this domain. Choose carefully. -# Good choices: major CDNs, banks, government sites, search engines. -# Bad choices: obscure sites, already-blocked domains. -# ------------------------------------------------------------------------------ - -tls_domain = "www.google.com" - -# ------------------------------------------------------------------------------ -# Active Probe Masking -# When someone connects but fails the MTProto handshake (wrong secret), -# they might be an ISP active prober testing if this is a proxy. -# -# mask = false: drop the connection (prober knows something is here) -# mask = true: transparently proxy them to mask_host (prober sees a real website) -# -# With mask enabled, your server is indistinguishable from a real web server -# to anyone who doesn't have the correct secret. -# ------------------------------------------------------------------------------ - -mask = true - -# The real web server to forward failed handshakes to. -# If omitted, defaults to tls_domain. -# mask_host = "www.google.com" - -# Port on the mask host to connect to. -mask_port = 443 - -# Inject PROXY protocol header when forwarding to mask host. -# 0 = disabled, 1 = v1, 2 = v2. Leave disabled unless mask_host expects it. -# mask_proxy_protocol = 0 - -# ------------------------------------------------------------------------------ -# TLS Certificate Emulation -# ------------------------------------------------------------------------------ - -# Size (bytes) of the locally generated fake TLS certificate. -# Only used when tls_emulation is disabled. -fake_cert_len = 2048 - -# KILLER FEATURE: Real-Time TLS Emulation. -# Telemt connects to tls_domain, fetches its actual TLS 1.3 certificate chain, -# and exactly replicates the byte sizes of ServerHello and Certificate records. -# Defeats DPI that uses TLS record length heuristics to detect proxies. -# Strongly recommended in censored environments. -tls_emulation = true - -# Directory to cache fetched TLS certificates. -tls_front_dir = "tlsfront" - -# ------------------------------------------------------------------------------ -# ServerHello Timing -# Real web servers take 30-150ms to respond to ClientHello due to network -# latency and crypto processing. A proxy responding in <1ms is suspicious. -# These settings add realistic delay to mimic genuine server behavior. -# ------------------------------------------------------------------------------ - -# Minimum delay before sending ServerHello (milliseconds). -server_hello_delay_min_ms = 50 - -# Maximum delay before sending ServerHello (milliseconds). -server_hello_delay_max_ms = 150 - -# ------------------------------------------------------------------------------ -# TLS Session Tickets -# Real TLS 1.3 servers send 1-2 NewSessionTicket messages after handshake. -# A server that sends zero tickets is anomalous and may trigger DPI flags. -# Set this to match your tls_domain's behavior (usually 2). -# ------------------------------------------------------------------------------ - -# tls_new_session_tickets = 0 - -# ------------------------------------------------------------------------------ -# Full Certificate Frequency -# When tls_emulation is enabled, this controls how often (per client IP) -# to send the complete emulated certificate chain. -# -# > 0: Subsequent connections within TTL seconds get a smaller cached version. -# Saves bandwidth but creates a detectable size difference between -# first and repeat connections. -# -# = 0: Every connection gets the full certificate. More bandwidth but -# perfectly consistent behavior, no anomalies for DPI to detect. -# ------------------------------------------------------------------------------ - -tls_full_cert_ttl_secs = 0 - -# ------------------------------------------------------------------------------ -# ALPN Enforcement -# Ensure ServerHello responds with the exact ALPN protocol the client requested. -# Mismatched ALPN (e.g., client asks h2, server says http/1.1) is a DPI red flag. -# ------------------------------------------------------------------------------ - -alpn_enforce = true - - -# ============================================================================== -# ACCESS CONTROL & USERS -# ============================================================================== - -[access] - -# ------------------------------------------------------------------------------ -# Replay Attack Protection -# DPI can record a legitimate user's handshake and replay it later to probe -# whether the server is a proxy. Telemt remembers recent handshake nonces -# and rejects duplicates. -# ------------------------------------------------------------------------------ - -# Number of nonce slots in the replay detection buffer. -replay_check_len = 65536 - -# How long (seconds) to remember nonces before expiring them. -replay_window_secs = 1800 - -# Allow clients with incorrect system clocks to connect. -# false = reject clients with significant time skew (more secure) -# true = accept anyone regardless of clock (more permissive) -ignore_time_skew = false - -# ------------------------------------------------------------------------------ -# User Secrets -# Each user needs a unique 32-character hex string as their secret. -# Generate with: openssl rand -hex 16 -# -# This secret is embedded in the tg:// link. Anyone with it can connect. -# Format: username = "hex_secret" -# ------------------------------------------------------------------------------ - -[access.users] -# alice = "0123456789abcdef0123456789abcdef" -# bob = "fedcba9876543210fedcba9876543210" - -# ------------------------------------------------------------------------------ -# Per-User Connection Limits -# Limits concurrent TCP connections per user to prevent secret sharing. -# Uncomment and set for each user as needed. -# ------------------------------------------------------------------------------ - -[access.user_max_tcp_conns] -# alice = 100 -# bob = 50 - -# ------------------------------------------------------------------------------ -# Per-User Expiration Dates -# Automatically revoke access after the specified date (ISO 8601 format). -# ------------------------------------------------------------------------------ - -[access.user_expirations] -# alice = "2025-12-31T23:59:59Z" -# bob = "2026-06-15T00:00:00Z" - -# ------------------------------------------------------------------------------ -# Per-User Data Quotas -# Maximum total bytes transferred per user. Connection refused after limit. -# ------------------------------------------------------------------------------ - -[access.user_data_quota] -# alice = 107374182400 -# bob = 53687091200 - -# ------------------------------------------------------------------------------ -# Per-User Unique IP Limits -# Maximum number of different IP addresses that can use this secret -# at the same time. Highly effective against secret leaking/sharing. -# Set to 1 for single-device, 2-3 for phone+desktop, etc. -# ------------------------------------------------------------------------------ - -[access.user_max_unique_ips] -# alice = 3 -# bob = 2 - - -# ============================================================================== -# UPSTREAM ROUTING -# Controls how Telemt connects to Telegram servers (or ME servers). -# If omitted entirely, uses the OS default route. -# ============================================================================== - -# ------------------------------------------------------------------------------ -# Direct upstream: use the server's own network interface. -# You can optionally bind to a specific interface or local IP. -# ------------------------------------------------------------------------------ - -# [[upstreams]] -# type = "direct" -# interface = "eth0" -# bind_addresses = ["192.0.2.10"] -# weight = 1 -# enabled = true -# scopes = "*" - -# ------------------------------------------------------------------------------ -# SOCKS5 upstream: route Telegram traffic through a SOCKS5 proxy. -# Useful if your server's IP is blocked from reaching Telegram DCs. -# ------------------------------------------------------------------------------ - -# [[upstreams]] -# type = "socks5" -# address = "198.51.100.30:1080" -# username = "proxy-user" -# password = "proxy-pass" -# weight = 1 -# enabled = true - - -# ============================================================================== -# DATACENTER OVERRIDES -# Force specific DC IDs to route to specific IP:Port combinations. -# DC 203 (CDN) is auto-injected by Telemt if not specified here. -# ============================================================================== - -# [dc_overrides] -# "201" = "149.154.175.50:443" -# "202" = ["149.154.167.51:443", "149.154.175.100:443"] From 5e3408e80b3d391d88ac1b13ecf348d6ea6019f7 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:08:14 +0300 Subject: [PATCH 193/261] Update LICENSE --- LICENSE | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 4b50acd..590e3ef 100644 --- a/LICENSE +++ b/LICENSE @@ -9,6 +9,18 @@ persons to whom the Software is furnished to do so, provided that all copyright notices, license terms, and conditions set forth in this License are preserved and complied with. +### Official Translations + +The canonical version of this License is the English version. + +Official translations are provided for informational purposes only +and for convenience, and do not have legal force. In case of any +discrepancy, the English version of this License shall prevail. + +Available translations: +- German: docs/LICENSE/LICENSE.de.md +- Russian: docs/LICENSE/LICENSE.ru.md + ### Definitions For the purposes of this License: @@ -127,4 +139,4 @@ USE OR OTHER DEALINGS IN THE SOFTWARE IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS -SHALL REMAIN IN FULL FORCE AND EFFECT \ No newline at end of file +SHALL REMAIN IN FULL FORCE AND EFFECT From c12d27f08aaa9743e7378bb1a541464c3eba9234 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:10:07 +0300 Subject: [PATCH 194/261] Middle-End docs --- docs/{ => middle-end/KDF-internals}/MIDDLE-END-KDF.de.md | 0 docs/{ => middle-end/KDF-internals}/MIDDLE-END-KDF.en.md | 0 docs/{ => middle-end/KDF-internals}/MIDDLE-END-KDF.ru.md | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename docs/{ => middle-end/KDF-internals}/MIDDLE-END-KDF.de.md (100%) rename docs/{ => middle-end/KDF-internals}/MIDDLE-END-KDF.en.md (100%) rename docs/{ => middle-end/KDF-internals}/MIDDLE-END-KDF.ru.md (100%) diff --git a/docs/MIDDLE-END-KDF.de.md b/docs/middle-end/KDF-internals/MIDDLE-END-KDF.de.md similarity index 100% rename from docs/MIDDLE-END-KDF.de.md rename to docs/middle-end/KDF-internals/MIDDLE-END-KDF.de.md diff --git a/docs/MIDDLE-END-KDF.en.md b/docs/middle-end/KDF-internals/MIDDLE-END-KDF.en.md similarity index 100% rename from docs/MIDDLE-END-KDF.en.md rename to docs/middle-end/KDF-internals/MIDDLE-END-KDF.en.md diff --git a/docs/MIDDLE-END-KDF.ru.md b/docs/middle-end/KDF-internals/MIDDLE-END-KDF.ru.md similarity index 100% rename from docs/MIDDLE-END-KDF.ru.md rename to docs/middle-end/KDF-internals/MIDDLE-END-KDF.ru.md From 01af2999bb24fd3dc3f79cbc7b25728ad133c166 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:10:46 +0300 Subject: [PATCH 195/261] Update LICENSE --- LICENSE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 590e3ef..649840d 100644 --- a/LICENSE +++ b/LICENSE @@ -17,7 +17,8 @@ Official translations are provided for informational purposes only and for convenience, and do not have legal force. In case of any discrepancy, the English version of this License shall prevail. -Available translations: +Available versions: +- English in Markdown: docs/LICENSE/LICENSE.md - German: docs/LICENSE/LICENSE.de.md - Russian: docs/LICENSE/LICENSE.ru.md From 2b058f7df7e3ea1b48739c5a2ca5c4cf80fb818a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:11:12 +0300 Subject: [PATCH 196/261] Create LICENSE.en.md --- docs/LICENSE/LICENSE.en.md | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 docs/LICENSE/LICENSE.en.md diff --git a/docs/LICENSE/LICENSE.en.md b/docs/LICENSE/LICENSE.en.md new file mode 100644 index 0000000..77796a3 --- /dev/null +++ b/docs/LICENSE/LICENSE.en.md @@ -0,0 +1,143 @@ +###### TELEMT Public License 3 ###### +##### Copyright (c) 2026 Telemt ##### + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this Software and associated documentation files (the "Software"), +to use, reproduce, modify, prepare derivative works of, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, provided that all +copyright notices, license terms, and conditions set forth in this License +are preserved and complied with. + +### Official Translations + +The canonical version of this License is the English version. + +Official translations are provided for informational purposes only +and for convenience, and do not have legal force. In case of any +discrepancy, the English version of this License shall prevail. + +Available versions: +- English in Markdown: docs/LICENSE/LICENSE.md +- German: docs/LICENSE/LICENSE.de.md +- Russian: docs/LICENSE/LICENSE.ru.md + +### Definitions + +For the purposes of this License: + +"Software" means the Telemt software, including source code, documentation, +and any associated files distributed under this License. + +"Contributor" means any person or entity that submits code, patches, +documentation, or other contributions to the Software that are accepted +into the Software by the maintainers. + +"Contribution" means any work of authorship intentionally submitted +to the Software for inclusion in the Software. + +"Modified Version" means any version of the Software that has been +changed, adapted, extended, or otherwise modified from the original +Software. + +"Maintainers" means the individuals or entities responsible for +the official Telemt project and its releases. + +#### 1 Attribution + +Redistributions of the Software, in source or binary form, MUST RETAIN the +above copyright notice, this license text, and any existing attribution +notices. + +#### 2 Modification Notice + +If you modify the Software, you MUST clearly state that the Software has been +modified and include a brief description of the changes made. + +Modified versions MUST NOT be presented as the original Telemt. + +#### 3 Trademark and Branding + +This license DOES NOT grant permission to use the name "Telemt", +the Telemt logo, or any Telemt trademarks or branding. + +Redistributed or modified versions of the Software MAY NOT use the Telemt +name in a way that suggests endorsement or official origin without explicit +permission from the Telemt maintainers. + +Use of the name "Telemt" to describe a modified version of the Software +is permitted only if the modified version is clearly identified as a +modified or unofficial version. + +Any distribution that could reasonably confuse users into believing that +the software is an official Telemt release is prohibited. + +#### 4 Binary Distribution Transparency + +If you distribute compiled binaries of the Software, +you are ENCOURAGED to provide access to the corresponding +source code and build instructions where reasonably possible. + +This helps preserve transparency and allows recipients to verify the +integrity and reproducibility of distributed builds. + +#### 5 Patent Grant and Defensive Termination Clause + +Each contributor grants you a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Software. + +This patent license applies only to those patent claims necessarily +infringed by the contributor’s contribution alone or by combination of +their contribution with the Software. + +If you initiate or participate in any patent litigation, including +cross-claims or counterclaims, alleging that the Software or any +contribution incorporated within the Software constitutes patent +infringement, then **all rights granted to you under this license shall +terminate immediately** as of the date such litigation is filed. + +Additionally, if you initiate legal action alleging that the +Software itself infringes your patent or other intellectual +property rights, then all rights granted to you under this +license SHALL TERMINATE automatically. + +#### 6 Contributions + +Unless you explicitly state otherwise, any Contribution intentionally +submitted for inclusion in the Software shall be licensed under the terms +of this License. + +By submitting a Contribution, you grant the Telemt maintainers and all +recipients of the Software the rights described in this License with +respect to that Contribution. + +#### 7 Network Use Attribution + +If the Software is used to provide a publicly accessible network service, +the operator of such service MUST provide attribution to Telemt in at least +one of the following locations: + +- service documentation +- service description +- an "About" or similar informational page +- other user-visible materials reasonably associated with the service + +Such attribution MUST NOT imply endorsement by the Telemt project or its +maintainers. + +#### 8 Disclaimer of Warranty and Severability Clause + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE + +IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, +SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT +OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS +SHALL REMAIN IN FULL FORCE AND EFFECT \ No newline at end of file From 75a654c766e9f680d7577c509e30bd3fb2b380d0 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:23:24 +0300 Subject: [PATCH 197/261] TELEMT-Lizenz 3 auf Deutsch --- docs/LICENSE/LICENSE.de.md | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/LICENSE/LICENSE.de.md diff --git a/docs/LICENSE/LICENSE.de.md b/docs/LICENSE/LICENSE.de.md new file mode 100644 index 0000000..35c8dcf --- /dev/null +++ b/docs/LICENSE/LICENSE.de.md @@ -0,0 +1,92 @@ +# Öffentliche TELEMT-Lizenz 3 + +***Alle Rechte vorbehalten (c) 2026 Telemt*** + +Hiermit wird jeder Person, die eine Kopie dieser Software und der dazugehörigen Dokumentation (nachfolgend "Software") erhält, unentgeltlich die Erlaubnis erteilt, die Software ohne Einschränkungen zu nutzen, einschließlich des Rechts, die Software zu verwenden, zu vervielfältigen, zu ändern, abgeleitete Werke zu erstellen, zu verbinden, zu veröffentlichen, zu verbreiten, zu unterlizenzieren und/oder Kopien der Software zu verkaufen sowie diese Rechte auch denjenigen einzuräumen, denen die Software zur Verfügung gestellt wird, vorausgesetzt, dass sämtliche Urheberrechtshinweise sowie die Bedingungen und Bestimmungen dieser Lizenz eingehalten werden. + +### Begriffsbestimmungen + +Für die Zwecke dieser Lizenz gelten die folgenden Definitionen: + +**"Software" (Software)** — die Telemt-Software einschließlich Quellcode, Dokumentation und sämtlicher zugehöriger Dateien, die unter den Bedingungen dieser Lizenz verbreitet werden. + +**"Contributor" (Contributor)** — jede natürliche oder juristische Person, die Code, Patches, Dokumentation oder andere Materialien eingereicht hat, die von den Maintainers des Projekts angenommen und in die Software aufgenommen wurden. + +**"Beitrag" (Contribution)** — jedes urheberrechtlich geschützte Werk, das bewusst zur Aufnahme in die Software eingereicht wurde. + +**"Modifizierte Version" (Modified Version)** — jede Version der Software, die gegenüber der ursprünglichen Software geändert, angepasst, erweitert oder anderweitig modifiziert wurde. + +**"Maintainers" (Maintainers)** — natürliche oder juristische Personen, die für das offizielle Telemt-Projekt und dessen offizielle Veröffentlichungen verantwortlich sind. + +### 1 Urheberrechtshinweis (Attribution) + +Bei der Weitergabe der Software, sowohl in Form des Quellcodes als auch in binärer Form, MÜSSEN folgende Elemente erhalten bleiben: + +- der oben genannte Urheberrechtshinweis; +- der vollständige Text dieser Lizenz; +- sämtliche bestehenden Hinweise auf Urheberschaft. + +### 2 Hinweis auf Modifikationen + +Wenn Änderungen an der Software vorgenommen werden, MUSS die Person, die diese Änderungen vorgenommen hat, eindeutig darauf hinweisen, dass die Software modifiziert wurde, und eine kurze Beschreibung der vorgenommenen Änderungen beifügen. + +Modifizierte Versionen der Software DÜRFEN NICHT als die originale Version von Telemt dargestellt werden. + +### 3 Marken und Bezeichnungen + +Diese Lizenz GEWÄHRT KEINE Rechte zur Nutzung der Bezeichnung **"Telemt"**, des Telemt-Logos oder sonstiger Marken, Kennzeichen oder Branding-Elemente von Telemt. + +Weiterverbreitete oder modifizierte Versionen der Software DÜRFEN die Bezeichnung Telemt nicht in einer Weise verwenden, die bei Nutzern den Eindruck eines offiziellen Ursprungs oder einer Billigung durch das Telemt-Projekt erwecken könnte, sofern hierfür keine ausdrückliche Genehmigung der Maintainers vorliegt. + +Die Verwendung der Bezeichnung **Telemt** zur Beschreibung einer modifizierten Version der Software ist nur zulässig, wenn diese Version eindeutig als modifiziert oder inoffiziell gekennzeichnet ist. + +Jegliche Verbreitung, die Nutzer vernünftigerweise darüber täuschen könnte, dass es sich um eine offizielle Veröffentlichung von Telemt handelt, ist untersagt. + +### 4 Transparenz bei der Verbreitung von Binärversionen + +Im Falle der Verbreitung kompilierter Binärversionen der Software wird der Verbreiter HIERMIT ERMUTIGT (encouraged), soweit dies vernünftigerweise möglich ist, Zugang zum entsprechenden Quellcode sowie zu den Build-Anweisungen bereitzustellen. + +Diese Praxis trägt zur Transparenz bei und ermöglicht es Empfängern, die Integrität und Reproduzierbarkeit der verbreiteten Builds zu überprüfen. + +## 5 Gewährung einer Patentlizenz und Beendigung von Rechten + +Jeder Contributor gewährt den Empfängern der Software eine unbefristete, weltweite, nicht-exklusive, unentgeltliche, lizenzgebührenfreie und unwiderrufliche Patentlizenz für: + +- die Herstellung, +- die Beauftragung der Herstellung, +- die Nutzung, +- das Anbieten zum Verkauf, +- den Verkauf, +- den Import, +- sowie jede sonstige Verbreitung der Software. + +Diese Patentlizenz erstreckt sich ausschließlich auf solche Patentansprüche, die notwendigerweise durch den jeweiligen Beitrag des Contributors allein oder in Kombination mit der Software verletzt würden. + +Leitet eine Person ein Patentverfahren ein oder beteiligt sich daran, einschließlich Gegenklagen oder Kreuzklagen, mit der Behauptung, dass die Software oder ein darin enthaltener Beitrag ein Patent verletzt, **erlöschen sämtliche durch diese Lizenz gewährten Rechte für diese Person unmittelbar mit Einreichung der Klage**. + +Darüber hinaus erlöschen alle durch diese Lizenz gewährten Rechte **automatisch**, wenn eine Person ein gerichtliches Verfahren einleitet, in dem behauptet wird, dass die Software selbst ein Patent oder andere Rechte des geistigen Eigentums verletzt. + +### 6 Beteiligung und Beiträge zur Entwicklung + +Sofern ein Contributor nicht ausdrücklich etwas anderes erklärt, gilt jeder Beitrag, der bewusst zur Aufnahme in die Software eingereicht wird, als unter den Bedingungen dieser Lizenz lizenziert. + +Durch die Einreichung eines Beitrags gewährt der Contributor den Maintainers des Telemt-Projekts sowie allen Empfängern der Software die in dieser Lizenz beschriebenen Rechte in Bezug auf diesen Beitrag. + +### 7 Urheberhinweis bei Netzwerk- und Servicenutzung + +Wird die Software zur Bereitstellung eines öffentlich zugänglichen Netzwerkdienstes verwendet, MUSS der Betreiber dieses Dienstes einen Hinweis auf die Urheberschaft von Telemt an mindestens einer der folgenden Stellen anbringen: + +* in der Servicedokumentation; +* in der Dienstbeschreibung; +* auf einer Seite "Über" oder einer vergleichbaren Informationsseite; +* in anderen für Nutzer zugänglichen Materialien, die in angemessenem Zusammenhang mit dem Dienst stehen. + +Ein solcher Hinweis DARF NICHT den Eindruck erwecken, dass der Dienst vom Telemt-Projekt oder dessen Maintainers unterstützt oder offiziell gebilligt wird. + +### 8 Haftungsausschluss und salvatorische Klausel + +DIE SOFTWARE WIRD "WIE BESEHEN" BEREITGESTELLT, OHNE JEGLICHE AUSDRÜCKLICHE ODER STILLSCHWEIGENDE GEWÄHRLEISTUNG, EINSCHLIESSLICH, ABER NICHT BESCHRÄNKT AUF GEWÄHRLEISTUNGEN DER MARKTGÄNGIGKEIT, DER EIGNUNG FÜR EINEN BESTIMMTEN ZWECK UND DER NICHTVERLETZUNG VON RECHTEN. + +IN KEINEM FALL HAFTEN DIE AUTOREN ODER RECHTEINHABER FÜR IRGENDWELCHE ANSPRÜCHE, SCHÄDEN ODER SONSTIGE HAFTUNG, DIE AUS VERTRAG, UNERLAUBTER HANDLUNG ODER AUF ANDERE WEISE AUS DER SOFTWARE ODER DER NUTZUNG DER SOFTWARE ENTSTEHEN. + +SOLLTE EINE BESTIMMUNG DIESER LIZENZ ALS UNWIRKSAM ODER NICHT DURCHSETZBAR ANGESEHEN WERDEN, IST DIESE BESTIMMUNG SO AUSZULEGEN, DASS SIE DEM URSPRÜNGLICHEN WILLEN DER PARTEIEN MÖGLICHST NAHEKOMMT; DIE ÜBRIGEN BESTIMMUNGEN BLEIBEN DAVON UNBERÜHRT UND IN VOLLER WIRKUNG. \ No newline at end of file From 58e5605f394d6363271947ed4c8bae40f02488d6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:23:41 +0300 Subject: [PATCH 198/261] =?UTF-8?q?Telemt=20PL=203=20=D0=BD=D0=B0=20=D1=80?= =?UTF-8?q?=D1=83=D1=81=D1=81=D0=BA=D0=BE=D0=BC=20=D1=8F=D0=B7=D1=8B=D0=BA?= =?UTF-8?q?=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/LICENSE/LICENSE.ru.md | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/LICENSE/LICENSE.ru.md diff --git a/docs/LICENSE/LICENSE.ru.md b/docs/LICENSE/LICENSE.ru.md new file mode 100644 index 0000000..b88d9da --- /dev/null +++ b/docs/LICENSE/LICENSE.ru.md @@ -0,0 +1,90 @@ +# Публичная лицензия TELEMT 3 + +***Все права защищёны (c) 2026 Telemt*** + +Настоящим любому лицу, получившему копию данного программного обеспечения и сопутствующей документации (далее — "Программное обеспечение"), безвозмездно предоставляется разрешение использовать Программное обеспечение без ограничений, включая право использовать, воспроизводить, изменять, создавать производные произведения, объединять, публиковать, распространять, сублицензировать и (или) продавать копии Программного обеспечения, а также предоставлять такие права лицам, которым предоставляется Программное обеспечение, при условии соблюдения всех уведомлений об авторских правах, условий и положений настоящей Лицензии. + +### Определения + +Для целей настоящей Лицензии применяются следующие определения: + +**"Программное обеспечение" (Software)** — программное обеспечение Telemt, включая исходный код, документацию и любые связанные файлы, распространяемые на условиях настоящей Лицензии. + +**"Контрибьютор" (Contributor)** — любое физическое или юридическое лицо, направившее код, исправления (патчи), документацию или иные материалы, которые были приняты мейнтейнерами проекта и включены в состав Программного обеспечения. + +**"Вклад" (Contribution)** — любое произведение авторского права, намеренно представленное для включения в состав Программного обеспечения. + +**"Модифицированная версия" (Modified Version)** — любая версия Программного обеспечения, которая была изменена, адаптирована, расширена или иным образом модифицирована по сравнению с исходным Программным обеспечением. + +**"Мейнтейнеры" (Maintainers)** — физические или юридические лица, ответственные за официальный проект Telemt и его официальные релизы. + +### 1 Указание авторства + +При распространении Программного обеспечения, как в форме исходного кода, так и в бинарной форме, ДОЛЖНЫ СОХРАНЯТЬСЯ: + +- указанное выше уведомление об авторских правах; +- текст настоящей Лицензии; +- любые существующие уведомления об авторстве. + +### 2 Уведомление о модификации + +В случае внесения изменений в Программное обеспечение лицо, осуществившее такие изменения, ОБЯЗАНО явно указать, что Программное обеспечение было модифицировано, а также включить краткое описание внесённых изменений. + +Модифицированные версии Программного обеспечения НЕ ДОЛЖНЫ представляться как оригинальная версия Telemt. + +### 3 Товарные знаки и обозначения + +Настоящая Лицензия НЕ ПРЕДОСТАВЛЯЕТ права использовать наименование **"Telemt"**, логотип Telemt, а также любые товарные знаки, фирменные обозначения или элементы бренда Telemt. + +Распространяемые или модифицированные версии Программного обеспечения НЕ ДОЛЖНЫ использовать наименование Telemt таким образом, который может создавать у пользователей впечатление официального происхождения либо одобрения со стороны проекта Telemt без явного разрешения мейнтейнеров проекта. + +Использование наименования **Telemt** для описания модифицированной версии Программного обеспечения допускается только при условии, что такая версия ясно обозначена как модифицированная или неофициальная. + +Запрещается любое распространение, которое может разумно вводить пользователей в заблуждение относительно того, что программное обеспечение является официальным релизом Telemt. + +### 4 Прозрачность распространения бинарных версий + +В случае распространения скомпилированных бинарных версий Программного обеспечения распространитель НАСТОЯЩИМ ПОБУЖДАЕТСЯ предоставлять доступ к соответствующему исходному коду и инструкциям по сборке, если это разумно возможно. + +Такая практика способствует прозрачности распространения и позволяет получателям проверять целостность и воспроизводимость распространяемых сборок. + +### 5 Предоставление патентной лицензии и прекращение прав + +Каждый контрибьютор предоставляет получателям Программного обеспечения бессрочную, всемирную, неисключительную, безвозмездную, не требующую выплаты роялти и безотзывную патентную лицензию на: + +- изготовление, +- поручение изготовления, +- использование, +- предложение к продаже, +- продажу, +- импорт, +- и иное распространение Программного обеспечения. + +Такая патентная лицензия распространяется исключительно на те патентные требования, которые неизбежно нарушаются соответствующим вкладом контрибьютора как таковым либо его сочетанием с Программным обеспечением. + +Если лицо инициирует либо участвует в каком-либо судебном разбирательстве по патентному спору, включая встречные или перекрёстные иски, утверждая, что Программное обеспечение либо любой вклад, включённый в него, нарушает патент, **все права, предоставленные такому лицу настоящей Лицензией, немедленно прекращаются** с даты подачи соответствующего иска. + +Кроме того, если лицо инициирует судебное разбирательство, утверждая, что само Программное обеспечение нарушает его патентные либо иные права интеллектуальной собственности, все права, предоставленные настоящей Лицензией, **автоматически прекращаются**. + +### 6 Участие и вклад в разработку + +Если контрибьютор явно не указал иное, любой Вклад, намеренно представленный для включения в Программное обеспечение, считается лицензированным на условиях настоящей Лицензии. +Путём предоставления Вклада контрибьютор предоставляет мейнтейнером проекта Telemt и всем получателям Программного обеспечения права, предусмотренные настоящей Лицензией, в отношении такого Вклада. + +### 7 Указание авторства при сетевом и сервисном использовании + +В случае использования Программного обеспечения для предоставления публично доступного сетевого сервиса оператор такого сервиса ОБЯЗАН обеспечить указание авторства Telemt как минимум в одном из следующих мест: +- документация сервиса; +- описание сервиса; +- страница "О программе" или аналогичная информационная страница; +- иные материалы, доступные пользователям и разумно связанные с данным сервисом. + +Такое указание авторства НЕ ДОЛЖНО создавать впечатление одобрения или официальной поддержки со стороны проекта Telemt либо его мейнтейнеров. + +### 8 Отказ от гарантий и делимость положений + +ПРОГРАММНОЕ ОБЕСПЕЧЕНИЕ ПРЕДОСТАВЛЯЕТСЯ "КАК ЕСТЬ", БЕЗ КАКИХ-ЛИБО ГАРАНТИЙ, ЯВНЫХ ИЛИ ПОДРАЗУМЕВАЕМЫХ, ВКЛЮЧАЯ, НО НЕ ОГРАНИЧИВАЯСЬ ГАРАНТИЯМИ КОММЕРЧЕСКОЙ ПРИГОДНОСТИ, ПРИГОДНОСТИ ДЛЯ КОНКРЕТНОЙ ЦЕЛИ И НЕНАРУШЕНИЯ ПРАВ. + +НИ ПРИ КАКИХ ОБСТОЯТЕЛЬСТВАХ АВТОРЫ ИЛИ ПРАВООБЛАДАТЕЛИ НЕ НЕСУТ ОТВЕТСТВЕННОСТИ ПО КАКИМ-ЛИБО ТРЕБОВАНИЯМ, УБЫТКАМ ИЛИ ИНОЙ ОТВЕТСТВЕННОСТИ, ВОЗНИКАЮЩЕЙ В РЕЗУЛЬТАТЕ ДОГОВОРА, ДЕЛИКТА ИЛИ ИНЫМ ОБРАЗОМ, СВЯЗАННЫМ С ПРОГРАММНЫМ ОБЕСПЕЧЕНИЕМ ИЛИ ЕГО ИСПОЛЬЗОВАНИЕМ. + +В СЛУЧАЕ ЕСЛИ КАКОЕ-ЛИБО ПОЛОЖЕНИЕ НАСТОЯЩЕЙ ЛИЦЕНЗИИ ПРИЗНАЁТСЯ НЕДЕЙСТВИТЕЛЬНЫМ ИЛИ НЕПРИМЕНИМЫМ, ТАКОЕ ПОЛОЖЕНИЕ ПОДЛЕЖИТ ТОЛКОВАНИЮ МАКСИМАЛЬНО БЛИЗКО К ИСХОДНОМУ НАМЕРЕНИЮ СТОРОН, ПРИ ЭТОМ ОСТАЛЬНЫЕ ПОЛОЖЕНИЯ СОХРАНЯЮТ ПОЛНУЮ ЮРИДИЧЕСКУЮ СИЛУ. From 37b1a0289e130cfec20fe406242877f861d78eba Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:56:31 +0300 Subject: [PATCH 199/261] Update LICENSE --- LICENSE | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/LICENSE b/LICENSE index 649840d..9035205 100644 --- a/LICENSE +++ b/LICENSE @@ -12,35 +12,57 @@ are preserved and complied with. ### Official Translations The canonical version of this License is the English version. - Official translations are provided for informational purposes only and for convenience, and do not have legal force. In case of any discrepancy, the English version of this License shall prevail. - Available versions: - English in Markdown: docs/LICENSE/LICENSE.md - German: docs/LICENSE/LICENSE.de.md - Russian: docs/LICENSE/LICENSE.ru.md +### License Versioning Policy + +This License is version 3 of the TELEMT Public License. +Each version of the Software is licensed under the License that +accompanies its corresponding source code distribution. + +Future versions of the Software may be distributed under a different +version of the TELEMT Public License or under a different license, +as determined by the Telemt maintainers. + +Any such change of license applies only to the versions of the +Software distributed with the new license and SHALL NOT retroactively +affect any previously released versions of the Software. + +Recipients of the Software are granted rights only under the License +provided with the version of the Software they received. + +Redistributions of the Software, including Modified Versions, MUST +preserve the copyright notices, license text, and conditions of this +License for all portions of the Software derived from Telemt. + +Additional terms or licenses may be applied to modifications or +additional code added by a redistributor, provided that such terms +do not restrict or alter the rights granted under this License for +the original Telemt Software. + +Nothing in this section limits the rights granted under this License +for versions of the Software already released. + ### Definitions For the purposes of this License: - -"Software" means the Telemt software, including source code, documentation, +- "Software" means the Telemt software, including source code, documentation, and any associated files distributed under this License. - -"Contributor" means any person or entity that submits code, patches, +- "Contributor" means any person or entity that submits code, patches, documentation, or other contributions to the Software that are accepted into the Software by the maintainers. - -"Contribution" means any work of authorship intentionally submitted +- "Contribution" means any work of authorship intentionally submitted to the Software for inclusion in the Software. - -"Modified Version" means any version of the Software that has been +- "Modified Version" means any version of the Software that has been changed, adapted, extended, or otherwise modified from the original Software. - -"Maintainers" means the individuals or entities responsible for +- "Maintainers" means the individuals or entities responsible for the official Telemt project and its releases. #### 1 Attribution From cb9144bdb3858fb5a135c41c4b58ec8bce8b02be Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 15:59:21 +0300 Subject: [PATCH 200/261] Update LICENSING.md --- LICENSING.md | 138 ++------------------------------------------------- 1 file changed, 3 insertions(+), 135 deletions(-) diff --git a/LICENSING.md b/LICENSING.md index 9b9ac49..d72906b 100644 --- a/LICENSING.md +++ b/LICENSING.md @@ -2,143 +2,11 @@ ## Licenses for Versions | Version ≥ | Version ≤ | License | |-----------|-----------|---------------| -| 1.0 | 2.0.0.1 | NO LICNESE | -| 3.0.0 | 3.4.0 | TELEMT PL 3 | +| 1.0 | 3.3.17 | NO LICNESE | +| 3.3.18 | 3.4.0 | TELEMT PL 3 | ### License Types - **NO LICENSE** = ***ALL RIGHT RESERVED*** - **TELEMT PL** - special Telemt Public License based on Apache License 2 principles -## Telemt Public License 3 -```text -###### TELEMT Public License 3 ###### -##### Copyright (c) 2026 Telemt ##### - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this Software and associated documentation files (the "Software"), -to use, reproduce, modify, prepare derivative works of, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to permit -persons to whom the Software is furnished to do so, provided that all -copyright notices, license terms, and conditions set forth in this License -are preserved and complied with. - -### Definitions - -For the purposes of this License: - -"Software" means the Telemt software, including source code, documentation, -and any associated files distributed under this License. - -"Contributor" means any person or entity that submits code, patches, -documentation, or other contributions to the Software that are accepted -into the Software by the maintainers. - -"Contribution" means any work of authorship intentionally submitted -to the Software for inclusion in the Software. - -"Modified Version" means any version of the Software that has been -changed, adapted, extended, or otherwise modified from the original -Software. - -"Maintainers" means the individuals or entities responsible for -the official Telemt project and its releases. - -#### 1 Attribution - -Redistributions of the Software, in source or binary form, MUST RETAIN the -above copyright notice, this license text, and any existing attribution -notices. - -#### 2 Modification Notice - -If you modify the Software, you MUST clearly state that the Software has been -modified and include a brief description of the changes made. - -Modified versions MUST NOT be presented as the original Telemt. - -#### 3 Trademark and Branding - -This license DOES NOT grant permission to use the name "Telemt", -the Telemt logo, or any Telemt trademarks or branding. - -Redistributed or modified versions of the Software MAY NOT use the Telemt -name in a way that suggests endorsement or official origin without explicit -permission from the Telemt maintainers. - -Use of the name "Telemt" to describe a modified version of the Software -is permitted only if the modified version is clearly identified as a -modified or unofficial version. - -Any distribution that could reasonably confuse users into believing that -the software is an official Telemt release is prohibited. - -#### 4 Binary Distribution Transparency - -If you distribute compiled binaries of the Software, -you are ENCOURAGED to provide access to the corresponding -source code and build instructions where reasonably possible. - -This helps preserve transparency and allows recipients to verify the -integrity and reproducibility of distributed builds. - -#### 5 Patent Grant and Defensive Termination Clause - -Each contributor grants you a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable patent license to make, have made, -use, offer to sell, sell, import, and otherwise transfer the Software. - -This patent license applies only to those patent claims necessarily -infringed by the contributor’s contribution alone or by combination of -their contribution with the Software. - -If you initiate or participate in any patent litigation, including -cross-claims or counterclaims, alleging that the Software or any -contribution incorporated within the Software constitutes patent -infringement, then **all rights granted to you under this license shall -terminate immediately** as of the date such litigation is filed. - -Additionally, if you initiate legal action alleging that the -Software itself infringes your patent or other intellectual -property rights, then all rights granted to you under this -license SHALL TERMINATE automatically. - -#### 6 Contributions - -Unless you explicitly state otherwise, any Contribution intentionally -submitted for inclusion in the Software shall be licensed under the terms -of this License. - -By submitting a Contribution, you grant the Telemt maintainers and all -recipients of the Software the rights described in this License with -respect to that Contribution. - -#### 7 Network Use Attribution - -If the Software is used to provide a publicly accessible network service, -the operator of such service MUST provide attribution to Telemt in at least -one of the following locations: - -- service documentation -- service description -- an "About" or similar informational page -- other user-visible materials reasonably associated with the service - -Such attribution MUST NOT imply endorsement by the Telemt project or its -maintainers. - -#### 8 Disclaimer of Warranty and Severability Clause - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -USE OR OTHER DEALINGS IN THE SOFTWARE - -IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, -SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT -OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS -SHALL REMAIN IN FULL FORCE AND EFFECT -``` +## [Telemt Public License 3](https://github.com/telemt/telemt/blob/main/LICENSE) From f7e3ddcdb6aa660d41a8a071f3b48f31fe4b2e18 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 16:02:40 +0300 Subject: [PATCH 201/261] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 9035205..15061fc 100644 --- a/LICENSE +++ b/LICENSE @@ -137,7 +137,7 @@ respect to that Contribution. #### 7 Network Use Attribution If the Software is used to provide a publicly accessible network service, -the operator of such service MUST provide attribution to Telemt in at least +the operator of such service SHOULD provide attribution to Telemt in at least one of the following locations: - service documentation From 8b0b47145d082315057352c6b2717e20700ad35e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 18:54:05 +0300 Subject: [PATCH 202/261] New Hot-Reload method --- src/config/hot_reload.rs | 457 +++++++++++++++++++++++++++++++-------- src/config/load.rs | 107 ++++++++- 2 files changed, 470 insertions(+), 94 deletions(-) diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 632ca8c..a375899 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -21,9 +21,11 @@ //! `network.*`, `use_middle_proxy`) are **not** applied; a warning is emitted. //! Non-hot changes are never mixed into the runtime config snapshot. +use std::collections::BTreeSet; use std::net::IpAddr; -use std::path::PathBuf; -use std::sync::Arc; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock as StdRwLock}; +use std::time::Duration; use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher}; use tokio::sync::{mpsc, watch}; @@ -33,7 +35,10 @@ use crate::config::{ LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel, MeWriterPickMode, }; -use super::load::ProxyConfig; +use super::load::{LoadedConfig, ProxyConfig}; + +const HOT_RELOAD_STABLE_SNAPSHOTS: u8 = 2; +const HOT_RELOAD_DEBOUNCE: Duration = Duration::from_millis(50); // ── Hot fields ──────────────────────────────────────────────────────────────── @@ -287,6 +292,149 @@ fn listeners_equal( }) } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct WatchManifest { + files: BTreeSet, + dirs: BTreeSet, +} + +impl WatchManifest { + fn from_source_files(source_files: &[PathBuf]) -> Self { + let mut files = BTreeSet::new(); + let mut dirs = BTreeSet::new(); + + for path in source_files { + let normalized = normalize_watch_path(path); + files.insert(normalized.clone()); + if let Some(parent) = normalized.parent() { + dirs.insert(parent.to_path_buf()); + } + } + + Self { files, dirs } + } + + fn matches_event_paths(&self, event_paths: &[PathBuf]) -> bool { + event_paths + .iter() + .map(|path| normalize_watch_path(path)) + .any(|path| self.files.contains(&path)) + } +} + +#[derive(Debug, Default)] +struct ReloadState { + applied_snapshot_hash: Option, + candidate_snapshot_hash: Option, + candidate_hits: u8, +} + +impl ReloadState { + fn new(applied_snapshot_hash: Option) -> Self { + Self { + applied_snapshot_hash, + candidate_snapshot_hash: None, + candidate_hits: 0, + } + } + + fn is_applied(&self, hash: u64) -> bool { + self.applied_snapshot_hash == Some(hash) + } + + fn observe_candidate(&mut self, hash: u64) -> u8 { + if self.candidate_snapshot_hash == Some(hash) { + self.candidate_hits = self.candidate_hits.saturating_add(1); + } else { + self.candidate_snapshot_hash = Some(hash); + self.candidate_hits = 1; + } + self.candidate_hits + } + + fn reset_candidate(&mut self) { + self.candidate_snapshot_hash = None; + self.candidate_hits = 0; + } + + fn mark_applied(&mut self, hash: u64) { + self.applied_snapshot_hash = Some(hash); + self.reset_candidate(); + } +} + +fn normalize_watch_path(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| { + if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .map(|cwd| cwd.join(path)) + .unwrap_or_else(|_| path.to_path_buf()) + } + }) +} + +fn sync_watch_paths( + watcher: &mut W, + current: &BTreeSet, + next: &BTreeSet, + recursive_mode: RecursiveMode, + kind: &str, +) { + for path in current.difference(next) { + if let Err(e) = watcher.unwatch(path) { + warn!(path = %path.display(), error = %e, "config watcher: failed to unwatch {kind}"); + } + } + + for path in next.difference(current) { + if let Err(e) = watcher.watch(path, recursive_mode) { + warn!(path = %path.display(), error = %e, "config watcher: failed to watch {kind}"); + } + } +} + +fn apply_watch_manifest( + notify_watcher: Option<&mut W1>, + poll_watcher: Option<&mut W2>, + manifest_state: &Arc>, + next_manifest: WatchManifest, +) { + let current_manifest = manifest_state + .read() + .map(|manifest| manifest.clone()) + .unwrap_or_default(); + + if current_manifest == next_manifest { + return; + } + + if let Some(watcher) = notify_watcher { + sync_watch_paths( + watcher, + ¤t_manifest.dirs, + &next_manifest.dirs, + RecursiveMode::NonRecursive, + "config directory", + ); + } + + if let Some(watcher) = poll_watcher { + sync_watch_paths( + watcher, + ¤t_manifest.files, + &next_manifest.files, + RecursiveMode::NonRecursive, + "config file", + ); + } + + if let Ok(mut manifest) = manifest_state.write() { + *manifest = next_manifest; + } +} + fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { let mut cfg = old.clone(); @@ -970,18 +1118,42 @@ fn reload_config( log_tx: &watch::Sender, detected_ip_v4: Option, detected_ip_v6: Option, -) { - let new_cfg = match ProxyConfig::load(config_path) { - Ok(c) => c, + reload_state: &mut ReloadState, +) -> Option { + let loaded = match ProxyConfig::load_with_metadata(config_path) { + Ok(loaded) => loaded, Err(e) => { + reload_state.reset_candidate(); error!("config reload: failed to parse {:?}: {}", config_path, e); - return; + return None; } }; + let LoadedConfig { + config: new_cfg, + source_files, + rendered_hash, + } = loaded; + let next_manifest = WatchManifest::from_source_files(&source_files); if let Err(e) = new_cfg.validate() { + reload_state.reset_candidate(); error!("config reload: validation failed: {}; keeping old config", e); - return; + return Some(next_manifest); + } + + if reload_state.is_applied(rendered_hash) { + return Some(next_manifest); + } + + let candidate_hits = reload_state.observe_candidate(rendered_hash); + if candidate_hits < HOT_RELOAD_STABLE_SNAPSHOTS { + info!( + snapshot_hash = rendered_hash, + candidate_hits, + required_hits = HOT_RELOAD_STABLE_SNAPSHOTS, + "config reload: candidate snapshot observed but not stable yet" + ); + return Some(next_manifest); } let old_cfg = config_tx.borrow().clone(); @@ -996,17 +1168,19 @@ fn reload_config( } if !hot_changed { - return; + reload_state.mark_applied(rendered_hash); + return Some(next_manifest); } if old_hot.dns_overrides != applied_hot.dns_overrides && let Err(e) = crate::network::dns_overrides::install_entries(&applied_hot.dns_overrides) { + reload_state.reset_candidate(); error!( "config reload: invalid network.dns_overrides: {}; keeping old config", e ); - return; + return Some(next_manifest); } log_changes( @@ -1018,6 +1192,8 @@ fn reload_config( detected_ip_v6, ); config_tx.send(Arc::new(applied_cfg)).ok(); + reload_state.mark_applied(rendered_hash); + Some(next_manifest) } // ── Public API ──────────────────────────────────────────────────────────────── @@ -1040,80 +1216,86 @@ pub fn spawn_config_watcher( let (config_tx, config_rx) = watch::channel(initial); let (log_tx, log_rx) = watch::channel(initial_level); - // Bridge: sync notify callbacks → async task via mpsc. - let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4); + let config_path = normalize_watch_path(&config_path); + let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok(); + let initial_manifest = initial_loaded + .as_ref() + .map(|loaded| WatchManifest::from_source_files(&loaded.source_files)) + .unwrap_or_else(|| WatchManifest::from_source_files(std::slice::from_ref(&config_path))); + let initial_snapshot_hash = initial_loaded.as_ref().map(|loaded| loaded.rendered_hash); - // Canonicalize so path matches what notify returns (absolute) in events. - let config_path = match config_path.canonicalize() { - Ok(p) => p, - Err(_) => config_path.to_path_buf(), - }; - - // Watch the parent directory rather than the file itself, because many - // editors (vim, nano) and systemd write via rename, which would cause - // inotify to lose track of the original inode. - let watch_dir = config_path - .parent() - .unwrap_or_else(|| std::path::Path::new(".")) - .to_path_buf(); - - // ── inotify watcher (instant on local fs) ──────────────────────────── - let config_file = config_path.clone(); - let tx_inotify = notify_tx.clone(); - let inotify_ok = match recommended_watcher(move |res: notify::Result| { - let Ok(event) = res else { return }; - let is_our_file = event.paths.iter().any(|p| p == &config_file); - if !is_our_file { return; } - if matches!(event.kind, EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)) { - let _ = tx_inotify.try_send(()); - } - }) { - Ok(mut w) => match w.watch(&watch_dir, RecursiveMode::NonRecursive) { - Ok(()) => { - info!("config watcher: inotify active on {:?}", config_path); - Box::leak(Box::new(w)); - true - } - Err(e) => { warn!("config watcher: inotify watch failed: {}", e); false } - }, - Err(e) => { warn!("config watcher: inotify unavailable: {}", e); false } - }; - - // ── poll watcher (always active, fixes Docker bind mounts / NFS) ───── - // inotify does not receive events for files mounted from the host into - // a container. PollWatcher compares file contents every 3 s and fires - // on any change regardless of the underlying fs. - let config_file2 = config_path.clone(); - let tx_poll = notify_tx.clone(); - match notify::poll::PollWatcher::new( - move |res: notify::Result| { - let Ok(event) = res else { return }; - let is_our_file = event.paths.iter().any(|p| p == &config_file2); - if !is_our_file { return; } - if matches!(event.kind, EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)) { - let _ = tx_poll.try_send(()); - } - }, - notify::Config::default() - .with_poll_interval(std::time::Duration::from_secs(3)) - .with_compare_contents(true), - ) { - Ok(mut w) => match w.watch(&config_path, RecursiveMode::NonRecursive) { - Ok(()) => { - if inotify_ok { - info!("config watcher: poll watcher also active (Docker/NFS safe)"); - } else { - info!("config watcher: poll watcher active on {:?} (3s interval)", config_path); - } - Box::leak(Box::new(w)); - } - Err(e) => warn!("config watcher: poll watch failed: {}", e), - }, - Err(e) => warn!("config watcher: poll watcher unavailable: {}", e), - } - - // ── event loop ─────────────────────────────────────────────────────── tokio::spawn(async move { + let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4); + let manifest_state = Arc::new(StdRwLock::new(WatchManifest::default())); + let mut reload_state = ReloadState::new(initial_snapshot_hash); + + let tx_inotify = notify_tx.clone(); + let manifest_for_inotify = manifest_state.clone(); + let mut inotify_watcher = match recommended_watcher(move |res: notify::Result| { + let Ok(event) = res else { return }; + if !matches!(event.kind, EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)) { + return; + } + let is_our_file = manifest_for_inotify + .read() + .map(|manifest| manifest.matches_event_paths(&event.paths)) + .unwrap_or(false); + if is_our_file { + let _ = tx_inotify.try_send(()); + } + }) { + Ok(watcher) => Some(watcher), + Err(e) => { + warn!("config watcher: inotify unavailable: {}", e); + None + } + }; + apply_watch_manifest( + inotify_watcher.as_mut(), + Option::<&mut notify::poll::PollWatcher>::None, + &manifest_state, + initial_manifest.clone(), + ); + if inotify_watcher.is_some() { + info!("config watcher: inotify active on {:?}", config_path); + } + + let tx_poll = notify_tx.clone(); + let manifest_for_poll = manifest_state.clone(); + let mut poll_watcher = match notify::poll::PollWatcher::new( + move |res: notify::Result| { + let Ok(event) = res else { return }; + if !matches!(event.kind, EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)) { + return; + } + let is_our_file = manifest_for_poll + .read() + .map(|manifest| manifest.matches_event_paths(&event.paths)) + .unwrap_or(false); + if is_our_file { + let _ = tx_poll.try_send(()); + } + }, + notify::Config::default() + .with_poll_interval(Duration::from_secs(3)) + .with_compare_contents(true), + ) { + Ok(watcher) => Some(watcher), + Err(e) => { + warn!("config watcher: poll watcher unavailable: {}", e); + None + } + }; + apply_watch_manifest( + Option::<&mut notify::RecommendedWatcher>::None, + poll_watcher.as_mut(), + &manifest_state, + initial_manifest.clone(), + ); + if poll_watcher.is_some() { + info!("config watcher: poll watcher active (Docker/NFS safe)"); + } + #[cfg(unix)] let mut sighup = { use tokio::signal::unix::{SignalKind, signal}; @@ -1133,11 +1315,25 @@ pub fn spawn_config_watcher( #[cfg(not(unix))] if notify_rx.recv().await.is_none() { break; } - // Debounce: drain extra events that arrive within 50 ms. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; + // Debounce: drain extra events that arrive within a short quiet window. + tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await; while notify_rx.try_recv().is_ok() {} - reload_config(&config_path, &config_tx, &log_tx, detected_ip_v4, detected_ip_v6); + if let Some(next_manifest) = reload_config( + &config_path, + &config_tx, + &log_tx, + detected_ip_v4, + detected_ip_v6, + &mut reload_state, + ) { + apply_watch_manifest( + inotify_watcher.as_mut(), + poll_watcher.as_mut(), + &manifest_state, + next_manifest, + ); + } } }); @@ -1152,6 +1348,40 @@ mod tests { ProxyConfig::default() } + fn write_reload_config(path: &Path, ad_tag: Option<&str>, server_port: Option) { + let mut config = String::from( + r#" + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#, + ); + + if ad_tag.is_some() { + config.push_str("\n[general]\n"); + if let Some(tag) = ad_tag { + config.push_str(&format!("ad_tag = \"{tag}\"\n")); + } + } + + if let Some(port) = server_port { + config.push_str("\n[server]\n"); + config.push_str(&format!("port = {port}\n")); + } + + std::fs::write(path, config).unwrap(); + } + + fn temp_config_path(prefix: &str) -> PathBuf { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("{prefix}_{nonce}.toml")) + } + #[test] fn overlay_applies_hot_and_preserves_non_hot() { let old = sample_config(); @@ -1219,4 +1449,61 @@ mod tests { assert_eq!(applied.general.use_middle_proxy, old.general.use_middle_proxy); assert!(!config_equal(&applied, &new)); } + + #[test] + fn reload_requires_stable_snapshot_before_hot_apply() { + let initial_tag = "11111111111111111111111111111111"; + let final_tag = "22222222222222222222222222222222"; + let path = temp_config_path("telemt_hot_reload_stable"); + + write_reload_config(&path, Some(initial_tag), None); + let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap()); + let initial_hash = ProxyConfig::load_with_metadata(&path).unwrap().rendered_hash; + let (config_tx, _config_rx) = watch::channel(initial_cfg.clone()); + let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone()); + let mut reload_state = ReloadState::new(Some(initial_hash)); + + write_reload_config(&path, None, None); + reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap(); + assert_eq!( + config_tx.borrow().general.ad_tag.as_deref(), + Some(initial_tag) + ); + + write_reload_config(&path, Some(final_tag), None); + reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap(); + assert_eq!( + config_tx.borrow().general.ad_tag.as_deref(), + Some(initial_tag) + ); + + reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap(); + assert_eq!(config_tx.borrow().general.ad_tag.as_deref(), Some(final_tag)); + + let _ = std::fs::remove_file(path); + } + + #[test] + fn reload_keeps_hot_apply_when_non_hot_fields_change() { + let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let final_tag = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let path = temp_config_path("telemt_hot_reload_mixed"); + + write_reload_config(&path, Some(initial_tag), None); + let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap()); + let initial_hash = ProxyConfig::load_with_metadata(&path).unwrap().rendered_hash; + let (config_tx, _config_rx) = watch::channel(initial_cfg.clone()); + let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone()); + let mut reload_state = ReloadState::new(Some(initial_hash)); + + write_reload_config(&path, Some(final_tag), Some(initial_cfg.server.port + 1)); + reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap(); + reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap(); + + let applied = config_tx.borrow().clone(); + assert_eq!(applied.general.ad_tag.as_deref(), Some(final_tag)); + assert_eq!(applied.server.port, initial_cfg.server.port); + + let _ = std::fs::remove_file(path); + } } diff --git a/src/config/load.rs b/src/config/load.rs index 3f1cd5c..ed3e303 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -1,8 +1,9 @@ #![allow(deprecated)] -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; +use std::hash::{DefaultHasher, Hash, Hasher}; use std::net::{IpAddr, SocketAddr}; -use std::path::Path; +use std::path::{Path, PathBuf}; use rand::Rng; use tracing::warn; @@ -13,7 +14,37 @@ use crate::error::{ProxyError, Result}; use super::defaults::*; use super::types::*; -fn preprocess_includes(content: &str, base_dir: &Path, depth: u8) -> Result { +#[derive(Debug, Clone)] +pub(crate) struct LoadedConfig { + pub(crate) config: ProxyConfig, + pub(crate) source_files: Vec, + pub(crate) rendered_hash: u64, +} + +fn normalize_config_path(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| { + if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .map(|cwd| cwd.join(path)) + .unwrap_or_else(|_| path.to_path_buf()) + } + }) +} + +fn hash_rendered_snapshot(rendered: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + rendered.hash(&mut hasher); + hasher.finish() +} + +fn preprocess_includes( + content: &str, + base_dir: &Path, + depth: u8, + source_files: &mut BTreeSet, +) -> Result { if depth > 10 { return Err(ProxyError::Config("Include depth > 10".into())); } @@ -25,10 +56,16 @@ fn preprocess_includes(content: &str, base_dir: &Path, depth: u8) -> Result>(path: P) -> Result { - let content = - std::fs::read_to_string(&path).map_err(|e| ProxyError::Config(e.to_string()))?; - let base_dir = path.as_ref().parent().unwrap_or(Path::new(".")); - let processed = preprocess_includes(&content, base_dir, 0)?; + Self::load_with_metadata(path).map(|loaded| loaded.config) + } + + pub(crate) fn load_with_metadata>(path: P) -> Result { + let path = path.as_ref(); + let content = std::fs::read_to_string(path).map_err(|e| ProxyError::Config(e.to_string()))?; + let base_dir = path.parent().unwrap_or(Path::new(".")); + let mut source_files = BTreeSet::new(); + source_files.insert(normalize_config_path(path)); + let processed = preprocess_includes(&content, base_dir, 0, &mut source_files)?; let parsed_toml: toml::Value = toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?; @@ -786,7 +829,11 @@ impl ProxyConfig { .entry("203".to_string()) .or_insert_with(|| vec!["91.105.192.100:443".to_string()]); - Ok(config) + Ok(LoadedConfig { + config, + source_files: source_files.into_iter().collect(), + rendered_hash: hash_rendered_snapshot(&processed), + }) } pub fn validate(&self) -> Result<()> { @@ -1111,6 +1158,48 @@ mod tests { ); } + #[test] + fn load_with_metadata_collects_include_files() { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("telemt_load_metadata_{nonce}")); + std::fs::create_dir_all(&dir).unwrap(); + let main_path = dir.join("config.toml"); + let include_path = dir.join("included.toml"); + + std::fs::write( + &include_path, + r#" + [access.users] + user = "00000000000000000000000000000000" + "#, + ) + .unwrap(); + std::fs::write( + &main_path, + r#" + include = "included.toml" + + [censorship] + tls_domain = "example.com" + "#, + ) + .unwrap(); + + let loaded = ProxyConfig::load_with_metadata(&main_path).unwrap(); + let main_normalized = normalize_config_path(&main_path); + let include_normalized = normalize_config_path(&include_path); + + assert!(loaded.source_files.contains(&main_normalized)); + assert!(loaded.source_files.contains(&include_normalized)); + + let _ = std::fs::remove_file(main_path); + let _ = std::fs::remove_file(include_path); + let _ = std::fs::remove_dir(dir); + } + #[test] fn dc_overrides_inject_dc203_default() { let toml = r#" From 4ddbb9790839d5014268a99b785cfbab1e3609d9 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 20:29:12 +0300 Subject: [PATCH 203/261] Create TLS-F-TCP-s.ru.md --- docs/fronting-splitting/TLS-F-TCP-s.ru.md | 274 ++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 docs/fronting-splitting/TLS-F-TCP-s.ru.md diff --git a/docs/fronting-splitting/TLS-F-TCP-s.ru.md b/docs/fronting-splitting/TLS-F-TCP-s.ru.md new file mode 100644 index 0000000..dd4d425 --- /dev/null +++ b/docs/fronting-splitting/TLS-F-TCP-s.ru.md @@ -0,0 +1,274 @@ +# TLS-F и TCP-S в Telemt + +## Общая архитектура + +**Telemt** - это прежде всего реализация **MTProxy**, через которую проходит payload Telegram + +Подсистема **TLS-Fronting / TCP-Splitting** служит **маскировочным транспортным слоем**, задача которого - сделать MTProxy-соединение внешне похожим на обычное TLS-подключение к легитимному сайту + +Таким образом: + +- **MTProxy** - основной функциональный слой Telemt для обработки Telegram-трафика +- **TLS-Fronting / TCP-Splitting** - подсистема маскировки транспорта + +С точки зрения сети Telemt ведёт себя как **TLS-сервер**, но фактически: + +- валидные MTProxy-клиенты остаются внутри контура Telemt +- любые другие TLS-клиенты проксируются на обычный HTTPS-сервер-заглушку + +# Базовый сценарий / Best-practice + +Предположим, у вас есть домен: + +``` +umweltschutz.de +``` + +### 1 DNS + +Вы создаёте A-запись: + +``` +umweltschutz.de -> A-запись 198.18.88.88 +``` + +где `198.18.88.88` - IP вашего сервера с telemt + +### 2 TLS-домен + +В конфигурации Telemt: + +``` +tls_domain = umweltschutz.de +``` + +Этот домен используется клиентом как SNI в ClientHello + +### 3. Сервер-заглушка + +Вы поднимаете обычный HTTPS-сервер, например **nginx**, с сертификатом для этого домена. + +Он может работать: + +- на том же сервере +- на другом сервере +- на другом порту + +В конфигурации Telemt: + +``` +mask_host = 127.0.0.1 +mask_port = 8443 +``` + +где `127.0.0.1` - IP сервера-заглушки, а 8443 - порт, который он слушает + +Этот сервер нужен **для обработки любых non-MTProxy запросов** + +### 4 Работа Telemt + +После запуска Telemt действует следующим образом: + +1) принимает входящее TCP-соединение +2) анализирует TLS-ClientHello +3) пытается определить, является ли соединение валидным **MTProxy FakeTLS** + +Далее работают два варианта логики: + +--- + +# Сценарий 1 - MTProxy клиент с валидным ключом + +Если клиент предъявил **валидный MTProxy-ключ**: + +- соединение **остаётся внутри Telemt** +- TLS используется только как **транспортная маскировка** +- далее запускается обычная логика **MTProxy** + +Для внешнего наблюдателя это выглядит как: + +``` +TLS connection -> umweltschutz.de +``` + +Хотя внутри передаётся **MTProto-трафик Telegram** + +# Сценарий 2 - обычный TLS-клиент (crawler, scanner, браузер) + +Если Telemt не обнаруживает валидный MTProxy-ключ: + +соединение **переключается в режим TCP-Splitting (TCP-Splicing)**. + +В этом режиме Telemt: + +1. открывает новое TCP-соединение к + +``` +mask_host:mask_port +``` + +2. начинает **проксировать TCP-трафик** + +Важно: + +* клиентский TLS-запрос **НЕ модифицируется** +* **ClientHello передаётся "как есть", без изменений** +* **SNI остаётся неизменным** +* Telemt **не завершает TLS-рукопожатие**, а только перенаправляет его на более низком уровне сетевого стека - L4 + +Таким образом upstream-сервер получает **оригинальное TLS-соединение клиента**: + +- если это nginx-заглушка, он просто отдаёт обычный сайт +- для внешнего наблюдателя это выглядит как обычный HTTPS-сервер + +# TCP-S / TCP-Splitting / TCP-Splicing + +Ключевые свойства механизма: + +**Telemt работает как TCP-переключатель:** + +1) принимает соединение +2️) определяет тип клиента +3) либо: + +- обрабатывает MTProxy внутри +- либо проксирует TCP-поток + +При проксировании: + +- Telemt **разрешает `mask_host` в IP** +- устанавливает TCP-соединение +- начинает **bidirectional TCP relay** + +При этом: + +- TLS-рукопожатие происходит **между клиентом и `mask_host`** +- Telemt выступает только **на уровне L4 - как TCP-релей**, такой же как HAProxy в TCP-режиме + +# Использование чужого домена + +Можно использовать и внешний сайт. + +Например: + +``` +tls_domain = github.com +mask_host = github.com +mask_port = 443 +``` + +или + +``` +mask_host = 140.82.121.4 +``` + +В этом случае: + +- цензор видит **TLS-подключение к github.com** +- обычные клиенты/краулер действительно получают **настоящий GitHub** + +Telemt просто **проксирует TCP-соединение на GitHub** + +# Что видит анализатор трафика? + +Для DPI это выглядит так: + +``` +client -> TLS -> github.com +``` + +или + +``` +client -> TLS -> umweltschutz.de +``` + +TLS-handshake выглядит валидным, SNI соответствует домену, сертификат корректный - от целевого `mask_host:mask_port` + +# Что видит сканер / краулер? + +Если сканер попытается подключиться: + +``` +openssl s_client -connect 198.18.88.88:443 -servername umweltschutz.de +``` + +он получит **обычный HTTPS-сайт-заглушку** + +Потому что: + +- он не предъявил MTProxy-ключ +- Telemt отправил соединение на `mask_host:mask_port`, на котором находится nginx + +# Какую проблему решает TLS-Fronting / TCP-Splitting? + +Эта архитектура решает сразу несколько проблем обхода цензуры. + +## 1 Закрытие плоскости MTProxy от активного сканирования + +Многие цензоры: + +- сканируют IP-адреса +- проверяют известные сигнатуры прокси + +Telemt отвечает на такие проверки **обычным HTTPS-сайтом**, поэтому прокси невозможно обнаружить простым сканированием + +--- + +## 2 Маскировка трафика под легитимный TLS + +Для DPI-систем соединение выглядит как: + +``` +обычный TLS-трафик к популярному домену +``` + +Это делает блокировку значительно сложнее и непредсказуемее + +--- + +## 3 Устойчивость к протокольному анализу + +MTProxy трафик проходит **внутри TLS-like-потока**, поэтому: + +- не видны характерные сигнатуры MTProto +- соединение выглядит как обычный HTTPS + +--- + +## 4 Правдоподобное поведение сервера + +Даже если краулер: + +- подключится сам +- выполнит TLS-handshake +- попытается получить HTTP-ответ + +он увидит **реальный сайт**, а не telemt + +Это устраняет один из главных признаков для антифрод-краулеров мобильных операторов + +# Схема + +```text + Client + │ + │ TCP + │ + V + Telemt + │ + ├── valid MTProxy key + │ │ + │ V + │ MTProxy logic + │ + └── обычный TLS клиент + │ + V + TCP-Splitting + │ + V + mask_host:mask_port +``` \ No newline at end of file From 4677b43c6e00407bb67c3c1a7884e124fef6ae89 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 20:38:24 +0300 Subject: [PATCH 204/261] TLS-F New Methods --- src/tls_front/cache.rs | 6 ++- src/tls_front/emulator.rs | 67 +++++++++++++++++++----- src/tls_front/fetcher.rs | 105 +++++++++++++++++++++++++++++++++----- src/tls_front/types.rs | 83 ++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 28 deletions(-) diff --git a/src/tls_front/cache.rs b/src/tls_front/cache.rs index 23e60db..0dc2b5d 100644 --- a/src/tls_front/cache.rs +++ b/src/tls_front/cache.rs @@ -8,7 +8,9 @@ use tokio::sync::RwLock; use tokio::time::sleep; use tracing::{debug, warn, info}; -use crate::tls_front::types::{CachedTlsData, ParsedServerHello, TlsFetchResult}; +use crate::tls_front::types::{ + CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, +}; /// Lightweight in-memory + optional on-disk cache for TLS fronting data. #[derive(Debug)] @@ -37,6 +39,7 @@ impl TlsFrontCache { cert_payload: None, app_data_records_sizes: vec![default_len], total_app_data_len: default_len, + behavior_profile: TlsBehaviorProfile::default(), fetched_at: SystemTime::now(), domain: "default".to_string(), }); @@ -189,6 +192,7 @@ impl TlsFrontCache { cert_payload: fetched.cert_payload, app_data_records_sizes: fetched.app_data_records_sizes.clone(), total_app_data_len: fetched.total_app_data_len, + behavior_profile: fetched.behavior_profile, fetched_at: SystemTime::now(), domain: domain.to_string(), }; diff --git a/src/tls_front/emulator.rs b/src/tls_front/emulator.rs index c8c18ac..3278f63 100644 --- a/src/tls_front/emulator.rs +++ b/src/tls_front/emulator.rs @@ -3,7 +3,7 @@ use crate::protocol::constants::{ TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE, TLS_VERSION, }; use crate::protocol::tls::{TLS_DIGEST_LEN, TLS_DIGEST_POS, gen_fake_x25519_key}; -use crate::tls_front::types::{CachedTlsData, ParsedCertificateInfo}; +use crate::tls_front::types::{CachedTlsData, ParsedCertificateInfo, TlsProfileSource}; const MIN_APP_DATA: usize = 64; const MAX_APP_DATA: usize = 16640; // RFC 8446 §5.2 allows up to 2^14 + 256 @@ -108,14 +108,12 @@ pub fn build_emulated_server_hello( ) -> Vec { // --- ServerHello --- let mut extensions = Vec::new(); - // KeyShare (x25519) let key = gen_fake_x25519_key(rng); - extensions.extend_from_slice(&0x0033u16.to_be_bytes()); // key_share - extensions.extend_from_slice(&(2 + 2 + 32u16).to_be_bytes()); // len - extensions.extend_from_slice(&0x001du16.to_be_bytes()); // X25519 + extensions.extend_from_slice(&0x0033u16.to_be_bytes()); + extensions.extend_from_slice(&(2 + 2 + 32u16).to_be_bytes()); + extensions.extend_from_slice(&0x001du16.to_be_bytes()); extensions.extend_from_slice(&(32u16).to_be_bytes()); extensions.extend_from_slice(&key); - // supported_versions (TLS1.3) extensions.extend_from_slice(&0x002bu16.to_be_bytes()); extensions.extend_from_slice(&(2u16).to_be_bytes()); extensions.extend_from_slice(&0x0304u16.to_be_bytes()); @@ -128,7 +126,6 @@ pub fn build_emulated_server_hello( extensions.push(alpn_proto.len() as u8); extensions.extend_from_slice(alpn_proto); } - let extensions_len = extensions.len() as u16; let body_len = 2 + // version @@ -173,11 +170,22 @@ pub fn build_emulated_server_hello( ]; // --- ApplicationData (fake encrypted records) --- - // Use the same number and sizes of ApplicationData records as the cached server. - let mut sizes = cached.app_data_records_sizes.clone(); - if sizes.is_empty() { - sizes.push(cached.total_app_data_len.max(1024)); - } + let sizes = match cached.behavior_profile.source { + TlsProfileSource::Raw | TlsProfileSource::Merged => cached + .app_data_records_sizes + .first() + .copied() + .or_else(|| cached.behavior_profile.app_data_record_sizes.first().copied()) + .map(|size| vec![size]) + .unwrap_or_else(|| vec![cached.total_app_data_len.max(1024)]), + _ => { + let mut sizes = cached.app_data_records_sizes.clone(); + if sizes.is_empty() { + sizes.push(cached.total_app_data_len.max(1024)); + } + sizes + } + }; let mut sizes = jitter_and_clamp_sizes(&sizes, rng); let compact_payload = cached .cert_info @@ -269,7 +277,9 @@ pub fn build_emulated_server_hello( mod tests { use std::time::SystemTime; - use crate::tls_front::types::{CachedTlsData, ParsedServerHello, TlsCertPayload}; + use crate::tls_front::types::{ + CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource, + }; use super::build_emulated_server_hello; use crate::crypto::SecureRandom; @@ -300,6 +310,7 @@ mod tests { cert_payload, app_data_records_sizes: vec![64], total_app_data_len: 64, + behavior_profile: TlsBehaviorProfile::default(), fetched_at: SystemTime::now(), domain: "example.com".to_string(), } @@ -385,4 +396,34 @@ mod tests { let payload = first_app_data_payload(&response); assert!(payload.starts_with(b"CN=example.com")); } + + #[test] + fn test_build_emulated_server_hello_ignores_tail_records_for_raw_profile() { + let mut cached = make_cached(None); + cached.app_data_records_sizes = vec![27, 3905, 537, 69]; + cached.total_app_data_len = 4538; + cached.behavior_profile.source = TlsProfileSource::Merged; + cached.behavior_profile.app_data_record_sizes = vec![27, 3905, 537]; + cached.behavior_profile.ticket_record_sizes = vec![69]; + + let rng = SecureRandom::new(); + let response = build_emulated_server_hello( + b"secret", + &[0x12; 32], + &[0x34; 16], + &cached, + false, + &rng, + None, + 0, + ); + + let hello_len = u16::from_be_bytes([response[3], response[4]]) as usize; + let ccs_start = 5 + hello_len; + let app_start = ccs_start + 6; + let app_len = u16::from_be_bytes([response[app_start + 3], response[app_start + 4]]) as usize; + + assert_eq!(response[app_start], TLS_RECORD_APPLICATION); + assert_eq!(app_start + 5 + app_len, response.len()); + } } diff --git a/src/tls_front/fetcher.rs b/src/tls_front/fetcher.rs index 4d9067c..38872af 100644 --- a/src/tls_front/fetcher.rs +++ b/src/tls_front/fetcher.rs @@ -21,14 +21,18 @@ use x509_parser::certificate::X509Certificate; use crate::crypto::SecureRandom; use crate::network::dns_overrides::resolve_socket_addr; -use crate::protocol::constants::{TLS_RECORD_APPLICATION, TLS_RECORD_HANDSHAKE}; +use crate::protocol::constants::{ + TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE, +}; use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder}; use crate::tls_front::types::{ ParsedCertificateInfo, ParsedServerHello, + TlsBehaviorProfile, TlsCertPayload, TlsExtension, TlsFetchResult, + TlsProfileSource, }; /// No-op verifier: accept any certificate (we only need lengths and metadata). @@ -282,6 +286,41 @@ fn parse_server_hello(body: &[u8]) -> Option { }) } +fn derive_behavior_profile(records: &[(u8, Vec)]) -> TlsBehaviorProfile { + let mut change_cipher_spec_count = 0u8; + let mut app_data_record_sizes = Vec::new(); + + for (record_type, body) in records { + match *record_type { + TLS_RECORD_CHANGE_CIPHER => { + change_cipher_spec_count = change_cipher_spec_count.saturating_add(1); + } + TLS_RECORD_APPLICATION => { + app_data_record_sizes.push(body.len()); + } + _ => {} + } + } + + let mut ticket_record_sizes = Vec::new(); + while app_data_record_sizes + .last() + .is_some_and(|size| *size <= 256 && ticket_record_sizes.len() < 2) + { + if let Some(size) = app_data_record_sizes.pop() { + ticket_record_sizes.push(size); + } + } + ticket_record_sizes.reverse(); + + TlsBehaviorProfile { + change_cipher_spec_count: change_cipher_spec_count.max(1), + app_data_record_sizes, + ticket_record_sizes, + source: TlsProfileSource::Raw, + } +} + fn parse_cert_info(certs: &[CertificateDer<'static>]) -> Option { let first = certs.first()?; let (_rem, cert) = X509Certificate::from_der(first.as_ref()).ok()?; @@ -443,39 +482,50 @@ where .await??; let mut records = Vec::new(); - // Read up to 4 records: ServerHello, CCS, and up to two ApplicationData. - for _ in 0..4 { + let mut app_records_seen = 0usize; + // Read a bounded encrypted flight: ServerHello, CCS, certificate-like data, + // and a small number of ticket-like tail records. + for _ in 0..8 { match timeout(connect_timeout, read_tls_record(&mut stream)).await { - Ok(Ok(rec)) => records.push(rec), + Ok(Ok(rec)) => { + if rec.0 == TLS_RECORD_APPLICATION { + app_records_seen += 1; + } + records.push(rec); + } Ok(Err(e)) => return Err(e), Err(_) => break, } - if records.len() >= 3 && records.iter().any(|(t, _)| *t == TLS_RECORD_APPLICATION) { + if app_records_seen >= 4 { break; } } - let mut app_sizes = Vec::new(); let mut server_hello = None; for (t, body) in &records { if *t == TLS_RECORD_HANDSHAKE && server_hello.is_none() { server_hello = parse_server_hello(body); - } else if *t == TLS_RECORD_APPLICATION { - app_sizes.push(body.len()); } } let parsed = server_hello.ok_or_else(|| anyhow!("ServerHello not received"))?; + let behavior_profile = derive_behavior_profile(&records); + let mut app_sizes = behavior_profile.app_data_record_sizes.clone(); + app_sizes.extend_from_slice(&behavior_profile.ticket_record_sizes); let total_app_data_len = app_sizes.iter().sum::().max(1024); + let app_data_records_sizes = behavior_profile + .app_data_record_sizes + .first() + .copied() + .or_else(|| behavior_profile.ticket_record_sizes.first().copied()) + .map(|size| vec![size]) + .unwrap_or_else(|| vec![total_app_data_len]); Ok(TlsFetchResult { server_hello_parsed: parsed, - app_data_records_sizes: if app_sizes.is_empty() { - vec![total_app_data_len] - } else { - app_sizes - }, + app_data_records_sizes, total_app_data_len, + behavior_profile, cert_info: None, cert_payload: None, }) @@ -608,6 +658,12 @@ where server_hello_parsed: parsed, app_data_records_sizes: app_data_records_sizes.clone(), total_app_data_len: app_data_records_sizes.iter().sum(), + behavior_profile: TlsBehaviorProfile { + change_cipher_spec_count: 1, + app_data_record_sizes: app_data_records_sizes, + ticket_record_sizes: Vec::new(), + source: TlsProfileSource::Rustls, + }, cert_info, cert_payload, }) @@ -706,6 +762,7 @@ pub async fn fetch_real_tls( if let Some(mut raw) = raw_result { raw.cert_info = rustls_result.cert_info; raw.cert_payload = rustls_result.cert_payload; + raw.behavior_profile.source = TlsProfileSource::Merged; debug!(sni = %sni, "Fetched TLS metadata via raw probe + rustls cert chain"); Ok(raw) } else { @@ -725,7 +782,11 @@ pub async fn fetch_real_tls( #[cfg(test)] mod tests { - use super::encode_tls13_certificate_message; + use super::{derive_behavior_profile, encode_tls13_certificate_message}; + use crate::protocol::constants::{ + TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE, + }; + use crate::tls_front::types::TlsProfileSource; fn read_u24(bytes: &[u8]) -> usize { ((bytes[0] as usize) << 16) | ((bytes[1] as usize) << 8) | (bytes[2] as usize) @@ -753,4 +814,20 @@ mod tests { fn test_encode_tls13_certificate_message_empty_chain() { assert!(encode_tls13_certificate_message(&[]).is_none()); } + + #[test] + fn test_derive_behavior_profile_splits_ticket_like_tail_records() { + let profile = derive_behavior_profile(&[ + (TLS_RECORD_HANDSHAKE, vec![0u8; 90]), + (TLS_RECORD_CHANGE_CIPHER, vec![0x01]), + (TLS_RECORD_APPLICATION, vec![0u8; 1400]), + (TLS_RECORD_APPLICATION, vec![0u8; 220]), + (TLS_RECORD_APPLICATION, vec![0u8; 180]), + ]); + + assert_eq!(profile.change_cipher_spec_count, 1); + assert_eq!(profile.app_data_record_sizes, vec![1400]); + assert_eq!(profile.ticket_record_sizes, vec![220, 180]); + assert_eq!(profile.source, TlsProfileSource::Raw); + } } diff --git a/src/tls_front/types.rs b/src/tls_front/types.rs index c411081..10aca05 100644 --- a/src/tls_front/types.rs +++ b/src/tls_front/types.rs @@ -39,6 +39,53 @@ pub struct TlsCertPayload { pub certificate_message: Vec, } +/// Provenance of the cached TLS behavior profile. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum TlsProfileSource { + /// Built from hardcoded defaults or legacy cache entries. + #[default] + Default, + /// Derived from raw TLS record capture only. + Raw, + /// Derived from rustls-only metadata fallback. + Rustls, + /// Merged from raw TLS capture and rustls certificate metadata. + Merged, +} + +/// Coarse-grained TLS response behavior captured per SNI. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TlsBehaviorProfile { + /// Number of ChangeCipherSpec records observed before encrypted flight. + #[serde(default = "default_change_cipher_spec_count")] + pub change_cipher_spec_count: u8, + /// Sizes of the primary encrypted flight records carrying cert-like payload. + #[serde(default)] + pub app_data_record_sizes: Vec, + /// Sizes of small tail ApplicationData records that look like tickets. + #[serde(default)] + pub ticket_record_sizes: Vec, + /// Source of this behavior profile. + #[serde(default)] + pub source: TlsProfileSource, +} + +fn default_change_cipher_spec_count() -> u8 { + 1 +} + +impl Default for TlsBehaviorProfile { + fn default() -> Self { + Self { + change_cipher_spec_count: default_change_cipher_spec_count(), + app_data_record_sizes: Vec::new(), + ticket_record_sizes: Vec::new(), + source: TlsProfileSource::Default, + } + } +} + /// Cached data per SNI used by the emulator. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CachedTlsData { @@ -48,6 +95,8 @@ pub struct CachedTlsData { pub cert_payload: Option, pub app_data_records_sizes: Vec, pub total_app_data_len: usize, + #[serde(default)] + pub behavior_profile: TlsBehaviorProfile, #[serde(default = "now_system_time", skip_serializing, skip_deserializing)] pub fetched_at: SystemTime, pub domain: String, @@ -63,6 +112,40 @@ pub struct TlsFetchResult { pub server_hello_parsed: ParsedServerHello, pub app_data_records_sizes: Vec, pub total_app_data_len: usize, + #[serde(default)] + pub behavior_profile: TlsBehaviorProfile, pub cert_info: Option, pub cert_payload: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cached_tls_data_deserializes_without_behavior_profile() { + let json = r#" + { + "server_hello_template": { + "version": [3, 3], + "random": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "session_id": [], + "cipher_suite": [19, 1], + "compression": 0, + "extensions": [] + }, + "cert_info": null, + "cert_payload": null, + "app_data_records_sizes": [1024], + "total_app_data_len": 1024, + "domain": "example.com" + } + "#; + + let cached: CachedTlsData = serde_json::from_str(json).unwrap(); + assert_eq!(cached.behavior_profile.change_cipher_spec_count, 1); + assert!(cached.behavior_profile.app_data_record_sizes.is_empty()); + assert!(cached.behavior_profile.ticket_record_sizes.is_empty()); + assert_eq!(cached.behavior_profile.source, TlsProfileSource::Default); + } +} From 56e38e8d00e4570c457bc2f93027e80d569bb52e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 20:41:14 +0300 Subject: [PATCH 205/261] Update TLS-F-TCP-s.ru.md --- docs/fronting-splitting/TLS-F-TCP-s.ru.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/fronting-splitting/TLS-F-TCP-s.ru.md b/docs/fronting-splitting/TLS-F-TCP-s.ru.md index dd4d425..6ae6f05 100644 --- a/docs/fronting-splitting/TLS-F-TCP-s.ru.md +++ b/docs/fronting-splitting/TLS-F-TCP-s.ru.md @@ -44,7 +44,7 @@ tls_domain = umweltschutz.de Этот домен используется клиентом как SNI в ClientHello -### 3. Сервер-заглушка +### 3 Сервер-заглушка Вы поднимаете обычный HTTPS-сервер, например **nginx**, с сертификатом для этого домена. @@ -93,11 +93,11 @@ TLS connection -> umweltschutz.de Хотя внутри передаётся **MTProto-трафик Telegram** -# Сценарий 2 - обычный TLS-клиент (crawler, scanner, браузер) +# Сценарий 2 - обычный TLS-клиент - crawler / scanner / browser Если Telemt не обнаруживает валидный MTProxy-ключ: -соединение **переключается в режим TCP-Splitting (TCP-Splicing)**. +соединение **переключается в режим TCP-Splitting / TCP-Splicing**. В этом режиме Telemt: @@ -271,4 +271,4 @@ MTProxy трафик проходит **внутри TLS-like-потока**, п │ V mask_host:mask_port -``` \ No newline at end of file +``` From 7d5e1cb9e850004fb0e6ed0bd43a4acc8264f8aa Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sat, 14 Mar 2026 20:42:21 +0300 Subject: [PATCH 206/261] Rename TLS-F-TCP-s.ru.md to TLS-F-TCP-S.ru.md --- docs/fronting-splitting/{TLS-F-TCP-s.ru.md => TLS-F-TCP-S.ru.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/fronting-splitting/{TLS-F-TCP-s.ru.md => TLS-F-TCP-S.ru.md} (100%) diff --git a/docs/fronting-splitting/TLS-F-TCP-s.ru.md b/docs/fronting-splitting/TLS-F-TCP-S.ru.md similarity index 100% rename from docs/fronting-splitting/TLS-F-TCP-s.ru.md rename to docs/fronting-splitting/TLS-F-TCP-S.ru.md From ac0698b772f00c11e26ae385115a3be1da9938ef Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 15 Mar 2026 00:17:54 +0300 Subject: [PATCH 207/261] ME Writer Rebinding - Lifecycle and Consistency fixes --- src/api/model.rs | 10 + src/api/runtime_stats.rs | 12 + src/transport/middle_proxy/health.rs | 54 ++++- src/transport/middle_proxy/pool_status.rs | 48 ++++ src/transport/middle_proxy/pool_writer.rs | 22 +- src/transport/middle_proxy/registry.rs | 256 +++++++++++++++++----- src/transport/middle_proxy/send.rs | 22 +- 7 files changed, 361 insertions(+), 63 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index 0bc52de..31233d7 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -236,6 +236,8 @@ pub(super) struct MeWritersSummary { pub(super) required_writers: usize, pub(super) alive_writers: usize, pub(super) coverage_pct: f64, + pub(super) fresh_alive_writers: usize, + pub(super) fresh_coverage_pct: f64, } #[derive(Serialize, Clone)] @@ -250,6 +252,12 @@ pub(super) struct MeWriterStatus { pub(super) bound_clients: usize, pub(super) idle_for_secs: Option, pub(super) rtt_ema_ms: Option, + pub(super) matches_active_generation: bool, + pub(super) in_desired_map: bool, + pub(super) allow_drain_fallback: bool, + pub(super) drain_started_at_epoch_secs: Option, + pub(super) drain_deadline_epoch_secs: Option, + pub(super) drain_over_ttl: bool, } #[derive(Serialize, Clone)] @@ -276,6 +284,8 @@ pub(super) struct DcStatus { pub(super) floor_capped: bool, pub(super) alive_writers: usize, pub(super) coverage_pct: f64, + pub(super) fresh_alive_writers: usize, + pub(super) fresh_coverage_pct: f64, pub(super) rtt_ms: Option, pub(super) load: usize, } diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 139a4c5..9260c40 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -314,6 +314,8 @@ async fn get_minimal_payload_cached( required_writers: status.required_writers, alive_writers: status.alive_writers, coverage_pct: status.coverage_pct, + fresh_alive_writers: status.fresh_alive_writers, + fresh_coverage_pct: status.fresh_coverage_pct, }, writers: status .writers @@ -329,6 +331,12 @@ async fn get_minimal_payload_cached( bound_clients: entry.bound_clients, idle_for_secs: entry.idle_for_secs, rtt_ema_ms: entry.rtt_ema_ms, + matches_active_generation: entry.matches_active_generation, + in_desired_map: entry.in_desired_map, + allow_drain_fallback: entry.allow_drain_fallback, + drain_started_at_epoch_secs: entry.drain_started_at_epoch_secs, + drain_deadline_epoch_secs: entry.drain_deadline_epoch_secs, + drain_over_ttl: entry.drain_over_ttl, }) .collect(), }; @@ -363,6 +371,8 @@ async fn get_minimal_payload_cached( floor_capped: entry.floor_capped, alive_writers: entry.alive_writers, coverage_pct: entry.coverage_pct, + fresh_alive_writers: entry.fresh_alive_writers, + fresh_coverage_pct: entry.fresh_coverage_pct, rtt_ms: entry.rtt_ms, load: entry.load, }) @@ -486,6 +496,8 @@ fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersDa required_writers: 0, alive_writers: 0, coverage_pct: 0.0, + fresh_alive_writers: 0, + fresh_coverage_pct: 0.0, }, writers: Vec::new(), } diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index b422dc6..3cc400f 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -62,6 +62,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c let mut adaptive_idle_since: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new(); + let mut drain_warn_next_allowed: HashMap = HashMap::new(); let mut degraded_interval = true; loop { let interval = if degraded_interval { @@ -71,7 +72,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c }; tokio::time::sleep(interval).await; pool.prune_closed_writers().await; - reap_draining_writers(&pool).await; + reap_draining_writers(&pool, &mut drain_warn_next_allowed).await; let v4_degraded = check_family( IpFamily::V4, &pool, @@ -110,17 +111,47 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c } } -async fn reap_draining_writers(pool: &Arc) { +async fn reap_draining_writers( + pool: &Arc, + warn_next_allowed: &mut HashMap, +) { let now_epoch_secs = MePool::now_epoch_secs(); + let now = Instant::now(); + let drain_ttl_secs = pool.me_pool_drain_ttl_secs.load(std::sync::atomic::Ordering::Relaxed); let writers = pool.writers.read().await.clone(); for writer in writers { if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) { continue; } - if pool.registry.is_writer_empty(writer.id).await { + let is_empty = pool.registry.is_writer_empty(writer.id).await; + if is_empty { pool.remove_writer_and_close_clients(writer.id).await; continue; } + let drain_started_at_epoch_secs = writer + .draining_started_at_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + if drain_ttl_secs > 0 + && drain_started_at_epoch_secs != 0 + && now_epoch_secs.saturating_sub(drain_started_at_epoch_secs) > drain_ttl_secs + && should_emit_writer_warn( + warn_next_allowed, + writer.id, + now, + pool.warn_rate_limit_duration(), + ) + { + warn!( + writer_id = writer.id, + writer_dc = writer.writer_dc, + endpoint = %writer.addr, + generation = writer.generation, + drain_ttl_secs, + force_close_secs = pool.me_pool_force_close_secs.load(std::sync::atomic::Ordering::Relaxed), + allow_drain_fallback = writer.allow_drain_fallback.load(std::sync::atomic::Ordering::Relaxed), + "ME draining writer remains non-empty past drain TTL" + ); + } let deadline_epoch_secs = writer .drain_deadline_epoch_secs .load(std::sync::atomic::Ordering::Relaxed); @@ -132,6 +163,23 @@ async fn reap_draining_writers(pool: &Arc) { } } +fn should_emit_writer_warn( + next_allowed: &mut HashMap, + writer_id: u64, + now: Instant, + cooldown: Duration, +) -> bool { + let Some(ready_at) = next_allowed.get(&writer_id).copied() else { + next_allowed.insert(writer_id, now + cooldown); + return true; + }; + if now >= ready_at { + next_allowed.insert(writer_id, now + cooldown); + return true; + } + false +} + async fn check_family( family: IpFamily, pool: &Arc, diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 6673cf2..99070a8 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -19,6 +19,12 @@ pub(crate) struct MeApiWriterStatusSnapshot { pub bound_clients: usize, pub idle_for_secs: Option, pub rtt_ema_ms: Option, + pub matches_active_generation: bool, + pub in_desired_map: bool, + pub allow_drain_fallback: bool, + pub drain_started_at_epoch_secs: Option, + pub drain_deadline_epoch_secs: Option, + pub drain_over_ttl: bool, } #[derive(Clone, Debug)] @@ -35,6 +41,8 @@ pub(crate) struct MeApiDcStatusSnapshot { pub floor_capped: bool, pub alive_writers: usize, pub coverage_pct: f64, + pub fresh_alive_writers: usize, + pub fresh_coverage_pct: f64, pub rtt_ms: Option, pub load: usize, } @@ -55,6 +63,8 @@ pub(crate) struct MeApiStatusSnapshot { pub required_writers: usize, pub alive_writers: usize, pub coverage_pct: f64, + pub fresh_alive_writers: usize, + pub fresh_coverage_pct: f64, pub writers: Vec, pub dcs: Vec, } @@ -213,6 +223,8 @@ impl MePool { pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot { let now_epoch_secs = Self::now_epoch_secs(); + let active_generation = self.current_generation(); + let drain_ttl_secs = self.me_pool_drain_ttl_secs.load(Ordering::Relaxed); let mut endpoints_by_dc = BTreeMap::>::new(); if self.decision.ipv4_me { @@ -239,6 +251,7 @@ impl MePool { let mut live_writers_by_dc_endpoint = HashMap::<(i16, SocketAddr), usize>::new(); let mut live_writers_by_dc = HashMap::::new(); + let mut fresh_writers_by_dc = HashMap::::new(); let mut dc_rtt_agg = HashMap::::new(); let mut writer_rows = Vec::::with_capacity(writers.len()); @@ -247,6 +260,10 @@ impl MePool { let dc = i16::try_from(writer.writer_dc).ok(); let draining = writer.draining.load(Ordering::Relaxed); let degraded = writer.degraded.load(Ordering::Relaxed); + let matches_active_generation = writer.generation == active_generation; + let in_desired_map = dc + .and_then(|dc_idx| endpoints_by_dc.get(&dc_idx)) + .is_some_and(|endpoints| endpoints.contains(&endpoint)); let bound_clients = activity .bound_clients_by_writer .get(&writer.id) @@ -256,6 +273,21 @@ impl MePool { .get(&writer.id) .map(|idle_ts| now_epoch_secs.saturating_sub(*idle_ts)); let rtt_ema_ms = rtt.get(&writer.id).map(|(_, ema)| *ema); + let allow_drain_fallback = writer.allow_drain_fallback.load(Ordering::Relaxed); + let drain_started_at_epoch_secs = writer + .draining_started_at_epoch_secs + .load(Ordering::Relaxed); + let drain_deadline_epoch_secs = writer + .drain_deadline_epoch_secs + .load(Ordering::Relaxed); + let drain_started_at_epoch_secs = + (drain_started_at_epoch_secs != 0).then_some(drain_started_at_epoch_secs); + let drain_deadline_epoch_secs = + (drain_deadline_epoch_secs != 0).then_some(drain_deadline_epoch_secs); + let drain_over_ttl = draining + && drain_ttl_secs > 0 + && drain_started_at_epoch_secs + .is_some_and(|started| now_epoch_secs.saturating_sub(started) > drain_ttl_secs); let state = match WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) { WriterContour::Warm => "warm", WriterContour::Active => "active", @@ -273,6 +305,9 @@ impl MePool { entry.0 += ema_ms; entry.1 += 1; } + if matches_active_generation && in_desired_map { + *fresh_writers_by_dc.entry(dc_idx).or_insert(0) += 1; + } } } @@ -287,6 +322,12 @@ impl MePool { bound_clients, idle_for_secs, rtt_ema_ms, + matches_active_generation, + in_desired_map, + allow_drain_fallback, + drain_started_at_epoch_secs, + drain_deadline_epoch_secs, + drain_over_ttl, }); } @@ -295,6 +336,7 @@ impl MePool { let mut dcs = Vec::::with_capacity(endpoints_by_dc.len()); let mut available_endpoints = 0usize; let mut alive_writers = 0usize; + let mut fresh_alive_writers = 0usize; let floor_mode = self.floor_mode(); let adaptive_cpu_cores = (self .me_adaptive_floor_cpu_cores_effective @@ -333,6 +375,7 @@ impl MePool { let floor_capped = matches!(floor_mode, MeFloorMode::Adaptive) && dc_required_writers < base_required; let dc_alive_writers = live_writers_by_dc.get(&dc).copied().unwrap_or(0); + let dc_fresh_alive_writers = fresh_writers_by_dc.get(&dc).copied().unwrap_or(0); let dc_load = activity .active_sessions_by_target_dc .get(&dc) @@ -344,6 +387,7 @@ impl MePool { available_endpoints += dc_available_endpoints; alive_writers += dc_alive_writers; + fresh_alive_writers += dc_fresh_alive_writers; dcs.push(MeApiDcStatusSnapshot { dc, @@ -367,6 +411,8 @@ impl MePool { floor_capped, alive_writers: dc_alive_writers, coverage_pct: ratio_pct(dc_alive_writers, dc_required_writers), + fresh_alive_writers: dc_fresh_alive_writers, + fresh_coverage_pct: ratio_pct(dc_fresh_alive_writers, dc_required_writers), rtt_ms: dc_rtt_ms, load: dc_load, }); @@ -381,6 +427,8 @@ impl MePool { required_writers, alive_writers, coverage_pct: ratio_pct(alive_writers, required_writers), + fresh_alive_writers, + fresh_coverage_pct: ratio_pct(fresh_alive_writers, required_writers), writers: writer_rows, dcs, } diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 64fb700..8ce3de3 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -178,6 +178,7 @@ impl MePool { allow_drain_fallback: allow_drain_fallback.clone(), }; self.writers.write().await.push(writer.clone()); + self.registry.register_writer(writer_id, tx.clone()).await; self.registry.mark_writer_idle(writer_id).await; self.conn_count.fetch_add(1, Ordering::Relaxed); self.writer_available.notify_one(); @@ -414,9 +415,15 @@ impl MePool { }; let (conn_id, mut service_rx) = pool.registry.register().await; - pool.registry - .bind_writer(conn_id, writer_id, tx_signal.clone(), meta.clone()) - .await; + if !pool + .registry + .bind_writer(conn_id, writer_id, meta.clone()) + .await + { + let _ = pool.registry.unregister(conn_id).await; + stats_signal.increment_me_rpc_proxy_req_signal_skipped_no_meta_total(); + continue; + } let payload = build_proxy_req_payload( conn_id, @@ -521,6 +528,12 @@ impl MePool { self.conn_count.fetch_sub(1, Ordering::Relaxed); } } + let conns = self.registry.writer_lost(writer_id).await; + { + let mut tracker = self.ping_tracker.lock().await; + tracker.retain(|_, (_, wid)| *wid != writer_id); + } + self.rtt_stats.lock().await.remove(&writer_id); if let Some(tx) = close_tx { let _ = tx.send(WriterCommand::Close).await; } @@ -533,8 +546,7 @@ impl MePool { } self.trigger_immediate_refill_for_dc(addr, writer_dc); } - self.rtt_stats.lock().await.remove(&writer_id); - self.registry.writer_lost(writer_id).await + conns } pub(crate) async fn mark_writer_draining_with_timeout( diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index ee04969..cc3028b 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -138,6 +138,15 @@ impl ConnRegistry { (id, rx) } + pub async fn register_writer(&self, writer_id: u64, tx: mpsc::Sender) { + let mut inner = self.inner.write().await; + inner.writers.insert(writer_id, tx); + inner + .conns_for_writer + .entry(writer_id) + .or_insert_with(HashSet::new); + } + /// Unregister connection, returning associated writer_id if any. pub async fn unregister(&self, id: u64) -> Option { let mut inner = self.inner.write().await; @@ -282,24 +291,39 @@ impl ConnRegistry { } } - pub async fn bind_writer( - &self, - conn_id: u64, - writer_id: u64, - tx: mpsc::Sender, - meta: ConnMeta, - ) { + pub async fn bind_writer(&self, conn_id: u64, writer_id: u64, meta: ConnMeta) -> bool { let mut inner = self.inner.write().await; - inner.meta.entry(conn_id).or_insert(meta.clone()); - inner.writer_for_conn.insert(conn_id, writer_id); + if !inner.writers.contains_key(&writer_id) { + return false; + } + + let previous_writer_id = inner.writer_for_conn.insert(conn_id, writer_id); + if let Some(previous_writer_id) = previous_writer_id + && previous_writer_id != writer_id + { + let became_empty = if let Some(set) = inner.conns_for_writer.get_mut(&previous_writer_id) + { + set.remove(&conn_id); + set.is_empty() + } else { + false + }; + if became_empty { + inner + .writer_idle_since_epoch_secs + .insert(previous_writer_id, Self::now_epoch_secs()); + } + } + + inner.meta.insert(conn_id, meta.clone()); inner.last_meta_for_writer.insert(writer_id, meta); inner.writer_idle_since_epoch_secs.remove(&writer_id); - inner.writers.entry(writer_id).or_insert_with(|| tx.clone()); inner .conns_for_writer .entry(writer_id) .or_insert_with(HashSet::new) .insert(conn_id); + true } pub async fn mark_writer_idle(&self, writer_id: u64) { @@ -384,6 +408,9 @@ impl ConnRegistry { let mut out = Vec::new(); for conn_id in conns { + if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) { + continue; + } inner.writer_for_conn.remove(&conn_id); if let Some(m) = inner.meta.get(&conn_id) { out.push(BoundConn { @@ -427,47 +454,52 @@ mod tests { let (conn_c, _rx_c) = registry.register().await; let (writer_tx_a, _writer_rx_a) = tokio::sync::mpsc::channel(8); let (writer_tx_b, _writer_rx_b) = tokio::sync::mpsc::channel(8); + registry.register_writer(10, writer_tx_a.clone()).await; + registry.register_writer(20, writer_tx_b.clone()).await; let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); - registry - .bind_writer( - conn_a, - 10, - writer_tx_a.clone(), - ConnMeta { - target_dc: 2, - client_addr: addr, - our_addr: addr, - proto_flags: 0, - }, - ) - .await; - registry - .bind_writer( - conn_b, - 10, - writer_tx_a, - ConnMeta { - target_dc: -2, - client_addr: addr, - our_addr: addr, - proto_flags: 0, - }, - ) - .await; - registry - .bind_writer( - conn_c, - 20, - writer_tx_b, - ConnMeta { - target_dc: 4, - client_addr: addr, - our_addr: addr, - proto_flags: 0, - }, - ) - .await; + assert!( + registry + .bind_writer( + conn_a, + 10, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + assert!( + registry + .bind_writer( + conn_b, + 10, + ConnMeta { + target_dc: -2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + assert!( + registry + .bind_writer( + conn_c, + 20, + ConnMeta { + target_dc: 4, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); let snapshot = registry.writer_activity_snapshot().await; assert_eq!(snapshot.bound_clients_by_writer.get(&10), Some(&2)); @@ -476,4 +508,130 @@ mod tests { assert_eq!(snapshot.active_sessions_by_target_dc.get(&-2), Some(&1)); assert_eq!(snapshot.active_sessions_by_target_dc.get(&4), Some(&1)); } + + #[tokio::test] + async fn bind_writer_rebinds_conn_atomically() { + let registry = ConnRegistry::new(); + let (conn_id, _rx) = registry.register().await; + let (writer_tx_a, _writer_rx_a) = tokio::sync::mpsc::channel(8); + let (writer_tx_b, _writer_rx_b) = tokio::sync::mpsc::channel(8); + registry.register_writer(10, writer_tx_a).await; + registry.register_writer(20, writer_tx_b).await; + + let client_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + let first_our_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 443); + let second_our_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(2, 2, 2, 2)), 443); + + assert!( + registry + .bind_writer( + conn_id, + 10, + ConnMeta { + target_dc: 2, + client_addr, + our_addr: first_our_addr, + proto_flags: 1, + }, + ) + .await + ); + assert!( + registry + .bind_writer( + conn_id, + 20, + ConnMeta { + target_dc: 2, + client_addr, + our_addr: second_our_addr, + proto_flags: 2, + }, + ) + .await + ); + + let writer = registry.get_writer(conn_id).await.expect("writer binding"); + assert_eq!(writer.writer_id, 20); + + let meta = registry.get_meta(conn_id).await.expect("conn meta"); + assert_eq!(meta.our_addr, second_our_addr); + assert_eq!(meta.proto_flags, 2); + + let snapshot = registry.writer_activity_snapshot().await; + assert_eq!(snapshot.bound_clients_by_writer.get(&10), Some(&0)); + assert_eq!(snapshot.bound_clients_by_writer.get(&20), Some(&1)); + assert!(registry.writer_idle_since_snapshot().await.contains_key(&10)); + } + + #[tokio::test] + async fn writer_lost_does_not_drop_rebound_conn() { + let registry = ConnRegistry::new(); + let (conn_id, _rx) = registry.register().await; + let (writer_tx_a, _writer_rx_a) = tokio::sync::mpsc::channel(8); + let (writer_tx_b, _writer_rx_b) = tokio::sync::mpsc::channel(8); + registry.register_writer(10, writer_tx_a).await; + registry.register_writer(20, writer_tx_b).await; + + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + assert!( + registry + .bind_writer( + conn_id, + 10, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + assert!( + registry + .bind_writer( + conn_id, + 20, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 1, + }, + ) + .await + ); + + let lost = registry.writer_lost(10).await; + assert!(lost.is_empty()); + assert_eq!(registry.get_writer(conn_id).await.expect("writer").writer_id, 20); + + let removed_writer = registry.unregister(conn_id).await; + assert_eq!(removed_writer, Some(20)); + assert!(registry.is_writer_empty(20).await); + } + + #[tokio::test] + async fn bind_writer_rejects_unregistered_writer() { + let registry = ConnRegistry::new(); + let (conn_id, _rx) = registry.register().await; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + + assert!( + !registry + .bind_writer( + conn_id, + 10, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + assert!(registry.get_writer(conn_id).await.is_none()); + } } diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index f63662b..0f9fed6 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -375,9 +375,14 @@ impl MePool { match w.tx.try_send(WriterCommand::Data(payload.clone())) { Ok(()) => { self.stats.increment_me_writer_pick_success_try_total(pick_mode); - self.registry - .bind_writer(conn_id, w.id, w.tx.clone(), meta) - .await; + if !self.registry.bind_writer(conn_id, w.id, meta).await { + debug!( + conn_id, + writer_id = w.id, + "ME writer disappeared before bind commit, retrying" + ); + continue; + } if w.generation < self.current_generation() { self.stats.increment_pool_stale_pick_total(); debug!( @@ -421,9 +426,14 @@ impl MePool { Ok(()) => { self.stats .increment_me_writer_pick_success_fallback_total(pick_mode); - self.registry - .bind_writer(conn_id, w.id, w.tx.clone(), meta) - .await; + if !self.registry.bind_writer(conn_id, w.id, meta).await { + debug!( + conn_id, + writer_id = w.id, + "ME writer disappeared before fallback bind commit, retrying" + ); + continue; + } if w.generation < self.current_generation() { self.stats.increment_pool_stale_pick_total(); } From 3d43ff6e57d67279a358deb52391cb1a5b7409b1 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 15 Mar 2026 00:32:57 +0300 Subject: [PATCH 208/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dd3e5fb..ac529af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.17" +version = "3.3.18" edition = "2024" [dependencies] From 58f26ba8a7b560ea83d434d8bc2c6e2fd208f21f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 15 Mar 2026 12:13:46 +0300 Subject: [PATCH 209/261] Configurable ME draining writer overflow threshold --- src/config/defaults.rs | 4 + src/config/hot_reload.rs | 10 + src/config/types.rs | 6 + src/maestro/me_startup.rs | 1 + src/transport/middle_proxy/config_updater.rs | 2 + src/transport/middle_proxy/health.rs | 221 +++++++++++++++++++ src/transport/middle_proxy/pool.rs | 6 + 7 files changed, 250 insertions(+) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 76479cf..dd15a87 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -584,6 +584,10 @@ pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 { 90 } +pub(crate) fn default_me_pool_drain_threshold() -> u64 { + 128 +} + pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 { default_me_pool_drain_ttl_secs() } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index a375899..d14e5e2 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -55,6 +55,7 @@ pub struct HotFields { pub me_reinit_coalesce_window_ms: u64, pub hardswap: bool, pub me_pool_drain_ttl_secs: u64, + pub me_pool_drain_threshold: u64, pub me_pool_min_fresh_ratio: f32, pub me_reinit_drain_timeout_secs: u64, pub me_hardswap_warmup_delay_min_ms: u64, @@ -135,6 +136,7 @@ impl HotFields { me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms, hardswap: cfg.general.hardswap, me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs, + me_pool_drain_threshold: cfg.general.me_pool_drain_threshold, me_pool_min_fresh_ratio: cfg.general.me_pool_min_fresh_ratio, me_reinit_drain_timeout_secs: cfg.general.me_reinit_drain_timeout_secs, me_hardswap_warmup_delay_min_ms: cfg.general.me_hardswap_warmup_delay_min_ms, @@ -450,6 +452,7 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms; cfg.general.hardswap = new.general.hardswap; cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs; + cfg.general.me_pool_drain_threshold = new.general.me_pool_drain_threshold; cfg.general.me_pool_min_fresh_ratio = new.general.me_pool_min_fresh_ratio; cfg.general.me_reinit_drain_timeout_secs = new.general.me_reinit_drain_timeout_secs; cfg.general.me_hardswap_warmup_delay_min_ms = new.general.me_hardswap_warmup_delay_min_ms; @@ -823,6 +826,13 @@ fn log_changes( ); } + if old_hot.me_pool_drain_threshold != new_hot.me_pool_drain_threshold { + info!( + "config reload: me_pool_drain_threshold: {} → {}", + old_hot.me_pool_drain_threshold, new_hot.me_pool_drain_threshold, + ); + } + if (old_hot.me_pool_min_fresh_ratio - new_hot.me_pool_min_fresh_ratio).abs() > f32::EPSILON { info!( "config reload: me_pool_min_fresh_ratio: {:.3} → {:.3}", diff --git a/src/config/types.rs b/src/config/types.rs index 24626a4..29f23e0 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -794,6 +794,11 @@ pub struct GeneralConfig { #[serde(default = "default_me_pool_drain_ttl_secs")] pub me_pool_drain_ttl_secs: u64, + /// Maximum allowed number of draining ME writers before oldest ones are force-closed in batches. + /// Set to 0 to disable threshold-based draining cleanup and keep timeout-only behavior. + #[serde(default = "default_me_pool_drain_threshold")] + pub me_pool_drain_threshold: u64, + /// Policy for new binds on stale draining writers. #[serde(default)] pub me_bind_stale_mode: MeBindStaleMode, @@ -973,6 +978,7 @@ impl Default for GeneralConfig { me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(), proxy_secret_len_max: default_proxy_secret_len_max(), me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(), + me_pool_drain_threshold: default_me_pool_drain_threshold(), me_bind_stale_mode: MeBindStaleMode::default(), me_bind_stale_ttl_secs: default_me_bind_stale_ttl_secs(), me_pool_min_fresh_ratio: default_me_pool_min_fresh_ratio(), diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs index 72fdd40..245c7a9 100644 --- a/src/maestro/me_startup.rs +++ b/src/maestro/me_startup.rs @@ -237,6 +237,7 @@ pub(crate) async fn initialize_me_pool( config.general.me_adaptive_floor_max_warm_writers_global, config.general.hardswap, config.general.me_pool_drain_ttl_secs, + config.general.me_pool_drain_threshold, config.general.effective_me_pool_force_close_secs(), config.general.me_pool_min_fresh_ratio, config.general.me_hardswap_warmup_delay_min_ms, diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 194da5b..b6a0160 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -298,6 +298,7 @@ async fn run_update_cycle( pool.update_runtime_reinit_policy( cfg.general.hardswap, cfg.general.me_pool_drain_ttl_secs, + cfg.general.me_pool_drain_threshold, cfg.general.effective_me_pool_force_close_secs(), cfg.general.me_pool_min_fresh_ratio, cfg.general.me_hardswap_warmup_delay_min_ms, @@ -524,6 +525,7 @@ pub async fn me_config_updater( pool.update_runtime_reinit_policy( cfg.general.hardswap, cfg.general.me_pool_drain_ttl_secs, + cfg.general.me_pool_drain_threshold, cfg.general.effective_me_pool_force_close_secs(), cfg.general.me_pool_min_fresh_ratio, cfg.general.me_hardswap_warmup_delay_min_ms, diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 3cc400f..e5f4260 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -118,7 +118,11 @@ async fn reap_draining_writers( let now_epoch_secs = MePool::now_epoch_secs(); let now = Instant::now(); let drain_ttl_secs = pool.me_pool_drain_ttl_secs.load(std::sync::atomic::Ordering::Relaxed); + let drain_threshold = pool + .me_pool_drain_threshold + .load(std::sync::atomic::Ordering::Relaxed); let writers = pool.writers.read().await.clone(); + let mut draining_writers = Vec::new(); for writer in writers { if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) { continue; @@ -128,6 +132,36 @@ async fn reap_draining_writers( pool.remove_writer_and_close_clients(writer.id).await; continue; } + draining_writers.push(writer); + } + + if drain_threshold > 0 && draining_writers.len() > drain_threshold as usize { + draining_writers.sort_by(|left, right| { + let left_started = left + .draining_started_at_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + let right_started = right + .draining_started_at_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + left_started + .cmp(&right_started) + .then_with(|| left.created_at.cmp(&right.created_at)) + .then_with(|| left.id.cmp(&right.id)) + }); + let overflow = draining_writers.len().saturating_sub(drain_threshold as usize); + warn!( + draining_writers = draining_writers.len(), + me_pool_drain_threshold = drain_threshold, + removing_writers = overflow, + "ME draining writer threshold exceeded, force-closing oldest draining writers" + ); + for writer in draining_writers.drain(..overflow) { + pool.stats.increment_pool_force_close_total(); + pool.remove_writer_and_close_clients(writer.id).await; + } + } + + for writer in draining_writers { let drain_started_at_epoch_secs = writer .draining_started_at_epoch_secs .load(std::sync::atomic::Ordering::Relaxed); @@ -1270,3 +1304,190 @@ async fn maybe_rotate_single_endpoint_shadow( "Single-endpoint shadow writer rotated" ); } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; + use std::time::{Duration, Instant}; + + use tokio::sync::mpsc; + use tokio_util::sync::CancellationToken; + + use super::reap_draining_writers; + use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; + use crate::crypto::SecureRandom; + use crate::network::probe::NetworkDecision; + use crate::stats::Stats; + use crate::transport::middle_proxy::codec::WriterCommand; + use crate::transport::middle_proxy::pool::{MePool, MeWriter, WriterContour}; + use crate::transport::middle_proxy::registry::ConnMeta; + + async fn make_pool(me_pool_drain_threshold: u64) -> Arc { + let general = GeneralConfig { + me_pool_drain_threshold, + ..GeneralConfig::default() + }; + MePool::new( + None, + vec![1u8; 32], + None, + false, + None, + Vec::new(), + 1, + None, + 12, + 1200, + HashMap::new(), + HashMap::new(), + None, + NetworkDecision::default(), + None, + Arc::new(SecureRandom::new()), + Arc::new(Stats::default()), + general.me_keepalive_enabled, + general.me_keepalive_interval_secs, + general.me_keepalive_jitter_secs, + general.me_keepalive_payload_random, + general.rpc_proxy_req_every, + general.me_warmup_stagger_enabled, + general.me_warmup_step_delay_ms, + general.me_warmup_step_jitter_ms, + general.me_reconnect_max_concurrent_per_dc, + general.me_reconnect_backoff_base_ms, + general.me_reconnect_backoff_cap_ms, + general.me_reconnect_fast_retry_count, + general.me_single_endpoint_shadow_writers, + general.me_single_endpoint_outage_mode_enabled, + general.me_single_endpoint_outage_disable_quarantine, + general.me_single_endpoint_outage_backoff_min_ms, + general.me_single_endpoint_outage_backoff_max_ms, + general.me_single_endpoint_shadow_rotate_every_secs, + general.me_floor_mode, + general.me_adaptive_floor_idle_secs, + general.me_adaptive_floor_min_writers_single_endpoint, + general.me_adaptive_floor_min_writers_multi_endpoint, + general.me_adaptive_floor_recover_grace_secs, + general.me_adaptive_floor_writers_per_core_total, + general.me_adaptive_floor_cpu_cores_override, + general.me_adaptive_floor_max_extra_writers_single_per_core, + general.me_adaptive_floor_max_extra_writers_multi_per_core, + general.me_adaptive_floor_max_active_writers_per_core, + general.me_adaptive_floor_max_warm_writers_per_core, + general.me_adaptive_floor_max_active_writers_global, + general.me_adaptive_floor_max_warm_writers_global, + general.hardswap, + general.me_pool_drain_ttl_secs, + general.me_pool_drain_threshold, + general.effective_me_pool_force_close_secs(), + general.me_pool_min_fresh_ratio, + general.me_hardswap_warmup_delay_min_ms, + general.me_hardswap_warmup_delay_max_ms, + general.me_hardswap_warmup_extra_passes, + general.me_hardswap_warmup_pass_backoff_base_ms, + general.me_bind_stale_mode, + general.me_bind_stale_ttl_secs, + general.me_secret_atomic_snapshot, + general.me_deterministic_writer_sort, + MeWriterPickMode::default(), + general.me_writer_pick_sample_size, + MeSocksKdfPolicy::default(), + general.me_writer_cmd_channel_capacity, + general.me_route_channel_capacity, + general.me_route_backpressure_base_timeout_ms, + general.me_route_backpressure_high_timeout_ms, + general.me_route_backpressure_high_watermark_pct, + general.me_reader_route_data_wait_ms, + general.me_health_interval_ms_unhealthy, + general.me_health_interval_ms_healthy, + general.me_warn_rate_limit_ms, + MeRouteNoWriterMode::default(), + general.me_route_no_writer_wait_ms, + general.me_route_inline_recovery_attempts, + general.me_route_inline_recovery_wait_ms, + ) + } + + async fn insert_draining_writer( + pool: &Arc, + writer_id: u64, + drain_started_at_epoch_secs: u64, + ) -> u64 { + let (conn_id, _rx) = pool.registry.register().await; + let (tx, _writer_rx) = mpsc::channel::(8); + let writer = MeWriter { + id: writer_id, + addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4000 + writer_id as u16), + source_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + writer_dc: 2, + generation: 1, + contour: Arc::new(AtomicU8::new(WriterContour::Draining.as_u8())), + created_at: Instant::now() - Duration::from_secs(writer_id), + tx: tx.clone(), + cancel: CancellationToken::new(), + degraded: Arc::new(AtomicBool::new(false)), + rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)), + draining: Arc::new(AtomicBool::new(true)), + draining_started_at_epoch_secs: Arc::new(AtomicU64::new(drain_started_at_epoch_secs)), + drain_deadline_epoch_secs: Arc::new(AtomicU64::new(0)), + allow_drain_fallback: Arc::new(AtomicBool::new(false)), + }; + pool.writers.write().await.push(writer); + pool.registry.register_writer(writer_id, tx).await; + pool.conn_count.fetch_add(1, Ordering::Relaxed); + assert!( + pool.registry + .bind_writer( + conn_id, + writer_id, + ConnMeta { + target_dc: 2, + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6000), + our_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443), + proto_flags: 0, + }, + ) + .await + ); + conn_id + } + + #[tokio::test] + async fn reap_draining_writers_force_closes_oldest_over_threshold() { + let pool = make_pool(2).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let conn_a = insert_draining_writer(&pool, 10, now_epoch_secs.saturating_sub(30)).await; + let conn_b = insert_draining_writer(&pool, 20, now_epoch_secs.saturating_sub(20)).await; + let conn_c = insert_draining_writer(&pool, 30, now_epoch_secs.saturating_sub(10)).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + let writer_ids: Vec = pool.writers.read().await.iter().map(|writer| writer.id).collect(); + assert_eq!(writer_ids, vec![20, 30]); + assert!(pool.registry.get_writer(conn_a).await.is_none()); + assert_eq!(pool.registry.get_writer(conn_b).await.unwrap().writer_id, 20); + assert_eq!(pool.registry.get_writer(conn_c).await.unwrap().writer_id, 30); + } + + #[tokio::test] + async fn reap_draining_writers_keeps_timeout_only_behavior_when_threshold_disabled() { + let pool = make_pool(0).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let conn_a = insert_draining_writer(&pool, 10, now_epoch_secs.saturating_sub(30)).await; + let conn_b = insert_draining_writer(&pool, 20, now_epoch_secs.saturating_sub(20)).await; + let conn_c = insert_draining_writer(&pool, 30, now_epoch_secs.saturating_sub(10)).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + let writer_ids: Vec = pool.writers.read().await.iter().map(|writer| writer.id).collect(); + assert_eq!(writer_ids, vec![10, 20, 30]); + assert_eq!(pool.registry.get_writer(conn_a).await.unwrap().writer_id, 10); + assert_eq!(pool.registry.get_writer(conn_b).await.unwrap().writer_id, 20); + assert_eq!(pool.registry.get_writer(conn_c).await.unwrap().writer_id, 30); + } +} diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 42cba81..2a65160 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -171,6 +171,7 @@ pub struct MePool { pub(super) endpoint_quarantine: Arc>>, pub(super) kdf_material_fingerprint: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, + pub(super) me_pool_drain_threshold: AtomicU64, pub(super) me_pool_force_close_secs: AtomicU64, pub(super) me_pool_min_fresh_ratio_permille: AtomicU32, pub(super) me_hardswap_warmup_delay_min_ms: AtomicU64, @@ -271,6 +272,7 @@ impl MePool { me_adaptive_floor_max_warm_writers_global: u32, hardswap: bool, me_pool_drain_ttl_secs: u64, + me_pool_drain_threshold: u64, me_pool_force_close_secs: u64, me_pool_min_fresh_ratio: f32, me_hardswap_warmup_delay_min_ms: u64, @@ -446,6 +448,7 @@ impl MePool { endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())), kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())), me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs), + me_pool_drain_threshold: AtomicU64::new(me_pool_drain_threshold), me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs), me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille( me_pool_min_fresh_ratio, @@ -492,6 +495,7 @@ impl MePool { &self, hardswap: bool, drain_ttl_secs: u64, + pool_drain_threshold: u64, force_close_secs: u64, min_fresh_ratio: f32, hardswap_warmup_delay_min_ms: u64, @@ -530,6 +534,8 @@ impl MePool { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs .store(drain_ttl_secs, Ordering::Relaxed); + self.me_pool_drain_threshold + .store(pool_drain_threshold, Ordering::Relaxed); self.me_pool_force_close_secs .store(force_close_secs, Ordering::Relaxed); self.me_pool_min_fresh_ratio_permille From 4028579068bf6a0855f73b5e3bbd149b06fbff49 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 15 Mar 2026 12:43:31 +0300 Subject: [PATCH 210/261] Inherited per-user unique IP limit --- src/api/runtime_zero.rs | 2 ++ src/api/users.rs | 11 +++++- src/config/defaults.rs | 4 +++ src/config/hot_reload.rs | 9 +++-- src/config/types.rs | 6 ++++ src/ip_tracker.rs | 65 ++++++++++++++++++++++++++++++++---- src/maestro/mod.rs | 15 ++++++--- src/maestro/runtime_tasks.rs | 16 +++++++-- src/metrics.rs | 33 ++++++++++++++++-- 9 files changed, 143 insertions(+), 18 deletions(-) diff --git a/src/api/runtime_zero.rs b/src/api/runtime_zero.rs index 93e3931..ba89302 100644 --- a/src/api/runtime_zero.rs +++ b/src/api/runtime_zero.rs @@ -90,6 +90,7 @@ pub(super) struct EffectiveMiddleProxyLimits { #[derive(Serialize)] pub(super) struct EffectiveUserIpPolicyLimits { + pub(super) global_each: usize, pub(super) mode: &'static str, pub(super) window_secs: u64, } @@ -262,6 +263,7 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD me2dc_fallback: cfg.general.me2dc_fallback, }, user_ip_policy: EffectiveUserIpPolicyLimits { + global_each: cfg.access.user_max_unique_ips_global_each, mode: user_max_unique_ips_mode_label(cfg.access.user_max_unique_ips_mode), window_secs: cfg.access.user_max_unique_ips_window_secs, }, diff --git a/src/api/users.rs b/src/api/users.rs index da360c7..f339806 100644 --- a/src/api/users.rs +++ b/src/api/users.rs @@ -386,7 +386,16 @@ pub(super) async fn users_from_config( .get(&username) .map(chrono::DateTime::::to_rfc3339), data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(), - max_unique_ips: cfg.access.user_max_unique_ips.get(&username).copied(), + max_unique_ips: cfg + .access + .user_max_unique_ips + .get(&username) + .copied() + .filter(|limit| *limit > 0) + .or( + (cfg.access.user_max_unique_ips_global_each > 0) + .then_some(cfg.access.user_max_unique_ips_global_each), + ), current_connections: stats.get_user_curr_connects(&username), active_unique_ips: active_ip_list.len(), active_unique_ips_list: active_ip_list, diff --git a/src/config/defaults.rs b/src/config/defaults.rs index dd15a87..82ec0b3 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -639,6 +639,10 @@ pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 { DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS } +pub(crate) fn default_user_max_unique_ips_global_each() -> usize { + 0 +} + // Custom deserializer helpers #[derive(Deserialize)] diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index d14e5e2..6f07a4b 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -119,6 +119,7 @@ pub struct HotFields { pub user_expirations: std::collections::HashMap>, pub user_data_quota: std::collections::HashMap, pub user_max_unique_ips: std::collections::HashMap, + pub user_max_unique_ips_global_each: usize, pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode, pub user_max_unique_ips_window_secs: u64, } @@ -234,6 +235,7 @@ impl HotFields { user_expirations: cfg.access.user_expirations.clone(), user_data_quota: cfg.access.user_data_quota.clone(), user_max_unique_ips: cfg.access.user_max_unique_ips.clone(), + user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each, user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode, user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs, } @@ -535,6 +537,7 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.access.user_expirations = new.access.user_expirations.clone(); cfg.access.user_data_quota = new.access.user_data_quota.clone(); cfg.access.user_max_unique_ips = new.access.user_max_unique_ips.clone(); + cfg.access.user_max_unique_ips_global_each = new.access.user_max_unique_ips_global_each; cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode; cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs; @@ -1109,12 +1112,14 @@ fn log_changes( new_hot.user_max_unique_ips.len() ); } - if old_hot.user_max_unique_ips_mode != new_hot.user_max_unique_ips_mode + if old_hot.user_max_unique_ips_global_each != new_hot.user_max_unique_ips_global_each + || old_hot.user_max_unique_ips_mode != new_hot.user_max_unique_ips_mode || old_hot.user_max_unique_ips_window_secs != new_hot.user_max_unique_ips_window_secs { info!( - "config reload: user_max_unique_ips policy mode={:?} window={}s", + "config reload: user_max_unique_ips policy global_each={} mode={:?} window={}s", + new_hot.user_max_unique_ips_global_each, new_hot.user_max_unique_ips_mode, new_hot.user_max_unique_ips_window_secs ); diff --git a/src/config/types.rs b/src/config/types.rs index 29f23e0..e523592 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1323,6 +1323,11 @@ pub struct AccessConfig { #[serde(default)] pub user_max_unique_ips: HashMap, + /// Global per-user unique IP limit applied when a user has no individual override. + /// `0` disables the inherited limit. + #[serde(default = "default_user_max_unique_ips_global_each")] + pub user_max_unique_ips_global_each: usize, + #[serde(default)] pub user_max_unique_ips_mode: UserMaxUniqueIpsMode, @@ -1348,6 +1353,7 @@ impl Default for AccessConfig { user_expirations: HashMap::new(), user_data_quota: HashMap::new(), user_max_unique_ips: HashMap::new(), + user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(), user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(), user_max_unique_ips_window_secs: default_user_max_unique_ips_window_secs(), replay_check_len: default_replay_check_len(), diff --git a/src/ip_tracker.rs b/src/ip_tracker.rs index d406d51..fce20b6 100644 --- a/src/ip_tracker.rs +++ b/src/ip_tracker.rs @@ -17,6 +17,7 @@ pub struct UserIpTracker { active_ips: Arc>>>, recent_ips: Arc>>>, max_ips: Arc>>, + default_max_ips: Arc>, limit_mode: Arc>, limit_window: Arc>, last_compact_epoch_secs: Arc, @@ -28,6 +29,7 @@ impl UserIpTracker { active_ips: Arc::new(RwLock::new(HashMap::new())), recent_ips: Arc::new(RwLock::new(HashMap::new())), max_ips: Arc::new(RwLock::new(HashMap::new())), + default_max_ips: Arc::new(RwLock::new(0)), limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)), limit_window: Arc::new(RwLock::new(Duration::from_secs(30))), last_compact_epoch_secs: Arc::new(AtomicU64::new(0)), @@ -100,7 +102,10 @@ impl UserIpTracker { limits.remove(username); } - pub async fn load_limits(&self, limits: &HashMap) { + pub async fn load_limits(&self, default_limit: usize, limits: &HashMap) { + let mut default_max_ips = self.default_max_ips.write().await; + *default_max_ips = default_limit; + drop(default_max_ips); let mut max_ips = self.max_ips.write().await; max_ips.clone_from(limits); } @@ -114,9 +119,14 @@ impl UserIpTracker { pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> { self.maybe_compact_empty_users().await; + let default_max_ips = *self.default_max_ips.read().await; let limit = { let max_ips = self.max_ips.read().await; - max_ips.get(username).copied() + max_ips + .get(username) + .copied() + .filter(|limit| *limit > 0) + .or((default_max_ips > 0).then_some(default_max_ips)) }; let mode = *self.limit_mode.read().await; let window = *self.limit_window.read().await; @@ -255,10 +265,16 @@ impl UserIpTracker { pub async fn get_stats(&self) -> Vec<(String, usize, usize)> { let active_ips = self.active_ips.read().await; let max_ips = self.max_ips.read().await; + let default_max_ips = *self.default_max_ips.read().await; let mut stats = Vec::new(); for (username, user_ips) in active_ips.iter() { - let limit = max_ips.get(username).copied().unwrap_or(0); + let limit = max_ips + .get(username) + .copied() + .filter(|limit| *limit > 0) + .or((default_max_ips > 0).then_some(default_max_ips)) + .unwrap_or(0); stats.push((username.clone(), user_ips.len(), limit)); } @@ -293,8 +309,13 @@ impl UserIpTracker { } pub async fn get_user_limit(&self, username: &str) -> Option { + let default_max_ips = *self.default_max_ips.read().await; let max_ips = self.max_ips.read().await; - max_ips.get(username).copied() + max_ips + .get(username) + .copied() + .filter(|limit| *limit > 0) + .or((default_max_ips > 0).then_some(default_max_ips)) } pub async fn format_stats(&self) -> String { @@ -546,7 +567,7 @@ mod tests { config_limits.insert("user1".to_string(), 5); config_limits.insert("user2".to_string(), 3); - tracker.load_limits(&config_limits).await; + tracker.load_limits(0, &config_limits).await; assert_eq!(tracker.get_user_limit("user1").await, Some(5)); assert_eq!(tracker.get_user_limit("user2").await, Some(3)); @@ -560,16 +581,46 @@ mod tests { let mut first = HashMap::new(); first.insert("user1".to_string(), 2); first.insert("user2".to_string(), 3); - tracker.load_limits(&first).await; + tracker.load_limits(0, &first).await; let mut second = HashMap::new(); second.insert("user2".to_string(), 5); - tracker.load_limits(&second).await; + tracker.load_limits(0, &second).await; assert_eq!(tracker.get_user_limit("user1").await, None); assert_eq!(tracker.get_user_limit("user2").await, Some(5)); } + #[tokio::test] + async fn test_global_each_limit_applies_without_user_override() { + let tracker = UserIpTracker::new(); + tracker.load_limits(2, &HashMap::new()).await; + + let ip1 = test_ipv4(172, 16, 0, 1); + let ip2 = test_ipv4(172, 16, 0, 2); + let ip3 = test_ipv4(172, 16, 0, 3); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + assert!(tracker.check_and_add("test_user", ip2).await.is_ok()); + assert!(tracker.check_and_add("test_user", ip3).await.is_err()); + assert_eq!(tracker.get_user_limit("test_user").await, Some(2)); + } + + #[tokio::test] + async fn test_user_override_wins_over_global_each_limit() { + let tracker = UserIpTracker::new(); + let mut limits = HashMap::new(); + limits.insert("test_user".to_string(), 1); + tracker.load_limits(3, &limits).await; + + let ip1 = test_ipv4(172, 17, 0, 1); + let ip2 = test_ipv4(172, 17, 0, 2); + + assert!(tracker.check_and_add("test_user", ip1).await.is_ok()); + assert!(tracker.check_and_add("test_user", ip2).await.is_err()); + assert_eq!(tracker.get_user_limit("test_user").await, Some(1)); + } + #[tokio::test] async fn test_time_window_mode_blocks_recent_ip_churn() { let tracker = UserIpTracker::new(); diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index 5f6c70a..3db7f02 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -168,17 +168,24 @@ pub async fn run() -> std::result::Result<(), Box> { stats.clone(), )); let ip_tracker = Arc::new(UserIpTracker::new()); - ip_tracker.load_limits(&config.access.user_max_unique_ips).await; + ip_tracker + .load_limits( + config.access.user_max_unique_ips_global_each, + &config.access.user_max_unique_ips, + ) + .await; ip_tracker .set_limit_policy( config.access.user_max_unique_ips_mode, config.access.user_max_unique_ips_window_secs, ) .await; - if !config.access.user_max_unique_ips.is_empty() { + if config.access.user_max_unique_ips_global_each > 0 || !config.access.user_max_unique_ips.is_empty() + { info!( - "IP limits configured for {} users", - config.access.user_max_unique_ips.len() + global_each_limit = config.access.user_max_unique_ips_global_each, + explicit_user_limits = config.access.user_max_unique_ips.len(), + "User unique IP limits configured" ); } if !config.network.dns_overrides.is_empty() { diff --git a/src/maestro/runtime_tasks.rs b/src/maestro/runtime_tasks.rs index c8aa534..329e267 100644 --- a/src/maestro/runtime_tasks.rs +++ b/src/maestro/runtime_tasks.rs @@ -131,6 +131,10 @@ pub(crate) async fn spawn_runtime_tasks( let mut config_rx_ip_limits = config_rx.clone(); tokio::spawn(async move { let mut prev_limits = config_rx_ip_limits.borrow().access.user_max_unique_ips.clone(); + let mut prev_global_each = config_rx_ip_limits + .borrow() + .access + .user_max_unique_ips_global_each; let mut prev_mode = config_rx_ip_limits.borrow().access.user_max_unique_ips_mode; let mut prev_window = config_rx_ip_limits .borrow() @@ -143,9 +147,17 @@ pub(crate) async fn spawn_runtime_tasks( } let cfg = config_rx_ip_limits.borrow_and_update().clone(); - if prev_limits != cfg.access.user_max_unique_ips { - ip_tracker_policy.load_limits(&cfg.access.user_max_unique_ips).await; + if prev_limits != cfg.access.user_max_unique_ips + || prev_global_each != cfg.access.user_max_unique_ips_global_each + { + ip_tracker_policy + .load_limits( + cfg.access.user_max_unique_ips_global_each, + &cfg.access.user_max_unique_ips, + ) + .await; prev_limits = cfg.access.user_max_unique_ips.clone(); + prev_global_each = cfg.access.user_max_unique_ips_global_each; } if prev_mode != cfg.access.user_max_unique_ips_mode diff --git a/src/metrics.rs b/src/metrics.rs index c24dc54..02edfd7 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1774,14 +1774,24 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp "# HELP telemt_user_unique_ips_recent_window Per-user unique IPs seen in configured observation window" ); let _ = writeln!(out, "# TYPE telemt_user_unique_ips_recent_window gauge"); - let _ = writeln!(out, "# HELP telemt_user_unique_ips_limit Per-user configured unique IP limit (0 means unlimited)"); + let _ = writeln!(out, "# HELP telemt_user_unique_ips_limit Effective per-user unique IP limit (0 means unlimited)"); let _ = writeln!(out, "# TYPE telemt_user_unique_ips_limit gauge"); let _ = writeln!(out, "# HELP telemt_user_unique_ips_utilization Per-user unique IP usage ratio (0 for unlimited)"); let _ = writeln!(out, "# TYPE telemt_user_unique_ips_utilization gauge"); for user in unique_users { let current = ip_counts.get(&user).copied().unwrap_or(0); - let limit = config.access.user_max_unique_ips.get(&user).copied().unwrap_or(0); + let limit = config + .access + .user_max_unique_ips + .get(&user) + .copied() + .filter(|limit| *limit > 0) + .or( + (config.access.user_max_unique_ips_global_each > 0) + .then_some(config.access.user_max_unique_ips_global_each), + ) + .unwrap_or(0); let utilization = if limit > 0 { current as f64 / limit as f64 } else { @@ -1904,6 +1914,25 @@ mod tests { assert!(output.contains("telemt_user_unique_ips_recent_window{user=")); } + #[tokio::test] + async fn test_render_uses_global_each_unique_ip_limit() { + let stats = Stats::new(); + stats.increment_user_connects("alice"); + stats.increment_user_curr_connects("alice"); + let tracker = UserIpTracker::new(); + tracker + .check_and_add("alice", "203.0.113.10".parse().unwrap()) + .await + .unwrap(); + let mut config = ProxyConfig::default(); + config.access.user_max_unique_ips_global_each = 2; + + let output = render_metrics(&stats, &config, &tracker).await; + + assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2")); + assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000")); + } + #[tokio::test] async fn test_render_has_type_annotations() { let stats = Stats::new(); From 2d7df3da6c194db0a4018b3a3868cd6733a4c6cd Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Sun, 15 Mar 2026 19:06:56 +0300 Subject: [PATCH 211/261] Update FAQ.ru.md --- docs/FAQ.ru.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index a2fcf1a..cb5db7f 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -55,7 +55,10 @@ user2 = "00000000000000000000000000000002" user3 = "00000000000000000000000000000003" ``` 4. Сохранить конфиг. Ctrl+S -> Ctrl+X. Перезапускать telemt не нужно. -5. Получить ссылки через `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` +5. Получить ссылки через +```bash +curl -s http://127.0.0.1:9091/v1/users | jq +``` ## Как посмотреть метрики From 3d5af3d248a9e7be97e95e69a10495271b543a74 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Sun, 15 Mar 2026 19:09:02 +0300 Subject: [PATCH 212/261] Update FAQ.en.md --- docs/FAQ.en.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/FAQ.en.md b/docs/FAQ.en.md index 49d1592..e3a6519 100644 --- a/docs/FAQ.en.md +++ b/docs/FAQ.en.md @@ -55,7 +55,10 @@ user2 = "00000000000000000000000000000002" user3 = "00000000000000000000000000000003" ``` 4. Save the config. Ctrl+S -> Ctrl+X. You don't need to restart telemt. -5. Get the links via `journalctl -u telemt -n -g "links" --no-pager -o cat | tac` +5. Get the links via +```bash +curl -s http://127.0.0.1:9091/v1/users | jq +``` ## How to view metrics From 3634fbd7e8e58e631dde54d8906ba28b74642e19 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Sun, 15 Mar 2026 23:00:17 +0300 Subject: [PATCH 213/261] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1990109..2102a3a 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ ### 🇷🇺 RU -#### Релиз 3.3.16 +#### Релиз 3.3.15 Semistable -[3.3.16](https://github.com/telemt/telemt/releases/tag/3.3.16)! +[3.3.15](https://github.com/telemt/telemt/releases/tag/3.3.15) по итогам работы в продакшн признан одним из самых стабильных и рекомендуется к использованию, когда cutting-edge фичи некритичны! Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **API**, **статистики**, **UX** @@ -40,9 +40,9 @@ ### 🇬🇧 EN -#### Release 3.3.16 +#### Release 3.3.15 Semistable -[3.3.16](https://github.com/telemt/telemt/releases/tag/3.3.16) +[3.3.15](https://github.com/telemt/telemt/releases/tag/3.3.15) is, based on the results of his work in production, recognized as one of the most stable and recommended for use when cutting-edge features are not so necessary! We are looking forward to your feedback and improvement proposals — especially regarding **API**, **statistics**, **UX** From c351e08c43c40e6b6b4767ae19b35f830e63768d Mon Sep 17 00:00:00 2001 From: Maxim Anisimov Date: Fri, 13 Mar 2026 14:36:28 +0300 Subject: [PATCH 214/261] add support for data path option This commit adds support for configuring the data path via a configuration file or command-line option. This may be useful on systems without systemd, such as OpenWrt or Alpine Linux. Signed-off-by: Maxim Anisimov --- src/config/types.rs | 5 +++++ src/maestro/helpers.rs | 19 +++++++++++++++++-- src/maestro/mod.rs | 30 +++++++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/config/types.rs b/src/config/types.rs index e523592..04a22ce 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -3,6 +3,7 @@ use ipnetwork::IpNetwork; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::IpAddr; +use std::path::PathBuf; use super::defaults::*; @@ -356,6 +357,9 @@ impl Default for NetworkConfig { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GeneralConfig { + #[serde(default)] + pub data_path: Option, + #[serde(default)] pub modes: ProxyModes, @@ -871,6 +875,7 @@ pub struct GeneralConfig { impl Default for GeneralConfig { fn default() -> Self { Self { + data_path: None, modes: ProxyModes::default(), prefer_ipv6: false, fast_mode: default_true(), diff --git a/src/maestro/helpers.rs b/src/maestro/helpers.rs index acaecdd..78f3ec4 100644 --- a/src/maestro/helpers.rs +++ b/src/maestro/helpers.rs @@ -1,4 +1,5 @@ use std::time::Duration; +use std::path::PathBuf; use tokio::sync::watch; use tracing::{debug, error, info, warn}; @@ -9,8 +10,9 @@ use crate::transport::middle_proxy::{ ProxyConfigData, fetch_proxy_config_with_raw, load_proxy_config_cache, save_proxy_config_cache, }; -pub(crate) fn parse_cli() -> (String, bool, Option) { +pub(crate) fn parse_cli() -> (String, Option, bool, Option) { let mut config_path = "config.toml".to_string(); + let mut data_path: Option = None; let mut silent = false; let mut log_level: Option = None; @@ -28,6 +30,18 @@ pub(crate) fn parse_cli() -> (String, bool, Option) { let mut i = 0; while i < args.len() { match args[i].as_str() { + "--data-path" => { + i += 1; + if i < args.len() { + data_path = Some(PathBuf::from(args[i].clone())); + } else { + eprintln!("Missing value for --data-path"); + std::process::exit(0); + } + } + s if s.starts_with("--data-path=") => { + data_path = Some(PathBuf::from(s.trim_start_matches("--data-path=").to_string())); + } "--silent" | "-s" => { silent = true; } @@ -44,6 +58,7 @@ pub(crate) fn parse_cli() -> (String, bool, Option) { eprintln!("Usage: telemt [config.toml] [OPTIONS]"); eprintln!(); eprintln!("Options:"); + eprintln!(" --data-path Set data directory (absolute path; overrides config value)"); eprintln!(" --silent, -s Suppress info logs"); eprintln!(" --log-level debug|verbose|normal|silent"); eprintln!(" --help, -h Show this help"); @@ -78,7 +93,7 @@ pub(crate) fn parse_cli() -> (String, bool, Option) { i += 1; } - (config_path, silent, log_level) + (config_path, data_path, silent, log_level) } pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) { diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index 3db7f02..047c204 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -58,7 +58,7 @@ pub async fn run() -> std::result::Result<(), Box> { startup_tracker .start_component(COMPONENT_CONFIG_LOAD, Some("load and validate config".to_string())) .await; - let (config_path, cli_silent, cli_log_level) = parse_cli(); + let (config_path, data_path, cli_silent, cli_log_level) = parse_cli(); let mut config = match ProxyConfig::load(&config_path) { Ok(c) => c, @@ -80,6 +80,34 @@ pub async fn run() -> std::result::Result<(), Box> { std::process::exit(1); } + if let Some(p) = data_path { + config.general.data_path = Some(p); + } + + if let Some(ref data_path) = config.general.data_path { + if !data_path.is_absolute() { + eprintln!("[telemt] data_path must be absolute: {}", data_path.display()); + std::process::exit(1); + } + + if data_path.exists() { + if !data_path.is_dir() { + eprintln!("[telemt] data_path exists but is not a directory: {}", data_path.display()); + std::process::exit(1); + } + } else { + if let Err(e) = std::fs::create_dir_all(data_path) { + eprintln!("[telemt] Can't create data_path {}: {}", data_path.display(), e); + std::process::exit(1); + } + } + + if let Err(e) = std::env::set_current_dir(data_path) { + eprintln!("[telemt] Can't use data_path {}: {}", data_path.display(), e); + std::process::exit(1); + } + } + if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) { eprintln!("[telemt] Invalid network.dns_overrides: {}", e); std::process::exit(1); From c4f77814ee6004163f9da4e38214a9333c536a07 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Mon, 16 Mar 2026 13:40:20 +0300 Subject: [PATCH 215/261] Update QUICK_START_GUIDE.ru.md --- docs/QUICK_START_GUIDE.ru.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index 67dcbe2..35fbd27 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -72,6 +72,9 @@ classic = false secure = false tls = true +[server] +port = 443 + [server.api] enabled = true # listen = "127.0.0.1:9091" From 0ae67db4927375f0ee3fd6ff79dda0655b177457 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Mon, 16 Mar 2026 13:40:50 +0300 Subject: [PATCH 216/261] Update QUICK_START_GUIDE.en.md --- docs/QUICK_START_GUIDE.en.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/QUICK_START_GUIDE.en.md b/docs/QUICK_START_GUIDE.en.md index e140cf3..c43a2dc 100644 --- a/docs/QUICK_START_GUIDE.en.md +++ b/docs/QUICK_START_GUIDE.en.md @@ -72,6 +72,9 @@ classic = false secure = false tls = true +[server] +port = 443 + [server.api] enabled = true # listen = "127.0.0.1:9091" From b6206a6dfebe92bf6689e03983e77a42e7c2ad10 Mon Sep 17 00:00:00 2001 From: kavore <161734431+kavore@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:40:10 +0300 Subject: [PATCH 217/261] feat: make max_connections configurable via [server] section The concurrent connection limit was hardcoded to 10,000. Add server.max_connections config option (default: 10000, 0 = unlimited). --- src/config/defaults.rs | 4 ++++ src/config/types.rs | 6 ++++++ src/maestro/mod.rs | 9 +++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 82ec0b3..ea9250d 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -147,6 +147,10 @@ pub(crate) fn default_proxy_protocol_header_timeout_ms() -> u64 { 500 } +pub(crate) fn default_server_max_connections() -> u32 { + 10_000 +} + pub(crate) fn default_prefer_4() -> u8 { 4 } diff --git a/src/config/types.rs b/src/config/types.rs index 04a22ce..f676f54 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1167,6 +1167,11 @@ pub struct ServerConfig { #[serde(default)] pub listeners: Vec, + + /// Maximum number of concurrent client connections. + /// 0 means unlimited. + #[serde(default = "default_server_max_connections")] + pub max_connections: u32, } impl Default for ServerConfig { @@ -1184,6 +1189,7 @@ impl Default for ServerConfig { metrics_whitelist: default_metrics_whitelist(), api: ApiConfig::default(), listeners: Vec::new(), + max_connections: default_server_max_connections(), } } } diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index 047c204..da00b40 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -349,8 +349,13 @@ pub async fn run() -> std::result::Result<(), Box> { let beobachten = Arc::new(BeobachtenStore::new()); let rng = Arc::new(SecureRandom::new()); - // Connection concurrency limit - let max_connections = Arc::new(Semaphore::new(10_000)); + // Connection concurrency limit (0 = unlimited) + let max_connections_limit = if config.server.max_connections == 0 { + Semaphore::MAX_PERMITS + } else { + config.server.max_connections as usize + }; + let max_connections = Arc::new(Semaphore::new(max_connections_limit)); let me2dc_fallback = config.general.me2dc_fallback; let me_init_retry_attempts = config.general.me_init_retry_attempts; From 2c10560795c4c25c8f8d235a439a927488f0f57c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:25:14 +0300 Subject: [PATCH 218/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ac529af..9374924 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.18" +version = "3.3.19" edition = "2024" [dependencies] From ee4d15fed64c56b12c8b06155b0894493961e7f5 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Mon, 16 Mar 2026 22:02:55 +0300 Subject: [PATCH 219/261] Update FAQ.ru.md --- docs/FAQ.ru.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index cb5db7f..4353e98 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -83,6 +83,13 @@ metrics_whitelist = ["127.0.0.1/32", "::1/128", "0.0.0.0/0"] public_host = "proxy.example.com" ``` +### Общий лимит подключений к серверу +Ограничивает общее число открытых подключений к серверу: +```toml +[server] +max_connections = 10000 # 0 - unlimited, 10000 - default +``` + ### Upstream Manager Чтобы указать апстрим, добавьте в секцию `[[upstreams]]` файла config.toml: #### Привязка к IP @@ -113,3 +120,4 @@ password = "pass" # Password for Auth on SOCKS-server weight = 1 # Set Weight for Scenarios enabled = true ``` + From e1ef192c1096a34945464017dc03eef62a110ad5 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Mon, 16 Mar 2026 22:03:28 +0300 Subject: [PATCH 220/261] Update FAQ.en.md --- docs/FAQ.en.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/FAQ.en.md b/docs/FAQ.en.md index e3a6519..25522ad 100644 --- a/docs/FAQ.en.md +++ b/docs/FAQ.en.md @@ -83,6 +83,13 @@ To specify a domain in the links, add to the `[general.links]` section of the co public_host = "proxy.example.com" ``` +### Server connection limit +Limits the total number of open connections to the server: +```toml +[server] +max_connections = 10000 # 0 - unlimited, 10000 - default +``` + ### Upstream Manager To specify an upstream, add to the `[[upstreams]]` section of the config.toml file: #### Binding to IP From a74def9561e3d6fe653ae1764b98db2589be1513 Mon Sep 17 00:00:00 2001 From: Sergey Kutovoy Date: Tue, 17 Mar 2026 12:58:40 +0500 Subject: [PATCH 221/261] Update metrics configuration to support custom listen address - Bump telemt dependency version from 3.3.15 to 3.3.19. - Add `metrics_listen` option to `config.toml` for specifying a custom address for the metrics endpoint. - Update `ServerConfig` struct to include `metrics_listen` and adjust logic in `spawn_metrics_if_configured` to prioritize this new option over `metrics_port`. - Enhance error handling for invalid listen addresses in metrics setup. --- Cargo.lock | 2 +- config.toml | 1 + src/config/types.rs | 9 +++++++++ src/maestro/runtime_tasks.rs | 28 +++++++++++++++++++++++++--- src/metrics.rs | 28 ++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06ea5c6..a704404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2087,7 +2087,7 @@ dependencies = [ [[package]] name = "telemt" -version = "3.3.15" +version = "3.3.19" dependencies = [ "aes", "anyhow", diff --git a/config.toml b/config.toml index 63fa4ae..f4eb3ae 100644 --- a/config.toml +++ b/config.toml @@ -32,6 +32,7 @@ show = "*" port = 443 # proxy_protocol = false # Enable if behind HAProxy/nginx with PROXY protocol # metrics_port = 9090 +# metrics_listen = "0.0.0.0:9090" # Listen address for metrics (overrides metrics_port) # metrics_whitelist = ["127.0.0.1", "::1", "0.0.0.0/0"] [server.api] diff --git a/src/config/types.rs b/src/config/types.rs index f676f54..7ea1fe7 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1156,9 +1156,17 @@ pub struct ServerConfig { #[serde(default = "default_proxy_protocol_header_timeout_ms")] pub proxy_protocol_header_timeout_ms: u64, + /// Port for the Prometheus-compatible metrics endpoint. + /// Enables metrics when set; binds on all interfaces (dual-stack) by default. #[serde(default)] pub metrics_port: Option, + /// Listen address for metrics in `IP:PORT` format (e.g. `"127.0.0.1:9090"`). + /// When set, takes precedence over `metrics_port` and binds on the specified address only. + #[serde(default)] + pub metrics_listen: Option, + + /// CIDR whitelist for the metrics endpoint. #[serde(default = "default_metrics_whitelist")] pub metrics_whitelist: Vec, @@ -1186,6 +1194,7 @@ impl Default for ServerConfig { proxy_protocol: false, proxy_protocol_header_timeout_ms: default_proxy_protocol_header_timeout_ms(), metrics_port: None, + metrics_listen: None, metrics_whitelist: default_metrics_whitelist(), api: ApiConfig::default(), listeners: Vec::new(), diff --git a/src/maestro/runtime_tasks.rs b/src/maestro/runtime_tasks.rs index 329e267..d9691a8 100644 --- a/src/maestro/runtime_tasks.rs +++ b/src/maestro/runtime_tasks.rs @@ -279,11 +279,32 @@ pub(crate) async fn spawn_metrics_if_configured( ip_tracker: Arc, config_rx: watch::Receiver>, ) { - if let Some(port) = config.server.metrics_port { + // metrics_listen takes precedence; fall back to metrics_port for backward compat. + let metrics_target: Option<(u16, Option)> = + if let Some(ref listen) = config.server.metrics_listen { + match listen.parse::() { + Ok(addr) => Some((addr.port(), Some(listen.clone()))), + Err(e) => { + startup_tracker + .skip_component( + COMPONENT_METRICS_START, + Some(format!("invalid metrics_listen \"{}\": {}", listen, e)), + ) + .await; + None + } + } + } else { + config.server.metrics_port.map(|p| (p, None)) + }; + + if let Some((port, listen)) = metrics_target { + let fallback_label = format!("port {}", port); + let label = listen.as_deref().unwrap_or(&fallback_label); startup_tracker .start_component( COMPONENT_METRICS_START, - Some(format!("spawn metrics endpoint on {}", port)), + Some(format!("spawn metrics endpoint on {}", label)), ) .await; let stats = stats.clone(); @@ -294,6 +315,7 @@ pub(crate) async fn spawn_metrics_if_configured( tokio::spawn(async move { metrics::serve( port, + listen, stats, beobachten, ip_tracker_metrics, @@ -308,7 +330,7 @@ pub(crate) async fn spawn_metrics_if_configured( Some("metrics task spawned".to_string()), ) .await; - } else { + } else if config.server.metrics_listen.is_none() { startup_tracker .skip_component( COMPONENT_METRICS_START, diff --git a/src/metrics.rs b/src/metrics.rs index 02edfd7..f4f8a2e 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -21,6 +21,7 @@ use crate::transport::{ListenOptions, create_listener}; pub async fn serve( port: u16, + listen: Option, stats: Arc, beobachten: Arc, ip_tracker: Arc, @@ -28,6 +29,33 @@ pub async fn serve( whitelist: Vec, ) { let whitelist = Arc::new(whitelist); + + // If `metrics_listen` is set, bind on that single address only. + if let Some(ref listen_addr) = listen { + let addr: SocketAddr = match listen_addr.parse() { + Ok(a) => a, + Err(e) => { + warn!(error = %e, "Invalid metrics_listen address: {}", listen_addr); + return; + } + }; + let is_ipv6 = addr.is_ipv6(); + match bind_metrics_listener(addr, is_ipv6) { + Ok(listener) => { + info!("Metrics endpoint: http://{}/metrics and /beobachten", addr); + serve_listener( + listener, stats, beobachten, ip_tracker, config_rx, whitelist, + ) + .await; + } + Err(e) => { + warn!(error = %e, "Failed to bind metrics on {}", addr); + } + } + return; + } + + // Fallback: bind on 0.0.0.0 and [::] using metrics_port. let mut listener_v4 = None; let mut listener_v6 = None; From bd0cefdb12d864ea8a25e5829515fa86eb376b3a Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Tue, 17 Mar 2026 11:56:56 +0300 Subject: [PATCH 222/261] Update TLS-F-TCP-S.ru.md --- docs/fronting-splitting/TLS-F-TCP-S.ru.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/fronting-splitting/TLS-F-TCP-S.ru.md b/docs/fronting-splitting/TLS-F-TCP-S.ru.md index 6ae6f05..1f9f872 100644 --- a/docs/fronting-splitting/TLS-F-TCP-S.ru.md +++ b/docs/fronting-splitting/TLS-F-TCP-S.ru.md @@ -38,8 +38,9 @@ umweltschutz.de -> A-запись 198.18.88.88 В конфигурации Telemt: -``` -tls_domain = umweltschutz.de +```toml +[censorship] +tls_domain = "umweltschutz.de" ``` Этот домен используется клиентом как SNI в ClientHello @@ -56,8 +57,9 @@ tls_domain = umweltschutz.de В конфигурации Telemt: -``` -mask_host = 127.0.0.1 +```toml +[censorship] +mask_host = "127.0.0.1" mask_port = 8443 ``` @@ -151,16 +153,18 @@ mask_host:mask_port Например: -``` -tls_domain = github.com -mask_host = github.com +```toml +[censorship] +tls_domain = "github.com" +mask_host = "github.com" mask_port = 443 ``` или -``` -mask_host = 140.82.121.4 +```toml +[censorship] +mask_host = "140.82.121.4" ``` В этом случае: From c9271d90837c03d684f734d46ee7f0d5539d1a5d Mon Sep 17 00:00:00 2001 From: David Osipov Date: Tue, 17 Mar 2026 17:11:51 +0400 Subject: [PATCH 223/261] Add health monitoring tests for draining writers - Introduced adversarial tests to validate the behavior of the health monitoring system under various conditions, including the management of draining writers. - Implemented integration tests to ensure the health monitor correctly handles expired and empty draining writers. - Added regression tests to verify the functionality of the draining writers' cleanup process, ensuring it adheres to the defined thresholds and budgets. - Updated the module structure to include the new test files for better organization and maintainability. --- src/ip_tracker_regression_tests.rs | 450 +++++++++++++++++ src/main.rs | 2 + src/transport/middle_proxy/health.rs | 77 ++- .../middle_proxy/health_adversarial_tests.rs | 437 +++++++++++++++++ .../middle_proxy/health_integration_tests.rs | 227 +++++++++ .../middle_proxy/health_regression_tests.rs | 462 ++++++++++++++++++ src/transport/middle_proxy/mod.rs | 6 + 7 files changed, 1653 insertions(+), 8 deletions(-) create mode 100644 src/ip_tracker_regression_tests.rs create mode 100644 src/transport/middle_proxy/health_adversarial_tests.rs create mode 100644 src/transport/middle_proxy/health_integration_tests.rs create mode 100644 src/transport/middle_proxy/health_regression_tests.rs diff --git a/src/ip_tracker_regression_tests.rs b/src/ip_tracker_regression_tests.rs new file mode 100644 index 0000000..5d6b358 --- /dev/null +++ b/src/ip_tracker_regression_tests.rs @@ -0,0 +1,450 @@ +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::Arc; +use std::time::Duration; + +use crate::config::UserMaxUniqueIpsMode; +use crate::ip_tracker::UserIpTracker; + +fn ip_from_idx(idx: u32) -> IpAddr { + let a = 10u8; + let b = ((idx / 65_536) % 256) as u8; + let c = ((idx / 256) % 256) as u8; + let d = (idx % 256) as u8; + IpAddr::V4(Ipv4Addr::new(a, b, c, d)) +} + +#[tokio::test] +async fn active_window_enforces_large_unique_ip_burst() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("burst_user", 64).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::ActiveWindow, 30) + .await; + + for idx in 0..64 { + assert!(tracker.check_and_add("burst_user", ip_from_idx(idx)).await.is_ok()); + } + assert!(tracker.check_and_add("burst_user", ip_from_idx(9_999)).await.is_err()); + assert_eq!(tracker.get_active_ip_count("burst_user").await, 64); +} + +#[tokio::test] +async fn global_limit_applies_across_many_users() { + let tracker = UserIpTracker::new(); + tracker.load_limits(3, &HashMap::new()).await; + + for user_idx in 0..150u32 { + let user = format!("u{}", user_idx); + assert!(tracker.check_and_add(&user, ip_from_idx(user_idx * 10)).await.is_ok()); + assert!(tracker + .check_and_add(&user, ip_from_idx(user_idx * 10 + 1)) + .await + .is_ok()); + assert!(tracker + .check_and_add(&user, ip_from_idx(user_idx * 10 + 2)) + .await + .is_ok()); + assert!(tracker + .check_and_add(&user, ip_from_idx(user_idx * 10 + 3)) + .await + .is_err()); + } + + assert_eq!(tracker.get_stats().await.len(), 150); +} + +#[tokio::test] +async fn user_zero_override_falls_back_to_global_limit() { + let tracker = UserIpTracker::new(); + let mut limits = HashMap::new(); + limits.insert("target".to_string(), 0); + tracker.load_limits(2, &limits).await; + + assert!(tracker.check_and_add("target", ip_from_idx(1)).await.is_ok()); + assert!(tracker.check_and_add("target", ip_from_idx(2)).await.is_ok()); + assert!(tracker.check_and_add("target", ip_from_idx(3)).await.is_err()); + assert_eq!(tracker.get_user_limit("target").await, Some(2)); +} + +#[tokio::test] +async fn remove_ip_is_idempotent_after_counter_reaches_zero() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("u", 2).await; + let ip = ip_from_idx(42); + + tracker.check_and_add("u", ip).await.unwrap(); + tracker.remove_ip("u", ip).await; + tracker.remove_ip("u", ip).await; + tracker.remove_ip("u", ip).await; + + assert_eq!(tracker.get_active_ip_count("u").await, 0); + assert!(!tracker.is_ip_active("u", ip).await); +} + +#[tokio::test] +async fn clear_user_ips_resets_active_and_recent() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("u", 10).await; + + for idx in 0..6 { + tracker.check_and_add("u", ip_from_idx(idx)).await.unwrap(); + } + + tracker.clear_user_ips("u").await; + + assert_eq!(tracker.get_active_ip_count("u").await, 0); + let counts = tracker + .get_recent_counts_for_users(&["u".to_string()]) + .await; + assert_eq!(counts.get("u").copied().unwrap_or(0), 0); +} + +#[tokio::test] +async fn clear_all_resets_multi_user_state() { + let tracker = UserIpTracker::new(); + + for user_idx in 0..80u32 { + let user = format!("u{}", user_idx); + for ip_idx in 0..3 { + tracker + .check_and_add(&user, ip_from_idx(user_idx * 100 + ip_idx)) + .await + .unwrap(); + } + } + + tracker.clear_all().await; + + assert!(tracker.get_stats().await.is_empty()); + let users = (0..80u32) + .map(|idx| format!("u{}", idx)) + .collect::>(); + let recent = tracker.get_recent_counts_for_users(&users).await; + assert!(recent.values().all(|count| *count == 0)); +} + +#[tokio::test] +async fn get_active_ips_for_users_are_sorted() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("user", 10).await; + + tracker + .check_and_add("user", IpAddr::V4(Ipv4Addr::new(10, 0, 0, 9))) + .await + .unwrap(); + tracker + .check_and_add("user", IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))) + .await + .unwrap(); + tracker + .check_and_add("user", IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5))) + .await + .unwrap(); + + let map = tracker + .get_active_ips_for_users(&["user".to_string()]) + .await; + let ips = map.get("user").cloned().unwrap_or_default(); + + assert_eq!( + ips, + vec![ + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 9)), + ] + ); +} + +#[tokio::test] +async fn get_recent_ips_for_users_are_sorted() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("user", 10).await; + + tracker + .check_and_add("user", IpAddr::V4(Ipv4Addr::new(10, 1, 0, 9))) + .await + .unwrap(); + tracker + .check_and_add("user", IpAddr::V4(Ipv4Addr::new(10, 1, 0, 1))) + .await + .unwrap(); + tracker + .check_and_add("user", IpAddr::V4(Ipv4Addr::new(10, 1, 0, 5))) + .await + .unwrap(); + + let map = tracker + .get_recent_ips_for_users(&["user".to_string()]) + .await; + let ips = map.get("user").cloned().unwrap_or_default(); + + assert_eq!( + ips, + vec![ + IpAddr::V4(Ipv4Addr::new(10, 1, 0, 1)), + IpAddr::V4(Ipv4Addr::new(10, 1, 0, 5)), + IpAddr::V4(Ipv4Addr::new(10, 1, 0, 9)), + ] + ); +} + +#[tokio::test] +async fn time_window_expires_for_large_rotation() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("tw", 1).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::TimeWindow, 1) + .await; + + tracker.check_and_add("tw", ip_from_idx(1)).await.unwrap(); + tracker.remove_ip("tw", ip_from_idx(1)).await; + assert!(tracker.check_and_add("tw", ip_from_idx(2)).await.is_err()); + + tokio::time::sleep(Duration::from_millis(1_100)).await; + assert!(tracker.check_and_add("tw", ip_from_idx(2)).await.is_ok()); +} + +#[tokio::test] +async fn combined_mode_blocks_recent_after_disconnect() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("cmb", 1).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::Combined, 2) + .await; + + tracker.check_and_add("cmb", ip_from_idx(11)).await.unwrap(); + tracker.remove_ip("cmb", ip_from_idx(11)).await; + + assert!(tracker.check_and_add("cmb", ip_from_idx(12)).await.is_err()); +} + +#[tokio::test] +async fn load_limits_replaces_large_limit_map() { + let tracker = UserIpTracker::new(); + let mut first = HashMap::new(); + let mut second = HashMap::new(); + + for idx in 0..300usize { + first.insert(format!("u{}", idx), 2usize); + } + for idx in 150..450usize { + second.insert(format!("u{}", idx), 4usize); + } + + tracker.load_limits(0, &first).await; + tracker.load_limits(0, &second).await; + + assert_eq!(tracker.get_user_limit("u20").await, None); + assert_eq!(tracker.get_user_limit("u200").await, Some(4)); + assert_eq!(tracker.get_user_limit("u420").await, Some(4)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_same_user_unique_ip_pressure_stays_bounded() { + let tracker = Arc::new(UserIpTracker::new()); + tracker.set_user_limit("hot", 32).await; + tracker + .set_limit_policy(UserMaxUniqueIpsMode::ActiveWindow, 30) + .await; + + let mut handles = Vec::new(); + for worker in 0..16u32 { + let tracker_cloned = tracker.clone(); + handles.push(tokio::spawn(async move { + let base = worker * 200; + for step in 0..200u32 { + let _ = tracker_cloned + .check_and_add("hot", ip_from_idx(base + step)) + .await; + } + })); + } + + for handle in handles { + handle.await.unwrap(); + } + + assert!(tracker.get_active_ip_count("hot").await <= 32); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_many_users_isolate_limits() { + let tracker = Arc::new(UserIpTracker::new()); + tracker.load_limits(4, &HashMap::new()).await; + + let mut handles = Vec::new(); + for user_idx in 0..120u32 { + let tracker_cloned = tracker.clone(); + handles.push(tokio::spawn(async move { + let user = format!("u{}", user_idx); + for ip_idx in 0..10u32 { + let _ = tracker_cloned + .check_and_add(&user, ip_from_idx(user_idx * 1_000 + ip_idx)) + .await; + } + })); + } + + for handle in handles { + handle.await.unwrap(); + } + + let stats = tracker.get_stats().await; + assert_eq!(stats.len(), 120); + assert!(stats.iter().all(|(_, active, limit)| *active <= 4 && *limit == 4)); +} + +#[tokio::test] +async fn same_ip_reconnect_high_frequency_keeps_single_unique() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("same", 2).await; + let ip = ip_from_idx(9); + + for _ in 0..2_000 { + tracker.check_and_add("same", ip).await.unwrap(); + } + + assert_eq!(tracker.get_active_ip_count("same").await, 1); + assert!(tracker.is_ip_active("same", ip).await); +} + +#[tokio::test] +async fn format_stats_contains_expected_limited_and_unlimited_markers() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("limited", 2).await; + tracker.check_and_add("limited", ip_from_idx(1)).await.unwrap(); + tracker.check_and_add("open", ip_from_idx(2)).await.unwrap(); + + let text = tracker.format_stats().await; + + assert!(text.contains("limited")); + assert!(text.contains("open")); + assert!(text.contains("unlimited")); +} + +#[tokio::test] +async fn stats_report_global_default_for_users_without_override() { + let tracker = UserIpTracker::new(); + tracker.load_limits(5, &HashMap::new()).await; + + tracker.check_and_add("a", ip_from_idx(1)).await.unwrap(); + tracker.check_and_add("b", ip_from_idx(2)).await.unwrap(); + + let stats = tracker.get_stats().await; + assert!(stats.iter().any(|(user, _, limit)| user == "a" && *limit == 5)); + assert!(stats.iter().any(|(user, _, limit)| user == "b" && *limit == 5)); +} + +#[tokio::test] +async fn stress_cycle_add_remove_clear_preserves_empty_end_state() { + let tracker = UserIpTracker::new(); + + for cycle in 0..50u32 { + let user = format!("cycle{}", cycle); + tracker.set_user_limit(&user, 128).await; + + for ip_idx in 0..128u32 { + tracker + .check_and_add(&user, ip_from_idx(cycle * 10_000 + ip_idx)) + .await + .unwrap(); + } + + for ip_idx in 0..128u32 { + tracker + .remove_ip(&user, ip_from_idx(cycle * 10_000 + ip_idx)) + .await; + } + + tracker.clear_user_ips(&user).await; + } + + assert!(tracker.get_stats().await.is_empty()); +} + +#[tokio::test] +async fn remove_unknown_user_or_ip_does_not_corrupt_state() { + let tracker = UserIpTracker::new(); + + tracker.remove_ip("no_user", ip_from_idx(1)).await; + tracker.check_and_add("x", ip_from_idx(2)).await.unwrap(); + tracker.remove_ip("x", ip_from_idx(3)).await; + + assert_eq!(tracker.get_active_ip_count("x").await, 1); + assert!(tracker.is_ip_active("x", ip_from_idx(2)).await); +} + +#[tokio::test] +async fn active_and_recent_views_match_after_mixed_workload() { + let tracker = UserIpTracker::new(); + tracker.set_user_limit("mix", 16).await; + + for ip_idx in 0..12u32 { + tracker.check_and_add("mix", ip_from_idx(ip_idx)).await.unwrap(); + } + for ip_idx in 0..6u32 { + tracker.remove_ip("mix", ip_from_idx(ip_idx)).await; + } + + let active = tracker + .get_active_ips_for_users(&["mix".to_string()]) + .await + .get("mix") + .cloned() + .unwrap_or_default(); + let recent_count = tracker + .get_recent_counts_for_users(&["mix".to_string()]) + .await + .get("mix") + .copied() + .unwrap_or(0); + + assert_eq!(active.len(), 6); + assert!(recent_count >= active.len()); + assert!(recent_count <= 12); +} + +#[tokio::test] +async fn global_limit_switch_updates_enforcement_immediately() { + let tracker = UserIpTracker::new(); + tracker.load_limits(2, &HashMap::new()).await; + + assert!(tracker.check_and_add("u", ip_from_idx(1)).await.is_ok()); + assert!(tracker.check_and_add("u", ip_from_idx(2)).await.is_ok()); + assert!(tracker.check_and_add("u", ip_from_idx(3)).await.is_err()); + + tracker.clear_user_ips("u").await; + tracker.load_limits(4, &HashMap::new()).await; + + assert!(tracker.check_and_add("u", ip_from_idx(1)).await.is_ok()); + assert!(tracker.check_and_add("u", ip_from_idx(2)).await.is_ok()); + assert!(tracker.check_and_add("u", ip_from_idx(3)).await.is_ok()); + assert!(tracker.check_and_add("u", ip_from_idx(4)).await.is_ok()); + assert!(tracker.check_and_add("u", ip_from_idx(5)).await.is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_reconnect_and_disconnect_preserves_non_negative_counts() { + let tracker = Arc::new(UserIpTracker::new()); + tracker.set_user_limit("cc", 8).await; + + let mut handles = Vec::new(); + for worker in 0..8u32 { + let tracker_cloned = tracker.clone(); + handles.push(tokio::spawn(async move { + let ip = ip_from_idx(50 + worker); + for _ in 0..500u32 { + let _ = tracker_cloned.check_and_add("cc", ip).await; + tracker_cloned.remove_ip("cc", ip).await; + } + })); + } + + for handle in handles { + handle.await.unwrap(); + } + + assert!(tracker.get_active_ip_count("cc").await <= 8); +} diff --git a/src/main.rs b/src/main.rs index 73ada8c..2cfbe28 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,8 @@ mod config; mod crypto; mod error; mod ip_tracker; +#[cfg(test)] +mod ip_tracker_regression_tests; mod maestro; mod metrics; mod network; diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index e5f4260..8ac6839 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -25,6 +25,9 @@ const HEALTH_RECONNECT_BUDGET_PER_CORE: usize = 2; const HEALTH_RECONNECT_BUDGET_PER_DC: usize = 1; const HEALTH_RECONNECT_BUDGET_MIN: usize = 4; const HEALTH_RECONNECT_BUDGET_MAX: usize = 128; +const HEALTH_DRAIN_CLOSE_BUDGET_PER_CORE: usize = 16; +const HEALTH_DRAIN_CLOSE_BUDGET_MIN: usize = 16; +const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256; #[derive(Debug, Clone)] struct DcFloorPlanEntry { @@ -111,7 +114,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c } } -async fn reap_draining_writers( +pub(super) async fn reap_draining_writers( pool: &Arc, warn_next_allowed: &mut HashMap, ) { @@ -122,14 +125,22 @@ async fn reap_draining_writers( .me_pool_drain_threshold .load(std::sync::atomic::Ordering::Relaxed); let writers = pool.writers.read().await.clone(); + let activity = pool.registry.writer_activity_snapshot().await; let mut draining_writers = Vec::new(); + let mut empty_writer_ids = Vec::::new(); + let mut force_close_writer_ids = Vec::::new(); for writer in writers { if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) { continue; } - let is_empty = pool.registry.is_writer_empty(writer.id).await; - if is_empty { - pool.remove_writer_and_close_clients(writer.id).await; + if activity + .bound_clients_by_writer + .get(&writer.id) + .copied() + .unwrap_or(0) + == 0 + { + empty_writer_ids.push(writer.id); continue; } draining_writers.push(writer); @@ -156,12 +167,13 @@ async fn reap_draining_writers( "ME draining writer threshold exceeded, force-closing oldest draining writers" ); for writer in draining_writers.drain(..overflow) { - pool.stats.increment_pool_force_close_total(); - pool.remove_writer_and_close_clients(writer.id).await; + force_close_writer_ids.push(writer.id); } } + let mut active_draining_writer_ids = HashSet::with_capacity(draining_writers.len()); for writer in draining_writers { + active_draining_writer_ids.insert(writer.id); let drain_started_at_epoch_secs = writer .draining_started_at_epoch_secs .load(std::sync::atomic::Ordering::Relaxed); @@ -191,10 +203,59 @@ async fn reap_draining_writers( .load(std::sync::atomic::Ordering::Relaxed); if deadline_epoch_secs != 0 && now_epoch_secs >= deadline_epoch_secs { warn!(writer_id = writer.id, "Drain timeout, force-closing"); - pool.stats.increment_pool_force_close_total(); - pool.remove_writer_and_close_clients(writer.id).await; + force_close_writer_ids.push(writer.id); + active_draining_writer_ids.remove(&writer.id); } } + + warn_next_allowed.retain(|writer_id, _| active_draining_writer_ids.contains(writer_id)); + + let close_budget = health_drain_close_budget(); + let requested_force_close = force_close_writer_ids.len(); + let requested_empty_close = empty_writer_ids.len(); + let requested_close_total = requested_force_close.saturating_add(requested_empty_close); + let mut closed_writer_ids = HashSet::::new(); + let mut closed_total = 0usize; + for writer_id in force_close_writer_ids { + if closed_total >= close_budget { + break; + } + if !closed_writer_ids.insert(writer_id) { + continue; + } + pool.stats.increment_pool_force_close_total(); + pool.remove_writer_and_close_clients(writer_id).await; + closed_total = closed_total.saturating_add(1); + } + for writer_id in empty_writer_ids { + if closed_total >= close_budget { + break; + } + if !closed_writer_ids.insert(writer_id) { + continue; + } + pool.remove_writer_and_close_clients(writer_id).await; + closed_total = closed_total.saturating_add(1); + } + + let pending_close_total = requested_close_total.saturating_sub(closed_total); + if pending_close_total > 0 { + warn!( + close_budget, + closed_total, + pending_close_total, + "ME draining close backlog deferred to next health cycle" + ); + } +} + +pub(super) fn health_drain_close_budget() -> usize { + let cpu_cores = std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1); + cpu_cores + .saturating_mul(HEALTH_DRAIN_CLOSE_BUDGET_PER_CORE) + .clamp(HEALTH_DRAIN_CLOSE_BUDGET_MIN, HEALTH_DRAIN_CLOSE_BUDGET_MAX) } fn should_emit_writer_warn( diff --git a/src/transport/middle_proxy/health_adversarial_tests.rs b/src/transport/middle_proxy/health_adversarial_tests.rs new file mode 100644 index 0000000..675005a --- /dev/null +++ b/src/transport/middle_proxy/health_adversarial_tests.rs @@ -0,0 +1,437 @@ +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::codec::WriterCommand; +use super::health::{health_drain_close_budget, reap_draining_writers}; +use super::pool::{MePool, MeWriter, WriterContour}; +use super::registry::ConnMeta; +use super::me_health_monitor; +use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; +use crate::crypto::SecureRandom; +use crate::network::probe::NetworkDecision; +use crate::stats::Stats; + +async fn make_pool( + me_pool_drain_threshold: u64, + me_health_interval_ms_unhealthy: u64, + me_health_interval_ms_healthy: u64, +) -> (Arc, Arc) { + let general = GeneralConfig { + me_pool_drain_threshold, + me_health_interval_ms_unhealthy, + me_health_interval_ms_healthy, + ..GeneralConfig::default() + }; + + let rng = Arc::new(SecureRandom::new()); + let pool = MePool::new( + None, + vec![1u8; 32], + None, + false, + None, + Vec::new(), + 1, + None, + 12, + 1200, + HashMap::new(), + HashMap::new(), + None, + NetworkDecision::default(), + None, + rng.clone(), + Arc::new(Stats::default()), + general.me_keepalive_enabled, + general.me_keepalive_interval_secs, + general.me_keepalive_jitter_secs, + general.me_keepalive_payload_random, + general.rpc_proxy_req_every, + general.me_warmup_stagger_enabled, + general.me_warmup_step_delay_ms, + general.me_warmup_step_jitter_ms, + general.me_reconnect_max_concurrent_per_dc, + general.me_reconnect_backoff_base_ms, + general.me_reconnect_backoff_cap_ms, + general.me_reconnect_fast_retry_count, + general.me_single_endpoint_shadow_writers, + general.me_single_endpoint_outage_mode_enabled, + general.me_single_endpoint_outage_disable_quarantine, + general.me_single_endpoint_outage_backoff_min_ms, + general.me_single_endpoint_outage_backoff_max_ms, + general.me_single_endpoint_shadow_rotate_every_secs, + general.me_floor_mode, + general.me_adaptive_floor_idle_secs, + general.me_adaptive_floor_min_writers_single_endpoint, + general.me_adaptive_floor_min_writers_multi_endpoint, + general.me_adaptive_floor_recover_grace_secs, + general.me_adaptive_floor_writers_per_core_total, + general.me_adaptive_floor_cpu_cores_override, + general.me_adaptive_floor_max_extra_writers_single_per_core, + general.me_adaptive_floor_max_extra_writers_multi_per_core, + general.me_adaptive_floor_max_active_writers_per_core, + general.me_adaptive_floor_max_warm_writers_per_core, + general.me_adaptive_floor_max_active_writers_global, + general.me_adaptive_floor_max_warm_writers_global, + general.hardswap, + general.me_pool_drain_ttl_secs, + general.me_pool_drain_threshold, + general.effective_me_pool_force_close_secs(), + general.me_pool_min_fresh_ratio, + general.me_hardswap_warmup_delay_min_ms, + general.me_hardswap_warmup_delay_max_ms, + general.me_hardswap_warmup_extra_passes, + general.me_hardswap_warmup_pass_backoff_base_ms, + general.me_bind_stale_mode, + general.me_bind_stale_ttl_secs, + general.me_secret_atomic_snapshot, + general.me_deterministic_writer_sort, + MeWriterPickMode::default(), + general.me_writer_pick_sample_size, + MeSocksKdfPolicy::default(), + general.me_writer_cmd_channel_capacity, + general.me_route_channel_capacity, + general.me_route_backpressure_base_timeout_ms, + general.me_route_backpressure_high_timeout_ms, + general.me_route_backpressure_high_watermark_pct, + general.me_reader_route_data_wait_ms, + general.me_health_interval_ms_unhealthy, + general.me_health_interval_ms_healthy, + general.me_warn_rate_limit_ms, + MeRouteNoWriterMode::default(), + general.me_route_no_writer_wait_ms, + general.me_route_inline_recovery_attempts, + general.me_route_inline_recovery_wait_ms, + ); + + (pool, rng) +} + +async fn insert_draining_writer( + pool: &Arc, + writer_id: u64, + drain_started_at_epoch_secs: u64, + bound_clients: usize, + drain_deadline_epoch_secs: u64, +) { + let (tx, _writer_rx) = mpsc::channel::(8); + let writer = MeWriter { + id: writer_id, + addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6000 + writer_id as u16), + source_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + writer_dc: 2, + generation: 1, + contour: Arc::new(AtomicU8::new(WriterContour::Draining.as_u8())), + created_at: Instant::now() - Duration::from_secs(writer_id), + tx: tx.clone(), + cancel: CancellationToken::new(), + degraded: Arc::new(AtomicBool::new(false)), + rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)), + draining: Arc::new(AtomicBool::new(true)), + draining_started_at_epoch_secs: Arc::new(AtomicU64::new(drain_started_at_epoch_secs)), + drain_deadline_epoch_secs: Arc::new(AtomicU64::new(drain_deadline_epoch_secs)), + allow_drain_fallback: Arc::new(AtomicBool::new(false)), + }; + + pool.writers.write().await.push(writer); + pool.registry.register_writer(writer_id, tx).await; + pool.conn_count.fetch_add(1, Ordering::Relaxed); + + for idx in 0..bound_clients { + let (conn_id, _rx) = pool.registry.register().await; + assert!( + pool.registry + .bind_writer( + conn_id, + writer_id, + ConnMeta { + target_dc: 2, + client_addr: SocketAddr::new( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 8000 + idx as u16, + ), + our_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443), + proto_flags: 0, + }, + ) + .await + ); + } +} + +async fn writer_count(pool: &Arc) -> usize { + pool.writers.read().await.len() +} + +async fn sorted_writer_ids(pool: &Arc) -> Vec { + let mut ids = pool + .writers + .read() + .await + .iter() + .map(|writer| writer.id) + .collect::>(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn reap_draining_writers_clears_warn_state_when_pool_empty() { + let (pool, _rng) = make_pool(128, 1, 1).await; + let mut warn_next_allowed = HashMap::new(); + warn_next_allowed.insert(11, Instant::now() + Duration::from_secs(5)); + warn_next_allowed.insert(22, Instant::now() + Duration::from_secs(5)); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert!(warn_next_allowed.is_empty()); +} + +#[tokio::test] +async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycles() { + let threshold = 3u64; + let (pool, _rng) = make_pool(threshold, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + + for writer_id in 1..=60u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(600).saturating_add(writer_id), + 1, + 0, + ) + .await; + } + + let mut warn_next_allowed = HashMap::new(); + for _ in 0..64 { + reap_draining_writers(&pool, &mut warn_next_allowed).await; + if writer_count(&pool).await <= threshold as usize { + break; + } + } + + assert_eq!(writer_count(&pool).await, threshold as usize); + assert_eq!(sorted_writer_ids(&pool).await, vec![58, 59, 60]); +} + +#[tokio::test] +async fn reap_draining_writers_handles_large_empty_writer_population() { + let (pool, _rng) = make_pool(128, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let total = health_drain_close_budget().saturating_mul(3).saturating_add(27); + + for writer_id in 1..=total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(120), + 0, + 0, + ) + .await; + } + + let mut warn_next_allowed = HashMap::new(); + for _ in 0..24 { + if writer_count(&pool).await == 0 { + break; + } + reap_draining_writers(&pool, &mut warn_next_allowed).await; + } + + assert_eq!(writer_count(&pool).await, 0); +} + +#[tokio::test] +async fn reap_draining_writers_processes_mass_deadline_expiry_without_unbounded_growth() { + let (pool, _rng) = make_pool(128, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let total = health_drain_close_budget().saturating_mul(4).saturating_add(31); + + for writer_id in 1..=total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(180), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + } + + let mut warn_next_allowed = HashMap::new(); + for _ in 0..40 { + if writer_count(&pool).await == 0 { + break; + } + reap_draining_writers(&pool, &mut warn_next_allowed).await; + } + + assert_eq!(writer_count(&pool).await, 0); +} + +#[tokio::test] +async fn reap_draining_writers_maintains_warn_state_subset_property_under_bulk_churn() { + let (pool, _rng) = make_pool(128, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let mut warn_next_allowed = HashMap::new(); + + for wave in 0..40u64 { + for offset in 0..8u64 { + insert_draining_writer( + &pool, + wave * 100 + offset, + now_epoch_secs.saturating_sub(400 + offset), + 1, + 0, + ) + .await; + } + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + assert!(warn_next_allowed.len() <= writer_count(&pool).await); + + let ids = sorted_writer_ids(&pool).await; + for writer_id in ids.into_iter().take(3) { + let _ = pool.remove_writer_and_close_clients(writer_id).await; + } + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + assert!(warn_next_allowed.len() <= writer_count(&pool).await); + } +} + +#[tokio::test] +async fn reap_draining_writers_budgeted_cleanup_never_increases_pool_size() { + let (pool, _rng) = make_pool(5, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + + for writer_id in 1..=200u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(240).saturating_add(writer_id), + 1, + 0, + ) + .await; + } + + let mut warn_next_allowed = HashMap::new(); + let mut previous = writer_count(&pool).await; + for _ in 0..32 { + reap_draining_writers(&pool, &mut warn_next_allowed).await; + let current = writer_count(&pool).await; + assert!(current <= previous); + previous = current; + } +} + +#[tokio::test] +async fn me_health_monitor_converges_to_threshold_under_live_injection_churn() { + let threshold = 7u64; + let (pool, rng) = make_pool(threshold, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + + for writer_id in 1..=40u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(300).saturating_add(writer_id), + 1, + 0, + ) + .await; + } + + let monitor = tokio::spawn(me_health_monitor(pool.clone(), rng, 0)); + + for wave in 0..8u64 { + for offset in 0..10u64 { + insert_draining_writer( + &pool, + 1000 + wave * 100 + offset, + now_epoch_secs.saturating_sub(120).saturating_add(offset), + 1, + 0, + ) + .await; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + + tokio::time::sleep(Duration::from_millis(120)).await; + monitor.abort(); + let _ = monitor.await; + + assert!(writer_count(&pool).await <= threshold as usize); +} + +#[tokio::test] +async fn me_health_monitor_drains_deadline_storm_with_budgeted_progress() { + let (pool, rng) = make_pool(128, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + + for writer_id in 1..=220u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(120), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + } + + let monitor = tokio::spawn(me_health_monitor(pool.clone(), rng, 0)); + tokio::time::sleep(Duration::from_millis(120)).await; + monitor.abort(); + let _ = monitor.await; + + assert_eq!(writer_count(&pool).await, 0); +} + +#[tokio::test] +async fn me_health_monitor_eliminates_mixed_empty_and_deadline_backlog() { + let threshold = 12u64; + let (pool, rng) = make_pool(threshold, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + + for writer_id in 1..=180u64 { + let bound_clients = if writer_id % 3 == 0 { 0 } else { 1 }; + let deadline = if writer_id % 2 == 0 { + now_epoch_secs.saturating_sub(1) + } else { + 0 + }; + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(250).saturating_add(writer_id), + bound_clients, + deadline, + ) + .await; + } + + let monitor = tokio::spawn(me_health_monitor(pool.clone(), rng, 0)); + tokio::time::sleep(Duration::from_millis(140)).await; + monitor.abort(); + let _ = monitor.await; + + assert!(writer_count(&pool).await <= threshold as usize); +} + +#[test] +fn health_drain_close_budget_is_within_expected_bounds() { + let budget = health_drain_close_budget(); + assert!((16..=256).contains(&budget)); +} diff --git a/src/transport/middle_proxy/health_integration_tests.rs b/src/transport/middle_proxy/health_integration_tests.rs new file mode 100644 index 0000000..70b6411 --- /dev/null +++ b/src/transport/middle_proxy/health_integration_tests.rs @@ -0,0 +1,227 @@ +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::codec::WriterCommand; +use super::health::health_drain_close_budget; +use super::pool::{MePool, MeWriter, WriterContour}; +use super::registry::ConnMeta; +use super::me_health_monitor; +use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; +use crate::crypto::SecureRandom; +use crate::network::probe::NetworkDecision; +use crate::stats::Stats; + +async fn make_pool( + me_pool_drain_threshold: u64, + me_health_interval_ms_unhealthy: u64, + me_health_interval_ms_healthy: u64, +) -> (Arc, Arc) { + let general = GeneralConfig { + me_pool_drain_threshold, + me_health_interval_ms_unhealthy, + me_health_interval_ms_healthy, + ..GeneralConfig::default() + }; + let rng = Arc::new(SecureRandom::new()); + let pool = MePool::new( + None, + vec![1u8; 32], + None, + false, + None, + Vec::new(), + 1, + None, + 12, + 1200, + HashMap::new(), + HashMap::new(), + None, + NetworkDecision::default(), + None, + rng.clone(), + Arc::new(Stats::default()), + general.me_keepalive_enabled, + general.me_keepalive_interval_secs, + general.me_keepalive_jitter_secs, + general.me_keepalive_payload_random, + general.rpc_proxy_req_every, + general.me_warmup_stagger_enabled, + general.me_warmup_step_delay_ms, + general.me_warmup_step_jitter_ms, + general.me_reconnect_max_concurrent_per_dc, + general.me_reconnect_backoff_base_ms, + general.me_reconnect_backoff_cap_ms, + general.me_reconnect_fast_retry_count, + general.me_single_endpoint_shadow_writers, + general.me_single_endpoint_outage_mode_enabled, + general.me_single_endpoint_outage_disable_quarantine, + general.me_single_endpoint_outage_backoff_min_ms, + general.me_single_endpoint_outage_backoff_max_ms, + general.me_single_endpoint_shadow_rotate_every_secs, + general.me_floor_mode, + general.me_adaptive_floor_idle_secs, + general.me_adaptive_floor_min_writers_single_endpoint, + general.me_adaptive_floor_min_writers_multi_endpoint, + general.me_adaptive_floor_recover_grace_secs, + general.me_adaptive_floor_writers_per_core_total, + general.me_adaptive_floor_cpu_cores_override, + general.me_adaptive_floor_max_extra_writers_single_per_core, + general.me_adaptive_floor_max_extra_writers_multi_per_core, + general.me_adaptive_floor_max_active_writers_per_core, + general.me_adaptive_floor_max_warm_writers_per_core, + general.me_adaptive_floor_max_active_writers_global, + general.me_adaptive_floor_max_warm_writers_global, + general.hardswap, + general.me_pool_drain_ttl_secs, + general.me_pool_drain_threshold, + general.effective_me_pool_force_close_secs(), + general.me_pool_min_fresh_ratio, + general.me_hardswap_warmup_delay_min_ms, + general.me_hardswap_warmup_delay_max_ms, + general.me_hardswap_warmup_extra_passes, + general.me_hardswap_warmup_pass_backoff_base_ms, + general.me_bind_stale_mode, + general.me_bind_stale_ttl_secs, + general.me_secret_atomic_snapshot, + general.me_deterministic_writer_sort, + MeWriterPickMode::default(), + general.me_writer_pick_sample_size, + MeSocksKdfPolicy::default(), + general.me_writer_cmd_channel_capacity, + general.me_route_channel_capacity, + general.me_route_backpressure_base_timeout_ms, + general.me_route_backpressure_high_timeout_ms, + general.me_route_backpressure_high_watermark_pct, + general.me_reader_route_data_wait_ms, + general.me_health_interval_ms_unhealthy, + general.me_health_interval_ms_healthy, + general.me_warn_rate_limit_ms, + MeRouteNoWriterMode::default(), + general.me_route_no_writer_wait_ms, + general.me_route_inline_recovery_attempts, + general.me_route_inline_recovery_wait_ms, + ); + (pool, rng) +} + +async fn insert_draining_writer( + pool: &Arc, + writer_id: u64, + drain_started_at_epoch_secs: u64, + bound_clients: usize, + drain_deadline_epoch_secs: u64, +) { + let (tx, _writer_rx) = mpsc::channel::(8); + let writer = MeWriter { + id: writer_id, + addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 5500 + writer_id as u16), + source_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + writer_dc: 2, + generation: 1, + contour: Arc::new(AtomicU8::new(WriterContour::Draining.as_u8())), + created_at: Instant::now() - Duration::from_secs(writer_id), + tx: tx.clone(), + cancel: CancellationToken::new(), + degraded: Arc::new(AtomicBool::new(false)), + rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)), + draining: Arc::new(AtomicBool::new(true)), + draining_started_at_epoch_secs: Arc::new(AtomicU64::new(drain_started_at_epoch_secs)), + drain_deadline_epoch_secs: Arc::new(AtomicU64::new(drain_deadline_epoch_secs)), + allow_drain_fallback: Arc::new(AtomicBool::new(false)), + }; + pool.writers.write().await.push(writer); + pool.registry.register_writer(writer_id, tx).await; + pool.conn_count.fetch_add(1, Ordering::Relaxed); + for idx in 0..bound_clients { + let (conn_id, _rx) = pool.registry.register().await; + assert!( + pool.registry + .bind_writer( + conn_id, + writer_id, + ConnMeta { + target_dc: 2, + client_addr: SocketAddr::new( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 7200 + idx as u16, + ), + our_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443), + proto_flags: 0, + }, + ) + .await + ); + } +} + +#[tokio::test] +async fn me_health_monitor_drains_expired_backlog_over_multiple_cycles() { + let (pool, rng) = make_pool(128, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let writer_total = health_drain_close_budget().saturating_mul(2).saturating_add(9); + for writer_id in 1..=writer_total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(120), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + } + + let monitor = tokio::spawn(me_health_monitor(pool.clone(), rng, 0)); + tokio::time::sleep(Duration::from_millis(60)).await; + monitor.abort(); + let _ = monitor.await; + + assert!(pool.writers.read().await.is_empty()); +} + +#[tokio::test] +async fn me_health_monitor_cleans_empty_draining_writers_without_force_close() { + let (pool, rng) = make_pool(128, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + for writer_id in 1..=24u64 { + insert_draining_writer(&pool, writer_id, now_epoch_secs.saturating_sub(60), 0, 0).await; + } + + let monitor = tokio::spawn(me_health_monitor(pool.clone(), rng, 0)); + tokio::time::sleep(Duration::from_millis(30)).await; + monitor.abort(); + let _ = monitor.await; + + assert!(pool.writers.read().await.is_empty()); +} + +#[tokio::test] +async fn me_health_monitor_converges_retry_like_threshold_backlog_to_empty() { + let threshold = 4u64; + let (pool, rng) = make_pool(threshold, 1, 1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let writer_total = threshold as usize + health_drain_close_budget().saturating_add(11); + for writer_id in 1..=writer_total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(300).saturating_add(writer_id), + 1, + 0, + ) + .await; + } + + let monitor = tokio::spawn(me_health_monitor(pool.clone(), rng, 0)); + tokio::time::sleep(Duration::from_millis(60)).await; + monitor.abort(); + let _ = monitor.await; + + assert!(pool.writers.read().await.is_empty()); +} diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs new file mode 100644 index 0000000..05a8e6a --- /dev/null +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -0,0 +1,462 @@ +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +use super::codec::WriterCommand; +use super::health::{health_drain_close_budget, reap_draining_writers}; +use super::pool::{MePool, MeWriter, WriterContour}; +use super::registry::ConnMeta; +use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; +use crate::crypto::SecureRandom; +use crate::network::probe::NetworkDecision; +use crate::stats::Stats; + +async fn make_pool(me_pool_drain_threshold: u64) -> Arc { + let general = GeneralConfig { + me_pool_drain_threshold, + ..GeneralConfig::default() + }; + + MePool::new( + None, + vec![1u8; 32], + None, + false, + None, + Vec::new(), + 1, + None, + 12, + 1200, + HashMap::new(), + HashMap::new(), + None, + NetworkDecision::default(), + None, + Arc::new(SecureRandom::new()), + Arc::new(Stats::default()), + general.me_keepalive_enabled, + general.me_keepalive_interval_secs, + general.me_keepalive_jitter_secs, + general.me_keepalive_payload_random, + general.rpc_proxy_req_every, + general.me_warmup_stagger_enabled, + general.me_warmup_step_delay_ms, + general.me_warmup_step_jitter_ms, + general.me_reconnect_max_concurrent_per_dc, + general.me_reconnect_backoff_base_ms, + general.me_reconnect_backoff_cap_ms, + general.me_reconnect_fast_retry_count, + general.me_single_endpoint_shadow_writers, + general.me_single_endpoint_outage_mode_enabled, + general.me_single_endpoint_outage_disable_quarantine, + general.me_single_endpoint_outage_backoff_min_ms, + general.me_single_endpoint_outage_backoff_max_ms, + general.me_single_endpoint_shadow_rotate_every_secs, + general.me_floor_mode, + general.me_adaptive_floor_idle_secs, + general.me_adaptive_floor_min_writers_single_endpoint, + general.me_adaptive_floor_min_writers_multi_endpoint, + general.me_adaptive_floor_recover_grace_secs, + general.me_adaptive_floor_writers_per_core_total, + general.me_adaptive_floor_cpu_cores_override, + general.me_adaptive_floor_max_extra_writers_single_per_core, + general.me_adaptive_floor_max_extra_writers_multi_per_core, + general.me_adaptive_floor_max_active_writers_per_core, + general.me_adaptive_floor_max_warm_writers_per_core, + general.me_adaptive_floor_max_active_writers_global, + general.me_adaptive_floor_max_warm_writers_global, + general.hardswap, + general.me_pool_drain_ttl_secs, + general.me_pool_drain_threshold, + general.effective_me_pool_force_close_secs(), + general.me_pool_min_fresh_ratio, + general.me_hardswap_warmup_delay_min_ms, + general.me_hardswap_warmup_delay_max_ms, + general.me_hardswap_warmup_extra_passes, + general.me_hardswap_warmup_pass_backoff_base_ms, + general.me_bind_stale_mode, + general.me_bind_stale_ttl_secs, + general.me_secret_atomic_snapshot, + general.me_deterministic_writer_sort, + MeWriterPickMode::default(), + general.me_writer_pick_sample_size, + MeSocksKdfPolicy::default(), + general.me_writer_cmd_channel_capacity, + general.me_route_channel_capacity, + general.me_route_backpressure_base_timeout_ms, + general.me_route_backpressure_high_timeout_ms, + general.me_route_backpressure_high_watermark_pct, + general.me_reader_route_data_wait_ms, + general.me_health_interval_ms_unhealthy, + general.me_health_interval_ms_healthy, + general.me_warn_rate_limit_ms, + MeRouteNoWriterMode::default(), + general.me_route_no_writer_wait_ms, + general.me_route_inline_recovery_attempts, + general.me_route_inline_recovery_wait_ms, + ) +} + +async fn insert_draining_writer( + pool: &Arc, + writer_id: u64, + drain_started_at_epoch_secs: u64, + bound_clients: usize, + drain_deadline_epoch_secs: u64, +) -> Vec { + let mut conn_ids = Vec::with_capacity(bound_clients); + let (tx, _writer_rx) = mpsc::channel::(8); + let writer = MeWriter { + id: writer_id, + addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4500 + writer_id as u16), + source_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + writer_dc: 2, + generation: 1, + contour: Arc::new(AtomicU8::new(WriterContour::Draining.as_u8())), + created_at: Instant::now() - Duration::from_secs(writer_id), + tx: tx.clone(), + cancel: CancellationToken::new(), + degraded: Arc::new(AtomicBool::new(false)), + rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)), + draining: Arc::new(AtomicBool::new(true)), + draining_started_at_epoch_secs: Arc::new(AtomicU64::new(drain_started_at_epoch_secs)), + drain_deadline_epoch_secs: Arc::new(AtomicU64::new(drain_deadline_epoch_secs)), + allow_drain_fallback: Arc::new(AtomicBool::new(false)), + }; + pool.writers.write().await.push(writer); + pool.registry.register_writer(writer_id, tx).await; + pool.conn_count.fetch_add(1, Ordering::Relaxed); + for idx in 0..bound_clients { + let (conn_id, _rx) = pool.registry.register().await; + assert!( + pool.registry + .bind_writer( + conn_id, + writer_id, + ConnMeta { + target_dc: 2, + client_addr: SocketAddr::new( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 6200 + idx as u16, + ), + our_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443), + proto_flags: 0, + }, + ) + .await + ); + conn_ids.push(conn_id); + } + conn_ids +} + +async fn current_writer_ids(pool: &Arc) -> Vec { + let mut writer_ids = pool + .writers + .read() + .await + .iter() + .map(|writer| writer.id) + .collect::>(); + writer_ids.sort_unstable(); + writer_ids +} + +#[tokio::test] +async fn reap_draining_writers_drops_warn_state_for_removed_writer() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let conn_ids = + insert_draining_writer(&pool, 7, now_epoch_secs.saturating_sub(180), 1, 0).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + assert!(warn_next_allowed.contains_key(&7)); + + let _ = pool.remove_writer_and_close_clients(7).await; + assert!(pool.registry.get_writer(conn_ids[0]).await.is_none()); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + assert!(!warn_next_allowed.contains_key(&7)); +} + +#[tokio::test] +async fn reap_draining_writers_removes_empty_draining_writers() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 1, now_epoch_secs.saturating_sub(40), 0, 0).await; + insert_draining_writer(&pool, 2, now_epoch_secs.saturating_sub(30), 0, 0).await; + insert_draining_writer(&pool, 3, now_epoch_secs.saturating_sub(20), 1, 0).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert_eq!(current_writer_ids(&pool).await, vec![3]); +} + +#[tokio::test] +async fn reap_draining_writers_overflow_closes_oldest_non_empty_writers() { + let pool = make_pool(2).await; + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 11, now_epoch_secs.saturating_sub(40), 1, 0).await; + insert_draining_writer(&pool, 22, now_epoch_secs.saturating_sub(30), 1, 0).await; + insert_draining_writer(&pool, 33, now_epoch_secs.saturating_sub(20), 1, 0).await; + insert_draining_writer(&pool, 44, now_epoch_secs.saturating_sub(10), 1, 0).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert_eq!(current_writer_ids(&pool).await, vec![33, 44]); +} + +#[tokio::test] +async fn reap_draining_writers_deadline_force_close_applies_under_threshold() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer( + &pool, + 50, + now_epoch_secs.saturating_sub(15), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert!(current_writer_ids(&pool).await.is_empty()); +} + +#[tokio::test] +async fn reap_draining_writers_limits_closes_per_health_tick() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let close_budget = health_drain_close_budget(); + let writer_total = close_budget.saturating_add(19); + for writer_id in 1..=writer_total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(20), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + } + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert_eq!(pool.writers.read().await.len(), writer_total - close_budget); +} + +#[tokio::test] +async fn reap_draining_writers_backlog_drains_across_ticks() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let close_budget = health_drain_close_budget(); + let writer_total = close_budget.saturating_mul(2).saturating_add(7); + for writer_id in 1..=writer_total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(20), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + } + let mut warn_next_allowed = HashMap::new(); + + for _ in 0..8 { + if pool.writers.read().await.is_empty() { + break; + } + reap_draining_writers(&pool, &mut warn_next_allowed).await; + } + + assert!(pool.writers.read().await.is_empty()); +} + +#[tokio::test] +async fn reap_draining_writers_threshold_backlog_converges_to_threshold() { + let threshold = 5u64; + let pool = make_pool(threshold).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let close_budget = health_drain_close_budget(); + let writer_total = threshold as usize + close_budget.saturating_add(12); + for writer_id in 1..=writer_total as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(200).saturating_add(writer_id), + 1, + 0, + ) + .await; + } + let mut warn_next_allowed = HashMap::new(); + + for _ in 0..16 { + reap_draining_writers(&pool, &mut warn_next_allowed).await; + if pool.writers.read().await.len() <= threshold as usize { + break; + } + } + + assert_eq!(pool.writers.read().await.len(), threshold as usize); +} + +#[tokio::test] +async fn reap_draining_writers_threshold_zero_preserves_non_expired_non_empty_writers() { + let pool = make_pool(0).await; + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 10, now_epoch_secs.saturating_sub(40), 1, 0).await; + insert_draining_writer(&pool, 20, now_epoch_secs.saturating_sub(30), 1, 0).await; + insert_draining_writer(&pool, 30, now_epoch_secs.saturating_sub(20), 1, 0).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert_eq!(current_writer_ids(&pool).await, vec![10, 20, 30]); +} + +#[tokio::test] +async fn reap_draining_writers_prioritizes_force_close_before_empty_cleanup() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let close_budget = health_drain_close_budget(); + for writer_id in 1..=close_budget as u64 { + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(20), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + } + let empty_writer_id = close_budget as u64 + 1; + insert_draining_writer(&pool, empty_writer_id, now_epoch_secs.saturating_sub(20), 0, 0).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert_eq!(current_writer_ids(&pool).await, vec![empty_writer_id]); +} + +#[tokio::test] +async fn reap_draining_writers_empty_cleanup_does_not_increment_force_close_metric() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 1, now_epoch_secs.saturating_sub(60), 0, 0).await; + insert_draining_writer(&pool, 2, now_epoch_secs.saturating_sub(50), 0, 0).await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert!(current_writer_ids(&pool).await.is_empty()); + assert_eq!(pool.stats.get_pool_force_close_total(), 0); +} + +#[tokio::test] +async fn reap_draining_writers_handles_duplicate_force_close_requests_for_same_writer() { + let pool = make_pool(1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer( + &pool, + 10, + now_epoch_secs.saturating_sub(30), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + insert_draining_writer( + &pool, + 20, + now_epoch_secs.saturating_sub(20), + 1, + now_epoch_secs.saturating_sub(1), + ) + .await; + let mut warn_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed).await; + + assert!(current_writer_ids(&pool).await.is_empty()); +} + +#[tokio::test] +async fn reap_draining_writers_warn_state_never_exceeds_live_draining_population_under_churn() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let mut warn_next_allowed = HashMap::new(); + + for wave in 0..12u64 { + for offset in 0..9u64 { + insert_draining_writer( + &pool, + wave * 100 + offset, + now_epoch_secs.saturating_sub(120 + offset), + 1, + 0, + ) + .await; + } + reap_draining_writers(&pool, &mut warn_next_allowed).await; + assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); + + let existing_writer_ids = current_writer_ids(&pool).await; + for writer_id in existing_writer_ids.into_iter().take(4) { + let _ = pool.remove_writer_and_close_clients(writer_id).await; + } + reap_draining_writers(&pool, &mut warn_next_allowed).await; + assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); + } +} + +#[tokio::test] +async fn reap_draining_writers_mixed_backlog_converges_without_leaking_warn_state() { + let pool = make_pool(6).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let mut warn_next_allowed = HashMap::new(); + + for writer_id in 1..=18u64 { + let bound_clients = if writer_id % 3 == 0 { 0 } else { 1 }; + let deadline = if writer_id % 2 == 0 { + now_epoch_secs.saturating_sub(1) + } else { + 0 + }; + insert_draining_writer( + &pool, + writer_id, + now_epoch_secs.saturating_sub(300).saturating_add(writer_id), + bound_clients, + deadline, + ) + .await; + } + + for _ in 0..16 { + reap_draining_writers(&pool, &mut warn_next_allowed).await; + if pool.writers.read().await.len() <= 6 { + break; + } + } + + assert!(pool.writers.read().await.len() <= 6); + assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); +} + +#[test] +fn general_config_default_drain_threshold_remains_enabled() { + assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 128); +} diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 92e222d..590c996 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -21,6 +21,12 @@ mod secret; mod selftest; mod wire; mod pool_status; +#[cfg(test)] +mod health_regression_tests; +#[cfg(test)] +mod health_integration_tests; +#[cfg(test)] +mod health_adversarial_tests; use bytes::Bytes; From 35bca7d4cc07db95721b39bae399ba9425ec7bbf Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Tue, 17 Mar 2026 16:31:32 +0300 Subject: [PATCH 224/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9374924..dad9cf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.19" +version = "3.3.20" edition = "2024" [dependencies] From 4739083f57bf88d4e5b0074a2f8e83a9a1229f7b Mon Sep 17 00:00:00 2001 From: TEMAndroid Date: Tue, 17 Mar 2026 18:24:57 +0300 Subject: [PATCH 225/261] feat(zabbix): add graphs to Telemt template - Add per-user graph prototypes (Connections, IPs, Traffic, Messages) - Add server-level graphs (Connections, Uptime, ME Keepalive, ME Reconnects, ME Route Drops, ME Writer Pool/Removals, Desync, Upstream, Refill) --- tools/zbx_telemt_template.yaml | 257 +++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) diff --git a/tools/zbx_telemt_template.yaml b/tools/zbx_telemt_template.yaml index 27995b9..fba8549 100644 --- a/tools/zbx_telemt_template.yaml +++ b/tools/zbx_telemt_template.yaml @@ -1165,6 +1165,60 @@ zabbix_export: tags: - tag: Application value: 'Users connections' + graph_prototypes: + - uuid: 4199de3dcea943d8a1ec62dc297b2e9f + name: 'User {#TELEMT_USER}: Connections' + graph_items: + - color: 1A7C11 + item: + host: Telemt + key: 'telemt.active_conn_[{#TELEMT_USER}]' + - color: F63100 + sortorder: '1' + item: + host: Telemt + key: 'telemt.total_conn_[{#TELEMT_USER}]' + - uuid: 84b8f22d891e49768891f497cac12fb3 + name: 'User {#TELEMT_USER}: IPs' + graph_items: + - color: 0080FF + item: + host: Telemt + key: 'telemt.ips_current_[{#TELEMT_USER}]' + - color: FF8000 + sortorder: '1' + item: + host: Telemt + key: 'telemt.ips_limit_[{#TELEMT_USER}]' + - color: AA00FF + sortorder: '2' + item: + host: Telemt + key: 'telemt.ips_utilization_[{#TELEMT_USER}]' + - uuid: 09dabe7125114e36a6ce40788a7cb888 + name: 'User {#TELEMT_USER}: Traffic' + graph_items: + - color: 00AA00 + item: + host: Telemt + key: 'telemt.octets_from_[{#TELEMT_USER}]' + - color: AA0000 + sortorder: '1' + item: + host: Telemt + key: 'telemt.octets_to_[{#TELEMT_USER}]' + - uuid: 367f458962574b0ab3c02278a4cd7ecb + name: 'User {#TELEMT_USER}: Messages' + graph_items: + - color: 00AAFF + item: + host: Telemt + key: 'telemt.msgs_from_[{#TELEMT_USER}]' + - color: FF5500 + sortorder: '1' + item: + host: Telemt + key: 'telemt.msgs_to_[{#TELEMT_USER}]' master_item: key: telemt.prom_metrics lld_macro_paths: @@ -1177,3 +1231,206 @@ zabbix_export: tags: - tag: target value: Telemt + graphs: + - uuid: f162658049ca4f50893c5cc02515ff10 + name: 'Telemt: Server Connections Overview' + graph_items: + - color: 1A7C11 + item: + host: Telemt + key: telemt.conn_total + - color: F63100 + sortorder: '1' + item: + host: Telemt + key: telemt.conn_bad_total + - color: FC6EA3 + sortorder: '2' + item: + host: Telemt + key: telemt.handshake_timeouts_total + - uuid: 759eca5e687142f19248f9d9343e1adf + name: 'Telemt: Uptime' + graph_items: + - color: 0080FF + item: + host: Telemt + key: telemt.uptime + - uuid: 0a27dbd0490d4a508c03ed39fa18545d + name: 'Telemt: ME Keepalive' + graph_items: + - color: 1A7C11 + item: + host: Telemt + key: telemt.me_keepalive_sent_total + - color: 00AA00 + sortorder: '1' + item: + host: Telemt + key: telemt.me_keepalive_pong_total + - color: F63100 + sortorder: '2' + item: + host: Telemt + key: telemt.me_keepalive_failed_total + - color: FF8000 + sortorder: '3' + item: + host: Telemt + key: telemt.me_keepalive_timeout_total + - uuid: 4015e24ff70b49f484e884d1dde687c0 + name: 'Telemt: ME Reconnects' + graph_items: + - color: 0080FF + item: + host: Telemt + key: telemt.me_reconnect_attempts_total + - color: 1A7C11 + sortorder: '1' + item: + host: Telemt + key: telemt.me_reconnect_success_total + - uuid: f3e3eeb0663c471aa26cf4b6872b0c50 + name: 'Telemt: ME Route Drops' + graph_items: + - color: F63100 + item: + host: Telemt + key: telemt.me_route_drop_channel_closed_total + - color: FF8000 + sortorder: '1' + item: + host: Telemt + key: telemt.me_route_drop_no_conn_total + - color: AA00FF + sortorder: '2' + item: + host: Telemt + key: telemt.me_route_drop_queue_full_total + - uuid: 49b51ed78a5943bdbd6d1d34fe28bf61 + name: 'Telemt: ME Writer Pool' + graph_items: + - color: 0080FF + item: + host: Telemt + key: telemt.pool_drain_active + - color: F63100 + sortorder: '1' + item: + host: Telemt + key: telemt.pool_force_close_total + - color: FF8000 + sortorder: '2' + item: + host: Telemt + key: telemt.pool_stale_pick_total + - color: 1A7C11 + sortorder: '3' + item: + host: Telemt + key: telemt.pool_swap_total + - uuid: a0779e6c979f4c1ab7ac4da7123a5ecb + name: 'Telemt: ME Writer Removals and Restores' + graph_items: + - color: F63100 + item: + host: Telemt + key: telemt.me_writer_removed_total + - color: FF8000 + sortorder: '1' + item: + host: Telemt + key: telemt.me_writer_removed_unexpected_total + - color: FFAA00 + sortorder: '2' + item: + host: Telemt + key: telemt.me_writer_removed_unexpected_minus_restored_total + - color: 1A7C11 + sortorder: '3' + item: + host: Telemt + key: telemt.me_writer_restored_same_endpoint_total + - color: 00AA00 + sortorder: '4' + item: + host: Telemt + key: telemt.me_writer_restored_fallback_total + - uuid: 4fead70290664953b026a228108bee0e + name: 'Telemt: Desync Detections' + graph_items: + - color: F63100 + item: + host: Telemt + key: telemt.desync_total + - color: 1A7C11 + sortorder: '1' + item: + host: Telemt + key: telemt.desync_full_logged_total + - color: FF8000 + sortorder: '2' + item: + host: Telemt + key: telemt.desync_suppressed_total + - uuid: 9f8c9f48cb534a66ac21b1bba1acb602 + name: 'Telemt: Upstream Connect Cycles' + graph_items: + - color: 0080FF + item: + host: Telemt + key: telemt.upstream_connect_attempt_total + - color: 1A7C11 + sortorder: '1' + item: + host: Telemt + key: telemt.upstream_connect_success_total + - color: F63100 + sortorder: '2' + item: + host: Telemt + key: telemt.upstream_connect_fail_total + - color: FF8000 + sortorder: '3' + item: + host: Telemt + key: telemt.upstream_connect_failfast_hard_error_total + - uuid: 05182057727547f8b8884b7e71e34f19 + name: 'Telemt: ME Single-Endpoint Outages' + graph_items: + - color: F63100 + item: + host: Telemt + key: telemt.me_single_endpoint_outage_enter_total + - color: 1A7C11 + sortorder: '1' + item: + host: Telemt + key: telemt.me_single_endpoint_outage_exit_total + - color: 0080FF + sortorder: '2' + item: + host: Telemt + key: telemt.me_single_endpoint_outage_reconnect_attempt_total + - color: 00AA00 + sortorder: '3' + item: + host: Telemt + key: telemt.me_single_endpoint_outage_reconnect_success_total + - uuid: 6892e8b7fbd2445d9ccc0574af58a354 + name: 'Telemt: ME Refill Activity' + graph_items: + - color: 0080FF + item: + host: Telemt + key: telemt.me_refill_triggered_total + - color: F63100 + sortorder: '1' + item: + host: Telemt + key: telemt.me_refill_failed_total + - color: FF8000 + sortorder: '2' + item: + host: Telemt + key: telemt.me_refill_skipped_inflight_total From 36b360dfb6e651d0d497e70a69568110d1b37480 Mon Sep 17 00:00:00 2001 From: TEMAndroid Date: Tue, 17 Mar 2026 20:09:36 +0300 Subject: [PATCH 226/261] feat: add Telemt Control API Python simple client with CLI Stdlib-only HTTP client covering all /v1 endpoints with argparse CLI. Supports If-Match concurrency, typed errors, user CRUD, and all runtime/stats routes. Usage: ./telemt_api.py help AI-Generated from API.md. Partially tested. Use with caution... --- tools/telemt_api.py | 728 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 tools/telemt_api.py diff --git a/tools/telemt_api.py b/tools/telemt_api.py new file mode 100644 index 0000000..36ba5e1 --- /dev/null +++ b/tools/telemt_api.py @@ -0,0 +1,728 @@ +""" +Telemt Control API Python Client +Full-coverage client for https://github.com/telemt/telemt + +Usage: + client = TelemtAPI("http://127.0.0.1:9091", auth_header="your-secret") + client.health() + client.create_user("alice", max_tcp_conns=10) + client.patch_user("alice", data_quota_bytes=1_000_000_000) + client.delete_user("alice") +""" + +from __future__ import annotations + +import json +import secrets +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + +class TememtAPIError(Exception): + """Raised when the API returns an error envelope or a transport error.""" + + def __init__(self, message: str, code: str | None = None, + http_status: int | None = None, request_id: int | None = None): + super().__init__(message) + self.code = code + self.http_status = http_status + self.request_id = request_id + + def __repr__(self) -> str: + return (f"TememtAPIError(message={str(self)!r}, code={self.code!r}, " + f"http_status={self.http_status}, request_id={self.request_id})") + + +# --------------------------------------------------------------------------- +# Response wrapper +# --------------------------------------------------------------------------- + +@dataclass +class APIResponse: + """Wraps a successful API response envelope.""" + ok: bool + data: Any + revision: str | None = None + + def __repr__(self) -> str: # pragma: no cover + return f"APIResponse(ok={self.ok}, revision={self.revision!r}, data={self.data!r})" + + +# --------------------------------------------------------------------------- +# Main client +# --------------------------------------------------------------------------- + +class TememtAPI: + """ + HTTP client for the Telemt Control API. + + Parameters + ---------- + base_url: + Scheme + host + port, e.g. ``"http://127.0.0.1:9091"``. + Trailing slash is stripped automatically. + auth_header: + Exact value for the ``Authorization`` header. + Leave *None* when ``auth_header`` is not configured server-side. + timeout: + Socket timeout in seconds for every request (default 10). + """ + + def __init__( + self, + base_url: str = "http://127.0.0.1:9091", + auth_header: str | None = None, + timeout: int = 10, + ) -> None: + self.base_url = base_url.rstrip("/") + self.auth_header = auth_header + self.timeout = timeout + + # ------------------------------------------------------------------ + # Low-level HTTP helpers + # ------------------------------------------------------------------ + + def _headers(self, extra: dict | None = None) -> dict: + h = {"Content-Type": "application/json; charset=utf-8", + "Accept": "application/json"} + if self.auth_header: + h["Authorization"] = self.auth_header + if extra: + h.update(extra) + return h + + def _request( + self, + method: str, + path: str, + body: dict | None = None, + if_match: str | None = None, + query: dict | None = None, + ) -> APIResponse: + url = self.base_url + path + if query: + qs = "&".join(f"{k}={v}" for k, v in query.items()) + url = f"{url}?{qs}" + + raw_body: bytes | None = None + if body is not None: + raw_body = json.dumps(body).encode() + + extra_headers: dict = {} + if if_match is not None: + extra_headers["If-Match"] = if_match + + req = Request( + url, + data=raw_body, + headers=self._headers(extra_headers), + method=method, + ) + + try: + with urlopen(req, timeout=self.timeout) as resp: + payload = json.loads(resp.read()) + except HTTPError as exc: + raw = exc.read() + try: + payload = json.loads(raw) + except Exception: + raise TememtAPIError( + str(exc), http_status=exc.code + ) from exc + err = payload.get("error", {}) + raise TememtAPIError( + err.get("message", str(exc)), + code=err.get("code"), + http_status=exc.code, + request_id=payload.get("request_id"), + ) from exc + except URLError as exc: + raise TememtAPIError(str(exc)) from exc + + if not payload.get("ok"): + err = payload.get("error", {}) + raise TememtAPIError( + err.get("message", "unknown error"), + code=err.get("code"), + request_id=payload.get("request_id"), + ) + + return APIResponse( + ok=True, + data=payload.get("data"), + revision=payload.get("revision"), + ) + + def _get(self, path: str, query: dict | None = None) -> APIResponse: + return self._request("GET", path, query=query) + + def _post(self, path: str, body: dict | None = None, + if_match: str | None = None) -> APIResponse: + return self._request("POST", path, body=body, if_match=if_match) + + def _patch(self, path: str, body: dict, + if_match: str | None = None) -> APIResponse: + return self._request("PATCH", path, body=body, if_match=if_match) + + def _delete(self, path: str, if_match: str | None = None) -> APIResponse: + return self._request("DELETE", path, if_match=if_match) + + # ------------------------------------------------------------------ + # Health & system + # ------------------------------------------------------------------ + + def health(self) -> APIResponse: + """GET /v1/health — liveness probe.""" + return self._get("/v1/health") + + def system_info(self) -> APIResponse: + """GET /v1/system/info — binary version, uptime, config hash.""" + return self._get("/v1/system/info") + + # ------------------------------------------------------------------ + # Runtime gates & initialization + # ------------------------------------------------------------------ + + def runtime_gates(self) -> APIResponse: + """GET /v1/runtime/gates — admission gates and startup progress.""" + return self._get("/v1/runtime/gates") + + def runtime_initialization(self) -> APIResponse: + """GET /v1/runtime/initialization — detailed startup timeline.""" + return self._get("/v1/runtime/initialization") + + # ------------------------------------------------------------------ + # Limits & security + # ------------------------------------------------------------------ + + def limits_effective(self) -> APIResponse: + """GET /v1/limits/effective — effective timeout/upstream/ME limits.""" + return self._get("/v1/limits/effective") + + def security_posture(self) -> APIResponse: + """GET /v1/security/posture — API auth, telemetry, log-level summary.""" + return self._get("/v1/security/posture") + + def security_whitelist(self) -> APIResponse: + """GET /v1/security/whitelist — current IP whitelist CIDRs.""" + return self._get("/v1/security/whitelist") + + # ------------------------------------------------------------------ + # Stats + # ------------------------------------------------------------------ + + def stats_summary(self) -> APIResponse: + """GET /v1/stats/summary — uptime, connection totals, user count.""" + return self._get("/v1/stats/summary") + + def stats_zero_all(self) -> APIResponse: + """GET /v1/stats/zero/all — zero-cost counters (core, upstream, ME, pool, desync).""" + return self._get("/v1/stats/zero/all") + + def stats_upstreams(self) -> APIResponse: + """GET /v1/stats/upstreams — upstream health + zero counters.""" + return self._get("/v1/stats/upstreams") + + def stats_minimal_all(self) -> APIResponse: + """GET /v1/stats/minimal/all — ME writers + DC snapshot (requires minimal_runtime_enabled).""" + return self._get("/v1/stats/minimal/all") + + def stats_me_writers(self) -> APIResponse: + """GET /v1/stats/me-writers — per-writer ME status (requires minimal_runtime_enabled).""" + return self._get("/v1/stats/me-writers") + + def stats_dcs(self) -> APIResponse: + """GET /v1/stats/dcs — per-DC coverage and writer counts (requires minimal_runtime_enabled).""" + return self._get("/v1/stats/dcs") + + # ------------------------------------------------------------------ + # Runtime deep-dive + # ------------------------------------------------------------------ + + def runtime_me_pool_state(self) -> APIResponse: + """GET /v1/runtime/me_pool_state — ME pool generation/writer/refill snapshot.""" + return self._get("/v1/runtime/me_pool_state") + + def runtime_me_quality(self) -> APIResponse: + """GET /v1/runtime/me_quality — ME KDF, route-drop, and per-DC RTT counters.""" + return self._get("/v1/runtime/me_quality") + + def runtime_upstream_quality(self) -> APIResponse: + """GET /v1/runtime/upstream_quality — per-upstream health, latency, DC preferences.""" + return self._get("/v1/runtime/upstream_quality") + + def runtime_nat_stun(self) -> APIResponse: + """GET /v1/runtime/nat_stun — NAT probe state, STUN servers, reflected IPs.""" + return self._get("/v1/runtime/nat_stun") + + def runtime_me_selftest(self) -> APIResponse: + """GET /v1/runtime/me-selftest — KDF/timeskew/IP/PID/BND health state.""" + return self._get("/v1/runtime/me-selftest") + + def runtime_connections_summary(self) -> APIResponse: + """GET /v1/runtime/connections/summary — live connection totals + top-N users (requires runtime_edge_enabled).""" + return self._get("/v1/runtime/connections/summary") + + def runtime_events_recent(self, limit: int | None = None) -> APIResponse: + """GET /v1/runtime/events/recent — recent ring-buffer events (requires runtime_edge_enabled). + + Parameters + ---------- + limit: + Optional cap on returned events (1–1000, server default 50). + """ + query = {"limit": str(limit)} if limit is not None else None + return self._get("/v1/runtime/events/recent", query=query) + + # ------------------------------------------------------------------ + # Users (read) + # ------------------------------------------------------------------ + + def list_users(self) -> APIResponse: + """GET /v1/users — list all users with connection/traffic info.""" + return self._get("/v1/users") + + def get_user(self, username: str) -> APIResponse: + """GET /v1/users/{username} — single user info.""" + return self._get(f"/v1/users/{_safe(username)}") + + # ------------------------------------------------------------------ + # Users (write) + # ------------------------------------------------------------------ + + def create_user( + self, + username: str, + *, + secret: str | None = None, + user_ad_tag: str | None = None, + max_tcp_conns: int | None = None, + expiration_rfc3339: str | None = None, + data_quota_bytes: int | None = None, + max_unique_ips: int | None = None, + if_match: str | None = None, + ) -> APIResponse: + """POST /v1/users — create a new user. + + Parameters + ---------- + username: + ``[A-Za-z0-9_.-]``, length 1–64. + secret: + Exactly 32 hex chars. Auto-generated if omitted. + user_ad_tag: + Exactly 32 hex chars. + max_tcp_conns: + Per-user concurrent TCP limit. + expiration_rfc3339: + RFC3339 expiration timestamp, e.g. ``"2025-12-31T23:59:59Z"``. + data_quota_bytes: + Per-user traffic quota in bytes. + max_unique_ips: + Per-user unique source IP limit. + if_match: + Optional ``If-Match`` revision for optimistic concurrency. + """ + body: Dict[str, Any] = {"username": username} + _opt(body, "secret", secret) + _opt(body, "user_ad_tag", user_ad_tag) + _opt(body, "max_tcp_conns", max_tcp_conns) + _opt(body, "expiration_rfc3339", expiration_rfc3339) + _opt(body, "data_quota_bytes", data_quota_bytes) + _opt(body, "max_unique_ips", max_unique_ips) + return self._post("/v1/users", body=body, if_match=if_match) + + def patch_user( + self, + username: str, + *, + secret: str | None = None, + user_ad_tag: str | None = None, + max_tcp_conns: int | None = None, + expiration_rfc3339: str | None = None, + data_quota_bytes: int | None = None, + max_unique_ips: int | None = None, + if_match: str | None = None, + ) -> APIResponse: + """PATCH /v1/users/{username} — partial update; only provided fields change. + + Parameters + ---------- + username: + Existing username to update. + secret: + New secret (32 hex chars). + user_ad_tag: + New ad tag (32 hex chars). + max_tcp_conns: + New TCP concurrency limit. + expiration_rfc3339: + New expiration timestamp. + data_quota_bytes: + New quota in bytes. + max_unique_ips: + New unique IP limit. + if_match: + Optional ``If-Match`` revision. + """ + body: Dict[str, Any] = {} + _opt(body, "secret", secret) + _opt(body, "user_ad_tag", user_ad_tag) + _opt(body, "max_tcp_conns", max_tcp_conns) + _opt(body, "expiration_rfc3339", expiration_rfc3339) + _opt(body, "data_quota_bytes", data_quota_bytes) + _opt(body, "max_unique_ips", max_unique_ips) + if not body: + raise ValueError("patch_user: at least one field must be provided") + return self._patch(f"/v1/users/{_safe(username)}", body=body, + if_match=if_match) + + def delete_user( + self, + username: str, + *, + if_match: str | None = None, + ) -> APIResponse: + """DELETE /v1/users/{username} — remove user; blocks deletion of last user. + + Parameters + ---------- + if_match: + Optional ``If-Match`` revision for optimistic concurrency. + """ + return self._delete(f"/v1/users/{_safe(username)}", if_match=if_match) + + # NOTE: POST /v1/users/{username}/rotate-secret currently returns 404 + # in the route matcher (documented limitation). The method is provided + # for completeness and future compatibility. + def rotate_secret( + self, + username: str, + *, + secret: str | None = None, + if_match: str | None = None, + ) -> APIResponse: + """POST /v1/users/{username}/rotate-secret — rotate user secret. + + .. warning:: + This endpoint currently returns ``404 not_found`` in all released + versions (documented route matcher limitation). The method is + included for future compatibility. + + Parameters + ---------- + secret: + New secret (32 hex chars). Auto-generated if omitted. + """ + body: Dict[str, Any] = {} + _opt(body, "secret", secret) + return self._post(f"/v1/users/{_safe(username)}/rotate-secret", + body=body or None, if_match=if_match) + + # ------------------------------------------------------------------ + # Convenience helpers + # ------------------------------------------------------------------ + + @staticmethod + def generate_secret() -> str: + """Generate a random 32-character hex secret suitable for user creation.""" + return secrets.token_hex(16) # 16 bytes → 32 hex chars + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _safe(username: str) -> str: + """Minimal guard: reject obvious path-injection attempts.""" + if "/" in username or "\\" in username: + raise ValueError(f"Invalid username: {username!r}") + return username + + +def _opt(d: dict, key: str, value: Any) -> None: + """Add key to dict only when value is not None.""" + if value is not None: + d[key] = value + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _print(resp: APIResponse) -> None: + print(json.dumps(resp.data, indent=2)) + if resp.revision: + print(f"# revision: {resp.revision}", flush=True) + + +def _build_parser(): + import argparse + + p = argparse.ArgumentParser( + prog="telemt_api.py", + description="Telemt Control API CLI", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +COMMANDS (read) + health Liveness check + info System info (version, uptime, config hash) + status Runtime gates + startup progress + init Runtime initialization timeline + limits Effective limits (timeouts, upstream, ME) + posture Security posture summary + whitelist IP whitelist entries + summary Stats summary (conns, uptime, users) + zero Zero-cost counters (core/upstream/ME/pool/desync) + upstreams Upstream health + zero counters + minimal ME writers + DC snapshot [minimal_runtime_enabled] + me-writers Per-writer ME status [minimal_runtime_enabled] + dcs Per-DC coverage [minimal_runtime_enabled] + me-pool ME pool generation/writer/refill snapshot + me-quality ME KDF, route-drops, per-DC RTT + upstream-quality Per-upstream health + latency + nat-stun NAT probe state + STUN servers + me-selftest KDF/timeskew/IP/PID/BND health + connections Live connection totals + top-N [runtime_edge_enabled] + events [--limit N] Recent ring-buffer events [runtime_edge_enabled] + +COMMANDS (users) + users List all users + user Get single user + create [OPTIONS] Create user + patch [OPTIONS] Partial update user + delete Delete user + secret [--secret S] Rotate secret (reserved; returns 404 in current release) + gen-secret Print a random 32-hex secret and exit + +USER OPTIONS (for create / patch) + --secret S 32 hex chars + --ad-tag S 32 hex chars (ad tag) + --max-conns N Max concurrent TCP connections + --expires DATETIME RFC3339 expiration (e.g. 2026-12-31T23:59:59Z) + --quota N Data quota in bytes + --max-ips N Max unique source IPs + +EXAMPLES + telemt_api.py health + telemt_api.py -u http://10.0.0.1:9091 -a mysecret users + telemt_api.py create alice --max-conns 5 --quota 10000000000 + telemt_api.py patch alice --expires 2027-01-01T00:00:00Z + telemt_api.py delete alice + telemt_api.py events --limit 20 + """, + ) + + p.add_argument("-u", "--url", default="http://127.0.0.1:9091", + metavar="URL", help="API base URL (default: http://127.0.0.1:9091)") + p.add_argument("-a", "--auth", default=None, metavar="TOKEN", + help="Authorization header value") + p.add_argument("-t", "--timeout", type=int, default=10, metavar="SEC", + help="Request timeout in seconds (default: 10)") + + p.add_argument("command", nargs="?", default="help", + help="Command to run (see COMMANDS below)") + p.add_argument("arg", nargs="?", default=None, metavar="USERNAME", + help="Username for user commands") + + # user create/patch fields + p.add_argument("--secret", default=None) + p.add_argument("--ad-tag", dest="ad_tag", default=None) + p.add_argument("--max-conns", dest="max_conns", type=int, default=None) + p.add_argument("--expires", default=None) + p.add_argument("--quota", type=int, default=None) + p.add_argument("--max-ips", dest="max_ips", type=int, default=None) + + # events + p.add_argument("--limit", type=int, default=None, + help="Max events for `events` command") + + # optimistic concurrency + p.add_argument("--if-match", dest="if_match", default=None, + metavar="REVISION", help="If-Match revision header") + + return p + + +if __name__ == "__main__": + import sys + + parser = _build_parser() + args = parser.parse_args() + + cmd = (args.command or "help").lower() + + if cmd in ("help", "--help"): + parser.print_help() + sys.exit(0) + + if cmd == "gen-secret": + print(TememtAPI.generate_secret()) + sys.exit(0) + + api = TememtAPI(args.url, auth_header=args.auth, timeout=args.timeout) + + try: + # -- read endpoints -------------------------------------------------- + if cmd == "health": + _print(api.health()) + + elif cmd == "info": + _print(api.system_info()) + + elif cmd == "status": + _print(api.runtime_gates()) + + elif cmd == "init": + _print(api.runtime_initialization()) + + elif cmd == "limits": + _print(api.limits_effective()) + + elif cmd == "posture": + _print(api.security_posture()) + + elif cmd == "whitelist": + _print(api.security_whitelist()) + + elif cmd == "summary": + _print(api.stats_summary()) + + elif cmd == "zero": + _print(api.stats_zero_all()) + + elif cmd == "upstreams": + _print(api.stats_upstreams()) + + elif cmd == "minimal": + _print(api.stats_minimal_all()) + + elif cmd == "me-writers": + _print(api.stats_me_writers()) + + elif cmd == "dcs": + _print(api.stats_dcs()) + + elif cmd == "me-pool": + _print(api.runtime_me_pool_state()) + + elif cmd == "me-quality": + _print(api.runtime_me_quality()) + + elif cmd == "upstream-quality": + _print(api.runtime_upstream_quality()) + + elif cmd == "nat-stun": + _print(api.runtime_nat_stun()) + + elif cmd == "me-selftest": + _print(api.runtime_me_selftest()) + + elif cmd == "connections": + _print(api.runtime_connections_summary()) + + elif cmd == "events": + _print(api.runtime_events_recent(limit=args.limit)) + + # -- user read ------------------------------------------------------- + elif cmd == "users": + resp = api.list_users() + users = resp.data or [] + if not users: + print("No users configured.") + else: + fmt = "{:<24} {:>7} {:>14} {}" + print(fmt.format("USERNAME", "CONNS", "OCTETS", "LINKS")) + print("-" * 72) + for u in users: + links = (u.get("links") or {}) + all_links = (links.get("classic") or []) + \ + (links.get("secure") or []) + \ + (links.get("tls") or []) + link_str = all_links[0] if all_links else "-" + print(fmt.format( + u["username"], + u.get("current_connections", 0), + u.get("total_octets", 0), + link_str, + )) + if resp.revision: + print(f"# revision: {resp.revision}") + + elif cmd == "user": + if not args.arg: + parser.error("user command requires ") + _print(api.get_user(args.arg)) + + # -- user write ------------------------------------------------------ + elif cmd == "create": + if not args.arg: + parser.error("create command requires ") + resp = api.create_user( + args.arg, + secret=args.secret, + user_ad_tag=args.ad_tag, + max_tcp_conns=args.max_conns, + expiration_rfc3339=args.expires, + data_quota_bytes=args.quota, + max_unique_ips=args.max_ips, + if_match=args.if_match, + ) + d = resp.data or {} + print(f"Created: {d.get('user', {}).get('username')}") + print(f"Secret: {d.get('secret')}") + links = (d.get("user") or {}).get("links") or {} + for kind, lst in links.items(): + for link in (lst or []): + print(f"Link ({kind}): {link}") + if resp.revision: + print(f"# revision: {resp.revision}") + + elif cmd == "patch": + if not args.arg: + parser.error("patch command requires ") + if not any([args.secret, args.ad_tag, args.max_conns, + args.expires, args.quota, args.max_ips]): + parser.error("patch requires at least one field (--secret, --max-conns, --expires, --quota, --max-ips, --ad-tag)") + _print(api.patch_user( + args.arg, + secret=args.secret, + user_ad_tag=args.ad_tag, + max_tcp_conns=args.max_conns, + expiration_rfc3339=args.expires, + data_quota_bytes=args.quota, + max_unique_ips=args.max_ips, + if_match=args.if_match, + )) + + elif cmd == "delete": + if not args.arg: + parser.error("delete command requires ") + resp = api.delete_user(args.arg, if_match=args.if_match) + print(f"Deleted: {resp.data}") + if resp.revision: + print(f"# revision: {resp.revision}") + + elif cmd == "secret": + if not args.arg: + parser.error("secret command requires ") + _print(api.rotate_secret(args.arg, secret=args.secret, + if_match=args.if_match)) + + else: + print(f"Unknown command: {cmd!r}\nRun with 'help' to see available commands.", + file=sys.stderr) + sys.exit(1) + + except TememtAPIError as exc: + print(f"API error [{exc.http_status}] {exc.code}: {exc}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(130) From 3739f38440b63d81005a621f0884eef535046f69 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:49:02 +0300 Subject: [PATCH 227/261] Adaptive Buffers + Session Eviction Method --- src/api/model.rs | 7 + src/api/runtime_stats.rs | 7 + src/config/defaults.rs | 33 +- src/config/hot_reload.rs | 42 ++ src/config/load.rs | 29 ++ src/config/types.rs | 27 ++ src/maestro/me_startup.rs | 5 + src/maestro/mod.rs | 2 +- src/metrics.rs | 151 +++++++ src/proxy/adaptive_buffers.rs | 383 ++++++++++++++++++ src/proxy/client.rs | 15 + src/proxy/direct_relay.rs | 17 +- src/proxy/middle_relay.rs | 38 +- src/proxy/mod.rs | 2 + src/proxy/relay.rs | 148 ++++++- src/proxy/session_eviction.rs | 46 +++ src/stats/mod.rs | 74 ++++ src/stream/buffer_pool.rs | 3 +- src/transport/middle_proxy/config_updater.rs | 10 + src/transport/middle_proxy/health.rs | 117 +++++- .../middle_proxy/health_adversarial_tests.rs | 27 +- .../middle_proxy/health_integration_tests.rs | 5 + .../middle_proxy/health_regression_tests.rs | 103 ++++- src/transport/middle_proxy/pool.rs | 68 ++++ src/transport/middle_proxy/pool_status.rs | 21 + src/transport/middle_proxy/registry.rs | 133 ++++++ src/transport/socket.rs | 16 +- 27 files changed, 1479 insertions(+), 50 deletions(-) create mode 100644 src/proxy/adaptive_buffers.rs create mode 100644 src/proxy/session_eviction.rs diff --git a/src/api/model.rs b/src/api/model.rs index 31233d7..6b6fd72 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -195,6 +195,8 @@ pub(super) struct ZeroPoolData { pub(super) pool_swap_total: u64, pub(super) pool_drain_active: u64, pub(super) pool_force_close_total: u64, + pub(super) pool_drain_soft_evict_total: u64, + pub(super) pool_drain_soft_evict_writer_total: u64, pub(super) pool_stale_pick_total: u64, pub(super) writer_removed_total: u64, pub(super) writer_removed_unexpected_total: u64, @@ -360,6 +362,11 @@ pub(super) struct MinimalMeRuntimeData { pub(super) me_reconnect_backoff_cap_ms: u64, pub(super) me_reconnect_fast_retry_count: u32, pub(super) me_pool_drain_ttl_secs: u64, + pub(super) me_pool_drain_soft_evict_enabled: bool, + pub(super) me_pool_drain_soft_evict_grace_secs: u64, + pub(super) me_pool_drain_soft_evict_per_writer: u8, + pub(super) me_pool_drain_soft_evict_budget_per_core: u16, + pub(super) me_pool_drain_soft_evict_cooldown_ms: u64, pub(super) me_pool_force_close_secs: u64, pub(super) me_pool_min_fresh_ratio: f32, pub(super) me_bind_stale_mode: &'static str, diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 9260c40..61c8a5a 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -96,6 +96,8 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer pool_swap_total: stats.get_pool_swap_total(), pool_drain_active: stats.get_pool_drain_active(), pool_force_close_total: stats.get_pool_force_close_total(), + pool_drain_soft_evict_total: stats.get_pool_drain_soft_evict_total(), + pool_drain_soft_evict_writer_total: stats.get_pool_drain_soft_evict_writer_total(), pool_stale_pick_total: stats.get_pool_stale_pick_total(), writer_removed_total: stats.get_me_writer_removed_total(), writer_removed_unexpected_total: stats.get_me_writer_removed_unexpected_total(), @@ -427,6 +429,11 @@ async fn get_minimal_payload_cached( me_reconnect_backoff_cap_ms: runtime.me_reconnect_backoff_cap_ms, me_reconnect_fast_retry_count: runtime.me_reconnect_fast_retry_count, me_pool_drain_ttl_secs: runtime.me_pool_drain_ttl_secs, + me_pool_drain_soft_evict_enabled: runtime.me_pool_drain_soft_evict_enabled, + me_pool_drain_soft_evict_grace_secs: runtime.me_pool_drain_soft_evict_grace_secs, + me_pool_drain_soft_evict_per_writer: runtime.me_pool_drain_soft_evict_per_writer, + me_pool_drain_soft_evict_budget_per_core: runtime.me_pool_drain_soft_evict_budget_per_core, + me_pool_drain_soft_evict_cooldown_ms: runtime.me_pool_drain_soft_evict_cooldown_ms, me_pool_force_close_secs: runtime.me_pool_force_close_secs, me_pool_min_fresh_ratio: runtime.me_pool_min_fresh_ratio, me_bind_stale_mode: runtime.me_bind_stale_mode, diff --git a/src/config/defaults.rs b/src/config/defaults.rs index ea9250d..7b5b4a8 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -27,8 +27,8 @@ const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024; const DEFAULT_ME_READER_ROUTE_DATA_WAIT_MS: u64 = 2; const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_FRAMES: usize = 32; const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_BYTES: usize = 128 * 1024; -const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_DELAY_US: u64 = 1500; -const DEFAULT_ME_D2C_ACK_FLUSH_IMMEDIATE: bool = false; +const DEFAULT_ME_D2C_FLUSH_BATCH_MAX_DELAY_US: u64 = 500; +const DEFAULT_ME_D2C_ACK_FLUSH_IMMEDIATE: bool = true; const DEFAULT_DIRECT_RELAY_COPY_BUF_C2S_BYTES: usize = 64 * 1024; const DEFAULT_DIRECT_RELAY_COPY_BUF_S2C_BYTES: usize = 256 * 1024; const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3; @@ -36,6 +36,11 @@ const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000; const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000; const DEFAULT_ME_ADMISSION_POLL_MS: u64 = 1000; const DEFAULT_ME_WARN_RATE_LIMIT_MS: u64 = 5000; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED: bool = true; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS: u64 = 30; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER: u8 = 1; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE: u16 = 8; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 5000; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; @@ -85,11 +90,11 @@ pub(crate) fn default_connect_timeout() -> u64 { } pub(crate) fn default_keepalive() -> u64 { - 60 + 15 } pub(crate) fn default_ack_timeout() -> u64 { - 300 + 90 } pub(crate) fn default_me_one_retry() -> u8 { 12 @@ -592,6 +597,26 @@ pub(crate) fn default_me_pool_drain_threshold() -> u64 { 128 } +pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool { + DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED +} + +pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 { + DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS +} + +pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 { + DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER +} + +pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 { + DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE +} + +pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 { + DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS +} + pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 { default_me_pool_drain_ttl_secs() } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 6f07a4b..c0ca98d 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -56,6 +56,11 @@ pub struct HotFields { pub hardswap: bool, pub me_pool_drain_ttl_secs: u64, pub me_pool_drain_threshold: u64, + pub me_pool_drain_soft_evict_enabled: bool, + pub me_pool_drain_soft_evict_grace_secs: u64, + pub me_pool_drain_soft_evict_per_writer: u8, + pub me_pool_drain_soft_evict_budget_per_core: u16, + pub me_pool_drain_soft_evict_cooldown_ms: u64, pub me_pool_min_fresh_ratio: f32, pub me_reinit_drain_timeout_secs: u64, pub me_hardswap_warmup_delay_min_ms: u64, @@ -138,6 +143,15 @@ impl HotFields { hardswap: cfg.general.hardswap, me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs, me_pool_drain_threshold: cfg.general.me_pool_drain_threshold, + me_pool_drain_soft_evict_enabled: cfg.general.me_pool_drain_soft_evict_enabled, + me_pool_drain_soft_evict_grace_secs: cfg.general.me_pool_drain_soft_evict_grace_secs, + me_pool_drain_soft_evict_per_writer: cfg.general.me_pool_drain_soft_evict_per_writer, + me_pool_drain_soft_evict_budget_per_core: cfg + .general + .me_pool_drain_soft_evict_budget_per_core, + me_pool_drain_soft_evict_cooldown_ms: cfg + .general + .me_pool_drain_soft_evict_cooldown_ms, me_pool_min_fresh_ratio: cfg.general.me_pool_min_fresh_ratio, me_reinit_drain_timeout_secs: cfg.general.me_reinit_drain_timeout_secs, me_hardswap_warmup_delay_min_ms: cfg.general.me_hardswap_warmup_delay_min_ms, @@ -455,6 +469,15 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.general.hardswap = new.general.hardswap; cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs; cfg.general.me_pool_drain_threshold = new.general.me_pool_drain_threshold; + cfg.general.me_pool_drain_soft_evict_enabled = new.general.me_pool_drain_soft_evict_enabled; + cfg.general.me_pool_drain_soft_evict_grace_secs = + new.general.me_pool_drain_soft_evict_grace_secs; + cfg.general.me_pool_drain_soft_evict_per_writer = + new.general.me_pool_drain_soft_evict_per_writer; + cfg.general.me_pool_drain_soft_evict_budget_per_core = + new.general.me_pool_drain_soft_evict_budget_per_core; + cfg.general.me_pool_drain_soft_evict_cooldown_ms = + new.general.me_pool_drain_soft_evict_cooldown_ms; cfg.general.me_pool_min_fresh_ratio = new.general.me_pool_min_fresh_ratio; cfg.general.me_reinit_drain_timeout_secs = new.general.me_reinit_drain_timeout_secs; cfg.general.me_hardswap_warmup_delay_min_ms = new.general.me_hardswap_warmup_delay_min_ms; @@ -835,6 +858,25 @@ fn log_changes( old_hot.me_pool_drain_threshold, new_hot.me_pool_drain_threshold, ); } + if old_hot.me_pool_drain_soft_evict_enabled != new_hot.me_pool_drain_soft_evict_enabled + || old_hot.me_pool_drain_soft_evict_grace_secs + != new_hot.me_pool_drain_soft_evict_grace_secs + || old_hot.me_pool_drain_soft_evict_per_writer + != new_hot.me_pool_drain_soft_evict_per_writer + || old_hot.me_pool_drain_soft_evict_budget_per_core + != new_hot.me_pool_drain_soft_evict_budget_per_core + || old_hot.me_pool_drain_soft_evict_cooldown_ms + != new_hot.me_pool_drain_soft_evict_cooldown_ms + { + info!( + "config reload: me_pool_drain_soft_evict: enabled={} grace={}s per_writer={} budget_per_core={} cooldown={}ms", + new_hot.me_pool_drain_soft_evict_enabled, + new_hot.me_pool_drain_soft_evict_grace_secs, + new_hot.me_pool_drain_soft_evict_per_writer, + new_hot.me_pool_drain_soft_evict_budget_per_core, + new_hot.me_pool_drain_soft_evict_cooldown_ms + ); + } if (old_hot.me_pool_min_fresh_ratio - new_hot.me_pool_min_fresh_ratio).abs() > f32::EPSILON { info!( diff --git a/src/config/load.rs b/src/config/load.rs index ed3e303..6fcbea3 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -406,6 +406,35 @@ impl ProxyConfig { )); } + if config.general.me_pool_drain_soft_evict_grace_secs > 3600 { + return Err(ProxyError::Config( + "general.me_pool_drain_soft_evict_grace_secs must be within [0, 3600]".to_string(), + )); + } + + if config.general.me_pool_drain_soft_evict_per_writer == 0 + || config.general.me_pool_drain_soft_evict_per_writer > 16 + { + return Err(ProxyError::Config( + "general.me_pool_drain_soft_evict_per_writer must be within [1, 16]".to_string(), + )); + } + + if config.general.me_pool_drain_soft_evict_budget_per_core == 0 + || config.general.me_pool_drain_soft_evict_budget_per_core > 64 + { + return Err(ProxyError::Config( + "general.me_pool_drain_soft_evict_budget_per_core must be within [1, 64]" + .to_string(), + )); + } + + if config.general.me_pool_drain_soft_evict_cooldown_ms == 0 { + return Err(ProxyError::Config( + "general.me_pool_drain_soft_evict_cooldown_ms must be > 0".to_string(), + )); + } + if config.access.user_max_unique_ips_window_secs == 0 { return Err(ProxyError::Config( "access.user_max_unique_ips_window_secs must be > 0".to_string(), diff --git a/src/config/types.rs b/src/config/types.rs index 7ea1fe7..e507044 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -803,6 +803,26 @@ pub struct GeneralConfig { #[serde(default = "default_me_pool_drain_threshold")] pub me_pool_drain_threshold: u64, + /// Enable staged client eviction for draining ME writers that remain non-empty past TTL. + #[serde(default = "default_me_pool_drain_soft_evict_enabled")] + pub me_pool_drain_soft_evict_enabled: bool, + + /// Extra grace in seconds after drain TTL before soft-eviction stage starts. + #[serde(default = "default_me_pool_drain_soft_evict_grace_secs")] + pub me_pool_drain_soft_evict_grace_secs: u64, + + /// Maximum number of client sessions to evict from one draining writer per health tick. + #[serde(default = "default_me_pool_drain_soft_evict_per_writer")] + pub me_pool_drain_soft_evict_per_writer: u8, + + /// Soft-eviction budget per CPU core for one health tick. + #[serde(default = "default_me_pool_drain_soft_evict_budget_per_core")] + pub me_pool_drain_soft_evict_budget_per_core: u16, + + /// Cooldown for repetitive soft-eviction on the same writer in milliseconds. + #[serde(default = "default_me_pool_drain_soft_evict_cooldown_ms")] + pub me_pool_drain_soft_evict_cooldown_ms: u64, + /// Policy for new binds on stale draining writers. #[serde(default)] pub me_bind_stale_mode: MeBindStaleMode, @@ -984,6 +1004,13 @@ impl Default for GeneralConfig { proxy_secret_len_max: default_proxy_secret_len_max(), me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(), me_pool_drain_threshold: default_me_pool_drain_threshold(), + me_pool_drain_soft_evict_enabled: default_me_pool_drain_soft_evict_enabled(), + me_pool_drain_soft_evict_grace_secs: default_me_pool_drain_soft_evict_grace_secs(), + me_pool_drain_soft_evict_per_writer: default_me_pool_drain_soft_evict_per_writer(), + me_pool_drain_soft_evict_budget_per_core: + default_me_pool_drain_soft_evict_budget_per_core(), + me_pool_drain_soft_evict_cooldown_ms: + default_me_pool_drain_soft_evict_cooldown_ms(), me_bind_stale_mode: MeBindStaleMode::default(), me_bind_stale_ttl_secs: default_me_bind_stale_ttl_secs(), me_pool_min_fresh_ratio: default_me_pool_min_fresh_ratio(), diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs index 245c7a9..94ae884 100644 --- a/src/maestro/me_startup.rs +++ b/src/maestro/me_startup.rs @@ -238,6 +238,11 @@ pub(crate) async fn initialize_me_pool( config.general.hardswap, config.general.me_pool_drain_ttl_secs, config.general.me_pool_drain_threshold, + config.general.me_pool_drain_soft_evict_enabled, + config.general.me_pool_drain_soft_evict_grace_secs, + config.general.me_pool_drain_soft_evict_per_writer, + config.general.me_pool_drain_soft_evict_budget_per_core, + config.general.me_pool_drain_soft_evict_cooldown_ms, config.general.effective_me_pool_force_close_secs(), config.general.me_pool_min_fresh_ratio, config.general.me_hardswap_warmup_delay_min_ms, diff --git a/src/maestro/mod.rs b/src/maestro/mod.rs index da00b40..dce421c 100644 --- a/src/maestro/mod.rs +++ b/src/maestro/mod.rs @@ -476,7 +476,7 @@ pub async fn run() -> std::result::Result<(), Box> { Duration::from_secs(config.access.replay_window_secs), )); - let buffer_pool = Arc::new(BufferPool::with_config(16 * 1024, 4096)); + let buffer_pool = Arc::new(BufferPool::with_config(64 * 1024, 4096)); connectivity::run_startup_connectivity( &config, diff --git a/src/metrics.rs b/src/metrics.rs index f4f8a2e..3de9896 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -292,6 +292,109 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp "telemt_connections_bad_total {}", if core_enabled { stats.get_connects_bad() } else { 0 } ); + let _ = writeln!(out, "# HELP telemt_connections_current Current active connections"); + let _ = writeln!(out, "# TYPE telemt_connections_current gauge"); + let _ = writeln!( + out, + "telemt_connections_current {}", + if core_enabled { + stats.get_current_connections_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_connections_direct_current Current active direct connections"); + let _ = writeln!(out, "# TYPE telemt_connections_direct_current gauge"); + let _ = writeln!( + out, + "telemt_connections_direct_current {}", + if core_enabled { + stats.get_current_connections_direct() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_connections_me_current Current active middle-end connections"); + let _ = writeln!(out, "# TYPE telemt_connections_me_current gauge"); + let _ = writeln!( + out, + "telemt_connections_me_current {}", + if core_enabled { + stats.get_current_connections_me() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_relay_adaptive_promotions_total Adaptive relay tier promotions" + ); + let _ = writeln!(out, "# TYPE telemt_relay_adaptive_promotions_total counter"); + let _ = writeln!( + out, + "telemt_relay_adaptive_promotions_total {}", + if core_enabled { + stats.get_relay_adaptive_promotions_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_relay_adaptive_demotions_total Adaptive relay tier demotions" + ); + let _ = writeln!(out, "# TYPE telemt_relay_adaptive_demotions_total counter"); + let _ = writeln!( + out, + "telemt_relay_adaptive_demotions_total {}", + if core_enabled { + stats.get_relay_adaptive_demotions_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_relay_adaptive_hard_promotions_total Adaptive relay hard promotions triggered by write pressure" + ); + let _ = writeln!( + out, + "# TYPE telemt_relay_adaptive_hard_promotions_total counter" + ); + let _ = writeln!( + out, + "telemt_relay_adaptive_hard_promotions_total {}", + if core_enabled { + stats.get_relay_adaptive_hard_promotions_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_reconnect_evict_total Reconnect-driven session evictions"); + let _ = writeln!(out, "# TYPE telemt_reconnect_evict_total counter"); + let _ = writeln!( + out, + "telemt_reconnect_evict_total {}", + if core_enabled { + stats.get_reconnect_evict_total() + } else { + 0 + } + ); + let _ = writeln!( + out, + "# HELP telemt_reconnect_stale_close_total Sessions closed because they became stale after reconnect" + ); + let _ = writeln!(out, "# TYPE telemt_reconnect_stale_close_total counter"); + let _ = writeln!( + out, + "telemt_reconnect_stale_close_total {}", + if core_enabled { + stats.get_reconnect_stale_close_total() + } else { + 0 + } + ); let _ = writeln!(out, "# HELP telemt_handshake_timeouts_total Handshake timeouts"); let _ = writeln!(out, "# TYPE telemt_handshake_timeouts_total counter"); @@ -1547,6 +1650,36 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_pool_drain_soft_evict_total Soft-evicted client sessions on stuck draining writers" + ); + let _ = writeln!(out, "# TYPE telemt_pool_drain_soft_evict_total counter"); + let _ = writeln!( + out, + "telemt_pool_drain_soft_evict_total {}", + if me_allows_normal { + stats.get_pool_drain_soft_evict_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_pool_drain_soft_evict_writer_total Draining writers with at least one soft eviction" + ); + let _ = writeln!(out, "# TYPE telemt_pool_drain_soft_evict_writer_total counter"); + let _ = writeln!( + out, + "telemt_pool_drain_soft_evict_writer_total {}", + if me_allows_normal { + stats.get_pool_drain_soft_evict_writer_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_pool_stale_pick_total Stale writer fallback picks for new binds"); let _ = writeln!(out, "# TYPE telemt_pool_stale_pick_total counter"); let _ = writeln!( @@ -1864,6 +1997,8 @@ mod tests { stats.increment_connects_all(); stats.increment_connects_all(); stats.increment_connects_bad(); + stats.increment_current_connections_direct(); + stats.increment_current_connections_me(); stats.increment_handshake_timeouts(); stats.increment_upstream_connect_attempt_total(); stats.increment_upstream_connect_attempt_total(); @@ -1895,6 +2030,9 @@ mod tests { assert!(output.contains("telemt_connections_total 2")); assert!(output.contains("telemt_connections_bad_total 1")); + assert!(output.contains("telemt_connections_current 2")); + assert!(output.contains("telemt_connections_direct_current 1")); + assert!(output.contains("telemt_connections_me_current 1")); assert!(output.contains("telemt_handshake_timeouts_total 1")); assert!(output.contains("telemt_upstream_connect_attempt_total 2")); assert!(output.contains("telemt_upstream_connect_success_total 1")); @@ -1937,6 +2075,9 @@ mod tests { let output = render_metrics(&stats, &config, &tracker).await; assert!(output.contains("telemt_connections_total 0")); assert!(output.contains("telemt_connections_bad_total 0")); + assert!(output.contains("telemt_connections_current 0")); + assert!(output.contains("telemt_connections_direct_current 0")); + assert!(output.contains("telemt_connections_me_current 0")); assert!(output.contains("telemt_handshake_timeouts_total 0")); assert!(output.contains("telemt_user_unique_ips_current{user=")); assert!(output.contains("telemt_user_unique_ips_recent_window{user=")); @@ -1970,11 +2111,21 @@ mod tests { assert!(output.contains("# TYPE telemt_uptime_seconds gauge")); assert!(output.contains("# TYPE telemt_connections_total counter")); assert!(output.contains("# TYPE telemt_connections_bad_total counter")); + assert!(output.contains("# TYPE telemt_connections_current gauge")); + assert!(output.contains("# TYPE telemt_connections_direct_current gauge")); + assert!(output.contains("# TYPE telemt_connections_me_current gauge")); + assert!(output.contains("# TYPE telemt_relay_adaptive_promotions_total counter")); + assert!(output.contains("# TYPE telemt_relay_adaptive_demotions_total counter")); + assert!(output.contains("# TYPE telemt_relay_adaptive_hard_promotions_total counter")); + assert!(output.contains("# TYPE telemt_reconnect_evict_total counter")); + assert!(output.contains("# TYPE telemt_reconnect_stale_close_total counter")); assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter")); assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter")); assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter")); assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter")); assert!(output.contains("# TYPE telemt_me_writer_removed_total counter")); + assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_total counter")); + assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_writer_total counter")); assert!(output.contains( "# TYPE telemt_me_writer_removed_unexpected_minus_restored_total gauge" )); diff --git a/src/proxy/adaptive_buffers.rs b/src/proxy/adaptive_buffers.rs new file mode 100644 index 0000000..3b1bce9 --- /dev/null +++ b/src/proxy/adaptive_buffers.rs @@ -0,0 +1,383 @@ +use dashmap::DashMap; +use std::cmp::max; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +const EMA_ALPHA: f64 = 0.2; +const PROFILE_TTL: Duration = Duration::from_secs(300); +const THROUGHPUT_UP_BPS: f64 = 8_000_000.0; +const THROUGHPUT_DOWN_BPS: f64 = 2_000_000.0; +const RATIO_CONFIRM_THRESHOLD: f64 = 1.12; +const TIER1_HOLD_TICKS: u32 = 8; +const TIER2_HOLD_TICKS: u32 = 4; +const QUIET_DEMOTE_TICKS: u32 = 480; +const HARD_COOLDOWN_TICKS: u32 = 20; +const HARD_PENDING_THRESHOLD: u32 = 3; +const HARD_PARTIAL_RATIO_THRESHOLD: f64 = 0.25; +const DIRECT_C2S_CAP_BYTES: usize = 128 * 1024; +const DIRECT_S2C_CAP_BYTES: usize = 512 * 1024; +const ME_FRAMES_CAP: usize = 96; +const ME_BYTES_CAP: usize = 384 * 1024; +const ME_DELAY_MIN_US: u64 = 150; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum AdaptiveTier { + Base = 0, + Tier1 = 1, + Tier2 = 2, + Tier3 = 3, +} + +impl AdaptiveTier { + pub fn promote(self) -> Self { + match self { + Self::Base => Self::Tier1, + Self::Tier1 => Self::Tier2, + Self::Tier2 => Self::Tier3, + Self::Tier3 => Self::Tier3, + } + } + + pub fn demote(self) -> Self { + match self { + Self::Base => Self::Base, + Self::Tier1 => Self::Base, + Self::Tier2 => Self::Tier1, + Self::Tier3 => Self::Tier2, + } + } + + fn ratio(self) -> (usize, usize) { + match self { + Self::Base => (1, 1), + Self::Tier1 => (5, 4), + Self::Tier2 => (3, 2), + Self::Tier3 => (2, 1), + } + } + + pub fn as_u8(self) -> u8 { + self as u8 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TierTransitionReason { + SoftConfirmed, + HardPressure, + QuietDemotion, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TierTransition { + pub from: AdaptiveTier, + pub to: AdaptiveTier, + pub reason: TierTransitionReason, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct RelaySignalSample { + pub c2s_bytes: u64, + pub s2c_requested_bytes: u64, + pub s2c_written_bytes: u64, + pub s2c_write_ops: u64, + pub s2c_partial_writes: u64, + pub s2c_consecutive_pending_writes: u32, +} + +#[derive(Debug, Clone, Copy)] +pub struct SessionAdaptiveController { + tier: AdaptiveTier, + max_tier_seen: AdaptiveTier, + throughput_ema_bps: f64, + incoming_ema_bps: f64, + outgoing_ema_bps: f64, + tier1_hold_ticks: u32, + tier2_hold_ticks: u32, + quiet_ticks: u32, + hard_cooldown_ticks: u32, +} + +impl SessionAdaptiveController { + pub fn new(initial_tier: AdaptiveTier) -> Self { + Self { + tier: initial_tier, + max_tier_seen: initial_tier, + throughput_ema_bps: 0.0, + incoming_ema_bps: 0.0, + outgoing_ema_bps: 0.0, + tier1_hold_ticks: 0, + tier2_hold_ticks: 0, + quiet_ticks: 0, + hard_cooldown_ticks: 0, + } + } + + pub fn max_tier_seen(&self) -> AdaptiveTier { + self.max_tier_seen + } + + pub fn observe(&mut self, sample: RelaySignalSample, tick_secs: f64) -> Option { + if tick_secs <= f64::EPSILON { + return None; + } + + if self.hard_cooldown_ticks > 0 { + self.hard_cooldown_ticks -= 1; + } + + let c2s_bps = (sample.c2s_bytes as f64 * 8.0) / tick_secs; + let incoming_bps = (sample.s2c_requested_bytes as f64 * 8.0) / tick_secs; + let outgoing_bps = (sample.s2c_written_bytes as f64 * 8.0) / tick_secs; + let throughput = c2s_bps.max(outgoing_bps); + + self.throughput_ema_bps = ema(self.throughput_ema_bps, throughput); + self.incoming_ema_bps = ema(self.incoming_ema_bps, incoming_bps); + self.outgoing_ema_bps = ema(self.outgoing_ema_bps, outgoing_bps); + + let tier1_now = self.throughput_ema_bps >= THROUGHPUT_UP_BPS; + if tier1_now { + self.tier1_hold_ticks = self.tier1_hold_ticks.saturating_add(1); + } else { + self.tier1_hold_ticks = 0; + } + + let ratio = if self.outgoing_ema_bps <= f64::EPSILON { + 0.0 + } else { + self.incoming_ema_bps / self.outgoing_ema_bps + }; + let tier2_now = ratio >= RATIO_CONFIRM_THRESHOLD; + if tier2_now { + self.tier2_hold_ticks = self.tier2_hold_ticks.saturating_add(1); + } else { + self.tier2_hold_ticks = 0; + } + + let partial_ratio = if sample.s2c_write_ops == 0 { + 0.0 + } else { + sample.s2c_partial_writes as f64 / sample.s2c_write_ops as f64 + }; + let hard_now = sample.s2c_consecutive_pending_writes >= HARD_PENDING_THRESHOLD + || partial_ratio >= HARD_PARTIAL_RATIO_THRESHOLD; + + if hard_now && self.hard_cooldown_ticks == 0 { + return self.promote(TierTransitionReason::HardPressure, HARD_COOLDOWN_TICKS); + } + + if self.tier1_hold_ticks >= TIER1_HOLD_TICKS && self.tier2_hold_ticks >= TIER2_HOLD_TICKS { + return self.promote(TierTransitionReason::SoftConfirmed, 0); + } + + let demote_candidate = self.throughput_ema_bps < THROUGHPUT_DOWN_BPS && !tier2_now && !hard_now; + if demote_candidate { + self.quiet_ticks = self.quiet_ticks.saturating_add(1); + if self.quiet_ticks >= QUIET_DEMOTE_TICKS { + self.quiet_ticks = 0; + return self.demote(TierTransitionReason::QuietDemotion); + } + } else { + self.quiet_ticks = 0; + } + + None + } + + fn promote( + &mut self, + reason: TierTransitionReason, + hard_cooldown_ticks: u32, + ) -> Option { + let from = self.tier; + let to = from.promote(); + if from == to { + return None; + } + self.tier = to; + self.max_tier_seen = max(self.max_tier_seen, to); + self.hard_cooldown_ticks = hard_cooldown_ticks; + self.tier1_hold_ticks = 0; + self.tier2_hold_ticks = 0; + self.quiet_ticks = 0; + Some(TierTransition { from, to, reason }) + } + + fn demote(&mut self, reason: TierTransitionReason) -> Option { + let from = self.tier; + let to = from.demote(); + if from == to { + return None; + } + self.tier = to; + self.tier1_hold_ticks = 0; + self.tier2_hold_ticks = 0; + Some(TierTransition { from, to, reason }) + } +} + +#[derive(Debug, Clone, Copy)] +struct UserAdaptiveProfile { + tier: AdaptiveTier, + seen_at: Instant, +} + +fn profiles() -> &'static DashMap { + static USER_PROFILES: OnceLock> = OnceLock::new(); + USER_PROFILES.get_or_init(DashMap::new) +} + +pub fn seed_tier_for_user(user: &str) -> AdaptiveTier { + let now = Instant::now(); + if let Some(entry) = profiles().get(user) { + let value = entry.value(); + if now.duration_since(value.seen_at) <= PROFILE_TTL { + return value.tier; + } + } + AdaptiveTier::Base +} + +pub fn record_user_tier(user: &str, tier: AdaptiveTier) { + let now = Instant::now(); + if let Some(mut entry) = profiles().get_mut(user) { + let existing = *entry; + let effective = if now.duration_since(existing.seen_at) > PROFILE_TTL { + tier + } else { + max(existing.tier, tier) + }; + *entry = UserAdaptiveProfile { + tier: effective, + seen_at: now, + }; + return; + } + profiles().insert( + user.to_string(), + UserAdaptiveProfile { tier, seen_at: now }, + ); +} + +pub fn direct_copy_buffers_for_tier( + tier: AdaptiveTier, + base_c2s: usize, + base_s2c: usize, +) -> (usize, usize) { + let (num, den) = tier.ratio(); + ( + scale(base_c2s, num, den, DIRECT_C2S_CAP_BYTES), + scale(base_s2c, num, den, DIRECT_S2C_CAP_BYTES), + ) +} + +pub fn me_flush_policy_for_tier( + tier: AdaptiveTier, + base_frames: usize, + base_bytes: usize, + base_delay: Duration, +) -> (usize, usize, Duration) { + let (num, den) = tier.ratio(); + let frames = scale(base_frames, num, den, ME_FRAMES_CAP).max(1); + let bytes = scale(base_bytes, num, den, ME_BYTES_CAP).max(4096); + let delay_us = base_delay.as_micros() as u64; + let adjusted_delay_us = match tier { + AdaptiveTier::Base => delay_us, + AdaptiveTier::Tier1 => (delay_us.saturating_mul(7)).saturating_div(10), + AdaptiveTier::Tier2 => delay_us.saturating_div(2), + AdaptiveTier::Tier3 => (delay_us.saturating_mul(3)).saturating_div(10), + } + .max(ME_DELAY_MIN_US) + .min(delay_us.max(ME_DELAY_MIN_US)); + (frames, bytes, Duration::from_micros(adjusted_delay_us)) +} + +fn ema(prev: f64, value: f64) -> f64 { + if prev <= f64::EPSILON { + value + } else { + (prev * (1.0 - EMA_ALPHA)) + (value * EMA_ALPHA) + } +} + +fn scale(base: usize, numerator: usize, denominator: usize, cap: usize) -> usize { + let scaled = base + .saturating_mul(numerator) + .saturating_div(denominator.max(1)); + scaled.min(cap).max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample( + c2s_bytes: u64, + s2c_requested_bytes: u64, + s2c_written_bytes: u64, + s2c_write_ops: u64, + s2c_partial_writes: u64, + s2c_consecutive_pending_writes: u32, + ) -> RelaySignalSample { + RelaySignalSample { + c2s_bytes, + s2c_requested_bytes, + s2c_written_bytes, + s2c_write_ops, + s2c_partial_writes, + s2c_consecutive_pending_writes, + } + } + + #[test] + fn test_soft_promotion_requires_tier1_and_tier2() { + let mut ctrl = SessionAdaptiveController::new(AdaptiveTier::Base); + let tick_secs = 0.25; + let mut promoted = None; + for _ in 0..8 { + promoted = ctrl.observe( + sample( + 300_000, // ~9.6 Mbps + 320_000, // incoming > outgoing to confirm tier2 + 250_000, + 10, + 0, + 0, + ), + tick_secs, + ); + } + + let transition = promoted.expect("expected soft promotion"); + assert_eq!(transition.from, AdaptiveTier::Base); + assert_eq!(transition.to, AdaptiveTier::Tier1); + assert_eq!(transition.reason, TierTransitionReason::SoftConfirmed); + } + + #[test] + fn test_hard_promotion_on_pending_pressure() { + let mut ctrl = SessionAdaptiveController::new(AdaptiveTier::Base); + let transition = ctrl + .observe( + sample(10_000, 20_000, 10_000, 4, 1, 3), + 0.25, + ) + .expect("expected hard promotion"); + assert_eq!(transition.reason, TierTransitionReason::HardPressure); + assert_eq!(transition.to, AdaptiveTier::Tier1); + } + + #[test] + fn test_quiet_demotion_is_slow_and_stepwise() { + let mut ctrl = SessionAdaptiveController::new(AdaptiveTier::Tier2); + let mut demotion = None; + for _ in 0..QUIET_DEMOTE_TICKS { + demotion = ctrl.observe(sample(1, 1, 1, 1, 0, 0), 0.25); + } + + let transition = demotion.expect("expected quiet demotion"); + assert_eq!(transition.from, AdaptiveTier::Tier2); + assert_eq!(transition.to, AdaptiveTier::Tier1); + assert_eq!(transition.reason, TierTransitionReason::QuietDemotion); + } +} diff --git a/src/proxy/client.rs b/src/proxy/client.rs index 99e6837..25e6cf9 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -40,6 +40,7 @@ use crate::proxy::handshake::{HandshakeSuccess, handle_mtproto_handshake, handle use crate::proxy::masking::handle_bad_client; use crate::proxy::middle_relay::handle_via_middle_proxy; use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController}; +use crate::proxy::session_eviction::register_session; fn beobachten_ttl(config: &ProxyConfig) -> Duration { Duration::from_secs(config.general.beobachten_minutes.saturating_mul(60)) @@ -731,6 +732,17 @@ impl RunningClientHandler { return Err(e); } + let registration = register_session(&user, success.dc_idx); + if registration.replaced_existing { + stats.increment_reconnect_evict_total(); + warn!( + user = %user, + dc = success.dc_idx, + "Reconnect detected: replacing active session for user+dc" + ); + } + let session_lease = registration.lease; + let route_snapshot = route_runtime.snapshot(); let session_id = rng.u64(); let relay_result = if config.general.use_middle_proxy @@ -750,6 +762,7 @@ impl RunningClientHandler { route_runtime.subscribe(), route_snapshot, session_id, + session_lease.clone(), ) .await } else { @@ -766,6 +779,7 @@ impl RunningClientHandler { route_runtime.subscribe(), route_snapshot, session_id, + session_lease.clone(), ) .await } @@ -783,6 +797,7 @@ impl RunningClientHandler { route_runtime.subscribe(), route_snapshot, session_id, + session_lease.clone(), ) .await }; diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index 7a7810a..b7a1fbf 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -18,6 +18,8 @@ use crate::proxy::route_mode::{ RelayRouteMode, RouteCutoverState, ROUTE_SWITCH_ERROR_MSG, affected_cutover_state, cutover_stagger_delay, }; +use crate::proxy::adaptive_buffers; +use crate::proxy::session_eviction::SessionLease; use crate::stats::Stats; use crate::stream::{BufferPool, CryptoReader, CryptoWriter}; use crate::transport::UpstreamManager; @@ -34,6 +36,7 @@ pub(crate) async fn handle_via_direct( mut route_rx: watch::Receiver, route_snapshot: RouteCutoverState, session_id: u64, + session_lease: SessionLease, ) -> Result<()> where R: AsyncRead + Unpin + Send + 'static, @@ -67,16 +70,26 @@ where stats.increment_user_curr_connects(user); stats.increment_current_connections_direct(); + let seed_tier = adaptive_buffers::seed_tier_for_user(user); + let (c2s_copy_buf, s2c_copy_buf) = adaptive_buffers::direct_copy_buffers_for_tier( + seed_tier, + config.general.direct_relay_copy_buf_c2s_bytes, + config.general.direct_relay_copy_buf_s2c_bytes, + ); + let relay_result = relay_bidirectional( client_reader, client_writer, tg_reader, tg_writer, - config.general.direct_relay_copy_buf_c2s_bytes, - config.general.direct_relay_copy_buf_s2c_bytes, + c2s_copy_buf, + s2c_copy_buf, user, + success.dc_idx, Arc::clone(&stats), buffer_pool, + session_lease, + seed_tier, ); tokio::pin!(relay_result); let relay_result = loop { diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index aaae1b3..4f70a17 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -20,6 +20,8 @@ use crate::proxy::route_mode::{ RelayRouteMode, RouteCutoverState, ROUTE_SWITCH_ERROR_MSG, affected_cutover_state, cutover_stagger_delay, }; +use crate::proxy::adaptive_buffers::{self, AdaptiveTier}; +use crate::proxy::session_eviction::SessionLease; use crate::stats::Stats; use crate::stream::{BufferPool, CryptoReader, CryptoWriter}; use crate::transport::middle_proxy::{MePool, MeResponse, proto_flags_for_tag}; @@ -59,8 +61,8 @@ struct MeD2cFlushPolicy { } impl MeD2cFlushPolicy { - fn from_config(config: &ProxyConfig) -> Self { - Self { + fn from_config(config: &ProxyConfig, tier: AdaptiveTier) -> Self { + let base = Self { max_frames: config .general .me_d2c_flush_batch_max_frames @@ -71,6 +73,18 @@ impl MeD2cFlushPolicy { .max(ME_D2C_FLUSH_BATCH_MAX_BYTES_MIN), max_delay: Duration::from_micros(config.general.me_d2c_flush_batch_max_delay_us), ack_flush_immediate: config.general.me_d2c_ack_flush_immediate, + }; + let (max_frames, max_bytes, max_delay) = adaptive_buffers::me_flush_policy_for_tier( + tier, + base.max_frames, + base.max_bytes, + base.max_delay, + ); + Self { + max_frames, + max_bytes, + max_delay, + ack_flush_immediate: base.ack_flush_immediate, } } } @@ -235,6 +249,7 @@ pub(crate) async fn handle_via_middle_proxy( mut route_rx: watch::Receiver, route_snapshot: RouteCutoverState, session_id: u64, + session_lease: SessionLease, ) -> Result<()> where R: AsyncRead + Unpin + Send + 'static, @@ -244,6 +259,7 @@ where let peer = success.peer; let proto_tag = success.proto_tag; let pool_generation = me_pool.current_generation(); + let seed_tier = adaptive_buffers::seed_tier_for_user(&user); debug!( user = %user, @@ -295,6 +311,15 @@ where return Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string())); } + if session_lease.is_stale() { + stats.increment_reconnect_stale_close_total(); + let _ = me_pool.send_close(conn_id).await; + me_pool.registry().unregister(conn_id).await; + stats.decrement_current_connections_me(); + stats.decrement_user_curr_connects(&user); + return Err(ProxyError::Proxy("Session evicted by reconnect".to_string())); + } + // Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable) let user_tag: Option> = config .access @@ -368,7 +393,7 @@ where let rng_clone = rng.clone(); let user_clone = user.clone(); let bytes_me2c_clone = bytes_me2c.clone(); - let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config); + let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config, seed_tier); let me_writer = tokio::spawn(async move { let mut writer = crypto_writer; let mut frame_buf = Vec::with_capacity(16 * 1024); @@ -528,6 +553,12 @@ where let mut frame_counter: u64 = 0; let mut route_watch_open = true; loop { + if session_lease.is_stale() { + stats.increment_reconnect_stale_close_total(); + let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; + main_result = Err(ProxyError::Proxy("Session evicted by reconnect".to_string())); + break; + } if let Some(cutover) = affected_cutover_state( &route_rx, RelayRouteMode::Middle, @@ -636,6 +667,7 @@ where frames_ok = frame_counter, "ME relay cleanup" ); + adaptive_buffers::record_user_tier(&user, seed_tier); me_pool.registry().unregister(conn_id).await; stats.decrement_current_connections_me(); stats.decrement_user_curr_connects(&user); diff --git a/src/proxy/mod.rs b/src/proxy/mod.rs index 1eed469..ab840f6 100644 --- a/src/proxy/mod.rs +++ b/src/proxy/mod.rs @@ -1,5 +1,6 @@ //! Proxy Defs +pub mod adaptive_buffers; pub mod client; pub mod direct_relay; pub mod handshake; @@ -7,6 +8,7 @@ pub mod masking; pub mod middle_relay; pub mod route_mode; pub mod relay; +pub mod session_eviction; pub use client::ClientHandler; #[allow(unused_imports)] diff --git a/src/proxy/relay.rs b/src/proxy/relay.rs index 06ce0d8..2b12d5a 100644 --- a/src/proxy/relay.rs +++ b/src/proxy/relay.rs @@ -63,6 +63,10 @@ use tokio::io::{ use tokio::time::Instant; use tracing::{debug, trace, warn}; use crate::error::Result; +use crate::proxy::adaptive_buffers::{ + self, AdaptiveTier, RelaySignalSample, SessionAdaptiveController, TierTransitionReason, +}; +use crate::proxy::session_eviction::SessionLease; use crate::stats::Stats; use crate::stream::BufferPool; @@ -79,6 +83,7 @@ const ACTIVITY_TIMEOUT: Duration = Duration::from_secs(1800); /// 10 seconds gives responsive timeout detection (±10s accuracy) /// without measurable overhead from atomic reads. const WATCHDOG_INTERVAL: Duration = Duration::from_secs(10); +const ADAPTIVE_TICK: Duration = Duration::from_millis(250); // ============= CombinedStream ============= @@ -155,6 +160,16 @@ struct SharedCounters { s2c_ops: AtomicU64, /// Milliseconds since relay epoch of last I/O activity last_activity_ms: AtomicU64, + /// Bytes requested to write to client (S→C direction). + s2c_requested_bytes: AtomicU64, + /// Total write operations for S→C direction. + s2c_write_ops: AtomicU64, + /// Number of partial writes to client. + s2c_partial_writes: AtomicU64, + /// Number of times S→C poll_write returned Pending. + s2c_pending_writes: AtomicU64, + /// Consecutive pending writes in S→C direction. + s2c_consecutive_pending_writes: AtomicU64, } impl SharedCounters { @@ -165,6 +180,11 @@ impl SharedCounters { c2s_ops: AtomicU64::new(0), s2c_ops: AtomicU64::new(0), last_activity_ms: AtomicU64::new(0), + s2c_requested_bytes: AtomicU64::new(0), + s2c_write_ops: AtomicU64::new(0), + s2c_partial_writes: AtomicU64::new(0), + s2c_pending_writes: AtomicU64::new(0), + s2c_consecutive_pending_writes: AtomicU64::new(0), } } @@ -259,9 +279,21 @@ impl AsyncWrite for StatsIo { buf: &[u8], ) -> Poll> { let this = self.get_mut(); + this.counters + .s2c_requested_bytes + .fetch_add(buf.len() as u64, Ordering::Relaxed); match Pin::new(&mut this.inner).poll_write(cx, buf) { Poll::Ready(Ok(n)) => { + this.counters.s2c_write_ops.fetch_add(1, Ordering::Relaxed); + this.counters + .s2c_consecutive_pending_writes + .store(0, Ordering::Relaxed); + if n < buf.len() { + this.counters + .s2c_partial_writes + .fetch_add(1, Ordering::Relaxed); + } if n > 0 { // S→C: data written to client this.counters.s2c_bytes.fetch_add(n as u64, Ordering::Relaxed); @@ -275,6 +307,15 @@ impl AsyncWrite for StatsIo { } Poll::Ready(Ok(n)) } + Poll::Pending => { + this.counters + .s2c_pending_writes + .fetch_add(1, Ordering::Relaxed); + this.counters + .s2c_consecutive_pending_writes + .fetch_add(1, Ordering::Relaxed); + Poll::Pending + } other => other, } } @@ -316,8 +357,11 @@ pub async fn relay_bidirectional( c2s_buf_size: usize, s2c_buf_size: usize, user: &str, + dc_idx: i16, stats: Arc, _buffer_pool: Arc, + session_lease: SessionLease, + seed_tier: AdaptiveTier, ) -> Result<()> where CR: AsyncRead + Unpin + Send + 'static, @@ -345,13 +389,33 @@ where // ── Watchdog: activity timeout + periodic rate logging ────────── let wd_counters = Arc::clone(&counters); let wd_user = user_owned.clone(); + let wd_dc = dc_idx; + let wd_stats = Arc::clone(&stats); + let wd_session = session_lease.clone(); let watchdog = async { - let mut prev_c2s: u64 = 0; - let mut prev_s2c: u64 = 0; + let mut prev_c2s_log: u64 = 0; + let mut prev_s2c_log: u64 = 0; + let mut prev_c2s_sample: u64 = 0; + let mut prev_s2c_requested_sample: u64 = 0; + let mut prev_s2c_written_sample: u64 = 0; + let mut prev_s2c_write_ops_sample: u64 = 0; + let mut prev_s2c_partial_sample: u64 = 0; + let mut accumulated_log = Duration::ZERO; + let mut adaptive = SessionAdaptiveController::new(seed_tier); loop { - tokio::time::sleep(WATCHDOG_INTERVAL).await; + tokio::time::sleep(ADAPTIVE_TICK).await; + + if wd_session.is_stale() { + wd_stats.increment_reconnect_stale_close_total(); + warn!( + user = %wd_user, + dc = wd_dc, + "Session evicted by reconnect" + ); + return; + } let now = Instant::now(); let idle = wd_counters.idle_duration(now, epoch); @@ -370,11 +434,80 @@ where return; // Causes select! to cancel copy_bidirectional } + let c2s_total = wd_counters.c2s_bytes.load(Ordering::Relaxed); + let s2c_requested_total = wd_counters + .s2c_requested_bytes + .load(Ordering::Relaxed); + let s2c_written_total = wd_counters.s2c_bytes.load(Ordering::Relaxed); + let s2c_write_ops_total = wd_counters + .s2c_write_ops + .load(Ordering::Relaxed); + let s2c_partial_total = wd_counters + .s2c_partial_writes + .load(Ordering::Relaxed); + let consecutive_pending = wd_counters + .s2c_consecutive_pending_writes + .load(Ordering::Relaxed) as u32; + + let sample = RelaySignalSample { + c2s_bytes: c2s_total.saturating_sub(prev_c2s_sample), + s2c_requested_bytes: s2c_requested_total + .saturating_sub(prev_s2c_requested_sample), + s2c_written_bytes: s2c_written_total + .saturating_sub(prev_s2c_written_sample), + s2c_write_ops: s2c_write_ops_total + .saturating_sub(prev_s2c_write_ops_sample), + s2c_partial_writes: s2c_partial_total + .saturating_sub(prev_s2c_partial_sample), + s2c_consecutive_pending_writes: consecutive_pending, + }; + + if let Some(transition) = adaptive.observe(sample, ADAPTIVE_TICK.as_secs_f64()) { + match transition.reason { + TierTransitionReason::SoftConfirmed => { + wd_stats.increment_relay_adaptive_promotions_total(); + } + TierTransitionReason::HardPressure => { + wd_stats.increment_relay_adaptive_promotions_total(); + wd_stats.increment_relay_adaptive_hard_promotions_total(); + } + TierTransitionReason::QuietDemotion => { + wd_stats.increment_relay_adaptive_demotions_total(); + } + } + adaptive_buffers::record_user_tier(&wd_user, adaptive.max_tier_seen()); + debug!( + user = %wd_user, + dc = wd_dc, + from_tier = transition.from.as_u8(), + to_tier = transition.to.as_u8(), + reason = ?transition.reason, + throughput_ema_bps = sample + .c2s_bytes + .max(sample.s2c_written_bytes) + .saturating_mul(8) + .saturating_mul(4), + "Adaptive relay tier transition" + ); + } + + prev_c2s_sample = c2s_total; + prev_s2c_requested_sample = s2c_requested_total; + prev_s2c_written_sample = s2c_written_total; + prev_s2c_write_ops_sample = s2c_write_ops_total; + prev_s2c_partial_sample = s2c_partial_total; + + accumulated_log = accumulated_log.saturating_add(ADAPTIVE_TICK); + if accumulated_log < WATCHDOG_INTERVAL { + continue; + } + accumulated_log = Duration::ZERO; + // ── Periodic rate logging ─────────────────────────────── let c2s = wd_counters.c2s_bytes.load(Ordering::Relaxed); let s2c = wd_counters.s2c_bytes.load(Ordering::Relaxed); - let c2s_delta = c2s - prev_c2s; - let s2c_delta = s2c - prev_s2c; + let c2s_delta = c2s.saturating_sub(prev_c2s_log); + let s2c_delta = s2c.saturating_sub(prev_s2c_log); if c2s_delta > 0 || s2c_delta > 0 { let secs = WATCHDOG_INTERVAL.as_secs_f64(); @@ -388,8 +521,8 @@ where ); } - prev_c2s = c2s; - prev_s2c = s2c; + prev_c2s_log = c2s; + prev_s2c_log = s2c; } }; @@ -424,6 +557,7 @@ where let c2s_ops = counters.c2s_ops.load(Ordering::Relaxed); let s2c_ops = counters.s2c_ops.load(Ordering::Relaxed); let duration = epoch.elapsed(); + adaptive_buffers::record_user_tier(&user_owned, seed_tier); match copy_result { Some(Ok((c2s, s2c))) => { diff --git a/src/proxy/session_eviction.rs b/src/proxy/session_eviction.rs new file mode 100644 index 0000000..c735cae --- /dev/null +++ b/src/proxy/session_eviction.rs @@ -0,0 +1,46 @@ +/// Session eviction is intentionally disabled in runtime. +/// +/// The initial `user+dc` single-lease model caused valid parallel client +/// connections to evict each other. Keep the API shape for compatibility, +/// but make it a no-op until a safer policy is introduced. + +#[derive(Debug, Clone, Default)] +pub struct SessionLease; + +impl SessionLease { + pub fn is_stale(&self) -> bool { + false + } + + #[allow(dead_code)] + pub fn release(&self) {} +} + +pub struct RegistrationResult { + pub lease: SessionLease, + pub replaced_existing: bool, +} + +pub fn register_session(_user: &str, _dc_idx: i16) -> RegistrationResult { + RegistrationResult { + lease: SessionLease, + replaced_existing: false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_session_eviction_disabled_behavior() { + let first = register_session("alice", 2); + let second = register_session("alice", 2); + assert!(!first.replaced_existing); + assert!(!second.replaced_existing); + assert!(!first.lease.is_stale()); + assert!(!second.lease.is_stale()); + first.lease.release(); + second.lease.release(); + } +} diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 25905b2..83cd03d 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -120,6 +120,8 @@ pub struct Stats { pool_swap_total: AtomicU64, pool_drain_active: AtomicU64, pool_force_close_total: AtomicU64, + pool_drain_soft_evict_total: AtomicU64, + pool_drain_soft_evict_writer_total: AtomicU64, pool_stale_pick_total: AtomicU64, me_writer_removed_total: AtomicU64, me_writer_removed_unexpected_total: AtomicU64, @@ -133,6 +135,11 @@ pub struct Stats { me_inline_recovery_total: AtomicU64, ip_reservation_rollback_tcp_limit_total: AtomicU64, ip_reservation_rollback_quota_limit_total: AtomicU64, + relay_adaptive_promotions_total: AtomicU64, + relay_adaptive_demotions_total: AtomicU64, + relay_adaptive_hard_promotions_total: AtomicU64, + reconnect_evict_total: AtomicU64, + reconnect_stale_close_total: AtomicU64, telemetry_core_enabled: AtomicBool, telemetry_user_enabled: AtomicBool, telemetry_me_level: AtomicU8, @@ -285,6 +292,36 @@ impl Stats { pub fn decrement_current_connections_me(&self) { Self::decrement_atomic_saturating(&self.current_connections_me); } + pub fn increment_relay_adaptive_promotions_total(&self) { + if self.telemetry_core_enabled() { + self.relay_adaptive_promotions_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_relay_adaptive_demotions_total(&self) { + if self.telemetry_core_enabled() { + self.relay_adaptive_demotions_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_relay_adaptive_hard_promotions_total(&self) { + if self.telemetry_core_enabled() { + self.relay_adaptive_hard_promotions_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_reconnect_evict_total(&self) { + if self.telemetry_core_enabled() { + self.reconnect_evict_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_reconnect_stale_close_total(&self) { + if self.telemetry_core_enabled() { + self.reconnect_stale_close_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_handshake_timeouts(&self) { if self.telemetry_core_enabled() { self.handshake_timeouts.fetch_add(1, Ordering::Relaxed); @@ -680,6 +717,18 @@ impl Stats { self.pool_force_close_total.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_pool_drain_soft_evict_total(&self) { + if self.telemetry_me_allows_normal() { + self.pool_drain_soft_evict_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_pool_drain_soft_evict_writer_total(&self) { + if self.telemetry_me_allows_normal() { + self.pool_drain_soft_evict_writer_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_pool_stale_pick_total(&self) { if self.telemetry_me_allows_normal() { self.pool_stale_pick_total.fetch_add(1, Ordering::Relaxed); @@ -933,6 +982,22 @@ impl Stats { self.get_current_connections_direct() .saturating_add(self.get_current_connections_me()) } + pub fn get_relay_adaptive_promotions_total(&self) -> u64 { + self.relay_adaptive_promotions_total.load(Ordering::Relaxed) + } + pub fn get_relay_adaptive_demotions_total(&self) -> u64 { + self.relay_adaptive_demotions_total.load(Ordering::Relaxed) + } + pub fn get_relay_adaptive_hard_promotions_total(&self) -> u64 { + self.relay_adaptive_hard_promotions_total + .load(Ordering::Relaxed) + } + pub fn get_reconnect_evict_total(&self) -> u64 { + self.reconnect_evict_total.load(Ordering::Relaxed) + } + pub fn get_reconnect_stale_close_total(&self) -> u64 { + self.reconnect_stale_close_total.load(Ordering::Relaxed) + } pub fn get_me_keepalive_sent(&self) -> u64 { self.me_keepalive_sent.load(Ordering::Relaxed) } pub fn get_me_keepalive_failed(&self) -> u64 { self.me_keepalive_failed.load(Ordering::Relaxed) } pub fn get_me_keepalive_pong(&self) -> u64 { self.me_keepalive_pong.load(Ordering::Relaxed) } @@ -1185,6 +1250,12 @@ impl Stats { pub fn get_pool_force_close_total(&self) -> u64 { self.pool_force_close_total.load(Ordering::Relaxed) } + pub fn get_pool_drain_soft_evict_total(&self) -> u64 { + self.pool_drain_soft_evict_total.load(Ordering::Relaxed) + } + pub fn get_pool_drain_soft_evict_writer_total(&self) -> u64 { + self.pool_drain_soft_evict_writer_total.load(Ordering::Relaxed) + } pub fn get_pool_stale_pick_total(&self) -> u64 { self.pool_stale_pick_total.load(Ordering::Relaxed) } @@ -1258,6 +1329,9 @@ impl Stats { } pub fn decrement_user_curr_connects(&self, user: &str) { + if !self.telemetry_user_enabled() { + return; + } self.maybe_cleanup_user_stats(); if let Some(stats) = self.user_stats.get(user) { Self::touch_user_stats(stats.value()); diff --git a/src/stream/buffer_pool.rs b/src/stream/buffer_pool.rs index 9c46922..dac0fb5 100644 --- a/src/stream/buffer_pool.rs +++ b/src/stream/buffer_pool.rs @@ -14,8 +14,7 @@ use std::sync::Arc; // ============= Configuration ============= /// Default buffer size -/// CHANGED: Reduced from 64KB to 16KB to match TLS record size and prevent bufferbloat. -pub const DEFAULT_BUFFER_SIZE: usize = 16 * 1024; +pub const DEFAULT_BUFFER_SIZE: usize = 64 * 1024; /// Default maximum number of pooled buffers pub const DEFAULT_MAX_BUFFERS: usize = 1024; diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index b6a0160..43a3569 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -299,6 +299,11 @@ async fn run_update_cycle( cfg.general.hardswap, cfg.general.me_pool_drain_ttl_secs, cfg.general.me_pool_drain_threshold, + cfg.general.me_pool_drain_soft_evict_enabled, + cfg.general.me_pool_drain_soft_evict_grace_secs, + cfg.general.me_pool_drain_soft_evict_per_writer, + cfg.general.me_pool_drain_soft_evict_budget_per_core, + cfg.general.me_pool_drain_soft_evict_cooldown_ms, cfg.general.effective_me_pool_force_close_secs(), cfg.general.me_pool_min_fresh_ratio, cfg.general.me_hardswap_warmup_delay_min_ms, @@ -526,6 +531,11 @@ pub async fn me_config_updater( cfg.general.hardswap, cfg.general.me_pool_drain_ttl_secs, cfg.general.me_pool_drain_threshold, + cfg.general.me_pool_drain_soft_evict_enabled, + cfg.general.me_pool_drain_soft_evict_grace_secs, + cfg.general.me_pool_drain_soft_evict_per_writer, + cfg.general.me_pool_drain_soft_evict_budget_per_core, + cfg.general.me_pool_drain_soft_evict_cooldown_ms, cfg.general.effective_me_pool_force_close_secs(), cfg.general.me_pool_min_fresh_ratio, cfg.general.me_hardswap_warmup_delay_min_ms, diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 8ac6839..862e58a 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -28,6 +28,8 @@ const HEALTH_RECONNECT_BUDGET_MAX: usize = 128; const HEALTH_DRAIN_CLOSE_BUDGET_PER_CORE: usize = 16; const HEALTH_DRAIN_CLOSE_BUDGET_MIN: usize = 16; const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256; +const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN: usize = 8; +const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX: usize = 256; #[derive(Debug, Clone)] struct DcFloorPlanEntry { @@ -66,6 +68,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c let mut adaptive_recover_until: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut drain_warn_next_allowed: HashMap = HashMap::new(); + let mut drain_soft_evict_next_allowed: HashMap = HashMap::new(); let mut degraded_interval = true; loop { let interval = if degraded_interval { @@ -75,7 +78,12 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c }; tokio::time::sleep(interval).await; pool.prune_closed_writers().await; - reap_draining_writers(&pool, &mut drain_warn_next_allowed).await; + reap_draining_writers( + &pool, + &mut drain_warn_next_allowed, + &mut drain_soft_evict_next_allowed, + ) + .await; let v4_degraded = check_family( IpFamily::V4, &pool, @@ -117,6 +125,7 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c pub(super) async fn reap_draining_writers( pool: &Arc, warn_next_allowed: &mut HashMap, + soft_evict_next_allowed: &mut HashMap, ) { let now_epoch_secs = MePool::now_epoch_secs(); let now = Instant::now(); @@ -172,7 +181,7 @@ pub(super) async fn reap_draining_writers( } let mut active_draining_writer_ids = HashSet::with_capacity(draining_writers.len()); - for writer in draining_writers { + for writer in &draining_writers { active_draining_writer_ids.insert(writer.id); let drain_started_at_epoch_secs = writer .draining_started_at_epoch_secs @@ -209,6 +218,86 @@ pub(super) async fn reap_draining_writers( } warn_next_allowed.retain(|writer_id, _| active_draining_writer_ids.contains(writer_id)); + soft_evict_next_allowed.retain(|writer_id, _| active_draining_writer_ids.contains(writer_id)); + + if pool.drain_soft_evict_enabled() && drain_ttl_secs > 0 && !draining_writers.is_empty() { + let mut force_close_ids = HashSet::::with_capacity(force_close_writer_ids.len()); + for writer_id in &force_close_writer_ids { + force_close_ids.insert(*writer_id); + } + let soft_grace_secs = pool.drain_soft_evict_grace_secs(); + let soft_trigger_age_secs = drain_ttl_secs.saturating_add(soft_grace_secs); + let per_writer_limit = pool.drain_soft_evict_per_writer(); + let soft_budget = health_drain_soft_evict_budget(pool); + let soft_cooldown = pool.drain_soft_evict_cooldown(); + let mut soft_evicted_total = 0usize; + + for writer in &draining_writers { + if soft_evicted_total >= soft_budget { + break; + } + if force_close_ids.contains(&writer.id) { + continue; + } + if pool.writer_accepts_new_binding(writer) { + continue; + } + let started_epoch_secs = writer + .draining_started_at_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + if started_epoch_secs == 0 + || now_epoch_secs.saturating_sub(started_epoch_secs) < soft_trigger_age_secs + { + continue; + } + if !should_emit_writer_warn( + soft_evict_next_allowed, + writer.id, + now, + soft_cooldown, + ) { + continue; + } + + let remaining_budget = soft_budget.saturating_sub(soft_evicted_total); + let limit = per_writer_limit.min(remaining_budget); + if limit == 0 { + break; + } + let conn_ids = pool + .registry + .bound_conn_ids_for_writer_limited(writer.id, limit) + .await; + if conn_ids.is_empty() { + continue; + } + + let mut evicted_for_writer = 0usize; + for conn_id in conn_ids { + if pool.registry.evict_bound_conn_if_writer(conn_id, writer.id).await { + evicted_for_writer = evicted_for_writer.saturating_add(1); + soft_evicted_total = soft_evicted_total.saturating_add(1); + pool.stats.increment_pool_drain_soft_evict_total(); + if soft_evicted_total >= soft_budget { + break; + } + } + } + + if evicted_for_writer > 0 { + pool.stats.increment_pool_drain_soft_evict_writer_total(); + info!( + writer_id = writer.id, + writer_dc = writer.writer_dc, + endpoint = %writer.addr, + drained_connections = evicted_for_writer, + soft_budget, + soft_trigger_age_secs, + "ME draining writer soft-evicted bound clients" + ); + } + } + } let close_budget = health_drain_close_budget(); let requested_force_close = force_close_writer_ids.len(); @@ -258,6 +347,19 @@ pub(super) fn health_drain_close_budget() -> usize { .clamp(HEALTH_DRAIN_CLOSE_BUDGET_MIN, HEALTH_DRAIN_CLOSE_BUDGET_MAX) } +pub(super) fn health_drain_soft_evict_budget(pool: &MePool) -> usize { + let cpu_cores = std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1); + let per_core = pool.drain_soft_evict_budget_per_core(); + cpu_cores + .saturating_mul(per_core) + .clamp( + HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN, + HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX, + ) +} + fn should_emit_writer_warn( next_allowed: &mut HashMap, writer_id: u64, @@ -1443,6 +1545,11 @@ mod tests { general.hardswap, general.me_pool_drain_ttl_secs, general.me_pool_drain_threshold, + general.me_pool_drain_soft_evict_enabled, + general.me_pool_drain_soft_evict_grace_secs, + general.me_pool_drain_soft_evict_per_writer, + general.me_pool_drain_soft_evict_budget_per_core, + general.me_pool_drain_soft_evict_cooldown_ms, general.effective_me_pool_force_close_secs(), general.me_pool_min_fresh_ratio, general.me_hardswap_warmup_delay_min_ms, @@ -1524,8 +1631,9 @@ mod tests { let conn_b = insert_draining_writer(&pool, 20, now_epoch_secs.saturating_sub(20)).await; let conn_c = insert_draining_writer(&pool, 30, now_epoch_secs.saturating_sub(10)).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; let writer_ids: Vec = pool.writers.read().await.iter().map(|writer| writer.id).collect(); assert_eq!(writer_ids, vec![20, 30]); @@ -1542,8 +1650,9 @@ mod tests { let conn_b = insert_draining_writer(&pool, 20, now_epoch_secs.saturating_sub(20)).await; let conn_c = insert_draining_writer(&pool, 30, now_epoch_secs.saturating_sub(10)).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; let writer_ids: Vec = pool.writers.read().await.iter().map(|writer| writer.id).collect(); assert_eq!(writer_ids, vec![10, 20, 30]); diff --git a/src/transport/middle_proxy/health_adversarial_tests.rs b/src/transport/middle_proxy/health_adversarial_tests.rs index 675005a..dc1a0eb 100644 --- a/src/transport/middle_proxy/health_adversarial_tests.rs +++ b/src/transport/middle_proxy/health_adversarial_tests.rs @@ -82,6 +82,11 @@ async fn make_pool( general.hardswap, general.me_pool_drain_ttl_secs, general.me_pool_drain_threshold, + general.me_pool_drain_soft_evict_enabled, + general.me_pool_drain_soft_evict_grace_secs, + general.me_pool_drain_soft_evict_per_writer, + general.me_pool_drain_soft_evict_budget_per_core, + general.me_pool_drain_soft_evict_cooldown_ms, general.effective_me_pool_force_close_secs(), general.me_pool_min_fresh_ratio, general.me_hardswap_warmup_delay_min_ms, @@ -185,10 +190,11 @@ async fn sorted_writer_ids(pool: &Arc) -> Vec { async fn reap_draining_writers_clears_warn_state_when_pool_empty() { let (pool, _rng) = make_pool(128, 1, 1).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); warn_next_allowed.insert(11, Instant::now() + Duration::from_secs(5)); warn_next_allowed.insert(22, Instant::now() + Duration::from_secs(5)); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.is_empty()); } @@ -197,6 +203,8 @@ async fn reap_draining_writers_clears_warn_state_when_pool_empty() { async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycles() { let threshold = 3u64; let (pool, _rng) = make_pool(threshold, 1, 1).await; + pool.me_pool_drain_soft_evict_enabled + .store(false, Ordering::Relaxed); let now_epoch_secs = MePool::now_epoch_secs(); for writer_id in 1..=60u64 { @@ -211,8 +219,9 @@ async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycle } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for _ in 0..64 { - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; if writer_count(&pool).await <= threshold as usize { break; } @@ -240,11 +249,12 @@ async fn reap_draining_writers_handles_large_empty_writer_population() { } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for _ in 0..24 { if writer_count(&pool).await == 0 { break; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; } assert_eq!(writer_count(&pool).await, 0); @@ -268,11 +278,12 @@ async fn reap_draining_writers_processes_mass_deadline_expiry_without_unbounded_ } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for _ in 0..40 { if writer_count(&pool).await == 0 { break; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; } assert_eq!(writer_count(&pool).await, 0); @@ -283,6 +294,7 @@ async fn reap_draining_writers_maintains_warn_state_subset_property_under_bulk_c let (pool, _rng) = make_pool(128, 1, 1).await; let now_epoch_secs = MePool::now_epoch_secs(); let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for wave in 0..40u64 { for offset in 0..8u64 { @@ -296,7 +308,7 @@ async fn reap_draining_writers_maintains_warn_state_subset_property_under_bulk_c .await; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.len() <= writer_count(&pool).await); let ids = sorted_writer_ids(&pool).await; @@ -304,7 +316,7 @@ async fn reap_draining_writers_maintains_warn_state_subset_property_under_bulk_c let _ = pool.remove_writer_and_close_clients(writer_id).await; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.len() <= writer_count(&pool).await); } } @@ -326,9 +338,10 @@ async fn reap_draining_writers_budgeted_cleanup_never_increases_pool_size() { } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); let mut previous = writer_count(&pool).await; for _ in 0..32 { - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; let current = writer_count(&pool).await; assert!(current <= previous); previous = current; diff --git a/src/transport/middle_proxy/health_integration_tests.rs b/src/transport/middle_proxy/health_integration_tests.rs index 70b6411..4724851 100644 --- a/src/transport/middle_proxy/health_integration_tests.rs +++ b/src/transport/middle_proxy/health_integration_tests.rs @@ -81,6 +81,11 @@ async fn make_pool( general.hardswap, general.me_pool_drain_ttl_secs, general.me_pool_drain_threshold, + general.me_pool_drain_soft_evict_enabled, + general.me_pool_drain_soft_evict_grace_secs, + general.me_pool_drain_soft_evict_per_writer, + general.me_pool_drain_soft_evict_budget_per_core, + general.me_pool_drain_soft_evict_cooldown_ms, general.effective_me_pool_force_close_secs(), general.me_pool_min_fresh_ratio, general.me_hardswap_warmup_delay_min_ms, diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs index 05a8e6a..45a1eee 100644 --- a/src/transport/middle_proxy/health_regression_tests.rs +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -39,7 +39,7 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc { NetworkDecision::default(), None, Arc::new(SecureRandom::new()), - Arc::new(Stats::default()), + Arc::new(Stats::new()), general.me_keepalive_enabled, general.me_keepalive_interval_secs, general.me_keepalive_jitter_secs, @@ -74,6 +74,11 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc { general.hardswap, general.me_pool_drain_ttl_secs, general.me_pool_drain_threshold, + general.me_pool_drain_soft_evict_enabled, + general.me_pool_drain_soft_evict_grace_secs, + general.me_pool_drain_soft_evict_per_writer, + general.me_pool_drain_soft_evict_budget_per_core, + general.me_pool_drain_soft_evict_cooldown_ms, general.effective_me_pool_force_close_secs(), general.me_pool_min_fresh_ratio, general.me_hardswap_warmup_delay_min_ms, @@ -175,14 +180,15 @@ async fn reap_draining_writers_drops_warn_state_for_removed_writer() { let conn_ids = insert_draining_writer(&pool, 7, now_epoch_secs.saturating_sub(180), 1, 0).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.contains_key(&7)); let _ = pool.remove_writer_and_close_clients(7).await; assert!(pool.registry.get_writer(conn_ids[0]).await.is_none()); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(!warn_next_allowed.contains_key(&7)); } @@ -194,8 +200,9 @@ async fn reap_draining_writers_removes_empty_draining_writers() { insert_draining_writer(&pool, 2, now_epoch_secs.saturating_sub(30), 0, 0).await; insert_draining_writer(&pool, 3, now_epoch_secs.saturating_sub(20), 1, 0).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert_eq!(current_writer_ids(&pool).await, vec![3]); } @@ -209,8 +216,9 @@ async fn reap_draining_writers_overflow_closes_oldest_non_empty_writers() { insert_draining_writer(&pool, 33, now_epoch_secs.saturating_sub(20), 1, 0).await; insert_draining_writer(&pool, 44, now_epoch_secs.saturating_sub(10), 1, 0).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert_eq!(current_writer_ids(&pool).await, vec![33, 44]); } @@ -228,8 +236,9 @@ async fn reap_draining_writers_deadline_force_close_applies_under_threshold() { ) .await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(current_writer_ids(&pool).await.is_empty()); } @@ -251,8 +260,9 @@ async fn reap_draining_writers_limits_closes_per_health_tick() { .await; } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert_eq!(pool.writers.read().await.len(), writer_total - close_budget); } @@ -274,12 +284,13 @@ async fn reap_draining_writers_backlog_drains_across_ticks() { .await; } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for _ in 0..8 { if pool.writers.read().await.is_empty() { break; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; } assert!(pool.writers.read().await.is_empty()); @@ -303,9 +314,10 @@ async fn reap_draining_writers_threshold_backlog_converges_to_threshold() { .await; } let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for _ in 0..16 { - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; if pool.writers.read().await.len() <= threshold as usize { break; } @@ -322,8 +334,9 @@ async fn reap_draining_writers_threshold_zero_preserves_non_expired_non_empty_wr insert_draining_writer(&pool, 20, now_epoch_secs.saturating_sub(30), 1, 0).await; insert_draining_writer(&pool, 30, now_epoch_secs.saturating_sub(20), 1, 0).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert_eq!(current_writer_ids(&pool).await, vec![10, 20, 30]); } @@ -346,8 +359,9 @@ async fn reap_draining_writers_prioritizes_force_close_before_empty_cleanup() { let empty_writer_id = close_budget as u64 + 1; insert_draining_writer(&pool, empty_writer_id, now_epoch_secs.saturating_sub(20), 0, 0).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert_eq!(current_writer_ids(&pool).await, vec![empty_writer_id]); } @@ -359,8 +373,9 @@ async fn reap_draining_writers_empty_cleanup_does_not_increment_force_close_metr insert_draining_writer(&pool, 1, now_epoch_secs.saturating_sub(60), 0, 0).await; insert_draining_writer(&pool, 2, now_epoch_secs.saturating_sub(50), 0, 0).await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(current_writer_ids(&pool).await.is_empty()); assert_eq!(pool.stats.get_pool_force_close_total(), 0); @@ -387,8 +402,9 @@ async fn reap_draining_writers_handles_duplicate_force_close_requests_for_same_w ) .await; let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(current_writer_ids(&pool).await.is_empty()); } @@ -398,6 +414,7 @@ async fn reap_draining_writers_warn_state_never_exceeds_live_draining_population let pool = make_pool(128).await; let now_epoch_secs = MePool::now_epoch_secs(); let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for wave in 0..12u64 { for offset in 0..9u64 { @@ -410,14 +427,14 @@ async fn reap_draining_writers_warn_state_never_exceeds_live_draining_population ) .await; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); let existing_writer_ids = current_writer_ids(&pool).await; for writer_id in existing_writer_ids.into_iter().take(4) { let _ = pool.remove_writer_and_close_clients(writer_id).await; } - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); } } @@ -427,6 +444,7 @@ async fn reap_draining_writers_mixed_backlog_converges_without_leaking_warn_stat let pool = make_pool(6).await; let now_epoch_secs = MePool::now_epoch_secs(); let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); for writer_id in 1..=18u64 { let bound_clients = if writer_id % 3 == 0 { 0 } else { 1 }; @@ -446,7 +464,7 @@ async fn reap_draining_writers_mixed_backlog_converges_without_leaking_warn_stat } for _ in 0..16 { - reap_draining_writers(&pool, &mut warn_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; if pool.writers.read().await.len() <= 6 { break; } @@ -456,7 +474,60 @@ async fn reap_draining_writers_mixed_backlog_converges_without_leaking_warn_stat assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); } +#[tokio::test] +async fn reap_draining_writers_soft_evicts_stuck_writer_with_per_writer_cap() { + let pool = make_pool(128).await; + pool.me_pool_drain_soft_evict_enabled.store(true, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_grace_secs.store(0, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_per_writer.store(1, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_budget_per_core.store(8, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_cooldown_ms + .store(1, Ordering::Relaxed); + + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 77, now_epoch_secs.saturating_sub(240), 3, 0).await; + let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; + + let activity = pool.registry.writer_activity_snapshot().await; + assert_eq!(activity.bound_clients_by_writer.get(&77), Some(&2)); + assert_eq!(pool.stats.get_pool_drain_soft_evict_total(), 1); + assert_eq!(pool.stats.get_pool_drain_soft_evict_writer_total(), 1); + assert_eq!(current_writer_ids(&pool).await, vec![77]); +} + +#[tokio::test] +async fn reap_draining_writers_soft_evict_respects_cooldown_per_writer() { + let pool = make_pool(128).await; + pool.me_pool_drain_soft_evict_enabled.store(true, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_grace_secs.store(0, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_per_writer.store(1, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_budget_per_core.store(8, Ordering::Relaxed); + pool.me_pool_drain_soft_evict_cooldown_ms + .store(60_000, Ordering::Relaxed); + + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 88, now_epoch_secs.saturating_sub(240), 3, 0).await; + let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; + + let activity = pool.registry.writer_activity_snapshot().await; + assert_eq!(activity.bound_clients_by_writer.get(&88), Some(&2)); + assert_eq!(pool.stats.get_pool_drain_soft_evict_total(), 1); + assert_eq!(pool.stats.get_pool_drain_soft_evict_writer_total(), 1); +} + #[test] fn general_config_default_drain_threshold_remains_enabled() { assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 128); + assert!(GeneralConfig::default().me_pool_drain_soft_evict_enabled); + assert_eq!( + GeneralConfig::default().me_pool_drain_soft_evict_per_writer, + 1 + ); } diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 2a65160..f3cc817 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -172,6 +172,11 @@ pub struct MePool { pub(super) kdf_material_fingerprint: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, pub(super) me_pool_drain_threshold: AtomicU64, + pub(super) me_pool_drain_soft_evict_enabled: AtomicBool, + pub(super) me_pool_drain_soft_evict_grace_secs: AtomicU64, + pub(super) me_pool_drain_soft_evict_per_writer: AtomicU8, + pub(super) me_pool_drain_soft_evict_budget_per_core: AtomicU32, + pub(super) me_pool_drain_soft_evict_cooldown_ms: AtomicU64, pub(super) me_pool_force_close_secs: AtomicU64, pub(super) me_pool_min_fresh_ratio_permille: AtomicU32, pub(super) me_hardswap_warmup_delay_min_ms: AtomicU64, @@ -273,6 +278,11 @@ impl MePool { hardswap: bool, me_pool_drain_ttl_secs: u64, me_pool_drain_threshold: u64, + me_pool_drain_soft_evict_enabled: bool, + me_pool_drain_soft_evict_grace_secs: u64, + me_pool_drain_soft_evict_per_writer: u8, + me_pool_drain_soft_evict_budget_per_core: u16, + me_pool_drain_soft_evict_cooldown_ms: u64, me_pool_force_close_secs: u64, me_pool_min_fresh_ratio: f32, me_hardswap_warmup_delay_min_ms: u64, @@ -449,6 +459,17 @@ impl MePool { kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())), me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs), me_pool_drain_threshold: AtomicU64::new(me_pool_drain_threshold), + me_pool_drain_soft_evict_enabled: AtomicBool::new(me_pool_drain_soft_evict_enabled), + me_pool_drain_soft_evict_grace_secs: AtomicU64::new(me_pool_drain_soft_evict_grace_secs), + me_pool_drain_soft_evict_per_writer: AtomicU8::new( + me_pool_drain_soft_evict_per_writer.max(1), + ), + me_pool_drain_soft_evict_budget_per_core: AtomicU32::new( + me_pool_drain_soft_evict_budget_per_core.max(1) as u32, + ), + me_pool_drain_soft_evict_cooldown_ms: AtomicU64::new( + me_pool_drain_soft_evict_cooldown_ms.max(1), + ), me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs), me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille( me_pool_min_fresh_ratio, @@ -496,6 +517,11 @@ impl MePool { hardswap: bool, drain_ttl_secs: u64, pool_drain_threshold: u64, + pool_drain_soft_evict_enabled: bool, + pool_drain_soft_evict_grace_secs: u64, + pool_drain_soft_evict_per_writer: u8, + pool_drain_soft_evict_budget_per_core: u16, + pool_drain_soft_evict_cooldown_ms: u64, force_close_secs: u64, min_fresh_ratio: f32, hardswap_warmup_delay_min_ms: u64, @@ -536,6 +562,18 @@ impl MePool { .store(drain_ttl_secs, Ordering::Relaxed); self.me_pool_drain_threshold .store(pool_drain_threshold, Ordering::Relaxed); + self.me_pool_drain_soft_evict_enabled + .store(pool_drain_soft_evict_enabled, Ordering::Relaxed); + self.me_pool_drain_soft_evict_grace_secs + .store(pool_drain_soft_evict_grace_secs, Ordering::Relaxed); + self.me_pool_drain_soft_evict_per_writer + .store(pool_drain_soft_evict_per_writer.max(1), Ordering::Relaxed); + self.me_pool_drain_soft_evict_budget_per_core.store( + pool_drain_soft_evict_budget_per_core.max(1) as u32, + Ordering::Relaxed, + ); + self.me_pool_drain_soft_evict_cooldown_ms + .store(pool_drain_soft_evict_cooldown_ms.max(1), Ordering::Relaxed); self.me_pool_force_close_secs .store(force_close_secs, Ordering::Relaxed); self.me_pool_min_fresh_ratio_permille @@ -690,6 +728,36 @@ impl MePool { } } + pub(super) fn drain_soft_evict_enabled(&self) -> bool { + self.me_pool_drain_soft_evict_enabled + .load(Ordering::Relaxed) + } + + pub(super) fn drain_soft_evict_grace_secs(&self) -> u64 { + self.me_pool_drain_soft_evict_grace_secs + .load(Ordering::Relaxed) + } + + pub(super) fn drain_soft_evict_per_writer(&self) -> usize { + self.me_pool_drain_soft_evict_per_writer + .load(Ordering::Relaxed) + .max(1) as usize + } + + pub(super) fn drain_soft_evict_budget_per_core(&self) -> usize { + self.me_pool_drain_soft_evict_budget_per_core + .load(Ordering::Relaxed) + .max(1) as usize + } + + pub(super) fn drain_soft_evict_cooldown(&self) -> Duration { + Duration::from_millis( + self.me_pool_drain_soft_evict_cooldown_ms + .load(Ordering::Relaxed) + .max(1), + ) + } + pub(super) async fn key_selector(&self) -> u32 { self.proxy_secret.read().await.key_selector } diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 99070a8..d32835c 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -124,6 +124,11 @@ pub(crate) struct MeApiRuntimeSnapshot { pub me_reconnect_backoff_cap_ms: u64, pub me_reconnect_fast_retry_count: u32, pub me_pool_drain_ttl_secs: u64, + pub me_pool_drain_soft_evict_enabled: bool, + pub me_pool_drain_soft_evict_grace_secs: u64, + pub me_pool_drain_soft_evict_per_writer: u8, + pub me_pool_drain_soft_evict_budget_per_core: u16, + pub me_pool_drain_soft_evict_cooldown_ms: u64, pub me_pool_force_close_secs: u64, pub me_pool_min_fresh_ratio: f32, pub me_bind_stale_mode: &'static str, @@ -562,6 +567,22 @@ impl MePool { me_reconnect_backoff_cap_ms: self.me_reconnect_backoff_cap.as_millis() as u64, me_reconnect_fast_retry_count: self.me_reconnect_fast_retry_count, me_pool_drain_ttl_secs: self.me_pool_drain_ttl_secs.load(Ordering::Relaxed), + me_pool_drain_soft_evict_enabled: self + .me_pool_drain_soft_evict_enabled + .load(Ordering::Relaxed), + me_pool_drain_soft_evict_grace_secs: self + .me_pool_drain_soft_evict_grace_secs + .load(Ordering::Relaxed), + me_pool_drain_soft_evict_per_writer: self + .me_pool_drain_soft_evict_per_writer + .load(Ordering::Relaxed), + me_pool_drain_soft_evict_budget_per_core: self + .me_pool_drain_soft_evict_budget_per_core + .load(Ordering::Relaxed) + .min(u16::MAX as u32) as u16, + me_pool_drain_soft_evict_cooldown_ms: self + .me_pool_drain_soft_evict_cooldown_ms + .load(Ordering::Relaxed), me_pool_force_close_secs: self.me_pool_force_close_secs.load(Ordering::Relaxed), me_pool_min_fresh_ratio: Self::permille_to_ratio( self.me_pool_min_fresh_ratio_permille.load(Ordering::Relaxed), diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index cc3028b..b8a926e 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -394,6 +394,56 @@ impl ConnRegistry { inner.writer_for_conn.keys().copied().collect() } + pub(super) async fn bound_conn_ids_for_writer_limited( + &self, + writer_id: u64, + limit: usize, + ) -> Vec { + if limit == 0 { + return Vec::new(); + } + let inner = self.inner.read().await; + let Some(conn_ids) = inner.conns_for_writer.get(&writer_id) else { + return Vec::new(); + }; + let mut out = conn_ids.iter().copied().collect::>(); + out.sort_unstable(); + out.truncate(limit); + out + } + + pub(super) async fn evict_bound_conn_if_writer(&self, conn_id: u64, writer_id: u64) -> bool { + let maybe_client_tx = { + let mut inner = self.inner.write().await; + if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) { + return false; + } + + let client_tx = inner.map.get(&conn_id).cloned(); + inner.map.remove(&conn_id); + inner.meta.remove(&conn_id); + inner.writer_for_conn.remove(&conn_id); + + let became_empty = if let Some(set) = inner.conns_for_writer.get_mut(&writer_id) { + set.remove(&conn_id); + set.is_empty() + } else { + false + }; + if became_empty { + inner + .writer_idle_since_epoch_secs + .insert(writer_id, Self::now_epoch_secs()); + } + client_tx + }; + + if let Some(client_tx) = maybe_client_tx { + let _ = client_tx.try_send(MeResponse::Close); + } + true + } + pub async fn writer_lost(&self, writer_id: u64) -> Vec { let mut inner = self.inner.write().await; inner.writers.remove(&writer_id); @@ -444,6 +494,7 @@ mod tests { use super::ConnMeta; use super::ConnRegistry; + use super::MeResponse; #[tokio::test] async fn writer_activity_snapshot_tracks_writer_and_dc_load() { @@ -634,4 +685,86 @@ mod tests { ); assert!(registry.get_writer(conn_id).await.is_none()); } + + #[tokio::test] + async fn bound_conn_ids_for_writer_limited_is_sorted_and_bounded() { + let registry = ConnRegistry::new(); + let (writer_tx, _writer_rx) = tokio::sync::mpsc::channel(8); + registry.register_writer(10, writer_tx).await; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + let mut conn_ids = Vec::new(); + for _ in 0..5 { + let (conn_id, _rx) = registry.register().await; + assert!( + registry + .bind_writer( + conn_id, + 10, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + conn_ids.push(conn_id); + } + conn_ids.sort_unstable(); + + let limited = registry.bound_conn_ids_for_writer_limited(10, 3).await; + assert_eq!(limited.len(), 3); + assert_eq!(limited, conn_ids.into_iter().take(3).collect::>()); + } + + #[tokio::test] + async fn evict_bound_conn_if_writer_does_not_touch_rebound_conn() { + let registry = ConnRegistry::new(); + let (conn_id, mut rx) = registry.register().await; + let (writer_tx_a, _writer_rx_a) = tokio::sync::mpsc::channel(8); + let (writer_tx_b, _writer_rx_b) = tokio::sync::mpsc::channel(8); + registry.register_writer(10, writer_tx_a).await; + registry.register_writer(20, writer_tx_b).await; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + + assert!( + registry + .bind_writer( + conn_id, + 10, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + assert!( + registry + .bind_writer( + conn_id, + 20, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 1, + }, + ) + .await + ); + + let evicted = registry.evict_bound_conn_if_writer(conn_id, 10).await; + assert!(!evicted); + assert_eq!(registry.get_writer(conn_id).await.expect("writer").writer_id, 20); + assert!(rx.try_recv().is_err()); + + let evicted = registry.evict_bound_conn_if_writer(conn_id, 20).await; + assert!(evicted); + assert!(registry.get_writer(conn_id).await.is_none()); + assert!(matches!(rx.try_recv(), Ok(MeResponse::Close))); + } } diff --git a/src/transport/socket.rs b/src/transport/socket.rs index aa4dc01..3ff96a2 100644 --- a/src/transport/socket.rs +++ b/src/transport/socket.rs @@ -11,6 +11,8 @@ use tokio::net::TcpStream; use socket2::{Socket, TcpKeepalive, Domain, Type, Protocol}; use tracing::debug; +const DEFAULT_SOCKET_BUFFER_BYTES: usize = 256 * 1024; + /// Configure TCP socket with recommended settings for proxy use #[allow(dead_code)] pub fn configure_tcp_socket( @@ -34,10 +36,10 @@ pub fn configure_tcp_socket( socket.set_tcp_keepalive(&keepalive)?; } - - // CHANGED: Removed manual buffer size setting (was 256KB). - // Allowing the OS kernel to handle TCP window scaling (Autotuning) is critical - // for mobile clients to avoid bufferbloat and stalled connections during uploads. + + // Use explicit baseline buffers to reduce slow-start stalls on high RTT links. + socket.set_recv_buffer_size(DEFAULT_SOCKET_BUFFER_BYTES)?; + socket.set_send_buffer_size(DEFAULT_SOCKET_BUFFER_BYTES)?; Ok(()) } @@ -62,6 +64,10 @@ pub fn configure_client_socket( let keepalive = keepalive.with_interval(Duration::from_secs(keepalive_secs)); socket.set_tcp_keepalive(&keepalive)?; + + // Keep explicit baseline buffers for predictable throughput across busy hosts. + socket.set_recv_buffer_size(DEFAULT_SOCKET_BUFFER_BYTES)?; + socket.set_send_buffer_size(DEFAULT_SOCKET_BUFFER_BYTES)?; // Set TCP user timeout (Linux only) // NOTE: iOS does not support TCP_USER_TIMEOUT - application-level timeout @@ -124,6 +130,8 @@ pub fn create_outgoing_socket_bound(addr: SocketAddr, bind_addr: Option) // Disable Nagle socket.set_nodelay(true)?; + socket.set_recv_buffer_size(DEFAULT_SOCKET_BUFFER_BYTES)?; + socket.set_send_buffer_size(DEFAULT_SOCKET_BUFFER_BYTES)?; if let Some(bind_ip) = bind_addr { let bind_sock_addr = SocketAddr::new(bind_ip, 0); From 462c927da673a5f2da3a2fcfaf80506b00f0a8a2 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Wed, 18 Mar 2026 10:53:09 +0300 Subject: [PATCH 228/261] Create CONFIG_PARAMS.en.md --- docs/CONFIG_PARAMS.en.md | 289 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 docs/CONFIG_PARAMS.en.md diff --git a/docs/CONFIG_PARAMS.en.md b/docs/CONFIG_PARAMS.en.md new file mode 100644 index 0000000..ed89b3d --- /dev/null +++ b/docs/CONFIG_PARAMS.en.md @@ -0,0 +1,289 @@ +# Telemt Config Parameters Reference + +This document lists all configuration keys accepted by `config.toml`. + +> [!WARNING] +> +> The configuration parameters detailed in this document are intended for advanced users and fine-tuning purposes. Modifying these settings without a clear understanding of their function may lead to application instability or other unexpected behavior. Please proceed with caution and at your own risk. + +## Top-level keys + +| Parameter | Type | Description | +|---|---|---| +| include | `String` (special directive) | Includes another TOML file with `include = "relative/or/absolute/path.toml"`; includes are processed recursively before parsing. | +| show_link | `"*" \| String[]` | Legacy top-level link visibility selector (`"*"` for all users or explicit usernames list). | +| dc_overrides | `Map` | Overrides DC endpoints for non-standard DCs; key is DC id string, value is `ip:port` list. | +| default_dc | `u8` | Default DC index used for unmapped non-standard DCs. | + +## [general] + +| Parameter | Type | Description | +|---|---|---| +| data_path | `String` | Optional runtime data directory path. | +| prefer_ipv6 | `bool` | Prefer IPv6 where applicable in runtime logic. | +| fast_mode | `bool` | Enables fast-path optimizations for traffic processing. | +| use_middle_proxy | `bool` | Enables Middle Proxy mode. | +| proxy_secret_path | `String` | Path to proxy secret binary; can be auto-downloaded if absent. | +| proxy_config_v4_cache_path | `String` | Optional cache path for raw `getProxyConfig` (IPv4) snapshot. | +| proxy_config_v6_cache_path | `String` | Optional cache path for raw `getProxyConfigV6` (IPv6) snapshot. | +| ad_tag | `String` | Global fallback ad tag (32 hex characters). | +| middle_proxy_nat_ip | `IpAddr` | Explicit public IP override for NAT environments. | +| middle_proxy_nat_probe | `bool` | Enables NAT probing for Middle Proxy KDF/public address discovery. | +| middle_proxy_nat_stun | `String` | Deprecated legacy single STUN server for NAT probing. | +| middle_proxy_nat_stun_servers | `String[]` | Deprecated legacy STUN list for NAT probing fallback. | +| stun_nat_probe_concurrency | `usize` | Maximum concurrent STUN probes during NAT detection. | +| middle_proxy_pool_size | `usize` | Target size of active Middle Proxy writer pool. | +| middle_proxy_warm_standby | `usize` | Number of warm standby Middle-End connections. | +| me_init_retry_attempts | `u32` | Startup retries for ME pool initialization (`0` means unlimited). | +| me2dc_fallback | `bool` | Allows fallback from ME mode to direct DC when ME startup fails. | +| me_keepalive_enabled | `bool` | Enables ME keepalive padding frames. | +| me_keepalive_interval_secs | `u64` | Keepalive interval in seconds. | +| me_keepalive_jitter_secs | `u64` | Keepalive jitter in seconds. | +| me_keepalive_payload_random | `bool` | Randomizes keepalive payload bytes instead of zero payload. | +| rpc_proxy_req_every | `u64` | Interval for service `RPC_PROXY_REQ` activity signals (`0` disables). | +| me_writer_cmd_channel_capacity | `usize` | Capacity of per-writer command channel. | +| me_route_channel_capacity | `usize` | Capacity of per-connection ME response route channel. | +| me_c2me_channel_capacity | `usize` | Capacity of per-client command queue (client reader -> ME sender). | +| me_reader_route_data_wait_ms | `u64` | Bounded wait for routing ME DATA to per-connection queue (`0` = no wait). | +| me_d2c_flush_batch_max_frames | `usize` | Max ME->client frames coalesced before flush. | +| me_d2c_flush_batch_max_bytes | `usize` | Max ME->client payload bytes coalesced before flush. | +| me_d2c_flush_batch_max_delay_us | `u64` | Max microsecond wait for coalescing more ME->client frames (`0` disables timed coalescing). | +| me_d2c_ack_flush_immediate | `bool` | Flushes client writer immediately after quick-ack write. | +| direct_relay_copy_buf_c2s_bytes | `usize` | Copy buffer size for client->DC direction in direct relay. | +| direct_relay_copy_buf_s2c_bytes | `usize` | Copy buffer size for DC->client direction in direct relay. | +| crypto_pending_buffer | `usize` | Max pending ciphertext buffer per client writer (bytes). | +| max_client_frame | `usize` | Maximum allowed client MTProto frame size (bytes). | +| desync_all_full | `bool` | Emits full crypto-desync forensic logs for every event. | +| beobachten | `bool` | Enables per-IP forensic observation buckets. | +| beobachten_minutes | `u64` | Retention window (minutes) for per-IP observation buckets. | +| beobachten_flush_secs | `u64` | Snapshot flush interval (seconds) for observation output file. | +| beobachten_file | `String` | Observation snapshot output file path. | +| hardswap | `bool` | Enables hard-swap generation switching for ME pool updates. | +| me_warmup_stagger_enabled | `bool` | Enables staggered warmup for extra ME writers. | +| me_warmup_step_delay_ms | `u64` | Base delay between warmup connections (ms). | +| me_warmup_step_jitter_ms | `u64` | Jitter for warmup delay (ms). | +| me_reconnect_max_concurrent_per_dc | `u32` | Max concurrent reconnect attempts per DC. | +| me_reconnect_backoff_base_ms | `u64` | Base reconnect backoff in ms. | +| me_reconnect_backoff_cap_ms | `u64` | Cap reconnect backoff in ms. | +| me_reconnect_fast_retry_count | `u32` | Number of fast retry attempts before backoff. | +| me_single_endpoint_shadow_writers | `u8` | Additional reserve writers for one-endpoint DC groups. | +| me_single_endpoint_outage_mode_enabled | `bool` | Enables aggressive outage recovery for one-endpoint DC groups. | +| me_single_endpoint_outage_disable_quarantine | `bool` | Ignores endpoint quarantine in one-endpoint outage mode. | +| me_single_endpoint_outage_backoff_min_ms | `u64` | Minimum reconnect backoff in outage mode (ms). | +| me_single_endpoint_outage_backoff_max_ms | `u64` | Maximum reconnect backoff in outage mode (ms). | +| me_single_endpoint_shadow_rotate_every_secs | `u64` | Periodic shadow writer rotation interval (`0` disables). | +| me_floor_mode | `"static" \| "adaptive"` | Writer floor policy mode. | +| me_adaptive_floor_idle_secs | `u64` | Idle time before adaptive floor may reduce one-endpoint target. | +| me_adaptive_floor_min_writers_single_endpoint | `u8` | Minimum adaptive writer target for one-endpoint DC groups. | +| me_adaptive_floor_min_writers_multi_endpoint | `u8` | Minimum adaptive writer target for multi-endpoint DC groups. | +| me_adaptive_floor_recover_grace_secs | `u64` | Grace period to hold static floor after activity. | +| me_adaptive_floor_writers_per_core_total | `u16` | Global writer budget per logical CPU core in adaptive mode. | +| me_adaptive_floor_cpu_cores_override | `u16` | Manual CPU core count override (`0` uses auto-detection). | +| me_adaptive_floor_max_extra_writers_single_per_core | `u16` | Per-core max extra writers above base floor for one-endpoint DCs. | +| me_adaptive_floor_max_extra_writers_multi_per_core | `u16` | Per-core max extra writers above base floor for multi-endpoint DCs. | +| me_adaptive_floor_max_active_writers_per_core | `u16` | Hard cap for active ME writers per logical CPU core. | +| me_adaptive_floor_max_warm_writers_per_core | `u16` | Hard cap for warm ME writers per logical CPU core. | +| me_adaptive_floor_max_active_writers_global | `u32` | Hard global cap for active ME writers. | +| me_adaptive_floor_max_warm_writers_global | `u32` | Hard global cap for warm ME writers. | +| upstream_connect_retry_attempts | `u32` | Connect attempts for selected upstream before error/fallback. | +| upstream_connect_retry_backoff_ms | `u64` | Delay between upstream connect attempts (ms). | +| upstream_connect_budget_ms | `u64` | Total wall-clock budget for one upstream connect request (ms). | +| upstream_unhealthy_fail_threshold | `u32` | Consecutive failed requests before upstream is marked unhealthy. | +| upstream_connect_failfast_hard_errors | `bool` | Skips additional retries for hard non-transient connect errors. | +| stun_iface_mismatch_ignore | `bool` | Ignores STUN/interface mismatch and keeps Middle Proxy mode. | +| unknown_dc_log_path | `String` | File path for unknown-DC request logging (`null` disables file path). | +| unknown_dc_file_log_enabled | `bool` | Enables unknown-DC file logging. | +| log_level | `"debug" \| "verbose" \| "normal" \| "silent"` | Runtime logging verbosity. | +| disable_colors | `bool` | Disables ANSI colors in logs. | +| me_socks_kdf_policy | `"strict" \| "compat"` | SOCKS-bound KDF fallback policy for ME handshake. | +| me_route_backpressure_base_timeout_ms | `u64` | Base backpressure timeout for route-channel send (ms). | +| me_route_backpressure_high_timeout_ms | `u64` | High backpressure timeout when queue occupancy exceeds watermark (ms). | +| me_route_backpressure_high_watermark_pct | `u8` | Queue occupancy threshold (%) for high timeout mode. | +| me_health_interval_ms_unhealthy | `u64` | Health monitor interval while writer coverage is degraded (ms). | +| me_health_interval_ms_healthy | `u64` | Health monitor interval while writer coverage is healthy (ms). | +| me_admission_poll_ms | `u64` | Poll interval for conditional-admission checks (ms). | +| me_warn_rate_limit_ms | `u64` | Cooldown for repetitive ME warning logs (ms). | +| me_route_no_writer_mode | `"async_recovery_failfast" \| "inline_recovery_legacy" \| "hybrid_async_persistent"` | Route behavior when no writer is immediately available. | +| me_route_no_writer_wait_ms | `u64` | Max wait in async-recovery failfast mode (ms). | +| me_route_inline_recovery_attempts | `u32` | Inline recovery attempts in legacy mode. | +| me_route_inline_recovery_wait_ms | `u64` | Max inline recovery wait in legacy mode (ms). | +| fast_mode_min_tls_record | `usize` | Minimum TLS record size when fast-mode coalescing is enabled (`0` disables). | +| update_every | `u64` | Unified interval for config/secret updater tasks. | +| me_reinit_every_secs | `u64` | Periodic ME pool reinitialization interval (seconds). | +| me_hardswap_warmup_delay_min_ms | `u64` | Minimum delay between hardswap warmup connects (ms). | +| me_hardswap_warmup_delay_max_ms | `u64` | Maximum delay between hardswap warmup connects (ms). | +| me_hardswap_warmup_extra_passes | `u8` | Additional warmup passes per hardswap cycle. | +| me_hardswap_warmup_pass_backoff_base_ms | `u64` | Base backoff between hardswap warmup passes (ms). | +| me_config_stable_snapshots | `u8` | Number of identical config snapshots required before apply. | +| me_config_apply_cooldown_secs | `u64` | Cooldown between applied ME map updates (seconds). | +| me_snapshot_require_http_2xx | `bool` | Requires 2xx HTTP responses for applying config snapshots. | +| me_snapshot_reject_empty_map | `bool` | Rejects empty config snapshots. | +| me_snapshot_min_proxy_for_lines | `u32` | Minimum parsed `proxy_for` rows required to accept snapshot. | +| proxy_secret_stable_snapshots | `u8` | Number of identical secret snapshots required before runtime rotation. | +| proxy_secret_rotate_runtime | `bool` | Enables runtime proxy-secret rotation from remote source. | +| me_secret_atomic_snapshot | `bool` | Keeps selector and secret bytes from the same snapshot atomically. | +| proxy_secret_len_max | `usize` | Maximum allowed proxy-secret length (bytes). | +| me_pool_drain_ttl_secs | `u64` | Drain TTL for stale ME writers after endpoint-map changes (seconds). | +| me_pool_drain_threshold | `u64` | Max draining stale writers before batch force-close (`0` disables threshold cleanup). | +| me_bind_stale_mode | `"never" \| "ttl" \| "always"` | Policy for new binds on stale draining writers. | +| me_bind_stale_ttl_secs | `u64` | TTL for stale bind allowance when stale mode is `ttl`. | +| me_pool_min_fresh_ratio | `f32` | Minimum desired-DC fresh coverage ratio before draining stale writers. | +| me_reinit_drain_timeout_secs | `u64` | Force-close timeout for stale writers after endpoint-map changes (`0` disables force-close). | +| proxy_secret_auto_reload_secs | `u64` | Deprecated legacy secret reload interval (fallback when `update_every` is not set). | +| proxy_config_auto_reload_secs | `u64` | Deprecated legacy config reload interval (fallback when `update_every` is not set). | +| me_reinit_singleflight | `bool` | Serializes ME reinit cycles across trigger sources. | +| me_reinit_trigger_channel | `usize` | Trigger queue capacity for reinit scheduler. | +| me_reinit_coalesce_window_ms | `u64` | Trigger coalescing window before starting reinit (ms). | +| me_deterministic_writer_sort | `bool` | Enables deterministic candidate sort for writer binding path. | +| me_writer_pick_mode | `"sorted_rr" \| "p2c"` | Writer selection mode for route bind path. | +| me_writer_pick_sample_size | `u8` | Number of candidates sampled by picker in `p2c` mode. | +| ntp_check | `bool` | Enables NTP drift check at startup. | +| ntp_servers | `String[]` | NTP servers used for drift check. | +| auto_degradation_enabled | `bool` | Enables automatic degradation from ME to direct DC. | +| degradation_min_unavailable_dc_groups | `u8` | Minimum unavailable ME DC groups required before degrading. | + +## [general.modes] + +| Parameter | Type | Description | +|---|---|---| +| classic | `bool` | Enables classic MTProxy mode. | +| secure | `bool` | Enables secure mode. | +| tls | `bool` | Enables TLS mode. | + +## [general.links] + +| Parameter | Type | Description | +|---|---|---| +| show | `"*" \| String[]` | Selects users whose tg:// links are shown at startup. | +| public_host | `String` | Public hostname/IP override for generated tg:// links. | +| public_port | `u16` | Public port override for generated tg:// links. | + +## [general.telemetry] + +| Parameter | Type | Description | +|---|---|---| +| core_enabled | `bool` | Enables core hot-path telemetry counters. | +| user_enabled | `bool` | Enables per-user telemetry counters. | +| me_level | `"silent" \| "normal" \| "debug"` | Middle-End telemetry verbosity level. | + +## [network] + +| Parameter | Type | Description | +|---|---|---| +| ipv4 | `bool` | Enables IPv4 networking. | +| ipv6 | `bool` | Enables/disables IPv6 (`null` = auto-detect availability). | +| prefer | `u8` | Preferred IP family for selection (`4` or `6`). | +| multipath | `bool` | Enables multipath behavior where supported. | +| stun_use | `bool` | Global switch for STUN probing. | +| stun_servers | `String[]` | STUN server list for public IP detection. | +| stun_tcp_fallback | `bool` | Enables TCP STUN fallback when UDP STUN is blocked. | +| http_ip_detect_urls | `String[]` | HTTP endpoints used as fallback public IP detectors. | +| cache_public_ip_path | `String` | File path for caching detected public IP. | +| dns_overrides | `String[]` | Runtime DNS overrides in `host:port:ip` format. | + +## [server] + +| Parameter | Type | Description | +|---|---|---| +| port | `u16` | Main proxy listen port. | +| listen_addr_ipv4 | `String` | IPv4 bind address for TCP listener. | +| listen_addr_ipv6 | `String` | IPv6 bind address for TCP listener. | +| listen_unix_sock | `String` | Unix socket path for listener. | +| listen_unix_sock_perm | `String` | Unix socket permissions in octal string (e.g., `"0666"`). | +| listen_tcp | `bool` | Explicit TCP listener enable/disable override. | +| proxy_protocol | `bool` | Enables HAProxy PROXY protocol parsing on incoming client connections. | +| proxy_protocol_header_timeout_ms | `u64` | Timeout for PROXY protocol header read/parse (ms). | +| metrics_port | `u16` | Metrics endpoint port (enables metrics listener). | +| metrics_listen | `String` | Full metrics bind address (`IP:PORT`), overrides `metrics_port`. | +| metrics_whitelist | `IpNetwork[]` | CIDR whitelist for metrics endpoint access. | +| max_connections | `u32` | Max concurrent client connections (`0` = unlimited). | + +## [server.api] + +| Parameter | Type | Description | +|---|---|---| +| enabled | `bool` | Enables control-plane REST API. | +| listen | `String` | API bind address in `IP:PORT` format. | +| whitelist | `IpNetwork[]` | CIDR whitelist allowed to access API. | +| auth_header | `String` | Exact expected `Authorization` header value (empty = disabled). | +| request_body_limit_bytes | `usize` | Maximum accepted HTTP request body size. | +| minimal_runtime_enabled | `bool` | Enables minimal runtime snapshots endpoint logic. | +| minimal_runtime_cache_ttl_ms | `u64` | Cache TTL for minimal runtime snapshots (ms; `0` disables cache). | +| runtime_edge_enabled | `bool` | Enables runtime edge endpoints. | +| runtime_edge_cache_ttl_ms | `u64` | Cache TTL for runtime edge aggregation payloads (ms). | +| runtime_edge_top_n | `usize` | Top-N size for edge connection leaderboard. | +| runtime_edge_events_capacity | `usize` | Ring-buffer capacity for runtime edge events. | +| read_only | `bool` | Rejects mutating API endpoints when enabled. | + +## [[server.listeners]] + +| Parameter | Type | Description | +|---|---|---| +| ip | `IpAddr` | Listener bind IP. | +| announce | `String` | Public IP/domain announced in proxy links (priority over `announce_ip`). | +| announce_ip | `IpAddr` | Deprecated legacy announce IP (migrated to `announce` if needed). | +| proxy_protocol | `bool` | Per-listener override for PROXY protocol enable flag. | +| reuse_allow | `bool` | Enables `SO_REUSEPORT` for multi-instance bind sharing. | + +## [timeouts] + +| Parameter | Type | Description | +|---|---|---| +| client_handshake | `u64` | Client handshake timeout. | +| tg_connect | `u64` | Upstream Telegram connect timeout. | +| client_keepalive | `u64` | Client keepalive timeout. | +| client_ack | `u64` | Client ACK timeout. | +| me_one_retry | `u8` | Quick ME reconnect attempts for single-address DC. | +| me_one_timeout_ms | `u64` | Timeout per quick attempt for single-address DC (ms). | + +## [censorship] + +| Parameter | Type | Description | +|---|---|---| +| tls_domain | `String` | Primary TLS domain used in fake TLS handshake profile. | +| tls_domains | `String[]` | Additional TLS domains for generating multiple links. | +| mask | `bool` | Enables masking/fronting relay mode. | +| mask_host | `String` | Upstream mask host for TLS fronting relay. | +| mask_port | `u16` | Upstream mask port for TLS fronting relay. | +| mask_unix_sock | `String` | Unix socket path for mask backend instead of TCP host/port. | +| fake_cert_len | `usize` | Length of synthetic certificate payload when emulation data is unavailable. | +| tls_emulation | `bool` | Enables certificate/TLS behavior emulation from cached real fronts. | +| tls_front_dir | `String` | Directory path for TLS front cache storage. | +| server_hello_delay_min_ms | `u64` | Minimum server_hello delay for anti-fingerprint behavior (ms). | +| server_hello_delay_max_ms | `u64` | Maximum server_hello delay for anti-fingerprint behavior (ms). | +| tls_new_session_tickets | `u8` | Number of `NewSessionTicket` messages to emit after handshake. | +| tls_full_cert_ttl_secs | `u64` | TTL for sending full cert payload per (domain, client IP) tuple. | +| alpn_enforce | `bool` | Enforces ALPN echo behavior based on client preference. | +| mask_proxy_protocol | `u8` | PROXY protocol mode for mask backend (`0` disabled, `1` v1, `2` v2). | + +## [access] + +| Parameter | Type | Description | +|---|---|---| +| users | `Map` | Username -> 32-hex secret mapping. | +| user_ad_tags | `Map` | Per-user ad tags (32 hex chars). | +| user_max_tcp_conns | `Map` | Per-user maximum concurrent TCP connections. | +| user_expirations | `Map>` | Per-user account expiration timestamps. | +| user_data_quota | `Map` | Per-user data quota limits. | +| user_max_unique_ips | `Map` | Per-user unique source IP limits. | +| user_max_unique_ips_global_each | `usize` | Global fallback per-user unique IP limit when no per-user override exists. | +| user_max_unique_ips_mode | `"active_window" \| "time_window" \| "combined"` | Unique source IP limit accounting mode. | +| user_max_unique_ips_window_secs | `u64` | Recent-window size for unique IP accounting (seconds). | +| replay_check_len | `usize` | Replay check storage length. | +| replay_window_secs | `u64` | Replay protection time window in seconds. | +| ignore_time_skew | `bool` | Ignores client/server timestamp skew in replay validation. | + +## [[upstreams]] + +| Parameter | Type | Description | +|---|---|---| +| type | `"direct" \| "socks4" \| "socks5"` | Upstream transport type selector. | +| weight | `u16` | Weighted selection coefficient for this upstream. | +| enabled | `bool` | Enables/disables this upstream entry. | +| scopes | `String` | Comma-separated scope tags for routing. | +| interface | `String` | Optional outgoing interface name (`direct`, `socks4`, `socks5`). | +| bind_addresses | `String[]` | Optional source bind addresses for `direct` upstream. | +| address | `String` | Upstream proxy address (`host:port`) for SOCKS upstreams. | +| user_id | `String` | SOCKS4 user ID (only for `type = "socks4"`). | +| username | `String` | SOCKS5 username (only for `type = "socks5"`). | +| password | `String` | SOCKS5 password (only for `type = "socks5"`). | From 2aa7ea51379f8cd66f4a950ff504419de701ce7d Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:46:13 +0300 Subject: [PATCH 229/261] ME Coverage Ratio in API + as Draining Factor --- src/api/model.rs | 2 + src/api/runtime_min.rs | 2 + src/api/runtime_stats.rs | 3 + src/transport/middle_proxy/pool_reinit.rs | 92 ++++++++++++++++++----- src/transport/middle_proxy/pool_status.rs | 16 ++++ 5 files changed, 96 insertions(+), 19 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index 6b6fd72..ac4e297 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -237,6 +237,7 @@ pub(super) struct MeWritersSummary { pub(super) available_pct: f64, pub(super) required_writers: usize, pub(super) alive_writers: usize, + pub(super) coverage_ratio: f64, pub(super) coverage_pct: f64, pub(super) fresh_alive_writers: usize, pub(super) fresh_coverage_pct: f64, @@ -285,6 +286,7 @@ pub(super) struct DcStatus { pub(super) floor_max: usize, pub(super) floor_capped: bool, pub(super) alive_writers: usize, + pub(super) coverage_ratio: f64, pub(super) coverage_pct: f64, pub(super) fresh_alive_writers: usize, pub(super) fresh_coverage_pct: f64, diff --git a/src/api/runtime_min.rs b/src/api/runtime_min.rs index d3066a3..f334dd0 100644 --- a/src/api/runtime_min.rs +++ b/src/api/runtime_min.rs @@ -113,6 +113,7 @@ pub(super) struct RuntimeMeQualityDcRttData { pub(super) rtt_ema_ms: Option, pub(super) alive_writers: usize, pub(super) required_writers: usize, + pub(super) coverage_ratio: f64, pub(super) coverage_pct: f64, } @@ -388,6 +389,7 @@ pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> Runtime rtt_ema_ms: dc.rtt_ms, alive_writers: dc.alive_writers, required_writers: dc.required_writers, + coverage_ratio: dc.coverage_ratio, coverage_pct: dc.coverage_pct, }) .collect(), diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 61c8a5a..f8948d1 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -315,6 +315,7 @@ async fn get_minimal_payload_cached( available_pct: status.available_pct, required_writers: status.required_writers, alive_writers: status.alive_writers, + coverage_ratio: status.coverage_ratio, coverage_pct: status.coverage_pct, fresh_alive_writers: status.fresh_alive_writers, fresh_coverage_pct: status.fresh_coverage_pct, @@ -372,6 +373,7 @@ async fn get_minimal_payload_cached( floor_max: entry.floor_max, floor_capped: entry.floor_capped, alive_writers: entry.alive_writers, + coverage_ratio: entry.coverage_ratio, coverage_pct: entry.coverage_pct, fresh_alive_writers: entry.fresh_alive_writers, fresh_coverage_pct: entry.fresh_coverage_pct, @@ -502,6 +504,7 @@ fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersDa available_pct: 0.0, required_writers: 0, alive_writers: 0, + coverage_ratio: 0.0, coverage_pct: 0.0, fresh_alive_writers: 0, fresh_coverage_pct: 0.0, diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 3d9d679..0d5c6f4 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -70,10 +70,12 @@ impl MePool { let mut missing_dc = Vec::::new(); let mut covered = 0usize; + let mut total = 0usize; for (dc, endpoints) in desired_by_dc { if endpoints.is_empty() { continue; } + total += 1; if endpoints .iter() .any(|addr| active_writer_addrs.contains(&(*dc, *addr))) @@ -85,7 +87,9 @@ impl MePool { } missing_dc.sort_unstable(); - let total = desired_by_dc.len().max(1); + if total == 0 { + return (1.0, missing_dc); + } let ratio = (covered as f32) / (total as f32); (ratio, missing_dc) } @@ -399,29 +403,21 @@ impl MePool { } if hardswap { - let mut fresh_missing_dc = Vec::<(i32, usize, usize)>::new(); - for (dc, endpoints) in &desired_by_dc { - if endpoints.is_empty() { - continue; - } - let required = self.required_writers_for_dc(endpoints.len()); - let fresh_count = writers - .iter() - .filter(|w| !w.draining.load(Ordering::Relaxed)) - .filter(|w| w.generation == generation) - .filter(|w| w.writer_dc == *dc) - .filter(|w| endpoints.contains(&w.addr)) - .count(); - if fresh_count < required { - fresh_missing_dc.push((*dc, fresh_count, required)); - } - } + let fresh_writer_addrs: HashSet<(i32, SocketAddr)> = writers + .iter() + .filter(|w| !w.draining.load(Ordering::Relaxed)) + .filter(|w| w.generation == generation) + .map(|w| (w.writer_dc, w.addr)) + .collect(); + let (fresh_coverage_ratio, fresh_missing_dc) = + Self::coverage_ratio(&desired_by_dc, &fresh_writer_addrs); if !fresh_missing_dc.is_empty() { warn!( previous_generation, generation, + fresh_coverage_ratio = format_args!("{fresh_coverage_ratio:.3}"), missing_dc = ?fresh_missing_dc, - "ME hardswap pending: fresh generation coverage incomplete" + "ME hardswap pending: fresh generation DC coverage incomplete" ); return; } @@ -491,3 +487,61 @@ impl MePool { self.zero_downtime_reinit_after_map_change(rng).await; } } + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use super::MePool; + + fn addr(octet: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, octet)), port) + } + + #[test] + fn coverage_ratio_counts_dc_coverage_not_floor() { + let dc1 = addr(1, 2001); + let dc2 = addr(2, 2002); + + let mut desired_by_dc = HashMap::>::new(); + desired_by_dc.insert(1, HashSet::from([dc1])); + desired_by_dc.insert(2, HashSet::from([dc2])); + + let active_writer_addrs = HashSet::from([(1, dc1)]); + let (ratio, missing_dc) = MePool::coverage_ratio(&desired_by_dc, &active_writer_addrs); + + assert_eq!(ratio, 0.5); + assert_eq!(missing_dc, vec![2]); + } + + #[test] + fn coverage_ratio_ignores_empty_dc_groups() { + let dc1 = addr(1, 2001); + + let mut desired_by_dc = HashMap::>::new(); + desired_by_dc.insert(1, HashSet::from([dc1])); + desired_by_dc.insert(2, HashSet::new()); + + let active_writer_addrs = HashSet::from([(1, dc1)]); + let (ratio, missing_dc) = MePool::coverage_ratio(&desired_by_dc, &active_writer_addrs); + + assert_eq!(ratio, 1.0); + assert!(missing_dc.is_empty()); + } + + #[test] + fn coverage_ratio_reports_missing_dcs_sorted() { + let dc1 = addr(1, 2001); + let dc2 = addr(2, 2002); + + let mut desired_by_dc = HashMap::>::new(); + desired_by_dc.insert(2, HashSet::from([dc2])); + desired_by_dc.insert(1, HashSet::from([dc1])); + + let (ratio, missing_dc) = MePool::coverage_ratio(&desired_by_dc, &HashSet::new()); + + assert_eq!(ratio, 0.0); + assert_eq!(missing_dc, vec![1, 2]); + } +} diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index d32835c..214ee49 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -40,6 +40,7 @@ pub(crate) struct MeApiDcStatusSnapshot { pub floor_max: usize, pub floor_capped: bool, pub alive_writers: usize, + pub coverage_ratio: f64, pub coverage_pct: f64, pub fresh_alive_writers: usize, pub fresh_coverage_pct: f64, @@ -62,6 +63,7 @@ pub(crate) struct MeApiStatusSnapshot { pub available_pct: f64, pub required_writers: usize, pub alive_writers: usize, + pub coverage_ratio: f64, pub coverage_pct: f64, pub fresh_alive_writers: usize, pub fresh_coverage_pct: f64, @@ -342,6 +344,8 @@ impl MePool { let mut available_endpoints = 0usize; let mut alive_writers = 0usize; let mut fresh_alive_writers = 0usize; + let mut coverage_ratio_dcs_total = 0usize; + let mut coverage_ratio_dcs_covered = 0usize; let floor_mode = self.floor_mode(); let adaptive_cpu_cores = (self .me_adaptive_floor_cpu_cores_effective @@ -393,6 +397,12 @@ impl MePool { available_endpoints += dc_available_endpoints; alive_writers += dc_alive_writers; fresh_alive_writers += dc_fresh_alive_writers; + if endpoint_count > 0 { + coverage_ratio_dcs_total += 1; + if dc_alive_writers > 0 { + coverage_ratio_dcs_covered += 1; + } + } dcs.push(MeApiDcStatusSnapshot { dc, @@ -415,6 +425,11 @@ impl MePool { floor_max, floor_capped, alive_writers: dc_alive_writers, + coverage_ratio: if endpoint_count > 0 && dc_alive_writers > 0 { + 100.0 + } else { + 0.0 + }, coverage_pct: ratio_pct(dc_alive_writers, dc_required_writers), fresh_alive_writers: dc_fresh_alive_writers, fresh_coverage_pct: ratio_pct(dc_fresh_alive_writers, dc_required_writers), @@ -431,6 +446,7 @@ impl MePool { available_pct: ratio_pct(available_endpoints, configured_endpoints), required_writers, alive_writers, + coverage_ratio: ratio_pct(coverage_ratio_dcs_covered, coverage_ratio_dcs_total), coverage_pct: ratio_pct(alive_writers, required_writers), fresh_alive_writers, fresh_coverage_pct: ratio_pct(fresh_alive_writers, required_writers), From 42f946f29e645a16c1e8ba3b7246a1afeb099dc8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:57:09 +0300 Subject: [PATCH 230/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dad9cf0..788bc2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.20" +version = "3.3.21" edition = "2024" [dependencies] From def42f0baa03284fb2671e25b94560242342924a Mon Sep 17 00:00:00 2001 From: Maxim Myalin Date: Wed, 18 Mar 2026 12:36:44 +0300 Subject: [PATCH 231/261] Add Shadowsocks upstream support --- Cargo.lock | 602 ++++++++++++++++++++++++++++- Cargo.toml | 1 + docs/API.md | 10 +- docs/FAQ.en.md | 14 + docs/FAQ.ru.md | 13 + docs/TUNING.de.md | 20 +- docs/TUNING.en.md | 20 +- docs/TUNING.ru.md | 20 +- src/api/model.rs | 1 + src/api/runtime_min.rs | 31 +- src/api/runtime_selftest.rs | 9 +- src/api/runtime_stats.rs | 19 +- src/config/load.rs | 228 +++++++++-- src/config/types.rs | 78 ++-- src/proxy/direct_relay.rs | 42 +- src/tls_front/fetcher.rs | 85 ++-- src/transport/middle_proxy/ping.rs | 22 +- src/transport/mod.rs | 6 +- src/transport/shadowsocks.rs | 60 +++ src/transport/upstream.rs | 468 ++++++++++++++++------ 20 files changed, 1476 insertions(+), 273 deletions(-) create mode 100644 src/transport/shadowsocks.rs diff --git a/Cargo.lock b/Cargo.lock index a704404..a943322 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.8.4" @@ -13,6 +23,20 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -55,6 +79,27 @@ version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "asn1-rs" version = "0.5.2" @@ -94,6 +139,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -112,6 +168,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.8.0" @@ -139,6 +201,20 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -163,6 +239,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "byte_string" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11aade7a05aa8c3a351cedc44c3fc45806430543382fcc4743a9b757a2a0b4ed" + [[package]] name = "bytes" version = "1.11.1" @@ -212,6 +294,30 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.43" @@ -261,6 +367,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -288,6 +395,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -357,6 +476,12 @@ dependencies = [ "itertools", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -413,6 +538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -444,6 +570,16 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der-parser" version = "8.2.0" @@ -489,12 +625,54 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "dynosaur" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12303417f378f29ba12cb12fc78a9df0d8e16ccb1ad94abf04d48d96bdda532" +dependencies = [ + "dynosaur_derive", +] + +[[package]] +name = "dynosaur_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0713d5c1d52e774c5cd7bb8b043d7c0fc4f921abfb678556140bfbe6ab2364" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -709,6 +887,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "h2" version = "0.4.13" @@ -783,6 +971,61 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand", + "resolv-conf", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1055,6 +1298,17 @@ dependencies = [ "libc", ] +[[package]] +name = "inotify" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd5b3eaf1a28b758ac0faa5a4254e8ab2705605496f1b1f3fbbc3988ad73d199" +dependencies = [ + "bitflags 2.10.0", + "inotify-sys", + "libc", +] + [[package]] name = "inotify-sys" version = "0.1.5" @@ -1074,6 +1328,18 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.10", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1226,6 +1492,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lru_time_cache" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" + [[package]] name = "matchers" version = "0.2.0" @@ -1285,10 +1557,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "nix" version = "0.28.0" @@ -1322,7 +1612,7 @@ dependencies = [ "crossbeam-channel", "filetime", "fsevent-sys", - "inotify", + "inotify 0.9.6", "kqueue", "libc", "log", @@ -1331,6 +1621,33 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.10.0", + "fsevent-sys", + "inotify 0.11.1", + "kqueue", + "libc", + "log", + "mio 1.1.1", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1388,6 +1705,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "oorandom" @@ -1395,6 +1716,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1424,6 +1751,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1436,6 +1783,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "plotters" version = "0.3.7" @@ -1464,6 +1821,35 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.4" @@ -1609,7 +1995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1619,7 +2005,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -1637,7 +2032,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1745,6 +2140,12 @@ dependencies = [ "webpki-roots 1.0.6", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -1759,6 +2160,19 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ring-compat" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccce7bae150b815f0811db41b8312fcb74bffa4cab9cee5429ee00f356dd5bd4" +dependencies = [ + "aead", + "ed25519", + "generic-array", + "pkcs8", + "ring", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -1870,12 +2284,33 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sealed" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "semver" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +[[package]] +name = "sendfd" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b183bfd5b1bc64ab0c1ef3ee06b008a9ef1b68a7d3a99ba566fbfe7a7c6d745b" +dependencies = [ + "libc", + "tokio", +] + [[package]] name = "serde" version = "1.0.228" @@ -1962,6 +2397,64 @@ dependencies = [ "digest", ] +[[package]] +name = "shadowsocks" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482831bf9d55acf3c98e211b6c852c3dfdf1d1b0d23fdf1d887c5a4b2acad4e4" +dependencies = [ + "aes", + "arc-swap", + "base64", + "blake3", + "byte_string", + "bytes", + "cfg-if", + "dynosaur", + "futures", + "hickory-resolver", + "libc", + "log", + "lru_time_cache", + "notify 8.2.0", + "percent-encoding", + "pin-project", + "rand", + "sealed", + "sendfd", + "serde", + "serde_json", + "serde_urlencoded", + "shadowsocks-crypto", + "socket2 0.6.2", + "spin", + "thiserror 2.0.18", + "tokio", + "tokio-tfo", + "trait-variant", + "url", + "windows-sys 0.61.2", +] + +[[package]] +name = "shadowsocks-crypto" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d038a3d17586f1c1ab3c1c3b9e4d5ef8fba98fb3890ad740c8487038b2e2ca5" +dependencies = [ + "aes", + "aes-gcm", + "blake3", + "bytes", + "cfg-if", + "chacha20poly1305", + "hkdf", + "md-5", + "rand", + "ring-compat", + "sha1", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1987,6 +2480,12 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" + [[package]] name = "slab" version = "0.4.12" @@ -2019,6 +2518,25 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2085,9 +2603,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "telemt" -version = "3.3.19" +version = "3.3.20" dependencies = [ "aes", "anyhow", @@ -2113,7 +2637,7 @@ dependencies = [ "lru", "md-5", "nix", - "notify", + "notify 6.1.1", "num-bigint", "num-traits", "parking_lot", @@ -2126,6 +2650,7 @@ dependencies = [ "serde_json", "sha1", "sha2", + "shadowsocks", "socket2 0.5.10", "thiserror 2.0.18", "tokio", @@ -2330,6 +2855,23 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tokio-tfo" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ad2c3b3bb958ad992354a7ebc468fc0f7cdc9af4997bf4d3fd3cb28bad36dc" +dependencies = [ + "cfg-if", + "futures", + "libc", + "log", + "once_cell", + "pin-project", + "socket2 0.6.2", + "tokio", + "windows-sys 0.60.2", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2494,6 +3036,17 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -2524,6 +3077,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -2548,6 +3111,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +dependencies = [ + "getrandom 0.4.1", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2743,6 +3317,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3042,6 +3622,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/Cargo.toml b/Cargo.toml index dad9cf0..e43e6ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ zeroize = { version = "1.8", features = ["derive"] } # Network socket2 = { version = "0.5", features = ["all"] } nix = { version = "0.28", default-features = false, features = ["net"] } +shadowsocks = { version = "1.24", features = ["aead-cipher-2022"] } # Serialization serde = { version = "1.0", features = ["derive"] } diff --git a/docs/API.md b/docs/API.md index 9296aff..a1f0f4f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -497,13 +497,14 @@ Note: the request contract is defined, but the corresponding route currently ret | `direct_total` | `usize` | Direct-route upstream entries. | | `socks4_total` | `usize` | SOCKS4 upstream entries. | | `socks5_total` | `usize` | SOCKS5 upstream entries. | +| `shadowsocks_total` | `usize` | Shadowsocks upstream entries. | #### `RuntimeUpstreamQualityUpstreamData` | Field | Type | Description | | --- | --- | --- | | `upstream_id` | `usize` | Runtime upstream index. | -| `route_kind` | `string` | `direct`, `socks4`, `socks5`. | -| `address` | `string` | Upstream address (`direct` literal for direct route kind). | +| `route_kind` | `string` | `direct`, `socks4`, `socks5`, `shadowsocks`. | +| `address` | `string` | Upstream address (`direct` literal for direct route kind, `host:port` only for proxied upstreams). | | `weight` | `u16` | Selection weight. | | `scopes` | `string` | Configured scope selector. | | `healthy` | `bool` | Current health flag. | @@ -757,13 +758,14 @@ Note: the request contract is defined, but the corresponding route currently ret | `direct_total` | `usize` | Number of direct upstream entries. | | `socks4_total` | `usize` | Number of SOCKS4 upstream entries. | | `socks5_total` | `usize` | Number of SOCKS5 upstream entries. | +| `shadowsocks_total` | `usize` | Number of Shadowsocks upstream entries. | #### `UpstreamStatus` | Field | Type | Description | | --- | --- | --- | | `upstream_id` | `usize` | Runtime upstream index. | -| `route_kind` | `string` | Upstream route kind: `direct`, `socks4`, `socks5`. | -| `address` | `string` | Upstream address (`direct` for direct route kind). Authentication fields are intentionally omitted. | +| `route_kind` | `string` | Upstream route kind: `direct`, `socks4`, `socks5`, `shadowsocks`. | +| `address` | `string` | Upstream address (`direct` for direct route kind, `host:port` for Shadowsocks). Authentication fields are intentionally omitted. | | `weight` | `u16` | Selection weight. | | `scopes` | `string` | Configured scope selector string. | | `healthy` | `bool` | Current health flag. | diff --git a/docs/FAQ.en.md b/docs/FAQ.en.md index 25522ad..4af1c34 100644 --- a/docs/FAQ.en.md +++ b/docs/FAQ.en.md @@ -120,3 +120,17 @@ password = "pass" # Password for Auth on SOCKS-server weight = 1 # Set Weight for Scenarios enabled = true ``` + +#### Shadowsocks as Upstream +Requires `use_middle_proxy = false`. + +```toml +[general] +use_middle_proxy = false + +[[upstreams]] +type = "shadowsocks" +url = "ss://2022-blake3-aes-256-gcm:BASE64_KEY@1.2.3.4:8388" +weight = 1 +enabled = true +``` diff --git a/docs/FAQ.ru.md b/docs/FAQ.ru.md index 4353e98..ae38cab 100644 --- a/docs/FAQ.ru.md +++ b/docs/FAQ.ru.md @@ -121,3 +121,16 @@ weight = 1 # Set Weight for Scenarios enabled = true ``` +#### Shadowsocks как Upstream +Требует `use_middle_proxy = false`. + +```toml +[general] +use_middle_proxy = false + +[[upstreams]] +type = "shadowsocks" +url = "ss://2022-blake3-aes-256-gcm:BASE64_KEY@1.2.3.4:8388" +weight = 1 +enabled = true +``` diff --git a/docs/TUNING.de.md b/docs/TUNING.de.md index 8c3c950..3b0f31d 100644 --- a/docs/TUNING.de.md +++ b/docs/TUNING.de.md @@ -82,7 +82,7 @@ Die unten angegebenen `Default`-Werte sind Code-Defaults (bei fehlendem Schlüss | Feld | Gilt für | Typ | Pflicht | Default | Bedeutung | |---|---|---|---|---|---| -| `[[upstreams]].type` | alle Upstreams | `"direct" \| "socks4" \| "socks5"` | ja | n/a | Upstream-Transporttyp. | +| `[[upstreams]].type` | alle Upstreams | `"direct" \| "socks4" \| "socks5" \| "shadowsocks"` | ja | n/a | Upstream-Transporttyp. | | `[[upstreams]].weight` | alle Upstreams | `u16` | nein | `1` | Basisgewicht für weighted-random Auswahl. | | `[[upstreams]].enabled` | alle Upstreams | `bool` | nein | `true` | Deaktivierte Einträge werden beim Start ignoriert. | | `[[upstreams]].scopes` | alle Upstreams | `String` | nein | `""` | Komma-separierte Scope-Tags für Request-Routing. | @@ -95,6 +95,8 @@ Die unten angegebenen `Default`-Werte sind Code-Defaults (bei fehlendem Schlüss | `interface` | `socks5` | `Option` | nein | `null` | Wird nur genutzt, wenn `address` als `ip:port` angegeben ist. | | `username` | `socks5` | `Option` | nein | `null` | SOCKS5 Benutzername. | | `password` | `socks5` | `Option` | nein | `null` | SOCKS5 Passwort. | +| `url` | `shadowsocks` | `String` | ja | n/a | Shadowsocks-SIP002-URL (`ss://...`). In Runtime-APIs wird nur `host:port` offengelegt. | +| `interface` | `shadowsocks` | `Option` | nein | `null` | Optionales ausgehendes Bind-Interface oder lokale Literal-IP. | ### Runtime-Regeln (wichtig) @@ -115,6 +117,7 @@ Die unten angegebenen `Default`-Werte sind Code-Defaults (bei fehlendem Schlüss 8. Im ME-Modus wird der gewählte Upstream auch für den ME-TCP-Dial-Pfad verwendet. 9. Im ME-Modus ist bei `direct` mit bind/interface die STUN-Reflection bind-aware für KDF-Adressmaterial. 10. Im ME-Modus werden bei SOCKS-Upstream `BND.ADDR/BND.PORT` für KDF verwendet, wenn gültig/öffentlich und gleiche IP-Familie. +11. `shadowsocks`-Upstreams erfordern `general.use_middle_proxy = false`. Mit aktiviertem ME-Modus schlägt das Laden der Config sofort fehl. ## Upstream-Konfigurationsbeispiele @@ -150,7 +153,20 @@ weight = 2 enabled = true ``` -### Beispiel 4: Gemischte Upstreams mit Scopes +### Beispiel 4: Shadowsocks-Upstream + +```toml +[general] +use_middle_proxy = false + +[[upstreams]] +type = "shadowsocks" +url = "ss://2022-blake3-aes-256-gcm:BASE64_KEY@198.51.100.50:8388" +weight = 2 +enabled = true +``` + +### Beispiel 5: Gemischte Upstreams mit Scopes ```toml [[upstreams]] diff --git a/docs/TUNING.en.md b/docs/TUNING.en.md index 1bbc439..6a6a320 100644 --- a/docs/TUNING.en.md +++ b/docs/TUNING.en.md @@ -82,7 +82,7 @@ Defaults below are code defaults (used when a key is omitted), not necessarily v | Field | Applies to | Type | Required | Default | Meaning | |---|---|---|---|---|---| -| `[[upstreams]].type` | all upstreams | `"direct" \| "socks4" \| "socks5"` | yes | n/a | Upstream transport type. | +| `[[upstreams]].type` | all upstreams | `"direct" \| "socks4" \| "socks5" \| "shadowsocks"` | yes | n/a | Upstream transport type. | | `[[upstreams]].weight` | all upstreams | `u16` | no | `1` | Base weight for weighted-random selection. | | `[[upstreams]].enabled` | all upstreams | `bool` | no | `true` | Disabled entries are ignored at startup. | | `[[upstreams]].scopes` | all upstreams | `String` | no | `""` | Comma-separated scope tags for request-level routing. | @@ -95,6 +95,8 @@ Defaults below are code defaults (used when a key is omitted), not necessarily v | `interface` | `socks5` | `Option` | no | `null` | Used only for SOCKS server `ip:port` dial path. | | `username` | `socks5` | `Option` | no | `null` | SOCKS5 username auth. | | `password` | `socks5` | `Option` | no | `null` | SOCKS5 password auth. | +| `url` | `shadowsocks` | `String` | yes | n/a | Shadowsocks SIP002 URL (`ss://...`). Only `host:port` is exposed in runtime APIs. | +| `interface` | `shadowsocks` | `Option` | no | `null` | Optional outgoing bind interface or literal local IP. | ### Runtime rules (important) @@ -115,6 +117,7 @@ Defaults below are code defaults (used when a key is omitted), not necessarily v 8. In ME mode, the selected upstream is also used for ME TCP dial path. 9. In ME mode for `direct` upstream with bind/interface, STUN reflection logic is bind-aware for KDF source material. 10. In ME mode for SOCKS upstream, SOCKS `BND.ADDR/BND.PORT` is used for KDF when it is valid/public for the same family. +11. `shadowsocks` upstreams require `general.use_middle_proxy = false`. Config load fails fast if ME mode is enabled. ## Upstream Configuration Examples @@ -150,7 +153,20 @@ weight = 2 enabled = true ``` -### Example 4: Mixed upstreams with scopes +### Example 4: Shadowsocks upstream + +```toml +[general] +use_middle_proxy = false + +[[upstreams]] +type = "shadowsocks" +url = "ss://2022-blake3-aes-256-gcm:BASE64_KEY@198.51.100.50:8388" +weight = 2 +enabled = true +``` + +### Example 5: Mixed upstreams with scopes ```toml [[upstreams]] diff --git a/docs/TUNING.ru.md b/docs/TUNING.ru.md index 6ea4d69..bae8fdd 100644 --- a/docs/TUNING.ru.md +++ b/docs/TUNING.ru.md @@ -82,7 +82,7 @@ | Поле | Применимость | Тип | Обязательно | Default | Назначение | |---|---|---|---|---|---| -| `[[upstreams]].type` | все upstream | `"direct" \| "socks4" \| "socks5"` | да | n/a | Тип upstream транспорта. | +| `[[upstreams]].type` | все upstream | `"direct" \| "socks4" \| "socks5" \| "shadowsocks"` | да | n/a | Тип upstream транспорта. | | `[[upstreams]].weight` | все upstream | `u16` | нет | `1` | Базовый вес в weighted-random выборе. | | `[[upstreams]].enabled` | все upstream | `bool` | нет | `true` | Выключенные записи игнорируются на старте. | | `[[upstreams]].scopes` | все upstream | `String` | нет | `""` | Список scope-токенов через запятую для маршрутизации. | @@ -95,6 +95,8 @@ | `interface` | `socks5` | `Option` | нет | `null` | Используется только если `address` задан как `ip:port`. | | `username` | `socks5` | `Option` | нет | `null` | Логин SOCKS5 auth. | | `password` | `socks5` | `Option` | нет | `null` | Пароль SOCKS5 auth. | +| `url` | `shadowsocks` | `String` | да | n/a | Shadowsocks SIP002 URL (`ss://...`). В runtime API раскрывается только `host:port`. | +| `interface` | `shadowsocks` | `Option` | нет | `null` | Необязательный исходящий bind-интерфейс или literal локальный IP. | ### Runtime-правила @@ -115,6 +117,7 @@ 8. В ME-режиме выбранный upstream также используется для ME TCP dial path. 9. В ME-режиме для `direct` upstream с bind/interface STUN-рефлексия выполняется bind-aware для KDF материала. 10. В ME-режиме для SOCKS upstream используются `BND.ADDR/BND.PORT` для KDF, если адрес валиден/публичен и соответствует IP family. +11. `shadowsocks` upstream требует `general.use_middle_proxy = false`. При включенном ME-режиме конфиг отклоняется при загрузке. ## Примеры конфигурации Upstreams @@ -150,7 +153,20 @@ weight = 2 enabled = true ``` -### Пример 4: смешанные upstream с scopes +### Пример 4: Shadowsocks upstream + +```toml +[general] +use_middle_proxy = false + +[[upstreams]] +type = "shadowsocks" +url = "ss://2022-blake3-aes-256-gcm:BASE64_KEY@198.51.100.50:8388" +weight = 2 +enabled = true +``` + +### Пример 5: смешанные upstream с scopes ```toml [[upstreams]] diff --git a/src/api/model.rs b/src/api/model.rs index 6b6fd72..c484466 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -134,6 +134,7 @@ pub(super) struct UpstreamSummaryData { pub(super) direct_total: usize, pub(super) socks4_total: usize, pub(super) socks5_total: usize, + pub(super) shadowsocks_total: usize, } #[derive(Serialize, Clone)] diff --git a/src/api/runtime_min.rs b/src/api/runtime_min.rs index d3066a3..2336663 100644 --- a/src/api/runtime_min.rs +++ b/src/api/runtime_min.rs @@ -158,6 +158,7 @@ pub(super) struct RuntimeUpstreamQualitySummaryData { pub(super) direct_total: usize, pub(super) socks4_total: usize, pub(super) socks5_total: usize, + pub(super) shadowsocks_total: usize, } #[derive(Serialize)] @@ -404,7 +405,9 @@ pub(super) async fn build_runtime_upstream_quality_data( connect_attempt_total: shared.stats.get_upstream_connect_attempt_total(), connect_success_total: shared.stats.get_upstream_connect_success_total(), connect_fail_total: shared.stats.get_upstream_connect_fail_total(), - connect_failfast_hard_error_total: shared.stats.get_upstream_connect_failfast_hard_error_total(), + connect_failfast_hard_error_total: shared + .stats + .get_upstream_connect_failfast_hard_error_total(), }; let Some(snapshot) = shared.upstream_manager.try_api_snapshot() else { @@ -444,6 +447,7 @@ pub(super) async fn build_runtime_upstream_quality_data( direct_total: snapshot.summary.direct_total, socks4_total: snapshot.summary.socks4_total, socks5_total: snapshot.summary.socks5_total, + shadowsocks_total: snapshot.summary.shadowsocks_total, }), upstreams: Some( snapshot @@ -455,6 +459,7 @@ pub(super) async fn build_runtime_upstream_quality_data( crate::transport::UpstreamRouteKind::Direct => "direct", crate::transport::UpstreamRouteKind::Socks4 => "socks4", crate::transport::UpstreamRouteKind::Socks5 => "socks5", + crate::transport::UpstreamRouteKind::Shadowsocks => "shadowsocks", }, address: upstream.address, weight: upstream.weight, @@ -474,7 +479,9 @@ pub(super) async fn build_runtime_upstream_quality_data( crate::transport::upstream::IpPreference::PreferV6 => "prefer_v6", crate::transport::upstream::IpPreference::PreferV4 => "prefer_v4", crate::transport::upstream::IpPreference::BothWork => "both_work", - crate::transport::upstream::IpPreference::Unavailable => "unavailable", + crate::transport::upstream::IpPreference::Unavailable => { + "unavailable" + } }, }) .collect(), @@ -512,14 +519,18 @@ pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNa live_total: snapshot.live_servers.len(), }, reflection: RuntimeNatStunReflectionBlockData { - v4: snapshot.reflection_v4.map(|entry| RuntimeNatStunReflectionData { - addr: entry.addr.to_string(), - age_secs: entry.age_secs, - }), - v6: snapshot.reflection_v6.map(|entry| RuntimeNatStunReflectionData { - addr: entry.addr.to_string(), - age_secs: entry.age_secs, - }), + v4: snapshot + .reflection_v4 + .map(|entry| RuntimeNatStunReflectionData { + addr: entry.addr.to_string(), + age_secs: entry.age_secs, + }), + v6: snapshot + .reflection_v6 + .map(|entry| RuntimeNatStunReflectionData { + addr: entry.addr.to_string(), + age_secs: entry.age_secs, + }), }, stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms, }), diff --git a/src/api/runtime_selftest.rs b/src/api/runtime_selftest.rs index 0dce3dc..0a3bef6 100644 --- a/src/api/runtime_selftest.rs +++ b/src/api/runtime_selftest.rs @@ -1,5 +1,5 @@ -use std::net::IpAddr; use std::collections::HashMap; +use std::net::IpAddr; use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -7,8 +7,8 @@ use serde::Serialize; use crate::config::{ProxyConfig, UpstreamType}; use crate::network::probe::{detect_interface_ipv4, detect_interface_ipv6, is_bogon}; -use crate::transport::middle_proxy::{bnd_snapshot, timeskew_snapshot, upstream_bnd_snapshots}; use crate::transport::UpstreamRouteKind; +use crate::transport::middle_proxy::{bnd_snapshot, timeskew_snapshot, upstream_bnd_snapshots}; use super::ApiShared; @@ -262,8 +262,8 @@ fn update_kdf_ewma(now_epoch_secs: u64, total_errors: u64) -> f64 { let delta_errors = total_errors.saturating_sub(guard.last_total_errors); let instant_rate_per_min = (delta_errors as f64) * 60.0 / (dt_secs as f64); let alpha = 1.0 - f64::exp(-(dt_secs as f64) / KDF_EWMA_TAU_SECS); - guard.ewma_errors_per_min = guard.ewma_errors_per_min - + alpha * (instant_rate_per_min - guard.ewma_errors_per_min); + guard.ewma_errors_per_min = + guard.ewma_errors_per_min + alpha * (instant_rate_per_min - guard.ewma_errors_per_min); guard.last_epoch_secs = now_epoch_secs; guard.last_total_errors = total_errors; guard.ewma_errors_per_min @@ -284,6 +284,7 @@ fn map_route_kind(value: UpstreamRouteKind) -> &'static str { UpstreamRouteKind::Direct => "direct", UpstreamRouteKind::Socks4 => "socks4", UpstreamRouteKind::Socks5 => "socks5", + UpstreamRouteKind::Shadowsocks => "shadowsocks", } } diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index 61c8a5a..7e12ca7 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -2,8 +2,8 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::config::ApiConfig; use crate::stats::Stats; -use crate::transport::upstream::IpPreference; use crate::transport::UpstreamRouteKind; +use crate::transport::upstream::IpPreference; use super::ApiShared; use super::model::{ @@ -138,7 +138,8 @@ fn build_zero_upstream_data(stats: &Stats) -> ZeroUpstreamData { .get_upstream_connect_duration_success_bucket_501_1000ms(), connect_duration_success_bucket_gt_1000ms: stats .get_upstream_connect_duration_success_bucket_gt_1000ms(), - connect_duration_fail_bucket_le_100ms: stats.get_upstream_connect_duration_fail_bucket_le_100ms(), + connect_duration_fail_bucket_le_100ms: stats + .get_upstream_connect_duration_fail_bucket_le_100ms(), connect_duration_fail_bucket_101_500ms: stats .get_upstream_connect_duration_fail_bucket_101_500ms(), connect_duration_fail_bucket_501_1000ms: stats @@ -180,6 +181,7 @@ pub(super) fn build_upstreams_data(shared: &ApiShared, api_cfg: &ApiConfig) -> U direct_total: snapshot.summary.direct_total, socks4_total: snapshot.summary.socks4_total, socks5_total: snapshot.summary.socks5_total, + shadowsocks_total: snapshot.summary.shadowsocks_total, }; let upstreams = snapshot .upstreams @@ -393,8 +395,7 @@ async fn get_minimal_payload_cached( adaptive_floor_min_writers_multi_endpoint: runtime .adaptive_floor_min_writers_multi_endpoint, adaptive_floor_recover_grace_secs: runtime.adaptive_floor_recover_grace_secs, - adaptive_floor_writers_per_core_total: runtime - .adaptive_floor_writers_per_core_total, + adaptive_floor_writers_per_core_total: runtime.adaptive_floor_writers_per_core_total, adaptive_floor_cpu_cores_override: runtime.adaptive_floor_cpu_cores_override, adaptive_floor_max_extra_writers_single_per_core: runtime .adaptive_floor_max_extra_writers_single_per_core, @@ -402,12 +403,9 @@ async fn get_minimal_payload_cached( .adaptive_floor_max_extra_writers_multi_per_core, adaptive_floor_max_active_writers_per_core: runtime .adaptive_floor_max_active_writers_per_core, - adaptive_floor_max_warm_writers_per_core: runtime - .adaptive_floor_max_warm_writers_per_core, - adaptive_floor_max_active_writers_global: runtime - .adaptive_floor_max_active_writers_global, - adaptive_floor_max_warm_writers_global: runtime - .adaptive_floor_max_warm_writers_global, + adaptive_floor_max_warm_writers_per_core: runtime.adaptive_floor_max_warm_writers_per_core, + adaptive_floor_max_active_writers_global: runtime.adaptive_floor_max_active_writers_global, + adaptive_floor_max_warm_writers_global: runtime.adaptive_floor_max_warm_writers_global, adaptive_floor_cpu_cores_detected: runtime.adaptive_floor_cpu_cores_detected, adaptive_floor_cpu_cores_effective: runtime.adaptive_floor_cpu_cores_effective, adaptive_floor_global_cap_raw: runtime.adaptive_floor_global_cap_raw, @@ -524,6 +522,7 @@ fn map_route_kind(value: UpstreamRouteKind) -> &'static str { UpstreamRouteKind::Direct => "direct", UpstreamRouteKind::Socks4 => "socks4", UpstreamRouteKind::Socks5 => "socks5", + UpstreamRouteKind::Shadowsocks => "shadowsocks", } } diff --git a/src/config/load.rs b/src/config/load.rs index 6fcbea3..222069c 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -6,8 +6,9 @@ use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use rand::Rng; +use serde::{Deserialize, Serialize}; +use shadowsocks::config::ServerConfig as ShadowsocksServerConfig; use tracing::warn; -use serde::{Serialize, Deserialize}; use crate::error::{ProxyError, Result}; @@ -122,13 +123,37 @@ fn sanitize_ad_tag(ad_tag: &mut Option) { }; if !is_valid_ad_tag(tag) { - warn!( - "Invalid general.ad_tag value, expected exactly 32 hex chars; ad_tag is disabled" - ); + warn!("Invalid general.ad_tag value, expected exactly 32 hex chars; ad_tag is disabled"); *ad_tag = None; } } +fn validate_upstreams(config: &ProxyConfig) -> Result<()> { + let has_enabled_shadowsocks = config.upstreams.iter().any(|upstream| { + upstream.enabled && matches!(upstream.upstream_type, UpstreamType::Shadowsocks { .. }) + }); + + if has_enabled_shadowsocks && config.general.use_middle_proxy { + return Err(ProxyError::Config( + "shadowsocks upstreams require general.use_middle_proxy = false".to_string(), + )); + } + + for upstream in &config.upstreams { + if let UpstreamType::Shadowsocks { url, .. } = &upstream.upstream_type { + let parsed = ShadowsocksServerConfig::from_url(url) + .map_err(|error| ProxyError::Config(format!("invalid shadowsocks url: {error}")))?; + if parsed.plugin().is_some() { + return Err(ProxyError::Config( + "shadowsocks plugins are not supported".to_string(), + )); + } + } + } + + Ok(()) +} + // ============= Main Config ============= #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -180,7 +205,8 @@ impl ProxyConfig { pub(crate) fn load_with_metadata>(path: P) -> Result { let path = path.as_ref(); - let content = std::fs::read_to_string(path).map_err(|e| ProxyError::Config(e.to_string()))?; + let content = + std::fs::read_to_string(path).map_err(|e| ProxyError::Config(e.to_string()))?; let base_dir = path.parent().unwrap_or(Path::new(".")); let mut source_files = BTreeSet::new(); source_files.insert(normalize_config_path(path)); @@ -207,15 +233,17 @@ impl ProxyConfig { .map(|table| table.contains_key("stun_servers")) .unwrap_or(false); - let mut config: ProxyConfig = - parsed_toml.try_into().map_err(|e| ProxyError::Config(e.to_string()))?; + let mut config: ProxyConfig = parsed_toml + .try_into() + .map_err(|e| ProxyError::Config(e.to_string()))?; if !update_every_is_explicit && (legacy_secret_is_explicit || legacy_config_is_explicit) { config.general.update_every = None; } let legacy_nat_stun = config.general.middle_proxy_nat_stun.take(); - let legacy_nat_stun_servers = std::mem::take(&mut config.general.middle_proxy_nat_stun_servers); + let legacy_nat_stun_servers = + std::mem::take(&mut config.general.middle_proxy_nat_stun_servers); let legacy_nat_stun_used = legacy_nat_stun.is_some() || !legacy_nat_stun_servers.is_empty(); if stun_servers_is_explicit { let mut explicit_stun_servers = Vec::new(); @@ -225,7 +253,9 @@ impl ProxyConfig { config.network.stun_servers = explicit_stun_servers; if legacy_nat_stun_used { - warn!("general.middle_proxy_nat_stun and general.middle_proxy_nat_stun_servers are ignored because network.stun_servers is explicitly set"); + warn!( + "general.middle_proxy_nat_stun and general.middle_proxy_nat_stun_servers are ignored because network.stun_servers is explicitly set" + ); } } else { // Keep the default STUN pool unless network.stun_servers is explicitly overridden. @@ -240,7 +270,9 @@ impl ProxyConfig { config.network.stun_servers = unified_stun_servers; if legacy_nat_stun_used { - warn!("general.middle_proxy_nat_stun and general.middle_proxy_nat_stun_servers are deprecated; use network.stun_servers"); + warn!( + "general.middle_proxy_nat_stun and general.middle_proxy_nat_stun_servers are deprecated; use network.stun_servers" + ); } } @@ -372,13 +404,15 @@ impl ProxyConfig { if !(4096..=1024 * 1024).contains(&config.general.direct_relay_copy_buf_c2s_bytes) { return Err(ProxyError::Config( - "general.direct_relay_copy_buf_c2s_bytes must be within [4096, 1048576]".to_string(), + "general.direct_relay_copy_buf_c2s_bytes must be within [4096, 1048576]" + .to_string(), )); } if !(8192..=2 * 1024 * 1024).contains(&config.general.direct_relay_copy_buf_s2c_bytes) { return Err(ProxyError::Config( - "general.direct_relay_copy_buf_s2c_bytes must be within [8192, 2097152]".to_string(), + "general.direct_relay_copy_buf_s2c_bytes must be within [8192, 2097152]" + .to_string(), )); } @@ -617,7 +651,8 @@ impl ProxyConfig { if !(1..=100).contains(&config.general.me_route_backpressure_high_watermark_pct) { return Err(ProxyError::Config( - "general.me_route_backpressure_high_watermark_pct must be within [1, 100]".to_string(), + "general.me_route_backpressure_high_watermark_pct must be within [1, 100]" + .to_string(), )); } @@ -779,11 +814,15 @@ impl ProxyConfig { crate::network::dns_overrides::validate_entries(&config.network.dns_overrides)?; if config.general.use_middle_proxy && config.network.ipv6 == Some(true) { - warn!("IPv6 with Middle Proxy is experimental and may cause KDF address mismatch; consider disabling IPv6 or ME"); + warn!( + "IPv6 with Middle Proxy is experimental and may cause KDF address mismatch; consider disabling IPv6 or ME" + ); } // Random fake_cert_len only when default is in use. - if !config.censorship.tls_emulation && config.censorship.fake_cert_len == default_fake_cert_len() { + if !config.censorship.tls_emulation + && config.censorship.fake_cert_len == default_fake_cert_len() + { config.censorship.fake_cert_len = rand::rng().gen_range(1024..4096); } @@ -793,8 +832,7 @@ impl ProxyConfig { let listen_tcp = config.server.listen_tcp.unwrap_or_else(|| { if config.server.listen_unix_sock.is_some() { // Unix socket present: TCP only if user explicitly set addresses or listeners. - config.server.listen_addr_ipv4.is_some() - || !config.server.listeners.is_empty() + config.server.listen_addr_ipv4.is_some() || !config.server.listeners.is_empty() } else { true } @@ -802,7 +840,9 @@ impl ProxyConfig { // Migration: Populate listeners if empty (skip when listen_tcp = false). if config.server.listeners.is_empty() && listen_tcp { - let ipv4_str = config.server.listen_addr_ipv4 + let ipv4_str = config + .server + .listen_addr_ipv4 .as_deref() .unwrap_or("0.0.0.0"); if let Ok(ipv4) = ipv4_str.parse::() { @@ -844,7 +884,10 @@ impl ProxyConfig { // Migration: Populate upstreams if empty (Default Direct). if config.upstreams.is_empty() { config.upstreams.push(UpstreamConfig { - upstream_type: UpstreamType::Direct { interface: None, bind_addresses: None }, + upstream_type: UpstreamType::Direct { + interface: None, + bind_addresses: None, + }, weight: 1, enabled: true, scopes: String::new(), @@ -858,6 +901,8 @@ impl ProxyConfig { .entry("203".to_string()) .or_insert_with(|| vec!["91.105.192.100:443".to_string()]); + validate_upstreams(&config)?; + Ok(LoadedConfig { config, source_files: source_files.into_iter().collect(), @@ -904,6 +949,9 @@ impl ProxyConfig { mod tests { use super::*; + const TEST_SHADOWSOCKS_URL: &str = + "ss://2022-blake3-aes-256-gcm:MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE=@127.0.0.1:8388"; + #[test] fn serde_defaults_remain_unchanged_for_present_sections() { let toml = r#" @@ -933,10 +981,7 @@ mod tests { cfg.general.me_init_retry_attempts, default_me_init_retry_attempts() ); - assert_eq!( - cfg.general.me2dc_fallback, - default_me2dc_fallback() - ); + assert_eq!(cfg.general.me2dc_fallback, default_me2dc_fallback()); assert_eq!( cfg.general.proxy_config_v4_cache_path, default_proxy_config_v4_cache_path() @@ -1245,11 +1290,12 @@ mod tests { let path = dir.join("telemt_dc_override_test.toml"); std::fs::write(&path, toml).unwrap(); let cfg = ProxyConfig::load(&path).unwrap(); - assert!(cfg - .dc_overrides - .get("203") - .map(|v| v.contains(&"91.105.192.100:443".to_string())) - .unwrap_or(false)); + assert!( + cfg.dc_overrides + .get("203") + .map(|v| v.contains(&"91.105.192.100:443".to_string())) + .unwrap_or(false) + ); let _ = std::fs::remove_file(path); } @@ -1436,11 +1482,9 @@ mod tests { let path = dir.join("telemt_me_adaptive_floor_min_writers_out_of_range_test.toml"); std::fs::write(&path, toml).unwrap(); let err = ProxyConfig::load(&path).unwrap_err().to_string(); - assert!( - err.contains( - "general.me_adaptive_floor_min_writers_single_endpoint must be within [1, 32]" - ) - ); + assert!(err.contains( + "general.me_adaptive_floor_min_writers_single_endpoint must be within [1, 32]" + )); let _ = std::fs::remove_file(path); } @@ -2026,6 +2070,124 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn shadowsocks_upstream_url_loads_successfully() { + let toml = format!( + r#" + [general] + use_middle_proxy = false + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + + [[upstreams]] + type = "shadowsocks" + url = "{url}" + interface = "127.0.0.2" + "#, + url = TEST_SHADOWSOCKS_URL, + ); + let dir = std::env::temp_dir(); + let path = dir.join("telemt_shadowsocks_valid_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + + assert!(matches!( + &cfg.upstreams[0].upstream_type, + UpstreamType::Shadowsocks { url, interface } + if url == TEST_SHADOWSOCKS_URL && interface.as_deref() == Some("127.0.0.2") + )); + + let _ = std::fs::remove_file(path); + } + + #[test] + fn shadowsocks_requires_direct_mode() { + let toml = format!( + r#" + [general] + use_middle_proxy = true + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + + [[upstreams]] + type = "shadowsocks" + url = "{url}" + "#, + url = TEST_SHADOWSOCKS_URL, + ); + let dir = std::env::temp_dir(); + let path = dir.join("telemt_shadowsocks_me_reject_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + + assert!(err.contains("shadowsocks upstreams require general.use_middle_proxy = false")); + + let _ = std::fs::remove_file(path); + } + + #[test] + fn invalid_shadowsocks_url_is_rejected() { + let toml = r#" + [general] + use_middle_proxy = false + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + + [[upstreams]] + type = "shadowsocks" + url = "not-a-valid-ss-url" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_shadowsocks_invalid_url_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + + assert!(err.contains("invalid shadowsocks url")); + + let _ = std::fs::remove_file(path); + } + + #[test] + fn shadowsocks_plugins_are_rejected() { + let toml = format!( + r#" + [general] + use_middle_proxy = false + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + + [[upstreams]] + type = "shadowsocks" + url = "{url}?plugin=obfs-local%3Bobfs%3Dhttp" + "#, + url = TEST_SHADOWSOCKS_URL, + ); + let dir = std::env::temp_dir(); + let path = dir.join("telemt_shadowsocks_plugin_reject_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + + assert!(err.contains("shadowsocks plugins are not supported")); + + let _ = std::fs::remove_file(path); + } + #[test] fn invalid_user_ad_tag_reports_access_user_ad_tags_key() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index e507044..868ac87 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -936,24 +936,38 @@ impl Default for GeneralConfig { me_reconnect_backoff_cap_ms: default_reconnect_backoff_cap_ms(), me_reconnect_fast_retry_count: default_me_reconnect_fast_retry_count(), me_single_endpoint_shadow_writers: default_me_single_endpoint_shadow_writers(), - me_single_endpoint_outage_mode_enabled: default_me_single_endpoint_outage_mode_enabled(), - me_single_endpoint_outage_disable_quarantine: default_me_single_endpoint_outage_disable_quarantine(), - me_single_endpoint_outage_backoff_min_ms: default_me_single_endpoint_outage_backoff_min_ms(), - me_single_endpoint_outage_backoff_max_ms: default_me_single_endpoint_outage_backoff_max_ms(), - me_single_endpoint_shadow_rotate_every_secs: default_me_single_endpoint_shadow_rotate_every_secs(), + me_single_endpoint_outage_mode_enabled: default_me_single_endpoint_outage_mode_enabled( + ), + me_single_endpoint_outage_disable_quarantine: + default_me_single_endpoint_outage_disable_quarantine(), + me_single_endpoint_outage_backoff_min_ms: + default_me_single_endpoint_outage_backoff_min_ms(), + me_single_endpoint_outage_backoff_max_ms: + default_me_single_endpoint_outage_backoff_max_ms(), + me_single_endpoint_shadow_rotate_every_secs: + default_me_single_endpoint_shadow_rotate_every_secs(), me_floor_mode: MeFloorMode::default(), me_adaptive_floor_idle_secs: default_me_adaptive_floor_idle_secs(), - me_adaptive_floor_min_writers_single_endpoint: default_me_adaptive_floor_min_writers_single_endpoint(), - me_adaptive_floor_min_writers_multi_endpoint: default_me_adaptive_floor_min_writers_multi_endpoint(), + me_adaptive_floor_min_writers_single_endpoint: + default_me_adaptive_floor_min_writers_single_endpoint(), + me_adaptive_floor_min_writers_multi_endpoint: + default_me_adaptive_floor_min_writers_multi_endpoint(), me_adaptive_floor_recover_grace_secs: default_me_adaptive_floor_recover_grace_secs(), - me_adaptive_floor_writers_per_core_total: default_me_adaptive_floor_writers_per_core_total(), + me_adaptive_floor_writers_per_core_total: + default_me_adaptive_floor_writers_per_core_total(), me_adaptive_floor_cpu_cores_override: default_me_adaptive_floor_cpu_cores_override(), - me_adaptive_floor_max_extra_writers_single_per_core: default_me_adaptive_floor_max_extra_writers_single_per_core(), - me_adaptive_floor_max_extra_writers_multi_per_core: default_me_adaptive_floor_max_extra_writers_multi_per_core(), - me_adaptive_floor_max_active_writers_per_core: default_me_adaptive_floor_max_active_writers_per_core(), - me_adaptive_floor_max_warm_writers_per_core: default_me_adaptive_floor_max_warm_writers_per_core(), - me_adaptive_floor_max_active_writers_global: default_me_adaptive_floor_max_active_writers_global(), - me_adaptive_floor_max_warm_writers_global: default_me_adaptive_floor_max_warm_writers_global(), + me_adaptive_floor_max_extra_writers_single_per_core: + default_me_adaptive_floor_max_extra_writers_single_per_core(), + me_adaptive_floor_max_extra_writers_multi_per_core: + default_me_adaptive_floor_max_extra_writers_multi_per_core(), + me_adaptive_floor_max_active_writers_per_core: + default_me_adaptive_floor_max_active_writers_per_core(), + me_adaptive_floor_max_warm_writers_per_core: + default_me_adaptive_floor_max_warm_writers_per_core(), + me_adaptive_floor_max_active_writers_global: + default_me_adaptive_floor_max_active_writers_global(), + me_adaptive_floor_max_warm_writers_global: + default_me_adaptive_floor_max_warm_writers_global(), upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(), upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(), upstream_connect_budget_ms: default_upstream_connect_budget_ms(), @@ -968,7 +982,8 @@ impl Default for GeneralConfig { me_socks_kdf_policy: MeSocksKdfPolicy::Strict, me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(), me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(), - me_route_backpressure_high_watermark_pct: default_me_route_backpressure_high_watermark_pct(), + me_route_backpressure_high_watermark_pct: + default_me_route_backpressure_high_watermark_pct(), me_health_interval_ms_unhealthy: default_me_health_interval_ms_unhealthy(), me_health_interval_ms_healthy: default_me_health_interval_ms_healthy(), me_admission_poll_ms: default_me_admission_poll_ms(), @@ -992,7 +1007,8 @@ impl Default for GeneralConfig { me_hardswap_warmup_delay_min_ms: default_me_hardswap_warmup_delay_min_ms(), me_hardswap_warmup_delay_max_ms: default_me_hardswap_warmup_delay_max_ms(), me_hardswap_warmup_extra_passes: default_me_hardswap_warmup_extra_passes(), - me_hardswap_warmup_pass_backoff_base_ms: default_me_hardswap_warmup_pass_backoff_base_ms(), + me_hardswap_warmup_pass_backoff_base_ms: + default_me_hardswap_warmup_pass_backoff_base_ms(), me_config_stable_snapshots: default_me_config_stable_snapshots(), me_config_apply_cooldown_secs: default_me_config_apply_cooldown_secs(), me_snapshot_require_http_2xx: default_me_snapshot_require_http_2xx(), @@ -1035,8 +1051,10 @@ impl GeneralConfig { /// Resolve the active updater interval for ME infrastructure refresh tasks. /// `update_every` has priority, otherwise legacy proxy_*_auto_reload_secs are used. pub fn effective_update_every_secs(&self) -> u64 { - self.update_every - .unwrap_or_else(|| self.proxy_secret_auto_reload_secs.min(self.proxy_config_auto_reload_secs)) + self.update_every.unwrap_or_else(|| { + self.proxy_secret_auto_reload_secs + .min(self.proxy_config_auto_reload_secs) + }) } /// Resolve periodic zero-downtime reinit interval for ME writers. @@ -1437,6 +1455,11 @@ pub enum UpstreamType { #[serde(default)] password: Option, }, + Shadowsocks { + url: String, + #[serde(default)] + interface: Option, + }, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1517,7 +1540,10 @@ impl ShowLink { } impl Serialize for ShowLink { - fn serialize(&self, serializer: S) -> std::result::Result { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { match self { ShowLink::None => Vec::::new().serialize(serializer), ShowLink::All => serializer.serialize_str("*"), @@ -1527,7 +1553,9 @@ impl Serialize for ShowLink { } impl<'de> Deserialize<'de> for ShowLink { - fn deserialize>(deserializer: D) -> std::result::Result { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { use serde::de; struct ShowLinkVisitor; @@ -1543,14 +1571,14 @@ impl<'de> Deserialize<'de> for ShowLink { if v == "*" { Ok(ShowLink::All) } else { - Err(de::Error::invalid_value( - de::Unexpected::Str(v), - &r#""*""#, - )) + Err(de::Error::invalid_value(de::Unexpected::Str(v), &r#""*""#)) } } - fn visit_seq>(self, mut seq: A) -> std::result::Result { + fn visit_seq>( + self, + mut seq: A, + ) -> std::result::Result { let mut names = Vec::new(); while let Some(name) = seq.next_element::()? { names.push(name); diff --git a/src/proxy/direct_relay.rs b/src/proxy/direct_relay.rs index b7a1fbf..108949c 100644 --- a/src/proxy/direct_relay.rs +++ b/src/proxy/direct_relay.rs @@ -3,8 +3,7 @@ use std::io::Write; use std::net::SocketAddr; use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; -use tokio::net::TcpStream; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadHalf, WriteHalf, split}; use tokio::sync::watch; use tracing::{debug, info, warn}; @@ -15,7 +14,7 @@ use crate::protocol::constants::*; use crate::proxy::handshake::{HandshakeSuccess, encrypt_tg_nonce_with_ciphers, generate_tg_nonce}; use crate::proxy::relay::relay_bidirectional; use crate::proxy::route_mode::{ - RelayRouteMode, RouteCutoverState, ROUTE_SWITCH_ERROR_MSG, affected_cutover_state, + ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay, }; use crate::proxy::adaptive_buffers; @@ -56,7 +55,11 @@ where ); let tg_stream = upstream_manager - .connect(dc_addr, Some(success.dc_idx), user.strip_prefix("scope_").filter(|s| !s.is_empty())) + .connect( + dc_addr, + Some(success.dc_idx), + user.strip_prefix("scope_").filter(|s| !s.is_empty()), + ) .await?; debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake"); @@ -93,11 +96,9 @@ where ); tokio::pin!(relay_result); let relay_result = loop { - if let Some(cutover) = affected_cutover_state( - &route_rx, - RelayRouteMode::Direct, - route_snapshot.generation, - ) { + if let Some(cutover) = + affected_cutover_state(&route_rx, RelayRouteMode::Direct, route_snapshot.generation) + { let delay = cutover_stagger_delay(session_id, cutover.generation); warn!( user = %user, @@ -148,7 +149,9 @@ fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result { for addr_str in addrs { match addr_str.parse::() { Ok(addr) => parsed.push(addr), - Err(_) => warn!(dc_idx = dc_idx, addr_str = %addr_str, "Invalid DC override address in config, ignoring"), + Err(_) => { + warn!(dc_idx = dc_idx, addr_str = %addr_str, "Invalid DC override address in config, ignoring") + } } } @@ -170,7 +173,10 @@ fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result { // Unknown DC requested by client without override: log and fall back. if !config.dc_overrides.contains_key(&dc_key) { - warn!(dc_idx = dc_idx, "Requested non-standard DC with no override; falling back to default cluster"); + warn!( + dc_idx = dc_idx, + "Requested non-standard DC with no override; falling back to default cluster" + ); if config.general.unknown_dc_file_log_enabled && let Some(path) = &config.general.unknown_dc_log_path && let Ok(handle) = tokio::runtime::Handle::try_current() @@ -204,15 +210,15 @@ fn get_dc_addr_static(dc_idx: i16, config: &ProxyConfig) -> Result { )) } -async fn do_tg_handshake_static( - mut stream: TcpStream, +async fn do_tg_handshake_static( + mut stream: S, success: &HandshakeSuccess, config: &ProxyConfig, rng: &SecureRandom, -) -> Result<( - CryptoReader, - CryptoWriter, -)> { +) -> Result<(CryptoReader>, CryptoWriter>)> +where + S: AsyncRead + AsyncWrite + Unpin, +{ let (nonce, _tg_enc_key, _tg_enc_iv, _tg_dec_key, _tg_dec_iv) = generate_tg_nonce( success.proto_tag, success.dc_idx, @@ -235,7 +241,7 @@ async fn do_tg_handshake_static( stream.write_all(&encrypted_nonce).await?; stream.flush().await?; - let (read_half, write_half) = stream.into_split(); + let (read_half, write_half) = split(stream); let max_pending = config.general.crypto_pending_buffer; Ok(( diff --git a/src/tls_front/fetcher.rs b/src/tls_front/fetcher.rs index 38872af..1ee51a2 100644 --- a/src/tls_front/fetcher.rs +++ b/src/tls_front/fetcher.rs @@ -7,33 +7,29 @@ use tokio::net::TcpStream; #[cfg(unix)] use tokio::net::UnixStream; use tokio::time::timeout; -use tokio_rustls::client::TlsStream; use tokio_rustls::TlsConnector; +use tokio_rustls::client::TlsStream; use tracing::{debug, warn}; -use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; use rustls::client::ClientConfig; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; use rustls::{DigitallySignedStruct, Error as RustlsError}; -use x509_parser::prelude::FromDer; use x509_parser::certificate::X509Certificate; +use x509_parser::prelude::FromDer; use crate::crypto::SecureRandom; use crate::network::dns_overrides::resolve_socket_addr; use crate::protocol::constants::{ TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE, }; -use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder}; use crate::tls_front::types::{ - ParsedCertificateInfo, - ParsedServerHello, - TlsBehaviorProfile, - TlsCertPayload, - TlsExtension, - TlsFetchResult, - TlsProfileSource, + ParsedCertificateInfo, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsExtension, + TlsFetchResult, TlsProfileSource, }; +use crate::transport::UpstreamStream; +use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder}; /// No-op verifier: accept any certificate (we only need lengths and metadata). #[derive(Debug)] @@ -144,21 +140,27 @@ fn build_client_hello(sni: &str, rng: &SecureRandom) -> Vec { exts.extend_from_slice(&0x000au16.to_be_bytes()); exts.extend_from_slice(&((2 + groups.len() * 2) as u16).to_be_bytes()); exts.extend_from_slice(&(groups.len() as u16 * 2).to_be_bytes()); - for g in groups { exts.extend_from_slice(&g.to_be_bytes()); } + for g in groups { + exts.extend_from_slice(&g.to_be_bytes()); + } // signature_algorithms let sig_algs: [u16; 4] = [0x0804, 0x0805, 0x0403, 0x0503]; // rsa_pss_rsae_sha256/384, ecdsa_secp256r1_sha256, rsa_pkcs1_sha256 exts.extend_from_slice(&0x000du16.to_be_bytes()); exts.extend_from_slice(&((2 + sig_algs.len() * 2) as u16).to_be_bytes()); exts.extend_from_slice(&(sig_algs.len() as u16 * 2).to_be_bytes()); - for a in sig_algs { exts.extend_from_slice(&a.to_be_bytes()); } + for a in sig_algs { + exts.extend_from_slice(&a.to_be_bytes()); + } // supported_versions (TLS1.3 + TLS1.2) let versions: [u16; 2] = [0x0304, 0x0303]; exts.extend_from_slice(&0x002bu16.to_be_bytes()); exts.extend_from_slice(&((1 + versions.len() * 2) as u16).to_be_bytes()); exts.push((versions.len() * 2) as u8); - for v in versions { exts.extend_from_slice(&v.to_be_bytes()); } + for v in versions { + exts.extend_from_slice(&v.to_be_bytes()); + } // key_share (x25519) let key = gen_key_share(rng); @@ -273,7 +275,10 @@ fn parse_server_hello(body: &[u8]) -> Option { pos += 4; let data = body.get(pos..pos + elen)?.to_vec(); pos += elen; - extensions.push(TlsExtension { ext_type: etype, data }); + extensions.push(TlsExtension { + ext_type: etype, + data, + }); } Some(ParsedServerHello { @@ -394,7 +399,7 @@ async fn connect_tcp_with_upstream( port: u16, connect_timeout: Duration, upstream: Option>, -) -> Result { +) -> Result { if let Some(manager) = upstream { if let Some(addr) = resolve_socket_addr(host, port) { match manager.connect(addr, None, None).await { @@ -408,23 +413,25 @@ async fn connect_tcp_with_upstream( ); } } - } else if let Ok(mut addrs) = tokio::net::lookup_host((host, port)).await { - if let Some(addr) = addrs.find(|a| a.is_ipv4()) { - match manager.connect(addr, None, None).await { - Ok(stream) => return Ok(stream), - Err(e) => { - warn!( - host = %host, - port = port, - error = %e, - "Upstream connect failed, using direct connect" - ); - } + } else if let Ok(mut addrs) = tokio::net::lookup_host((host, port)).await + && let Some(addr) = addrs.find(|a| a.is_ipv4()) + { + match manager.connect(addr, None, None).await { + Ok(stream) => return Ok(stream), + Err(e) => { + warn!( + host = %host, + port = port, + error = %e, + "Upstream connect failed, using direct connect" + ); } } } } - connect_with_dns_override(host, port, connect_timeout).await + Ok(UpstreamStream::Tcp( + connect_with_dns_override(host, port, connect_timeout).await?, + )) } fn encode_tls13_certificate_message(cert_chain_der: &[Vec]) -> Option> { @@ -443,9 +450,7 @@ fn encode_tls13_certificate_message(cert_chain_der: &[Vec]) -> Option { warn!( @@ -616,12 +622,13 @@ where .map(|slice| slice.to_vec()) .unwrap_or_default(); let cert_chain_der: Vec> = certs.iter().map(|c| c.as_ref().to_vec()).collect(); - let cert_payload = encode_tls13_certificate_message(&cert_chain_der).map(|certificate_message| { - TlsCertPayload { - cert_chain_der: cert_chain_der.clone(), - certificate_message, - } - }); + let cert_payload = + encode_tls13_certificate_message(&cert_chain_der).map(|certificate_message| { + TlsCertPayload { + cert_chain_der: cert_chain_der.clone(), + certificate_message, + } + }); let total_cert_len = cert_payload .as_ref() diff --git a/src/transport/middle_proxy/ping.rs b/src/transport/middle_proxy/ping.rs index 2c76592..4432282 100644 --- a/src/transport/middle_proxy/ping.rs +++ b/src/transport/middle_proxy/ping.rs @@ -7,6 +7,7 @@ use tokio::net::UdpSocket; use crate::config::{UpstreamConfig, UpstreamType}; use crate::crypto::SecureRandom; use crate::error::ProxyError; +use crate::transport::shadowsocks::sanitize_shadowsocks_url; use crate::transport::{UpstreamEgressInfo, UpstreamRouteKind}; use super::MePool; @@ -40,7 +41,11 @@ pub fn format_sample_line(sample: &MePingSample) -> String { let sign = if sample.dc >= 0 { "+" } else { "-" }; let addr = format!("{}:{}", sample.addr.ip(), sample.addr.port()); - match (sample.connect_ms, sample.handshake_ms.as_ref(), sample.error.as_ref()) { + match ( + sample.connect_ms, + sample.handshake_ms.as_ref(), + sample.error.as_ref(), + ) { (Some(conn), Some(hs), None) => format!( " {sign} {addr}\tPing: {:.0} ms / RPC: {:.0} ms / OK", conn, hs @@ -121,6 +126,7 @@ fn route_from_egress(egress: Option) -> Option { None => route, }) } + UpstreamRouteKind::Shadowsocks => Some("shadowsocks".to_string()), } } @@ -232,6 +238,9 @@ pub async fn format_me_route( } UpstreamType::Socks4 { address, .. } => format!("socks4://{address}"), UpstreamType::Socks5 { address, .. } => format!("socks5://{address}"), + UpstreamType::Shadowsocks { url, .. } => sanitize_shadowsocks_url(url) + .map(|address| format!("shadowsocks://{address}")) + .unwrap_or_else(|_| "shadowsocks://invalid".to_string()), }; } @@ -254,6 +263,12 @@ pub async fn format_me_route( if has_socks5 { kinds.push("socks5"); } + if enabled_upstreams + .iter() + .any(|u| matches!(u.upstream_type, UpstreamType::Shadowsocks { .. })) + { + kinds.push("shadowsocks"); + } format!("mixed upstreams ({})", kinds.join(", ")) } @@ -335,7 +350,10 @@ pub async fn run_me_ping(pool: &Arc, rng: &SecureRandom) -> Vec { connect_ms = Some(conn_rtt); route = route_from_egress(upstream_egress); - match pool.handshake_only(stream, addr, upstream_egress, rng).await { + match pool + .handshake_only(stream, addr, upstream_egress, rng) + .await + { Ok(hs) => { handshake_ms = Some(hs.handshake_ms); // drop halves to close diff --git a/src/transport/mod.rs b/src/transport/mod.rs index cba5465..fd40105 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -2,6 +2,7 @@ pub mod pool; pub mod proxy_protocol; +pub mod shadowsocks; pub mod socket; pub mod socks; pub mod upstream; @@ -14,5 +15,8 @@ pub use socket::*; #[allow(unused_imports)] pub use socks::*; #[allow(unused_imports)] -pub use upstream::{DcPingResult, StartupPingResult, UpstreamEgressInfo, UpstreamManager, UpstreamRouteKind}; +pub use upstream::{ + DcPingResult, StartupPingResult, UpstreamEgressInfo, UpstreamManager, UpstreamRouteKind, + UpstreamStream, +}; pub mod middle_proxy; diff --git a/src/transport/shadowsocks.rs b/src/transport/shadowsocks.rs new file mode 100644 index 0000000..5211b20 --- /dev/null +++ b/src/transport/shadowsocks.rs @@ -0,0 +1,60 @@ +use std::net::{IpAddr, SocketAddr}; +use std::time::Duration; + +use shadowsocks::{ + ProxyClientStream, + config::{ServerConfig, ServerType}, + context::Context, + net::ConnectOpts, +}; + +use crate::error::{ProxyError, Result}; + +pub(crate) type ShadowsocksStream = ProxyClientStream; + +fn parse_server_config(url: &str, connect_timeout: Duration) -> Result { + let mut config = ServerConfig::from_url(url) + .map_err(|error| ProxyError::Config(format!("invalid shadowsocks url: {error}")))?; + + if config.plugin().is_some() { + return Err(ProxyError::Config( + "shadowsocks plugins are not supported".to_string(), + )); + } + + config.set_timeout(connect_timeout); + Ok(config) +} + +pub(crate) fn sanitize_shadowsocks_url(url: &str) -> Result { + Ok(parse_server_config(url, Duration::from_secs(1))? + .addr() + .to_string()) +} + +fn connect_opts_for_interface(interface: &Option) -> ConnectOpts { + let mut opts = ConnectOpts::default(); + if let Some(interface) = interface { + if let Ok(ip) = interface.parse::() { + opts.bind_local_addr = Some(SocketAddr::new(ip, 0)); + } else { + opts.bind_interface = Some(interface.clone()); + } + } + opts +} + +pub(crate) async fn connect_shadowsocks( + url: &str, + interface: &Option, + target: SocketAddr, + connect_timeout: Duration, +) -> Result { + let config = parse_server_config(url, connect_timeout)?; + let context = Context::new_shared(ServerType::Local); + let opts = connect_opts_for_interface(interface); + + ProxyClientStream::connect_with_opts(context, &config, target, &opts) + .await + .map_err(ProxyError::Io) +} diff --git a/src/transport/upstream.rs b/src/transport/upstream.rs index 8360e1e..b0d82b1 100644 --- a/src/transport/upstream.rs +++ b/src/transport/upstream.rs @@ -4,22 +4,28 @@ #![allow(deprecated)] +use rand::Rng; use std::collections::{BTreeSet, HashMap}; -use std::net::{SocketAddr, IpAddr}; +use std::net::{IpAddr, SocketAddr}; +use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::task::{Context, Poll}; use std::time::Duration; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::net::TcpStream; use tokio::sync::RwLock; use tokio::time::Instant; -use rand::Rng; -use tracing::{debug, warn, info, trace}; +use tracing::{debug, info, trace, warn}; use crate::config::{UpstreamConfig, UpstreamType}; -use crate::error::{Result, ProxyError}; +use crate::error::{ProxyError, Result}; use crate::network::dns_overrides::{resolve_socket_addr, split_host_port}; -use crate::protocol::constants::{TG_DATACENTERS_V4, TG_DATACENTERS_V6, TG_DATACENTER_PORT}; +use crate::protocol::constants::{TG_DATACENTER_PORT, TG_DATACENTERS_V4, TG_DATACENTERS_V6}; use crate::stats::Stats; +use crate::transport::shadowsocks::{ + ShadowsocksStream, connect_shadowsocks, sanitize_shadowsocks_url, +}; use crate::transport::socket::{create_outgoing_socket_bound, resolve_interface_ip}; use crate::transport::socks::{connect_socks4, connect_socks5}; @@ -47,7 +53,10 @@ struct LatencyEma { impl LatencyEma { const fn new(alpha: f64) -> Self { - Self { value_ms: None, alpha } + Self { + value_ms: None, + alpha, + } } fn update(&mut self, sample_ms: f64) { @@ -131,11 +140,17 @@ impl UpstreamState { return Some(ms); } - let (sum, count) = self.dc_latency.iter() + let (sum, count) = self + .dc_latency + .iter() .filter_map(|l| l.get()) .fold((0.0, 0u32), |(s, c), v| (s + v, c + 1)); - if count > 0 { Some(sum / count as f64) } else { None } + if count > 0 { + Some(sum / count as f64) + } else { + None + } } } @@ -158,11 +173,78 @@ pub struct StartupPingResult { pub both_available: bool, } +pub enum UpstreamStream { + Tcp(TcpStream), + Shadowsocks(Box), +} + +impl std::fmt::Debug for UpstreamStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Tcp(_) => f.write_str("UpstreamStream::Tcp(..)"), + Self::Shadowsocks(_) => f.write_str("UpstreamStream::Shadowsocks(..)"), + } + } +} + +impl UpstreamStream { + pub fn into_tcp(self) -> Result { + match self { + Self::Tcp(stream) => Ok(stream), + Self::Shadowsocks(_) => Err(ProxyError::Config( + "shadowsocks upstreams are not supported when general.use_middle_proxy = true" + .to_string(), + )), + } + } +} + +impl AsyncRead for UpstreamStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + match self.get_mut() { + Self::Tcp(stream) => Pin::new(stream).poll_read(cx, buf), + Self::Shadowsocks(stream) => Pin::new(stream.as_mut()).poll_read(cx, buf), + } + } +} + +impl AsyncWrite for UpstreamStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match self.get_mut() { + Self::Tcp(stream) => Pin::new(stream).poll_write(cx, buf), + Self::Shadowsocks(stream) => Pin::new(stream.as_mut()).poll_write(cx, buf), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut() { + Self::Tcp(stream) => Pin::new(stream).poll_flush(cx), + Self::Shadowsocks(stream) => Pin::new(stream.as_mut()).poll_flush(cx), + } + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut() { + Self::Tcp(stream) => Pin::new(stream).poll_shutdown(cx), + Self::Shadowsocks(stream) => Pin::new(stream.as_mut()).poll_shutdown(cx), + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum UpstreamRouteKind { Direct, Socks4, Socks5, + Shadowsocks, } #[derive(Debug, Clone)] @@ -194,6 +276,7 @@ pub struct UpstreamApiSummarySnapshot { pub direct_total: usize, pub socks4_total: usize, pub socks5_total: usize, + pub shadowsocks_total: usize, } #[derive(Debug, Clone)] @@ -253,7 +336,8 @@ impl UpstreamManager { connect_failfast_hard_errors: bool, stats: Arc, ) -> Self { - let states = configs.into_iter() + let states = configs + .into_iter() .filter(|c| c.enabled) .map(UpstreamState::new) .collect(); @@ -311,20 +395,13 @@ impl UpstreamManager { summary.unhealthy_total += 1; } - let (route_kind, address) = match &upstream.config.upstream_type { - UpstreamType::Direct { .. } => { - summary.direct_total += 1; - (UpstreamRouteKind::Direct, "direct".to_string()) - } - UpstreamType::Socks4 { address, .. } => { - summary.socks4_total += 1; - (UpstreamRouteKind::Socks4, address.clone()) - } - UpstreamType::Socks5 { address, .. } => { - summary.socks5_total += 1; - (UpstreamRouteKind::Socks5, address.clone()) - } - }; + let (route_kind, address) = Self::describe_upstream(&upstream.config.upstream_type); + match route_kind { + UpstreamRouteKind::Direct => summary.direct_total += 1, + UpstreamRouteKind::Socks4 => summary.socks4_total += 1, + UpstreamRouteKind::Socks5 => summary.socks5_total += 1, + UpstreamRouteKind::Shadowsocks => summary.shadowsocks_total += 1, + } let mut dc = Vec::with_capacity(NUM_DCS); for dc_idx in 0..NUM_DCS { @@ -352,6 +429,18 @@ impl UpstreamManager { Some(UpstreamApiSnapshot { summary, upstreams }) } + fn describe_upstream(upstream_type: &UpstreamType) -> (UpstreamRouteKind, String) { + match upstream_type { + UpstreamType::Direct { .. } => (UpstreamRouteKind::Direct, "direct".to_string()), + UpstreamType::Socks4 { address, .. } => (UpstreamRouteKind::Socks4, address.clone()), + UpstreamType::Socks5 { address, .. } => (UpstreamRouteKind::Socks5, address.clone()), + UpstreamType::Shadowsocks { url, .. } => ( + UpstreamRouteKind::Shadowsocks, + sanitize_shadowsocks_url(url).unwrap_or_else(|_| "invalid".to_string()), + ), + } + } + pub fn api_policy_snapshot(&self) -> UpstreamApiPolicySnapshot { UpstreamApiPolicySnapshot { connect_retry_attempts: self.connect_retry_attempts, @@ -539,44 +628,44 @@ impl UpstreamManager { // Scope filter: // If scope is set: only scoped and matched items // If scope is not set: only unscoped items - let filtered_upstreams : Vec = upstreams.iter() + let filtered_upstreams: Vec = upstreams + .iter() .enumerate() .filter(|(_, u)| { - scope.map_or( - u.config.scopes.is_empty(), - |req_scope| { - u.config.scopes - .split(',') - .map(str::trim) - .any(|s| s == req_scope) - } - ) + scope.map_or(u.config.scopes.is_empty(), |req_scope| { + u.config + .scopes + .split(',') + .map(str::trim) + .any(|s| s == req_scope) + }) }) .map(|(i, _)| i) .collect(); // Healthy filter - let healthy: Vec = filtered_upstreams.iter() + let healthy: Vec = filtered_upstreams + .iter() .filter(|&&i| upstreams[i].healthy) .copied() .collect(); if filtered_upstreams.is_empty() { - if Self::should_emit_warn( - self.no_upstreams_warn_epoch_ms.as_ref(), - 5_000, - ) { - warn!(scope = scope, "No upstreams available! Using first (direct?)"); + if Self::should_emit_warn(self.no_upstreams_warn_epoch_ms.as_ref(), 5_000) { + warn!( + scope = scope, + "No upstreams available! Using first (direct?)" + ); } return None; } if healthy.is_empty() { - if Self::should_emit_warn( - self.no_healthy_warn_epoch_ms.as_ref(), - 5_000, - ) { - warn!(scope = scope, "No healthy upstreams available! Using random."); + if Self::should_emit_warn(self.no_healthy_warn_epoch_ms.as_ref(), 5_000) { + warn!( + scope = scope, + "No healthy upstreams available! Using random." + ); } return Some(filtered_upstreams[rand::rng().gen_range(0..filtered_upstreams.len())]); } @@ -585,14 +674,18 @@ impl UpstreamManager { return Some(healthy[0]); } - let weights: Vec<(usize, f64)> = healthy.iter().map(|&i| { - let base = upstreams[i].config.weight as f64; - let latency_factor = upstreams[i].effective_latency(dc_idx) - .map(|ms| if ms > 1.0 { 1000.0 / ms } else { 1000.0 }) - .unwrap_or(1.0); + let weights: Vec<(usize, f64)> = healthy + .iter() + .map(|&i| { + let base = upstreams[i].config.weight as f64; + let latency_factor = upstreams[i] + .effective_latency(dc_idx) + .map(|ms| if ms > 1.0 { 1000.0 / ms } else { 1000.0 }) + .unwrap_or(1.0); - (i, base * latency_factor) - }).collect(); + (i, base * latency_factor) + }) + .collect(); let total: f64 = weights.iter().map(|(_, w)| w).sum(); @@ -620,8 +713,34 @@ impl UpstreamManager { } /// Connect to target through a selected upstream. - pub async fn connect(&self, target: SocketAddr, dc_idx: Option, scope: Option<&str>) -> Result { - let (stream, _) = self.connect_with_details(target, dc_idx, scope).await?; + pub async fn connect( + &self, + target: SocketAddr, + dc_idx: Option, + scope: Option<&str>, + ) -> Result { + let idx = self + .select_upstream(dc_idx, scope) + .await + .ok_or_else(|| ProxyError::Config("No upstreams available".to_string()))?; + + let mut upstream = { + let guard = self.upstreams.read().await; + guard[idx].config.clone() + }; + + if let Some(s) = scope { + upstream.selected_scope = s.to_string(); + } + + let bind_rr = { + let guard = self.upstreams.read().await; + guard.get(idx).map(|u| u.bind_rr.clone()) + }; + + let (stream, _) = self + .connect_selected_upstream(idx, upstream, target, dc_idx, bind_rr) + .await?; Ok(stream) } @@ -632,7 +751,9 @@ impl UpstreamManager { dc_idx: Option, scope: Option<&str>, ) -> Result<(TcpStream, UpstreamEgressInfo)> { - let idx = self.select_upstream(dc_idx, scope).await + let idx = self + .select_upstream(dc_idx, scope) + .await .ok_or_else(|| ProxyError::Config("No upstreams available".to_string()))?; let mut upstream = { @@ -650,6 +771,20 @@ impl UpstreamManager { guard.get(idx).map(|u| u.bind_rr.clone()) }; + let (stream, egress) = self + .connect_selected_upstream(idx, upstream, target, dc_idx, bind_rr) + .await?; + Ok((stream.into_tcp()?, egress)) + } + + async fn connect_selected_upstream( + &self, + idx: usize, + upstream: UpstreamConfig, + target: SocketAddr, + dc_idx: Option, + bind_rr: Option>, + ) -> Result<(UpstreamStream, UpstreamEgressInfo)> { let connect_started_at = Instant::now(); let mut last_error: Option = None; let mut attempts_used = 0u32; @@ -662,8 +797,8 @@ impl UpstreamManager { break; } let remaining_budget = self.connect_budget.saturating_sub(elapsed); - let attempt_timeout = Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS) - .min(remaining_budget); + let attempt_timeout = + Duration::from_secs(DIRECT_CONNECT_TIMEOUT_SECS).min(remaining_budget); if attempt_timeout.is_zero() { last_error = Some(ProxyError::ConnectionTimeout { addr: target.to_string(), @@ -786,9 +921,12 @@ impl UpstreamManager { target: SocketAddr, bind_rr: Option>, connect_timeout: Duration, - ) -> Result<(TcpStream, UpstreamEgressInfo)> { + ) -> Result<(UpstreamStream, UpstreamEgressInfo)> { match &config.upstream_type { - UpstreamType::Direct { interface, bind_addresses } => { + UpstreamType::Direct { + interface, + bind_addresses, + } => { let bind_ip = Self::resolve_bind_address( interface, bind_addresses, @@ -796,9 +934,7 @@ impl UpstreamManager { bind_rr.as_deref(), true, ); - if bind_ip.is_none() - && bind_addresses.as_ref().is_some_and(|v| !v.is_empty()) - { + if bind_ip.is_none() && bind_addresses.as_ref().is_some_and(|v| !v.is_empty()) { return Err(ProxyError::Config(format!( "No valid bind_addresses for target family {target}" ))); @@ -813,8 +949,10 @@ impl UpstreamManager { socket.set_nonblocking(true)?; match socket.connect(&target.into()) { - Ok(()) => {}, - Err(err) if err.raw_os_error() == Some(libc::EINPROGRESS) || err.kind() == std::io::ErrorKind::WouldBlock => {}, + Ok(()) => {} + Err(err) + if err.raw_os_error() == Some(libc::EINPROGRESS) + || err.kind() == std::io::ErrorKind::WouldBlock => {} Err(err) => return Err(ProxyError::Io(err)), } @@ -836,7 +974,7 @@ impl UpstreamManager { let local_addr = stream.local_addr().ok(); Ok(( - stream, + UpstreamStream::Tcp(stream), UpstreamEgressInfo { upstream_id, route_kind: UpstreamRouteKind::Direct, @@ -846,8 +984,12 @@ impl UpstreamManager { socks_proxy_addr: None, }, )) - }, - UpstreamType::Socks4 { address, interface, user_id } => { + } + UpstreamType::Socks4 { + address, + interface, + user_id, + } => { // Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port let mut stream = if let Ok(proxy_addr) = address.parse::() { // IP:port format - use socket with optional interface binding @@ -863,8 +1005,10 @@ impl UpstreamManager { socket.set_nonblocking(true)?; match socket.connect(&proxy_addr.into()) { - Ok(()) => {}, - Err(err) if err.raw_os_error() == Some(libc::EINPROGRESS) || err.kind() == std::io::ErrorKind::WouldBlock => {}, + Ok(()) => {} + Err(err) + if err.raw_os_error() == Some(libc::EINPROGRESS) + || err.kind() == std::io::ErrorKind::WouldBlock => {} Err(err) => return Err(ProxyError::Io(err)), } @@ -888,14 +1032,16 @@ impl UpstreamManager { // Hostname:port format - use tokio DNS resolution // Note: interface binding is not supported for hostnames if interface.is_some() { - warn!("SOCKS4 interface binding is not supported for hostname addresses, ignoring"); + warn!( + "SOCKS4 interface binding is not supported for hostname addresses, ignoring" + ); } Self::connect_hostname_with_dns_override(address, connect_timeout).await? }; // replace socks user_id with config.selected_scope, if set - let scope: Option<&str> = Some(config.selected_scope.as_str()) - .filter(|s| !s.is_empty()); + let scope: Option<&str> = + Some(config.selected_scope.as_str()).filter(|s| !s.is_empty()); let _user_id: Option<&str> = scope.or(user_id.as_deref()); let bound = match tokio::time::timeout( @@ -915,7 +1061,7 @@ impl UpstreamManager { let local_addr = stream.local_addr().ok(); let socks_proxy_addr = stream.peer_addr().ok(); Ok(( - stream, + UpstreamStream::Tcp(stream), UpstreamEgressInfo { upstream_id, route_kind: UpstreamRouteKind::Socks4, @@ -925,8 +1071,13 @@ impl UpstreamManager { socks_proxy_addr, }, )) - }, - UpstreamType::Socks5 { address, interface, username, password } => { + } + UpstreamType::Socks5 { + address, + interface, + username, + password, + } => { // Try to parse as SocketAddr first (IP:port), otherwise treat as hostname:port let mut stream = if let Ok(proxy_addr) = address.parse::() { // IP:port format - use socket with optional interface binding @@ -942,8 +1093,10 @@ impl UpstreamManager { socket.set_nonblocking(true)?; match socket.connect(&proxy_addr.into()) { - Ok(()) => {}, - Err(err) if err.raw_os_error() == Some(libc::EINPROGRESS) || err.kind() == std::io::ErrorKind::WouldBlock => {}, + Ok(()) => {} + Err(err) + if err.raw_os_error() == Some(libc::EINPROGRESS) + || err.kind() == std::io::ErrorKind::WouldBlock => {} Err(err) => return Err(ProxyError::Io(err)), } @@ -967,15 +1120,17 @@ impl UpstreamManager { // Hostname:port format - use tokio DNS resolution // Note: interface binding is not supported for hostnames if interface.is_some() { - warn!("SOCKS5 interface binding is not supported for hostname addresses, ignoring"); + warn!( + "SOCKS5 interface binding is not supported for hostname addresses, ignoring" + ); } Self::connect_hostname_with_dns_override(address, connect_timeout).await? }; debug!(config = ?config, "Socks5 connection"); // replace socks user:pass with config.selected_scope, if set - let scope: Option<&str> = Some(config.selected_scope.as_str()) - .filter(|s| !s.is_empty()); + let scope: Option<&str> = + Some(config.selected_scope.as_str()).filter(|s| !s.is_empty()); let _username: Option<&str> = scope.or(username.as_deref()); let _password: Option<&str> = scope.or(password.as_deref()); @@ -996,7 +1151,7 @@ impl UpstreamManager { let local_addr = stream.local_addr().ok(); let socks_proxy_addr = stream.peer_addr().ok(); Ok(( - stream, + UpstreamStream::Tcp(stream), UpstreamEgressInfo { upstream_id, route_kind: UpstreamRouteKind::Socks5, @@ -1006,7 +1161,22 @@ impl UpstreamManager { socks_proxy_addr, }, )) - }, + } + UpstreamType::Shadowsocks { url, interface } => { + let stream = connect_shadowsocks(url, interface, target, connect_timeout).await?; + let local_addr = stream.get_ref().local_addr().ok(); + Ok(( + UpstreamStream::Shadowsocks(Box::new(stream)), + UpstreamEgressInfo { + upstream_id, + route_kind: UpstreamRouteKind::Shadowsocks, + local_addr, + direct_bind_ip: None, + socks_bound_addr: None, + socks_proxy_addr: None, + }, + )) + } } } @@ -1023,7 +1193,9 @@ impl UpstreamManager { ) -> Vec { let upstreams: Vec<(usize, UpstreamConfig, Arc)> = { let guard = self.upstreams.read().await; - guard.iter().enumerate() + guard + .iter() + .enumerate() .map(|(i, u)| (i, u.config.clone(), u.bind_rr.clone())) .collect() }; @@ -1051,6 +1223,11 @@ impl UpstreamManager { } UpstreamType::Socks4 { address, .. } => format!("socks4://{}", address), UpstreamType::Socks5 { address, .. } => format!("socks5://{}", address), + UpstreamType::Shadowsocks { url, .. } => { + let address = + sanitize_shadowsocks_url(url).unwrap_or_else(|_| "invalid".to_string()); + format!("shadowsocks://{address}") + } }; let mut v6_results = Vec::with_capacity(NUM_DCS); @@ -1061,8 +1238,14 @@ impl UpstreamManager { let result = tokio::time::timeout( Duration::from_secs(DC_PING_TIMEOUT_SECS), - self.ping_single_dc(*upstream_idx, upstream_config, Some(bind_rr.clone()), addr_v6) - ).await; + self.ping_single_dc( + *upstream_idx, + upstream_config, + Some(bind_rr.clone()), + addr_v6, + ), + ) + .await; let ping_result = match result { Ok(Ok(rtt_ms)) => { @@ -1112,8 +1295,14 @@ impl UpstreamManager { let result = tokio::time::timeout( Duration::from_secs(DC_PING_TIMEOUT_SECS), - self.ping_single_dc(*upstream_idx, upstream_config, Some(bind_rr.clone()), addr_v4) - ).await; + self.ping_single_dc( + *upstream_idx, + upstream_config, + Some(bind_rr.clone()), + addr_v4, + ), + ) + .await; let ping_result = match result { Ok(Ok(rtt_ms)) => { @@ -1162,7 +1351,7 @@ impl UpstreamManager { Err(_) => { warn!(dc = %dc_key, "Invalid dc_overrides key, skipping"); continue; - }, + } _ => continue, }; let dc_idx = dc_num as usize; @@ -1175,8 +1364,14 @@ impl UpstreamManager { } let result = tokio::time::timeout( Duration::from_secs(DC_PING_TIMEOUT_SECS), - self.ping_single_dc(*upstream_idx, upstream_config, Some(bind_rr.clone()), addr) - ).await; + self.ping_single_dc( + *upstream_idx, + upstream_config, + Some(bind_rr.clone()), + addr, + ), + ) + .await; let ping_result = match result { Ok(Ok(rtt_ms)) => DcPingResult { @@ -1205,7 +1400,9 @@ impl UpstreamManager { v4_results.push(ping_result); } } - Err(_) => warn!(dc = %dc_idx, addr = %addr_str, "Invalid dc_overrides address, skipping"), + Err(_) => { + warn!(dc = %dc_idx, addr = %addr_str, "Invalid dc_overrides address, skipping") + } } } } @@ -1381,12 +1578,8 @@ impl UpstreamManager { ipv6_enabled: bool, dc_overrides: HashMap>, ) { - let groups = Self::build_health_check_groups( - prefer_ipv6, - ipv4_enabled, - ipv6_enabled, - &dc_overrides, - ); + let groups = + Self::build_health_check_groups(prefer_ipv6, ipv4_enabled, ipv6_enabled, &dc_overrides); let required_healthy_groups = Self::required_healthy_group_count(groups.len()); let mut endpoint_rotation: HashMap<(usize, i16, bool), usize> = HashMap::new(); @@ -1416,13 +1609,16 @@ impl UpstreamManager { let mut group_ok = false; let mut group_rtt_ms = None; - for (is_primary, endpoints) in [(true, &group.primary), (false, &group.fallback)] { + for (is_primary, endpoints) in + [(true, &group.primary), (false, &group.fallback)] + { if endpoints.is_empty() { continue; } let rotation_key = (i, group.dc_idx, is_primary); - let start_idx = *endpoint_rotation.entry(rotation_key).or_insert(0) % endpoints.len(); + let start_idx = + *endpoint_rotation.entry(rotation_key).or_insert(0) % endpoints.len(); let mut next_idx = (start_idx + 1) % endpoints.len(); for step in 0..endpoints.len() { @@ -1544,8 +1740,7 @@ impl UpstreamManager { return None; } - UpstreamState::dc_array_idx(dc_idx) - .map(|idx| guard[0].dc_ip_pref[idx]) + UpstreamState::dc_array_idx(dc_idx).map(|idx| guard[0].dc_ip_pref[idx]) } /// Get preferred DC address based on config preference @@ -1566,6 +1761,12 @@ impl UpstreamManager { #[cfg(test)] mod tests { use super::*; + use std::sync::Arc; + + use crate::stats::Stats; + + const TEST_SHADOWSOCKS_URL: &str = + "ss://2022-blake3-aes-256-gcm:MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE=@127.0.0.1:8388"; #[test] fn required_healthy_group_count_applies_three_group_threshold() { @@ -1596,15 +1797,18 @@ mod tests { assert!(dc2.primary.iter().all(|addr| addr.is_ipv6())); assert!(dc2.fallback.iter().all(|addr| addr.is_ipv4())); - assert!(dc2 - .primary - .contains(&"[2001:db8::10]:443".parse::().unwrap())); - assert!(dc2 - .fallback - .contains(&"203.0.113.10:443".parse::().unwrap())); - assert!(dc2 - .fallback - .contains(&"203.0.113.11:443".parse::().unwrap())); + assert!( + dc2.primary + .contains(&"[2001:db8::10]:443".parse::().unwrap()) + ); + assert!( + dc2.fallback + .contains(&"203.0.113.10:443".parse::().unwrap()) + ); + assert!( + dc2.fallback + .contains(&"203.0.113.11:443".parse::().unwrap()) + ); } #[test] @@ -1626,12 +1830,14 @@ mod tests { .expect("override-only dc group must be present"); assert_eq!(dc9.primary.len(), 2); - assert!(dc9 - .primary - .contains(&"198.51.100.1:443".parse::().unwrap())); - assert!(dc9 - .primary - .contains(&"198.51.100.2:443".parse::().unwrap())); + assert!( + dc9.primary + .contains(&"198.51.100.1:443".parse::().unwrap()) + ); + assert!( + dc9.primary + .contains(&"198.51.100.2:443".parse::().unwrap()) + ); assert!(dc9.fallback.is_empty()); } @@ -1678,4 +1884,36 @@ mod tests { assert_eq!(bind, None); } + + #[test] + fn api_snapshot_reports_shadowsocks_as_sanitized_route() { + let manager = UpstreamManager::new( + vec![UpstreamConfig { + upstream_type: UpstreamType::Shadowsocks { + url: TEST_SHADOWSOCKS_URL.to_string(), + interface: None, + }, + weight: 2, + enabled: true, + scopes: String::new(), + selected_scope: String::new(), + }], + 1, + 100, + 1000, + 1, + false, + Arc::new(Stats::new()), + ); + + let snapshot = manager.try_api_snapshot().expect("snapshot"); + assert_eq!(snapshot.summary.configured_total, 1); + assert_eq!(snapshot.summary.shadowsocks_total, 1); + assert_eq!(snapshot.upstreams.len(), 1); + assert_eq!( + snapshot.upstreams[0].route_kind, + UpstreamRouteKind::Shadowsocks + ); + assert_eq!(snapshot.upstreams[0].address, "127.0.0.1:8388"); + } } From a5983c17d3656c6790a97e01a4bd81a74071d417 Mon Sep 17 00:00:00 2001 From: Maxim Myalin Date: Wed, 18 Mar 2026 12:36:48 +0300 Subject: [PATCH 232/261] Add Docker build context ignore file --- .dockerignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..eea4983 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.github +target +.kilocode +cache +tlsfront +*.tar +*.tar.gz From 31f6258c4794c3eadd7c9245ca18bfb1013706b8 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:54:59 +0300 Subject: [PATCH 233/261] Hot-Reload fixes Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/hot_reload.rs | 111 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index c0ca98d..fdf06fa 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -39,6 +39,7 @@ use super::load::{LoadedConfig, ProxyConfig}; const HOT_RELOAD_STABLE_SNAPSHOTS: u8 = 2; const HOT_RELOAD_DEBOUNCE: Duration = Duration::from_millis(50); +const HOT_RELOAD_STABLE_RECHECK: Duration = Duration::from_millis(75); // ── Hot fields ──────────────────────────────────────────────────────────────── @@ -379,6 +380,14 @@ impl ReloadState { self.applied_snapshot_hash = Some(hash); self.reset_candidate(); } + + fn pending_candidate(&self) -> Option<(u64, u8)> { + let hash = self.candidate_snapshot_hash?; + if self.candidate_hits < HOT_RELOAD_STABLE_SNAPSHOTS { + return Some((hash, self.candidate_hits)); + } + None + } } fn normalize_watch_path(path: &Path) -> PathBuf { @@ -1253,6 +1262,73 @@ fn reload_config( Some(next_manifest) } +async fn reload_with_internal_stable_rechecks( + config_path: &PathBuf, + config_tx: &watch::Sender>, + log_tx: &watch::Sender, + detected_ip_v4: Option, + detected_ip_v6: Option, + reload_state: &mut ReloadState, +) -> Option { + let mut next_manifest = reload_config( + config_path, + config_tx, + log_tx, + detected_ip_v4, + detected_ip_v6, + reload_state, + ); + let mut rechecks_left = HOT_RELOAD_STABLE_SNAPSHOTS.saturating_sub(1); + + while rechecks_left > 0 { + let Some((snapshot_hash, candidate_hits)) = reload_state.pending_candidate() else { + break; + }; + + info!( + snapshot_hash, + candidate_hits, + required_hits = HOT_RELOAD_STABLE_SNAPSHOTS, + rechecks_left, + recheck_delay_ms = HOT_RELOAD_STABLE_RECHECK.as_millis(), + "config reload: scheduling internal stable recheck" + ); + tokio::time::sleep(HOT_RELOAD_STABLE_RECHECK).await; + + let recheck_manifest = reload_config( + config_path, + config_tx, + log_tx, + detected_ip_v4, + detected_ip_v6, + reload_state, + ); + if recheck_manifest.is_some() { + next_manifest = recheck_manifest; + } + + if reload_state.is_applied(snapshot_hash) { + info!( + snapshot_hash, + "config reload: applied after internal stable recheck" + ); + break; + } + + if reload_state.pending_candidate().is_none() { + info!( + snapshot_hash, + "config reload: internal stable recheck aborted" + ); + break; + } + + rechecks_left = rechecks_left.saturating_sub(1); + } + + next_manifest +} + // ── Public API ──────────────────────────────────────────────────────────────── /// Spawn the hot-reload watcher task. @@ -1376,14 +1452,16 @@ pub fn spawn_config_watcher( tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await; while notify_rx.try_recv().is_ok() {} - if let Some(next_manifest) = reload_config( + if let Some(next_manifest) = reload_with_internal_stable_rechecks( &config_path, &config_tx, &log_tx, detected_ip_v4, detected_ip_v6, &mut reload_state, - ) { + ) + .await + { apply_watch_manifest( inotify_watcher.as_mut(), poll_watcher.as_mut(), @@ -1540,6 +1618,35 @@ mod tests { let _ = std::fs::remove_file(path); } + #[tokio::test] + async fn reload_cycle_applies_after_single_external_event() { + let initial_tag = "10101010101010101010101010101010"; + let final_tag = "20202020202020202020202020202020"; + let path = temp_config_path("telemt_hot_reload_single_event"); + + write_reload_config(&path, Some(initial_tag), None); + let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap()); + let initial_hash = ProxyConfig::load_with_metadata(&path).unwrap().rendered_hash; + let (config_tx, _config_rx) = watch::channel(initial_cfg.clone()); + let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone()); + let mut reload_state = ReloadState::new(Some(initial_hash)); + + write_reload_config(&path, Some(final_tag), None); + reload_with_internal_stable_rechecks( + &path, + &config_tx, + &log_tx, + None, + None, + &mut reload_state, + ) + .await + .unwrap(); + + assert_eq!(config_tx.borrow().general.ad_tag.as_deref(), Some(final_tag)); + let _ = std::fs::remove_file(path); + } + #[test] fn reload_keeps_hot_apply_when_non_hot_fields_change() { let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; From 85295a9961bb90b8bd6c7452aed84337f029c6a5 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:58:27 +0300 Subject: [PATCH 234/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 788bc2e..eba2e2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.21" +version = "3.3.22" edition = "2024" [dependencies] From 193545525650a0d960cecf950b97ac05e5028ff2 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Wed, 18 Mar 2026 18:20:23 +0300 Subject: [PATCH 235/261] Update CONFIG_PARAMS.en.md --- docs/CONFIG_PARAMS.en.md | 491 ++++++++++++++++++++------------------- 1 file changed, 248 insertions(+), 243 deletions(-) diff --git a/docs/CONFIG_PARAMS.en.md b/docs/CONFIG_PARAMS.en.md index ed89b3d..90da08a 100644 --- a/docs/CONFIG_PARAMS.en.md +++ b/docs/CONFIG_PARAMS.en.md @@ -8,282 +8,287 @@ This document lists all configuration keys accepted by `config.toml`. ## Top-level keys -| Parameter | Type | Description | -|---|---|---| -| include | `String` (special directive) | Includes another TOML file with `include = "relative/or/absolute/path.toml"`; includes are processed recursively before parsing. | -| show_link | `"*" \| String[]` | Legacy top-level link visibility selector (`"*"` for all users or explicit usernames list). | -| dc_overrides | `Map` | Overrides DC endpoints for non-standard DCs; key is DC id string, value is `ip:port` list. | -| default_dc | `u8` | Default DC index used for unmapped non-standard DCs. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| include | `String` (special directive) | `null` | — | Includes another TOML file with `include = "relative/or/absolute/path.toml"`; includes are processed recursively before parsing. | +| show_link | `"*" \| String[]` | `[]` (`ShowLink::None`) | — | Legacy top-level link visibility selector (`"*"` for all users or explicit usernames list). | +| dc_overrides | `Map` | `{}` | — | Overrides DC endpoints for non-standard DCs; key is DC id string, value is `ip:port` list. | +| default_dc | `u8 \| null` | `null` (effective fallback: `2` in ME routing) | — | Default DC index used for unmapped non-standard DCs. | ## [general] -| Parameter | Type | Description | -|---|---|---| -| data_path | `String` | Optional runtime data directory path. | -| prefer_ipv6 | `bool` | Prefer IPv6 where applicable in runtime logic. | -| fast_mode | `bool` | Enables fast-path optimizations for traffic processing. | -| use_middle_proxy | `bool` | Enables Middle Proxy mode. | -| proxy_secret_path | `String` | Path to proxy secret binary; can be auto-downloaded if absent. | -| proxy_config_v4_cache_path | `String` | Optional cache path for raw `getProxyConfig` (IPv4) snapshot. | -| proxy_config_v6_cache_path | `String` | Optional cache path for raw `getProxyConfigV6` (IPv6) snapshot. | -| ad_tag | `String` | Global fallback ad tag (32 hex characters). | -| middle_proxy_nat_ip | `IpAddr` | Explicit public IP override for NAT environments. | -| middle_proxy_nat_probe | `bool` | Enables NAT probing for Middle Proxy KDF/public address discovery. | -| middle_proxy_nat_stun | `String` | Deprecated legacy single STUN server for NAT probing. | -| middle_proxy_nat_stun_servers | `String[]` | Deprecated legacy STUN list for NAT probing fallback. | -| stun_nat_probe_concurrency | `usize` | Maximum concurrent STUN probes during NAT detection. | -| middle_proxy_pool_size | `usize` | Target size of active Middle Proxy writer pool. | -| middle_proxy_warm_standby | `usize` | Number of warm standby Middle-End connections. | -| me_init_retry_attempts | `u32` | Startup retries for ME pool initialization (`0` means unlimited). | -| me2dc_fallback | `bool` | Allows fallback from ME mode to direct DC when ME startup fails. | -| me_keepalive_enabled | `bool` | Enables ME keepalive padding frames. | -| me_keepalive_interval_secs | `u64` | Keepalive interval in seconds. | -| me_keepalive_jitter_secs | `u64` | Keepalive jitter in seconds. | -| me_keepalive_payload_random | `bool` | Randomizes keepalive payload bytes instead of zero payload. | -| rpc_proxy_req_every | `u64` | Interval for service `RPC_PROXY_REQ` activity signals (`0` disables). | -| me_writer_cmd_channel_capacity | `usize` | Capacity of per-writer command channel. | -| me_route_channel_capacity | `usize` | Capacity of per-connection ME response route channel. | -| me_c2me_channel_capacity | `usize` | Capacity of per-client command queue (client reader -> ME sender). | -| me_reader_route_data_wait_ms | `u64` | Bounded wait for routing ME DATA to per-connection queue (`0` = no wait). | -| me_d2c_flush_batch_max_frames | `usize` | Max ME->client frames coalesced before flush. | -| me_d2c_flush_batch_max_bytes | `usize` | Max ME->client payload bytes coalesced before flush. | -| me_d2c_flush_batch_max_delay_us | `u64` | Max microsecond wait for coalescing more ME->client frames (`0` disables timed coalescing). | -| me_d2c_ack_flush_immediate | `bool` | Flushes client writer immediately after quick-ack write. | -| direct_relay_copy_buf_c2s_bytes | `usize` | Copy buffer size for client->DC direction in direct relay. | -| direct_relay_copy_buf_s2c_bytes | `usize` | Copy buffer size for DC->client direction in direct relay. | -| crypto_pending_buffer | `usize` | Max pending ciphertext buffer per client writer (bytes). | -| max_client_frame | `usize` | Maximum allowed client MTProto frame size (bytes). | -| desync_all_full | `bool` | Emits full crypto-desync forensic logs for every event. | -| beobachten | `bool` | Enables per-IP forensic observation buckets. | -| beobachten_minutes | `u64` | Retention window (minutes) for per-IP observation buckets. | -| beobachten_flush_secs | `u64` | Snapshot flush interval (seconds) for observation output file. | -| beobachten_file | `String` | Observation snapshot output file path. | -| hardswap | `bool` | Enables hard-swap generation switching for ME pool updates. | -| me_warmup_stagger_enabled | `bool` | Enables staggered warmup for extra ME writers. | -| me_warmup_step_delay_ms | `u64` | Base delay between warmup connections (ms). | -| me_warmup_step_jitter_ms | `u64` | Jitter for warmup delay (ms). | -| me_reconnect_max_concurrent_per_dc | `u32` | Max concurrent reconnect attempts per DC. | -| me_reconnect_backoff_base_ms | `u64` | Base reconnect backoff in ms. | -| me_reconnect_backoff_cap_ms | `u64` | Cap reconnect backoff in ms. | -| me_reconnect_fast_retry_count | `u32` | Number of fast retry attempts before backoff. | -| me_single_endpoint_shadow_writers | `u8` | Additional reserve writers for one-endpoint DC groups. | -| me_single_endpoint_outage_mode_enabled | `bool` | Enables aggressive outage recovery for one-endpoint DC groups. | -| me_single_endpoint_outage_disable_quarantine | `bool` | Ignores endpoint quarantine in one-endpoint outage mode. | -| me_single_endpoint_outage_backoff_min_ms | `u64` | Minimum reconnect backoff in outage mode (ms). | -| me_single_endpoint_outage_backoff_max_ms | `u64` | Maximum reconnect backoff in outage mode (ms). | -| me_single_endpoint_shadow_rotate_every_secs | `u64` | Periodic shadow writer rotation interval (`0` disables). | -| me_floor_mode | `"static" \| "adaptive"` | Writer floor policy mode. | -| me_adaptive_floor_idle_secs | `u64` | Idle time before adaptive floor may reduce one-endpoint target. | -| me_adaptive_floor_min_writers_single_endpoint | `u8` | Minimum adaptive writer target for one-endpoint DC groups. | -| me_adaptive_floor_min_writers_multi_endpoint | `u8` | Minimum adaptive writer target for multi-endpoint DC groups. | -| me_adaptive_floor_recover_grace_secs | `u64` | Grace period to hold static floor after activity. | -| me_adaptive_floor_writers_per_core_total | `u16` | Global writer budget per logical CPU core in adaptive mode. | -| me_adaptive_floor_cpu_cores_override | `u16` | Manual CPU core count override (`0` uses auto-detection). | -| me_adaptive_floor_max_extra_writers_single_per_core | `u16` | Per-core max extra writers above base floor for one-endpoint DCs. | -| me_adaptive_floor_max_extra_writers_multi_per_core | `u16` | Per-core max extra writers above base floor for multi-endpoint DCs. | -| me_adaptive_floor_max_active_writers_per_core | `u16` | Hard cap for active ME writers per logical CPU core. | -| me_adaptive_floor_max_warm_writers_per_core | `u16` | Hard cap for warm ME writers per logical CPU core. | -| me_adaptive_floor_max_active_writers_global | `u32` | Hard global cap for active ME writers. | -| me_adaptive_floor_max_warm_writers_global | `u32` | Hard global cap for warm ME writers. | -| upstream_connect_retry_attempts | `u32` | Connect attempts for selected upstream before error/fallback. | -| upstream_connect_retry_backoff_ms | `u64` | Delay between upstream connect attempts (ms). | -| upstream_connect_budget_ms | `u64` | Total wall-clock budget for one upstream connect request (ms). | -| upstream_unhealthy_fail_threshold | `u32` | Consecutive failed requests before upstream is marked unhealthy. | -| upstream_connect_failfast_hard_errors | `bool` | Skips additional retries for hard non-transient connect errors. | -| stun_iface_mismatch_ignore | `bool` | Ignores STUN/interface mismatch and keeps Middle Proxy mode. | -| unknown_dc_log_path | `String` | File path for unknown-DC request logging (`null` disables file path). | -| unknown_dc_file_log_enabled | `bool` | Enables unknown-DC file logging. | -| log_level | `"debug" \| "verbose" \| "normal" \| "silent"` | Runtime logging verbosity. | -| disable_colors | `bool` | Disables ANSI colors in logs. | -| me_socks_kdf_policy | `"strict" \| "compat"` | SOCKS-bound KDF fallback policy for ME handshake. | -| me_route_backpressure_base_timeout_ms | `u64` | Base backpressure timeout for route-channel send (ms). | -| me_route_backpressure_high_timeout_ms | `u64` | High backpressure timeout when queue occupancy exceeds watermark (ms). | -| me_route_backpressure_high_watermark_pct | `u8` | Queue occupancy threshold (%) for high timeout mode. | -| me_health_interval_ms_unhealthy | `u64` | Health monitor interval while writer coverage is degraded (ms). | -| me_health_interval_ms_healthy | `u64` | Health monitor interval while writer coverage is healthy (ms). | -| me_admission_poll_ms | `u64` | Poll interval for conditional-admission checks (ms). | -| me_warn_rate_limit_ms | `u64` | Cooldown for repetitive ME warning logs (ms). | -| me_route_no_writer_mode | `"async_recovery_failfast" \| "inline_recovery_legacy" \| "hybrid_async_persistent"` | Route behavior when no writer is immediately available. | -| me_route_no_writer_wait_ms | `u64` | Max wait in async-recovery failfast mode (ms). | -| me_route_inline_recovery_attempts | `u32` | Inline recovery attempts in legacy mode. | -| me_route_inline_recovery_wait_ms | `u64` | Max inline recovery wait in legacy mode (ms). | -| fast_mode_min_tls_record | `usize` | Minimum TLS record size when fast-mode coalescing is enabled (`0` disables). | -| update_every | `u64` | Unified interval for config/secret updater tasks. | -| me_reinit_every_secs | `u64` | Periodic ME pool reinitialization interval (seconds). | -| me_hardswap_warmup_delay_min_ms | `u64` | Minimum delay between hardswap warmup connects (ms). | -| me_hardswap_warmup_delay_max_ms | `u64` | Maximum delay between hardswap warmup connects (ms). | -| me_hardswap_warmup_extra_passes | `u8` | Additional warmup passes per hardswap cycle. | -| me_hardswap_warmup_pass_backoff_base_ms | `u64` | Base backoff between hardswap warmup passes (ms). | -| me_config_stable_snapshots | `u8` | Number of identical config snapshots required before apply. | -| me_config_apply_cooldown_secs | `u64` | Cooldown between applied ME map updates (seconds). | -| me_snapshot_require_http_2xx | `bool` | Requires 2xx HTTP responses for applying config snapshots. | -| me_snapshot_reject_empty_map | `bool` | Rejects empty config snapshots. | -| me_snapshot_min_proxy_for_lines | `u32` | Minimum parsed `proxy_for` rows required to accept snapshot. | -| proxy_secret_stable_snapshots | `u8` | Number of identical secret snapshots required before runtime rotation. | -| proxy_secret_rotate_runtime | `bool` | Enables runtime proxy-secret rotation from remote source. | -| me_secret_atomic_snapshot | `bool` | Keeps selector and secret bytes from the same snapshot atomically. | -| proxy_secret_len_max | `usize` | Maximum allowed proxy-secret length (bytes). | -| me_pool_drain_ttl_secs | `u64` | Drain TTL for stale ME writers after endpoint-map changes (seconds). | -| me_pool_drain_threshold | `u64` | Max draining stale writers before batch force-close (`0` disables threshold cleanup). | -| me_bind_stale_mode | `"never" \| "ttl" \| "always"` | Policy for new binds on stale draining writers. | -| me_bind_stale_ttl_secs | `u64` | TTL for stale bind allowance when stale mode is `ttl`. | -| me_pool_min_fresh_ratio | `f32` | Minimum desired-DC fresh coverage ratio before draining stale writers. | -| me_reinit_drain_timeout_secs | `u64` | Force-close timeout for stale writers after endpoint-map changes (`0` disables force-close). | -| proxy_secret_auto_reload_secs | `u64` | Deprecated legacy secret reload interval (fallback when `update_every` is not set). | -| proxy_config_auto_reload_secs | `u64` | Deprecated legacy config reload interval (fallback when `update_every` is not set). | -| me_reinit_singleflight | `bool` | Serializes ME reinit cycles across trigger sources. | -| me_reinit_trigger_channel | `usize` | Trigger queue capacity for reinit scheduler. | -| me_reinit_coalesce_window_ms | `u64` | Trigger coalescing window before starting reinit (ms). | -| me_deterministic_writer_sort | `bool` | Enables deterministic candidate sort for writer binding path. | -| me_writer_pick_mode | `"sorted_rr" \| "p2c"` | Writer selection mode for route bind path. | -| me_writer_pick_sample_size | `u8` | Number of candidates sampled by picker in `p2c` mode. | -| ntp_check | `bool` | Enables NTP drift check at startup. | -| ntp_servers | `String[]` | NTP servers used for drift check. | -| auto_degradation_enabled | `bool` | Enables automatic degradation from ME to direct DC. | -| degradation_min_unavailable_dc_groups | `u8` | Minimum unavailable ME DC groups required before degrading. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| data_path | `String \| null` | `null` | — | Optional runtime data directory path. | +| prefer_ipv6 | `bool` | `false` | — | Prefer IPv6 where applicable in runtime logic. | +| fast_mode | `bool` | `true` | — | Enables fast-path optimizations for traffic processing. | +| use_middle_proxy | `bool` | `true` | none | Enables ME transport mode; if `false`, runtime falls back to direct DC routing. | +| proxy_secret_path | `String \| null` | `"proxy-secret"` | Path may be `null`. | Path to Telegram infrastructure proxy-secret file used by ME handshake logic. | +| proxy_config_v4_cache_path | `String \| null` | `"cache/proxy-config-v4.txt"` | — | Optional cache path for raw `getProxyConfig` (IPv4) snapshot. | +| proxy_config_v6_cache_path | `String \| null` | `"cache/proxy-config-v6.txt"` | — | Optional cache path for raw `getProxyConfigV6` (IPv6) snapshot. | +| ad_tag | `String \| null` | `null` | — | Global fallback ad tag (32 hex characters). | +| middle_proxy_nat_ip | `IpAddr \| null` | `null` | Must be a valid IP when set. | Manual public NAT IP override used as ME address material when set. | +| middle_proxy_nat_probe | `bool` | `true` | Auto-forced to `true` when `use_middle_proxy = true`. | Enables ME NAT probing; runtime may force it on when ME mode is active. | +| middle_proxy_nat_stun | `String \| null` | `null` | Deprecated. Use `network.stun_servers`. | Deprecated legacy single STUN server for NAT probing. | +| middle_proxy_nat_stun_servers | `String[]` | `[]` | Deprecated. Use `network.stun_servers`. | Deprecated legacy STUN list for NAT probing fallback. | +| stun_nat_probe_concurrency | `usize` | `8` | Must be `> 0`. | Maximum number of parallel STUN probes during NAT/public endpoint discovery. | +| middle_proxy_pool_size | `usize` | `8` | none | Target size of active ME writer pool. | +| middle_proxy_warm_standby | `usize` | `16` | none | Reserved compatibility field in current runtime revision. | +| me_init_retry_attempts | `u32` | `0` | `0..=1_000_000`. | Startup retries for ME pool initialization (`0` means unlimited). | +| me2dc_fallback | `bool` | `true` | — | Allows fallback from ME mode to direct DC when ME startup fails. | +| me_keepalive_enabled | `bool` | `true` | none | Enables periodic ME keepalive/ping traffic. | +| me_keepalive_interval_secs | `u64` | `8` | none | Base ME keepalive interval in seconds. | +| me_keepalive_jitter_secs | `u64` | `2` | none | Keepalive jitter in seconds to reduce synchronized bursts. | +| me_keepalive_payload_random | `bool` | `true` | none | Randomizes keepalive payload bytes instead of fixed zero payload. | +| rpc_proxy_req_every | `u64` | `0` | `0` or `10..=300`. | Interval for service `RPC_PROXY_REQ` activity signals (`0` disables). | +| me_writer_cmd_channel_capacity | `usize` | `4096` | Must be `> 0`. | Capacity of per-writer command channel. | +| me_route_channel_capacity | `usize` | `768` | Must be `> 0`. | Capacity of per-connection ME response route channel. | +| me_c2me_channel_capacity | `usize` | `1024` | Must be `> 0`. | Capacity of per-client command queue (client reader -> ME sender). | +| me_reader_route_data_wait_ms | `u64` | `2` | `0..=20`. | Bounded wait for routing ME DATA to per-connection queue (`0` = no wait). | +| me_d2c_flush_batch_max_frames | `usize` | `32` | `1..=512`. | Max ME->client frames coalesced before flush. | +| me_d2c_flush_batch_max_bytes | `usize` | `131072` | `4096..=2_097_152`. | Max ME->client payload bytes coalesced before flush. | +| me_d2c_flush_batch_max_delay_us | `u64` | `500` | `0..=5000`. | Max microsecond wait for coalescing more ME->client frames (`0` disables timed coalescing). | +| me_d2c_ack_flush_immediate | `bool` | `true` | — | Flushes client writer immediately after quick-ack write. | +| direct_relay_copy_buf_c2s_bytes | `usize` | `65536` | `4096..=1_048_576`. | Copy buffer size for client->DC direction in direct relay. | +| direct_relay_copy_buf_s2c_bytes | `usize` | `262144` | `8192..=2_097_152`. | Copy buffer size for DC->client direction in direct relay. | +| crypto_pending_buffer | `usize` | `262144` | — | Max pending ciphertext buffer per client writer (bytes). | +| max_client_frame | `usize` | `16777216` | — | Maximum allowed client MTProto frame size (bytes). | +| desync_all_full | `bool` | `false` | — | Emits full crypto-desync forensic logs for every event. | +| beobachten | `bool` | `true` | — | Enables per-IP forensic observation buckets. | +| beobachten_minutes | `u64` | `10` | Must be `> 0`. | Retention window (minutes) for per-IP observation buckets. | +| beobachten_flush_secs | `u64` | `15` | Must be `> 0`. | Snapshot flush interval (seconds) for observation output file. | +| beobachten_file | `String` | `"cache/beobachten.txt"` | — | Observation snapshot output file path. | +| hardswap | `bool` | `true` | none | Enables generation-based ME hardswap strategy. | +| me_warmup_stagger_enabled | `bool` | `true` | none | Staggers extra ME warmup dials to avoid connection spikes. | +| me_warmup_step_delay_ms | `u64` | `500` | none | Base delay in milliseconds between warmup dial steps. | +| me_warmup_step_jitter_ms | `u64` | `300` | none | Additional random delay in milliseconds for warmup steps. | +| me_reconnect_max_concurrent_per_dc | `u32` | `8` | none | Limits concurrent reconnect workers per DC during health recovery. | +| me_reconnect_backoff_base_ms | `u64` | `500` | none | Initial reconnect backoff in milliseconds. | +| me_reconnect_backoff_cap_ms | `u64` | `30000` | none | Maximum reconnect backoff cap in milliseconds. | +| me_reconnect_fast_retry_count | `u32` | `16` | none | Immediate retry budget before long backoff behavior applies. | +| me_single_endpoint_shadow_writers | `u8` | `2` | `0..=32`. | Additional reserve writers for one-endpoint DC groups. | +| me_single_endpoint_outage_mode_enabled | `bool` | `true` | — | Enables aggressive outage recovery for one-endpoint DC groups. | +| me_single_endpoint_outage_disable_quarantine | `bool` | `true` | — | Ignores endpoint quarantine in one-endpoint outage mode. | +| me_single_endpoint_outage_backoff_min_ms | `u64` | `250` | Must be `> 0`; also `<= me_single_endpoint_outage_backoff_max_ms`. | Minimum reconnect backoff in outage mode (ms). | +| me_single_endpoint_outage_backoff_max_ms | `u64` | `3000` | Must be `> 0`; also `>= me_single_endpoint_outage_backoff_min_ms`. | Maximum reconnect backoff in outage mode (ms). | +| me_single_endpoint_shadow_rotate_every_secs | `u64` | `900` | — | Periodic shadow writer rotation interval (`0` disables). | +| me_floor_mode | `"static" \| "adaptive"` | `"adaptive"` | — | Writer floor policy mode. | +| me_adaptive_floor_idle_secs | `u64` | `90` | — | Idle time before adaptive floor may reduce one-endpoint target. | +| me_adaptive_floor_min_writers_single_endpoint | `u8` | `1` | `1..=32`. | Minimum adaptive writer target for one-endpoint DC groups. | +| me_adaptive_floor_min_writers_multi_endpoint | `u8` | `1` | `1..=32`. | Minimum adaptive writer target for multi-endpoint DC groups. | +| me_adaptive_floor_recover_grace_secs | `u64` | `180` | — | Grace period to hold static floor after activity. | +| me_adaptive_floor_writers_per_core_total | `u16` | `48` | Must be `> 0`. | Global writer budget per logical CPU core in adaptive mode. | +| me_adaptive_floor_cpu_cores_override | `u16` | `0` | — | Manual CPU core count override (`0` uses auto-detection). | +| me_adaptive_floor_max_extra_writers_single_per_core | `u16` | `1` | — | Per-core max extra writers above base floor for one-endpoint DCs. | +| me_adaptive_floor_max_extra_writers_multi_per_core | `u16` | `2` | — | Per-core max extra writers above base floor for multi-endpoint DCs. | +| me_adaptive_floor_max_active_writers_per_core | `u16` | `64` | Must be `> 0`. | Hard cap for active ME writers per logical CPU core. | +| me_adaptive_floor_max_warm_writers_per_core | `u16` | `64` | Must be `> 0`. | Hard cap for warm ME writers per logical CPU core. | +| me_adaptive_floor_max_active_writers_global | `u32` | `256` | Must be `> 0`. | Hard global cap for active ME writers. | +| me_adaptive_floor_max_warm_writers_global | `u32` | `256` | Must be `> 0`. | Hard global cap for warm ME writers. | +| upstream_connect_retry_attempts | `u32` | `2` | Must be `> 0`. | Connect attempts for selected upstream before error/fallback. | +| upstream_connect_retry_backoff_ms | `u64` | `100` | — | Delay between upstream connect attempts (ms). | +| upstream_connect_budget_ms | `u64` | `3000` | Must be `> 0`. | Total wall-clock budget for one upstream connect request (ms). | +| upstream_unhealthy_fail_threshold | `u32` | `5` | Must be `> 0`. | Consecutive failed requests before upstream is marked unhealthy. | +| upstream_connect_failfast_hard_errors | `bool` | `false` | — | Skips additional retries for hard non-transient connect errors. | +| stun_iface_mismatch_ignore | `bool` | `false` | none | Reserved compatibility flag in current runtime revision. | +| unknown_dc_log_path | `String \| null` | `"unknown-dc.txt"` | — | File path for unknown-DC request logging (`null` disables file path). | +| unknown_dc_file_log_enabled | `bool` | `false` | — | Enables unknown-DC file logging. | +| log_level | `"debug" \| "verbose" \| "normal" \| "silent"` | `"normal"` | — | Runtime logging verbosity. | +| disable_colors | `bool` | `false` | — | Disables ANSI colors in logs. | +| me_socks_kdf_policy | `"strict" \| "compat"` | `"strict"` | — | SOCKS-bound KDF fallback policy for ME handshake. | +| me_route_backpressure_base_timeout_ms | `u64` | `25` | Must be `> 0`. | Base backpressure timeout for route-channel send (ms). | +| me_route_backpressure_high_timeout_ms | `u64` | `120` | Must be `>= me_route_backpressure_base_timeout_ms`. | High backpressure timeout when queue occupancy exceeds watermark (ms). | +| me_route_backpressure_high_watermark_pct | `u8` | `80` | `1..=100`. | Queue occupancy threshold (%) for high timeout mode. | +| me_health_interval_ms_unhealthy | `u64` | `1000` | Must be `> 0`. | Health monitor interval while writer coverage is degraded (ms). | +| me_health_interval_ms_healthy | `u64` | `3000` | Must be `> 0`. | Health monitor interval while writer coverage is healthy (ms). | +| me_admission_poll_ms | `u64` | `1000` | Must be `> 0`. | Poll interval for conditional-admission checks (ms). | +| me_warn_rate_limit_ms | `u64` | `5000` | Must be `> 0`. | Cooldown for repetitive ME warning logs (ms). | +| me_route_no_writer_mode | `"async_recovery_failfast" \| "inline_recovery_legacy" \| "hybrid_async_persistent"` | `"hybrid_async_persistent"` | — | Route behavior when no writer is immediately available. | +| me_route_no_writer_wait_ms | `u64` | `250` | `10..=5000`. | Max wait in async-recovery failfast mode (ms). | +| me_route_inline_recovery_attempts | `u32` | `3` | Must be `> 0`. | Inline recovery attempts in legacy mode. | +| me_route_inline_recovery_wait_ms | `u64` | `3000` | `10..=30000`. | Max inline recovery wait in legacy mode (ms). | +| fast_mode_min_tls_record | `usize` | `0` | — | Minimum TLS record size when fast-mode coalescing is enabled (`0` disables). | +| update_every | `u64 \| null` | `300` | If set: must be `> 0`; if `null`: legacy fallback path is used. | Unified refresh interval for ME config and proxy-secret updater tasks. | +| me_reinit_every_secs | `u64` | `900` | Must be `> 0`. | Periodic interval for zero-downtime ME reinit cycle. | +| me_hardswap_warmup_delay_min_ms | `u64` | `1000` | Must be `<= me_hardswap_warmup_delay_max_ms`. | Lower bound for hardswap warmup dial spacing. | +| me_hardswap_warmup_delay_max_ms | `u64` | `2000` | Must be `> 0`. | Upper bound for hardswap warmup dial spacing. | +| me_hardswap_warmup_extra_passes | `u8` | `3` | Must be within `[0, 10]`. | Additional warmup passes after the base pass in one hardswap cycle. | +| me_hardswap_warmup_pass_backoff_base_ms | `u64` | `500` | Must be `> 0`. | Base backoff between extra hardswap warmup passes. | +| me_config_stable_snapshots | `u8` | `2` | Must be `> 0`. | Number of identical ME config snapshots required before apply. | +| me_config_apply_cooldown_secs | `u64` | `300` | none | Cooldown between applied ME endpoint-map updates. | +| me_snapshot_require_http_2xx | `bool` | `true` | — | Requires 2xx HTTP responses for applying config snapshots. | +| me_snapshot_reject_empty_map | `bool` | `true` | — | Rejects empty config snapshots. | +| me_snapshot_min_proxy_for_lines | `u32` | `1` | Must be `> 0`. | Minimum parsed `proxy_for` rows required to accept snapshot. | +| proxy_secret_stable_snapshots | `u8` | `2` | Must be `> 0`. | Number of identical proxy-secret snapshots required before rotation. | +| proxy_secret_rotate_runtime | `bool` | `true` | none | Enables runtime proxy-secret rotation from updater snapshots. | +| me_secret_atomic_snapshot | `bool` | `true` | — | Keeps selector and secret bytes from the same snapshot atomically. | +| proxy_secret_len_max | `usize` | `256` | Must be within `[32, 4096]`. | Upper length limit for accepted proxy-secret bytes. | +| me_pool_drain_ttl_secs | `u64` | `90` | none | Time window where stale writers remain fallback-eligible after map change. | +| me_pool_drain_threshold | `u64` | `128` | — | Max draining stale writers before batch force-close (`0` disables threshold cleanup). | +| me_pool_drain_soft_evict_enabled | `bool` | `true` | — | Enables gradual soft-eviction of stale writers during drain/reinit instead of immediate hard close. | +| me_pool_drain_soft_evict_grace_secs | `u64` | `30` | `0..=3600`. | Grace period before stale writers become soft-evict candidates. | +| me_pool_drain_soft_evict_per_writer | `u8` | `1` | `1..=16`. | Maximum stale routes soft-evicted per writer in one eviction pass. | +| me_pool_drain_soft_evict_budget_per_core | `u16` | `8` | `1..=64`. | Per-core budget limiting aggregate soft-eviction work per pass. | +| me_pool_drain_soft_evict_cooldown_ms | `u64` | `5000` | Must be `> 0`. | Cooldown between consecutive soft-eviction passes (ms). | +| me_bind_stale_mode | `"never" \| "ttl" \| "always"` | `"ttl"` | — | Policy for new binds on stale draining writers. | +| me_bind_stale_ttl_secs | `u64` | `90` | — | TTL for stale bind allowance when stale mode is `ttl`. | +| me_pool_min_fresh_ratio | `f32` | `0.8` | Must be within `[0.0, 1.0]`. | Minimum fresh desired-DC coverage ratio before stale writers are drained. | +| me_reinit_drain_timeout_secs | `u64` | `120` | `0` disables force-close; if `> 0` and `< me_pool_drain_ttl_secs`, runtime bumps it to TTL. | Force-close timeout for draining stale writers (`0` keeps indefinite draining). | +| proxy_secret_auto_reload_secs | `u64` | `3600` | Deprecated. Use `general.update_every`. | Deprecated legacy secret reload interval (fallback when `update_every` is not set). | +| proxy_config_auto_reload_secs | `u64` | `3600` | Deprecated. Use `general.update_every`. | Deprecated legacy config reload interval (fallback when `update_every` is not set). | +| me_reinit_singleflight | `bool` | `true` | — | Serializes ME reinit cycles across trigger sources. | +| me_reinit_trigger_channel | `usize` | `64` | Must be `> 0`. | Trigger queue capacity for reinit scheduler. | +| me_reinit_coalesce_window_ms | `u64` | `200` | — | Trigger coalescing window before starting reinit (ms). | +| me_deterministic_writer_sort | `bool` | `true` | — | Enables deterministic candidate sort for writer binding path. | +| me_writer_pick_mode | `"sorted_rr" \| "p2c"` | `"p2c"` | — | Writer selection mode for route bind path. | +| me_writer_pick_sample_size | `u8` | `3` | `2..=4`. | Number of candidates sampled by picker in `p2c` mode. | +| ntp_check | `bool` | `true` | — | Enables NTP drift check at startup. | +| ntp_servers | `String[]` | `["pool.ntp.org"]` | — | NTP servers used for drift check. | +| auto_degradation_enabled | `bool` | `true` | none | Reserved compatibility flag in current runtime revision. | +| degradation_min_unavailable_dc_groups | `u8` | `2` | none | Reserved compatibility threshold in current runtime revision. | ## [general.modes] -| Parameter | Type | Description | -|---|---|---| -| classic | `bool` | Enables classic MTProxy mode. | -| secure | `bool` | Enables secure mode. | -| tls | `bool` | Enables TLS mode. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| classic | `bool` | `false` | — | Enables classic MTProxy mode. | +| secure | `bool` | `false` | — | Enables secure mode. | +| tls | `bool` | `true` | — | Enables TLS mode. | ## [general.links] -| Parameter | Type | Description | -|---|---|---| -| show | `"*" \| String[]` | Selects users whose tg:// links are shown at startup. | -| public_host | `String` | Public hostname/IP override for generated tg:// links. | -| public_port | `u16` | Public port override for generated tg:// links. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| show | `"*" \| String[]` | `"*"` | — | Selects users whose tg:// links are shown at startup. | +| public_host | `String \| null` | `null` | — | Public hostname/IP override for generated tg:// links. | +| public_port | `u16 \| null` | `null` | — | Public port override for generated tg:// links. | ## [general.telemetry] -| Parameter | Type | Description | -|---|---|---| -| core_enabled | `bool` | Enables core hot-path telemetry counters. | -| user_enabled | `bool` | Enables per-user telemetry counters. | -| me_level | `"silent" \| "normal" \| "debug"` | Middle-End telemetry verbosity level. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| core_enabled | `bool` | `true` | — | Enables core hot-path telemetry counters. | +| user_enabled | `bool` | `true` | — | Enables per-user telemetry counters. | +| me_level | `"silent" \| "normal" \| "debug"` | `"normal"` | — | Middle-End telemetry verbosity level. | ## [network] -| Parameter | Type | Description | -|---|---|---| -| ipv4 | `bool` | Enables IPv4 networking. | -| ipv6 | `bool` | Enables/disables IPv6 (`null` = auto-detect availability). | -| prefer | `u8` | Preferred IP family for selection (`4` or `6`). | -| multipath | `bool` | Enables multipath behavior where supported. | -| stun_use | `bool` | Global switch for STUN probing. | -| stun_servers | `String[]` | STUN server list for public IP detection. | -| stun_tcp_fallback | `bool` | Enables TCP STUN fallback when UDP STUN is blocked. | -| http_ip_detect_urls | `String[]` | HTTP endpoints used as fallback public IP detectors. | -| cache_public_ip_path | `String` | File path for caching detected public IP. | -| dns_overrides | `String[]` | Runtime DNS overrides in `host:port:ip` format. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| ipv4 | `bool` | `true` | — | Enables IPv4 networking. | +| ipv6 | `bool` | `false` | — | Enables/disables IPv6 when set | +| prefer | `u8` | `4` | Must be `4` or `6`. | Preferred IP family for selection (`4` or `6`). | +| multipath | `bool` | `false` | — | Enables multipath behavior where supported. | +| stun_use | `bool` | `true` | none | Global STUN switch; when `false`, STUN probing path is disabled. | +| stun_servers | `String[]` | Built-in STUN list (13 hosts) | Deduplicated; empty values are removed. | Primary STUN server list for NAT/public endpoint discovery. | +| stun_tcp_fallback | `bool` | `true` | none | Enables TCP fallback for STUN when UDP path is blocked. | +| http_ip_detect_urls | `String[]` | `["https://ifconfig.me/ip", "https://api.ipify.org"]` | none | HTTP fallback endpoints for public IP detection when STUN is unavailable. | +| cache_public_ip_path | `String` | `"cache/public_ip.txt"` | — | File path for caching detected public IP. | +| dns_overrides | `String[]` | `[]` | Must match `host:port:ip`; IPv6 must be bracketed. | Runtime DNS overrides in `host:port:ip` format. | ## [server] -| Parameter | Type | Description | -|---|---|---| -| port | `u16` | Main proxy listen port. | -| listen_addr_ipv4 | `String` | IPv4 bind address for TCP listener. | -| listen_addr_ipv6 | `String` | IPv6 bind address for TCP listener. | -| listen_unix_sock | `String` | Unix socket path for listener. | -| listen_unix_sock_perm | `String` | Unix socket permissions in octal string (e.g., `"0666"`). | -| listen_tcp | `bool` | Explicit TCP listener enable/disable override. | -| proxy_protocol | `bool` | Enables HAProxy PROXY protocol parsing on incoming client connections. | -| proxy_protocol_header_timeout_ms | `u64` | Timeout for PROXY protocol header read/parse (ms). | -| metrics_port | `u16` | Metrics endpoint port (enables metrics listener). | -| metrics_listen | `String` | Full metrics bind address (`IP:PORT`), overrides `metrics_port`. | -| metrics_whitelist | `IpNetwork[]` | CIDR whitelist for metrics endpoint access. | -| max_connections | `u32` | Max concurrent client connections (`0` = unlimited). | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| port | `u16` | `443` | — | Main proxy listen port. | +| listen_addr_ipv4 | `String \| null` | `"0.0.0.0"` | — | IPv4 bind address for TCP listener. | +| listen_addr_ipv6 | `String \| null` | `"::"` | — | IPv6 bind address for TCP listener. | +| listen_unix_sock | `String \| null` | `null` | — | Unix socket path for listener. | +| listen_unix_sock_perm | `String \| null` | `null` | — | Unix socket permissions in octal string (e.g., `"0666"`). | +| listen_tcp | `bool \| null` | `null` (auto) | — | Explicit TCP listener enable/disable override. | +| proxy_protocol | `bool` | `false` | — | Enables HAProxy PROXY protocol parsing on incoming client connections. | +| proxy_protocol_header_timeout_ms | `u64` | `500` | Must be `> 0`. | Timeout for PROXY protocol header read/parse (ms). | +| metrics_port | `u16 \| null` | `null` | — | Metrics endpoint port (enables metrics listener). | +| metrics_listen | `String \| null` | `null` | — | Full metrics bind address (`IP:PORT`), overrides `metrics_port`. | +| metrics_whitelist | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` | — | CIDR whitelist for metrics endpoint access. | +| max_connections | `u32` | `10000` | — | Max concurrent client connections (`0` = unlimited). | ## [server.api] -| Parameter | Type | Description | -|---|---|---| -| enabled | `bool` | Enables control-plane REST API. | -| listen | `String` | API bind address in `IP:PORT` format. | -| whitelist | `IpNetwork[]` | CIDR whitelist allowed to access API. | -| auth_header | `String` | Exact expected `Authorization` header value (empty = disabled). | -| request_body_limit_bytes | `usize` | Maximum accepted HTTP request body size. | -| minimal_runtime_enabled | `bool` | Enables minimal runtime snapshots endpoint logic. | -| minimal_runtime_cache_ttl_ms | `u64` | Cache TTL for minimal runtime snapshots (ms; `0` disables cache). | -| runtime_edge_enabled | `bool` | Enables runtime edge endpoints. | -| runtime_edge_cache_ttl_ms | `u64` | Cache TTL for runtime edge aggregation payloads (ms). | -| runtime_edge_top_n | `usize` | Top-N size for edge connection leaderboard. | -| runtime_edge_events_capacity | `usize` | Ring-buffer capacity for runtime edge events. | -| read_only | `bool` | Rejects mutating API endpoints when enabled. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| enabled | `bool` | `true` | — | Enables control-plane REST API. | +| listen | `String` | `"0.0.0.0:9091"` | Must be valid `IP:PORT`. | API bind address in `IP:PORT` format. | +| whitelist | `IpNetwork[]` | `["127.0.0.0/8"]` | — | CIDR whitelist allowed to access API. | +| auth_header | `String` | `""` | — | Exact expected `Authorization` header value (empty = disabled). | +| request_body_limit_bytes | `usize` | `65536` | Must be `> 0`. | Maximum accepted HTTP request body size. | +| minimal_runtime_enabled | `bool` | `true` | — | Enables minimal runtime snapshots endpoint logic. | +| minimal_runtime_cache_ttl_ms | `u64` | `1000` | `0..=60000`. | Cache TTL for minimal runtime snapshots (ms; `0` disables cache). | +| runtime_edge_enabled | `bool` | `false` | — | Enables runtime edge endpoints. | +| runtime_edge_cache_ttl_ms | `u64` | `1000` | `0..=60000`. | Cache TTL for runtime edge aggregation payloads (ms). | +| runtime_edge_top_n | `usize` | `10` | `1..=1000`. | Top-N size for edge connection leaderboard. | +| runtime_edge_events_capacity | `usize` | `256` | `16..=4096`. | Ring-buffer capacity for runtime edge events. | +| read_only | `bool` | `false` | — | Rejects mutating API endpoints when enabled. | ## [[server.listeners]] -| Parameter | Type | Description | -|---|---|---| -| ip | `IpAddr` | Listener bind IP. | -| announce | `String` | Public IP/domain announced in proxy links (priority over `announce_ip`). | -| announce_ip | `IpAddr` | Deprecated legacy announce IP (migrated to `announce` if needed). | -| proxy_protocol | `bool` | Per-listener override for PROXY protocol enable flag. | -| reuse_allow | `bool` | Enables `SO_REUSEPORT` for multi-instance bind sharing. | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| ip | `IpAddr` | — | — | Listener bind IP. | +| announce | `String \| null` | — | — | Public IP/domain announced in proxy links (priority over `announce_ip`). | +| announce_ip | `IpAddr \| null` | — | — | Deprecated legacy announce IP (migrated to `announce` if needed). | +| proxy_protocol | `bool \| null` | `null` | — | Per-listener override for PROXY protocol enable flag. | +| reuse_allow | `bool` | `false` | — | Enables `SO_REUSEPORT` for multi-instance bind sharing. | ## [timeouts] -| Parameter | Type | Description | -|---|---|---| -| client_handshake | `u64` | Client handshake timeout. | -| tg_connect | `u64` | Upstream Telegram connect timeout. | -| client_keepalive | `u64` | Client keepalive timeout. | -| client_ack | `u64` | Client ACK timeout. | -| me_one_retry | `u8` | Quick ME reconnect attempts for single-address DC. | -| me_one_timeout_ms | `u64` | Timeout per quick attempt for single-address DC (ms). | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| client_handshake | `u64` | `30` | — | Client handshake timeout. | +| tg_connect | `u64` | `10` | — | Upstream Telegram connect timeout. | +| client_keepalive | `u64` | `15` | — | Client keepalive timeout. | +| client_ack | `u64` | `90` | — | Client ACK timeout. | +| me_one_retry | `u8` | `12` | none | Fast reconnect attempts budget for single-endpoint DC scenarios. | +| me_one_timeout_ms | `u64` | `1200` | none | Timeout in milliseconds for each quick single-endpoint reconnect attempt. | ## [censorship] -| Parameter | Type | Description | -|---|---|---| -| tls_domain | `String` | Primary TLS domain used in fake TLS handshake profile. | -| tls_domains | `String[]` | Additional TLS domains for generating multiple links. | -| mask | `bool` | Enables masking/fronting relay mode. | -| mask_host | `String` | Upstream mask host for TLS fronting relay. | -| mask_port | `u16` | Upstream mask port for TLS fronting relay. | -| mask_unix_sock | `String` | Unix socket path for mask backend instead of TCP host/port. | -| fake_cert_len | `usize` | Length of synthetic certificate payload when emulation data is unavailable. | -| tls_emulation | `bool` | Enables certificate/TLS behavior emulation from cached real fronts. | -| tls_front_dir | `String` | Directory path for TLS front cache storage. | -| server_hello_delay_min_ms | `u64` | Minimum server_hello delay for anti-fingerprint behavior (ms). | -| server_hello_delay_max_ms | `u64` | Maximum server_hello delay for anti-fingerprint behavior (ms). | -| tls_new_session_tickets | `u8` | Number of `NewSessionTicket` messages to emit after handshake. | -| tls_full_cert_ttl_secs | `u64` | TTL for sending full cert payload per (domain, client IP) tuple. | -| alpn_enforce | `bool` | Enforces ALPN echo behavior based on client preference. | -| mask_proxy_protocol | `u8` | PROXY protocol mode for mask backend (`0` disabled, `1` v1, `2` v2). | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| tls_domain | `String` | `"petrovich.ru"` | — | Primary TLS domain used in fake TLS handshake profile. | +| tls_domains | `String[]` | `[]` | — | Additional TLS domains for generating multiple links. | +| mask | `bool` | `true` | — | Enables masking/fronting relay mode. | +| mask_host | `String \| null` | `null` | — | Upstream mask host for TLS fronting relay. | +| mask_port | `u16` | `443` | — | Upstream mask port for TLS fronting relay. | +| mask_unix_sock | `String \| null` | `null` | — | Unix socket path for mask backend instead of TCP host/port. | +| fake_cert_len | `usize` | `2048` | — | Length of synthetic certificate payload when emulation data is unavailable. | +| tls_emulation | `bool` | `true` | — | Enables certificate/TLS behavior emulation from cached real fronts. | +| tls_front_dir | `String` | `"tlsfront"` | — | Directory path for TLS front cache storage. | +| server_hello_delay_min_ms | `u64` | `0` | — | Minimum server_hello delay for anti-fingerprint behavior (ms). | +| server_hello_delay_max_ms | `u64` | `0` | — | Maximum server_hello delay for anti-fingerprint behavior (ms). | +| tls_new_session_tickets | `u8` | `0` | — | Number of `NewSessionTicket` messages to emit after handshake. | +| tls_full_cert_ttl_secs | `u64` | `90` | — | TTL for sending full cert payload per (domain, client IP) tuple. | +| alpn_enforce | `bool` | `true` | — | Enforces ALPN echo behavior based on client preference. | +| mask_proxy_protocol | `u8` | `0` | — | PROXY protocol mode for mask backend (`0` disabled, `1` v1, `2` v2). | ## [access] -| Parameter | Type | Description | -|---|---|---| -| users | `Map` | Username -> 32-hex secret mapping. | -| user_ad_tags | `Map` | Per-user ad tags (32 hex chars). | -| user_max_tcp_conns | `Map` | Per-user maximum concurrent TCP connections. | -| user_expirations | `Map>` | Per-user account expiration timestamps. | -| user_data_quota | `Map` | Per-user data quota limits. | -| user_max_unique_ips | `Map` | Per-user unique source IP limits. | -| user_max_unique_ips_global_each | `usize` | Global fallback per-user unique IP limit when no per-user override exists. | -| user_max_unique_ips_mode | `"active_window" \| "time_window" \| "combined"` | Unique source IP limit accounting mode. | -| user_max_unique_ips_window_secs | `u64` | Recent-window size for unique IP accounting (seconds). | -| replay_check_len | `usize` | Replay check storage length. | -| replay_window_secs | `u64` | Replay protection time window in seconds. | -| ignore_time_skew | `bool` | Ignores client/server timestamp skew in replay validation. | +| Parameter | Type | Default | Constraints / validation | TOML shape example | Description | +|---|---|---|---|---|---| +| users | `Map` | `{"default": "000…000"}` | Secret must be 32 hex characters. | `[access.users]`
`user = "32-hex secret"`
`user2 = "32-hex secret"` | User credentials map used for client authentication. | +| user_ad_tags | `Map` | `{}` | Every value must be exactly 32 hex characters. | `[access.user_ad_tags]`
`user = "32-hex ad_tag"` | Per-user ad tags used as override over `general.ad_tag`. | +| user_max_tcp_conns | `Map` | `{}` | — | `[access.user_max_tcp_conns]`
`user = 500` | Per-user maximum concurrent TCP connections. | +| user_expirations | `Map>` | `{}` | Timestamp must be valid RFC3339/ISO-8601 datetime. | `[access.user_expirations]`
`user = "2026-12-31T23:59:59Z"` | Per-user account expiration timestamps. | +| user_data_quota | `Map` | `{}` | — | `[access.user_data_quota]`
`user = 1073741824` | Per-user traffic quota in bytes. | +| user_max_unique_ips | `Map` | `{}` | — | `[access.user_max_unique_ips]`
`user = 16` | Per-user unique source IP limits. | +| user_max_unique_ips_global_each | `usize` | `0` | — | `user_max_unique_ips_global_each = 0` | Global fallback used when `[access.user_max_unique_ips]` has no per-user override. | +| user_max_unique_ips_mode | `"active_window" \| "time_window" \| "combined"` | `"active_window"` | — | `user_max_unique_ips_mode = "active_window"` | Unique source IP limit accounting mode. | +| user_max_unique_ips_window_secs | `u64` | `30` | Must be `> 0`. | `user_max_unique_ips_window_secs = 30` | Window size (seconds) used by unique-IP accounting modes that use time windows. | +| replay_check_len | `usize` | `65536` | — | `replay_check_len = 65536` | Replay-protection storage length. | +| replay_window_secs | `u64` | `1800` | — | `replay_window_secs = 1800` | Replay-protection window in seconds. | +| ignore_time_skew | `bool` | `false` | — | `ignore_time_skew = false` | Disables client/server timestamp skew checks in replay validation when enabled. | ## [[upstreams]] -| Parameter | Type | Description | -|---|---|---| -| type | `"direct" \| "socks4" \| "socks5"` | Upstream transport type selector. | -| weight | `u16` | Weighted selection coefficient for this upstream. | -| enabled | `bool` | Enables/disables this upstream entry. | -| scopes | `String` | Comma-separated scope tags for routing. | -| interface | `String` | Optional outgoing interface name (`direct`, `socks4`, `socks5`). | -| bind_addresses | `String[]` | Optional source bind addresses for `direct` upstream. | -| address | `String` | Upstream proxy address (`host:port`) for SOCKS upstreams. | -| user_id | `String` | SOCKS4 user ID (only for `type = "socks4"`). | -| username | `String` | SOCKS5 username (only for `type = "socks5"`). | -| password | `String` | SOCKS5 password (only for `type = "socks5"`). | +| Parameter | Type | Default | Constraints / validation | Description | +|---|---|---|---|---| +| type | `"direct" \| "socks4" \| "socks5"` | — | Required field. | Upstream transport type selector. | +| weight | `u16` | `1` | none | Base weight used by weighted-random upstream selection. | +| enabled | `bool` | `true` | none | Disabled entries are excluded from upstream selection at runtime. | +| scopes | `String` | `""` | none | Comma-separated scope tags used for request-level upstream filtering. | +| interface | `String \| null` | `null` | Optional; type-specific runtime rules apply. | Optional outbound interface/local bind hint (supported with type-specific rules). | +| bind_addresses | `String[] \| null` | `null` | Applies to `type = "direct"`. | Optional explicit local source bind addresses for `type = "direct"`. | +| address | `String` | — | Required for `type = "socks4"` and `type = "socks5"`. | SOCKS server endpoint (`host:port` or `ip:port`) for SOCKS upstream types. | +| user_id | `String \| null` | `null` | Only for `type = "socks4"`. | SOCKS4 CONNECT user ID (`type = "socks4"` only). | +| username | `String \| null` | `null` | Only for `type = "socks5"`. | SOCKS5 username (`type = "socks5"` only). | +| password | `String \| null` | `null` | Only for `type = "socks5"`. | SOCKS5 password (`type = "socks5"` only). | From 89e5668c7e000ac7ffd5c3512280236f64862902 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 22:33:41 +0300 Subject: [PATCH 236/261] Runtime guardrails Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 20 ++++ src/config/hot_reload.rs | 9 ++ src/config/load.rs | 24 +++++ src/config/types.rs | 23 +++++ src/maestro/helpers.rs | 1 + src/maestro/listeners.rs | 98 +++++++++++++++---- src/maestro/me_startup.rs | 2 + src/proxy/middle_relay.rs | 82 +++++++++++++--- src/transport/middle_proxy/health.rs | 2 + .../middle_proxy/health_adversarial_tests.rs | 2 + .../middle_proxy/health_integration_tests.rs | 2 + .../middle_proxy/health_regression_tests.rs | 2 + src/transport/middle_proxy/pool.rs | 8 ++ src/transport/middle_proxy/pool_writer.rs | 63 ++++++------ src/transport/middle_proxy/send.rs | 84 ++++++++++++++-- 15 files changed, 345 insertions(+), 77 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 7b5b4a8..54a53b3 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -36,12 +36,16 @@ const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000; const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000; const DEFAULT_ME_ADMISSION_POLL_MS: u64 = 1000; const DEFAULT_ME_WARN_RATE_LIMIT_MS: u64 = 5000; +const DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS: u64 = 3000; +const DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS: u64 = 250; +const DEFAULT_ME_C2ME_SEND_TIMEOUT_MS: u64 = 4000; const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED: bool = true; const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS: u64 = 30; const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER: u8 = 1; const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE: u16 = 8; const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 5000; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; +const DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS: u64 = 250; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5; const DEFAULT_UPSTREAM_CONNECT_BUDGET_MS: u64 = 3000; @@ -156,6 +160,10 @@ pub(crate) fn default_server_max_connections() -> u32 { 10_000 } +pub(crate) fn default_accept_permit_timeout_ms() -> u64 { + DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS +} + pub(crate) fn default_prefer_4() -> u8 { 4 } @@ -380,6 +388,18 @@ pub(crate) fn default_me_warn_rate_limit_ms() -> u64 { DEFAULT_ME_WARN_RATE_LIMIT_MS } +pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 { + DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS +} + +pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 { + DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS +} + +pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 { + DEFAULT_ME_C2ME_SEND_TIMEOUT_MS +} + pub(crate) fn default_upstream_connect_retry_attempts() -> u32 { DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index fdf06fa..7b94999 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -612,6 +612,8 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b || old.server.listen_tcp != new.server.listen_tcp || old.server.listen_unix_sock != new.server.listen_unix_sock || old.server.listen_unix_sock_perm != new.server.listen_unix_sock_perm + || old.server.max_connections != new.server.max_connections + || old.server.accept_permit_timeout_ms != new.server.accept_permit_timeout_ms { warned = true; warn!("config reload: server listener settings changed; restart required"); @@ -671,6 +673,9 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b } if old.general.me_route_no_writer_mode != new.general.me_route_no_writer_mode || old.general.me_route_no_writer_wait_ms != new.general.me_route_no_writer_wait_ms + || old.general.me_route_hybrid_max_wait_ms != new.general.me_route_hybrid_max_wait_ms + || old.general.me_route_blocking_send_timeout_ms + != new.general.me_route_blocking_send_timeout_ms || old.general.me_route_inline_recovery_attempts != new.general.me_route_inline_recovery_attempts || old.general.me_route_inline_recovery_wait_ms @@ -679,6 +684,10 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b warned = true; warn!("config reload: general.me_route_no_writer_* changed; restart required"); } + if old.general.me_c2me_send_timeout_ms != new.general.me_c2me_send_timeout_ms { + warned = true; + warn!("config reload: general.me_c2me_send_timeout_ms changed; restart required"); + } if old.general.unknown_dc_log_path != new.general.unknown_dc_log_path || old.general.unknown_dc_file_log_enabled != new.general.unknown_dc_file_log_enabled { diff --git a/src/config/load.rs b/src/config/load.rs index 6fcbea3..0635f80 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -346,6 +346,12 @@ impl ProxyConfig { )); } + if config.general.me_c2me_send_timeout_ms > 60_000 { + return Err(ProxyError::Config( + "general.me_c2me_send_timeout_ms must be within [0, 60000]".to_string(), + )); + } + if config.general.me_reader_route_data_wait_ms > 20 { return Err(ProxyError::Config( "general.me_reader_route_data_wait_ms must be within [0, 20]".to_string(), @@ -627,6 +633,18 @@ impl ProxyConfig { )); } + if !(50..=60_000).contains(&config.general.me_route_hybrid_max_wait_ms) { + return Err(ProxyError::Config( + "general.me_route_hybrid_max_wait_ms must be within [50, 60000]".to_string(), + )); + } + + if config.general.me_route_blocking_send_timeout_ms > 5000 { + return Err(ProxyError::Config( + "general.me_route_blocking_send_timeout_ms must be within [0, 5000]".to_string(), + )); + } + if !(2..=4).contains(&config.general.me_writer_pick_sample_size) { return Err(ProxyError::Config( "general.me_writer_pick_sample_size must be within [2, 4]".to_string(), @@ -687,6 +705,12 @@ impl ProxyConfig { )); } + if config.server.accept_permit_timeout_ms > 60_000 { + return Err(ProxyError::Config( + "server.accept_permit_timeout_ms must be within [0, 60000]".to_string(), + )); + } + if config.general.effective_me_pool_force_close_secs() > 0 && config.general.effective_me_pool_force_close_secs() < config.general.me_pool_drain_ttl_secs diff --git a/src/config/types.rs b/src/config/types.rs index e507044..047f3c2 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -462,6 +462,11 @@ pub struct GeneralConfig { #[serde(default = "default_me_c2me_channel_capacity")] pub me_c2me_channel_capacity: usize, + /// Maximum wait in milliseconds for enqueueing C2ME commands when the queue is full. + /// `0` keeps legacy unbounded wait behavior. + #[serde(default = "default_me_c2me_send_timeout_ms")] + pub me_c2me_send_timeout_ms: u64, + /// Bounded wait in milliseconds for routing ME DATA to per-connection queue. /// `0` keeps legacy no-wait behavior. #[serde(default = "default_me_reader_route_data_wait_ms")] @@ -716,6 +721,15 @@ pub struct GeneralConfig { #[serde(default = "default_me_route_no_writer_wait_ms")] pub me_route_no_writer_wait_ms: u64, + /// Maximum cumulative wait in milliseconds for hybrid no-writer mode before failfast. + #[serde(default = "default_me_route_hybrid_max_wait_ms")] + pub me_route_hybrid_max_wait_ms: u64, + + /// Maximum wait in milliseconds for blocking ME writer channel send fallback. + /// `0` keeps legacy unbounded wait behavior. + #[serde(default = "default_me_route_blocking_send_timeout_ms")] + pub me_route_blocking_send_timeout_ms: u64, + /// Number of inline recovery attempts in legacy mode. #[serde(default = "default_me_route_inline_recovery_attempts")] pub me_route_inline_recovery_attempts: u32, @@ -921,6 +935,7 @@ impl Default for GeneralConfig { me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(), me_route_channel_capacity: default_me_route_channel_capacity(), me_c2me_channel_capacity: default_me_c2me_channel_capacity(), + me_c2me_send_timeout_ms: default_me_c2me_send_timeout_ms(), me_reader_route_data_wait_ms: default_me_reader_route_data_wait_ms(), me_d2c_flush_batch_max_frames: default_me_d2c_flush_batch_max_frames(), me_d2c_flush_batch_max_bytes: default_me_d2c_flush_batch_max_bytes(), @@ -975,6 +990,8 @@ impl Default for GeneralConfig { me_warn_rate_limit_ms: default_me_warn_rate_limit_ms(), me_route_no_writer_mode: MeRouteNoWriterMode::default(), me_route_no_writer_wait_ms: default_me_route_no_writer_wait_ms(), + me_route_hybrid_max_wait_ms: default_me_route_hybrid_max_wait_ms(), + me_route_blocking_send_timeout_ms: default_me_route_blocking_send_timeout_ms(), me_route_inline_recovery_attempts: default_me_route_inline_recovery_attempts(), me_route_inline_recovery_wait_ms: default_me_route_inline_recovery_wait_ms(), links: LinksConfig::default(), @@ -1207,6 +1224,11 @@ pub struct ServerConfig { /// 0 means unlimited. #[serde(default = "default_server_max_connections")] pub max_connections: u32, + + /// Maximum wait in milliseconds while acquiring a connection slot permit. + /// `0` keeps legacy unbounded wait behavior. + #[serde(default = "default_accept_permit_timeout_ms")] + pub accept_permit_timeout_ms: u64, } impl Default for ServerConfig { @@ -1226,6 +1248,7 @@ impl Default for ServerConfig { api: ApiConfig::default(), listeners: Vec::new(), max_connections: default_server_max_connections(), + accept_permit_timeout_ms: default_accept_permit_timeout_ms(), } } } diff --git a/src/maestro/helpers.rs b/src/maestro/helpers.rs index 78f3ec4..f43e308 100644 --- a/src/maestro/helpers.rs +++ b/src/maestro/helpers.rs @@ -205,6 +205,7 @@ pub(crate) fn format_uptime(total_secs: u64) -> String { format!("{} / {} seconds", parts.join(", "), total_secs) } +#[allow(dead_code)] pub(crate) async fn wait_until_admission_open(admission_rx: &mut watch::Receiver) -> bool { loop { if *admission_rx.borrow() { diff --git a/src/maestro/listeners.rs b/src/maestro/listeners.rs index 6296fd7..fe041d9 100644 --- a/src/maestro/listeners.rs +++ b/src/maestro/listeners.rs @@ -24,7 +24,7 @@ use crate::transport::{ ListenOptions, UpstreamManager, create_listener, find_listener_processes, }; -use super::helpers::{is_expected_handshake_eof, print_proxy_links, wait_until_admission_open}; +use super::helpers::{is_expected_handshake_eof, print_proxy_links}; pub(crate) struct BoundListeners { pub(crate) listeners: Vec<(TcpListener, bool)>, @@ -195,7 +195,7 @@ pub(crate) async fn bind_listeners( has_unix_listener = true; let mut config_rx_unix: watch::Receiver> = config_rx.clone(); - let mut admission_rx_unix = admission_rx.clone(); + let admission_rx_unix = admission_rx.clone(); let stats = stats.clone(); let upstream_manager = upstream_manager.clone(); let replay_checker = replay_checker.clone(); @@ -212,17 +212,44 @@ pub(crate) async fn bind_listeners( let unix_conn_counter = Arc::new(std::sync::atomic::AtomicU64::new(1)); loop { - if !wait_until_admission_open(&mut admission_rx_unix).await { - warn!("Conditional-admission gate channel closed for unix listener"); - break; - } match unix_listener.accept().await { Ok((stream, _)) => { - let permit = match max_connections_unix.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => { - error!("Connection limiter is closed"); - break; + if !*admission_rx_unix.borrow() { + drop(stream); + continue; + } + let accept_permit_timeout_ms = config_rx_unix + .borrow() + .server + .accept_permit_timeout_ms; + let permit = if accept_permit_timeout_ms == 0 { + match max_connections_unix.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + error!("Connection limiter is closed"); + break; + } + } + } else { + match tokio::time::timeout( + Duration::from_millis(accept_permit_timeout_ms), + max_connections_unix.clone().acquire_owned(), + ) + .await + { + Ok(Ok(permit)) => permit, + Ok(Err(_)) => { + error!("Connection limiter is closed"); + break; + } + Err(_) => { + debug!( + timeout_ms = accept_permit_timeout_ms, + "Dropping accepted unix connection: permit wait timeout" + ); + drop(stream); + continue; + } } }; let conn_id = @@ -312,7 +339,7 @@ pub(crate) fn spawn_tcp_accept_loops( ) { for (listener, listener_proxy_protocol) in listeners { let mut config_rx: watch::Receiver> = config_rx.clone(); - let mut admission_rx_tcp = admission_rx.clone(); + let admission_rx_tcp = admission_rx.clone(); let stats = stats.clone(); let upstream_manager = upstream_manager.clone(); let replay_checker = replay_checker.clone(); @@ -327,17 +354,46 @@ pub(crate) fn spawn_tcp_accept_loops( tokio::spawn(async move { loop { - if !wait_until_admission_open(&mut admission_rx_tcp).await { - warn!("Conditional-admission gate channel closed for tcp listener"); - break; - } match listener.accept().await { Ok((stream, peer_addr)) => { - let permit = match max_connections_tcp.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => { - error!("Connection limiter is closed"); - break; + if !*admission_rx_tcp.borrow() { + debug!(peer = %peer_addr, "Admission gate closed, dropping connection"); + drop(stream); + continue; + } + let accept_permit_timeout_ms = config_rx + .borrow() + .server + .accept_permit_timeout_ms; + let permit = if accept_permit_timeout_ms == 0 { + match max_connections_tcp.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + error!("Connection limiter is closed"); + break; + } + } + } else { + match tokio::time::timeout( + Duration::from_millis(accept_permit_timeout_ms), + max_connections_tcp.clone().acquire_owned(), + ) + .await + { + Ok(Ok(permit)) => permit, + Ok(Err(_)) => { + error!("Connection limiter is closed"); + break; + } + Err(_) => { + debug!( + peer = %peer_addr, + timeout_ms = accept_permit_timeout_ms, + "Dropping accepted connection: permit wait timeout" + ); + drop(stream); + continue; + } } }; let config = config_rx.borrow_and_update().clone(); diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs index 94ae884..827b00c 100644 --- a/src/maestro/me_startup.rs +++ b/src/maestro/me_startup.rs @@ -267,6 +267,8 @@ pub(crate) async fn initialize_me_pool( config.general.me_warn_rate_limit_ms, config.general.me_route_no_writer_mode, config.general.me_route_no_writer_wait_ms, + config.general.me_route_hybrid_max_wait_ms, + config.general.me_route_blocking_send_timeout_ms, config.general.me_route_inline_recovery_attempts, config.general.me_route_inline_recovery_wait_ms, ); diff --git a/src/proxy/middle_relay.rs b/src/proxy/middle_relay.rs index 4f70a17..102b06c 100644 --- a/src/proxy/middle_relay.rs +++ b/src/proxy/middle_relay.rs @@ -222,6 +222,7 @@ fn should_yield_c2me_sender(sent_since_yield: usize, has_backlog: bool) -> bool async fn enqueue_c2me_command( tx: &mpsc::Sender, cmd: C2MeCommand, + send_timeout: Duration, ) -> std::result::Result<(), mpsc::error::SendError> { match tx.try_send(cmd) { Ok(()) => Ok(()), @@ -231,7 +232,17 @@ async fn enqueue_c2me_command( if tx.capacity() <= C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS { tokio::task::yield_now().await; } - tx.send(cmd).await + if send_timeout.is_zero() { + return tx.send(cmd).await; + } + match tokio::time::timeout(send_timeout, tx.reserve()).await { + Ok(Ok(permit)) => { + permit.send(cmd); + Ok(()) + } + Ok(Err(_)) => Err(mpsc::error::SendError(cmd)), + Err(_) => Err(mpsc::error::SendError(cmd)), + } } } } @@ -355,6 +366,7 @@ where .general .me_c2me_channel_capacity .max(C2ME_CHANNEL_CAPACITY_FALLBACK); + let c2me_send_timeout = Duration::from_millis(config.general.me_c2me_send_timeout_ms); let (c2me_tx, mut c2me_rx) = mpsc::channel::(c2me_channel_capacity); let me_pool_c2me = me_pool.clone(); let effective_tag = effective_tag; @@ -363,15 +375,42 @@ where while let Some(cmd) = c2me_rx.recv().await { match cmd { C2MeCommand::Data { payload, flags } => { - me_pool_c2me.send_proxy_req( - conn_id, - success.dc_idx, - peer, - translated_local_addr, - payload.as_ref(), - flags, - effective_tag.as_deref(), - ).await?; + if c2me_send_timeout.is_zero() { + me_pool_c2me + .send_proxy_req( + conn_id, + success.dc_idx, + peer, + translated_local_addr, + payload.as_ref(), + flags, + effective_tag.as_deref(), + ) + .await?; + } else { + match tokio::time::timeout( + c2me_send_timeout, + me_pool_c2me.send_proxy_req( + conn_id, + success.dc_idx, + peer, + translated_local_addr, + payload.as_ref(), + flags, + effective_tag.as_deref(), + ), + ) + .await + { + Ok(send_result) => send_result?, + Err(_) => { + return Err(ProxyError::Proxy(format!( + "ME send timeout after {}ms", + c2me_send_timeout.as_millis() + ))); + } + } + } sent_since_yield = sent_since_yield.saturating_add(1); if should_yield_c2me_sender(sent_since_yield, !c2me_rx.is_empty()) { sent_since_yield = 0; @@ -555,7 +594,7 @@ where loop { if session_lease.is_stale() { stats.increment_reconnect_stale_close_total(); - let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; + let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close, c2me_send_timeout).await; main_result = Err(ProxyError::Proxy("Session evicted by reconnect".to_string())); break; } @@ -573,7 +612,7 @@ where "Cutover affected middle session, closing client connection" ); tokio::time::sleep(delay).await; - let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; + let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close, c2me_send_timeout).await; main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string())); break; } @@ -607,9 +646,13 @@ where flags |= RPC_FLAG_NOT_ENCRYPTED; } // Keep client read loop lightweight: route heavy ME send path via a dedicated task. - if enqueue_c2me_command(&c2me_tx, C2MeCommand::Data { payload, flags }) - .await - .is_err() + if enqueue_c2me_command( + &c2me_tx, + C2MeCommand::Data { payload, flags }, + c2me_send_timeout, + ) + .await + .is_err() { main_result = Err(ProxyError::Proxy("ME sender channel closed".into())); break; @@ -618,7 +661,12 @@ where Ok(None) => { debug!(conn_id, "Client EOF"); client_closed = true; - let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await; + let _ = enqueue_c2me_command( + &c2me_tx, + C2MeCommand::Close, + c2me_send_timeout, + ) + .await; break; } Err(e) => { @@ -993,6 +1041,7 @@ mod tests { payload: Bytes::from_static(&[1, 2, 3]), flags: 0, }, + TokioDuration::from_millis(50), ) .await .unwrap(); @@ -1028,6 +1077,7 @@ mod tests { payload: Bytes::from_static(&[7, 7]), flags: 7, }, + TokioDuration::from_millis(100), ) .await .unwrap(); diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 862e58a..0b9b749 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -1574,6 +1574,8 @@ mod tests { general.me_warn_rate_limit_ms, MeRouteNoWriterMode::default(), general.me_route_no_writer_wait_ms, + general.me_route_hybrid_max_wait_ms, + general.me_route_blocking_send_timeout_ms, general.me_route_inline_recovery_attempts, general.me_route_inline_recovery_wait_ms, ) diff --git a/src/transport/middle_proxy/health_adversarial_tests.rs b/src/transport/middle_proxy/health_adversarial_tests.rs index dc1a0eb..3f182e4 100644 --- a/src/transport/middle_proxy/health_adversarial_tests.rs +++ b/src/transport/middle_proxy/health_adversarial_tests.rs @@ -111,6 +111,8 @@ async fn make_pool( general.me_warn_rate_limit_ms, MeRouteNoWriterMode::default(), general.me_route_no_writer_wait_ms, + general.me_route_hybrid_max_wait_ms, + general.me_route_blocking_send_timeout_ms, general.me_route_inline_recovery_attempts, general.me_route_inline_recovery_wait_ms, ); diff --git a/src/transport/middle_proxy/health_integration_tests.rs b/src/transport/middle_proxy/health_integration_tests.rs index 4724851..7f99d2a 100644 --- a/src/transport/middle_proxy/health_integration_tests.rs +++ b/src/transport/middle_proxy/health_integration_tests.rs @@ -110,6 +110,8 @@ async fn make_pool( general.me_warn_rate_limit_ms, MeRouteNoWriterMode::default(), general.me_route_no_writer_wait_ms, + general.me_route_hybrid_max_wait_ms, + general.me_route_blocking_send_timeout_ms, general.me_route_inline_recovery_attempts, general.me_route_inline_recovery_wait_ms, ); diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs index 45a1eee..606f7e5 100644 --- a/src/transport/middle_proxy/health_regression_tests.rs +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -103,6 +103,8 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc { general.me_warn_rate_limit_ms, MeRouteNoWriterMode::default(), general.me_route_no_writer_wait_ms, + general.me_route_hybrid_max_wait_ms, + general.me_route_blocking_send_timeout_ms, general.me_route_inline_recovery_attempts, general.me_route_inline_recovery_wait_ms, ) diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index f3cc817..d09f07c 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -193,6 +193,8 @@ pub struct MePool { pub(super) me_reader_route_data_wait_ms: Arc, pub(super) me_route_no_writer_mode: AtomicU8, pub(super) me_route_no_writer_wait: Duration, + pub(super) me_route_hybrid_max_wait: Duration, + pub(super) me_route_blocking_send_timeout: Duration, pub(super) me_route_inline_recovery_attempts: u32, pub(super) me_route_inline_recovery_wait: Duration, pub(super) me_health_interval_ms_unhealthy: AtomicU64, @@ -307,6 +309,8 @@ impl MePool { me_warn_rate_limit_ms: u64, me_route_no_writer_mode: MeRouteNoWriterMode, me_route_no_writer_wait_ms: u64, + me_route_hybrid_max_wait_ms: u64, + me_route_blocking_send_timeout_ms: u64, me_route_inline_recovery_attempts: u32, me_route_inline_recovery_wait_ms: u64, ) -> Arc { @@ -490,6 +494,10 @@ impl MePool { me_reader_route_data_wait_ms: Arc::new(AtomicU64::new(me_reader_route_data_wait_ms)), me_route_no_writer_mode: AtomicU8::new(me_route_no_writer_mode.as_u8()), me_route_no_writer_wait: Duration::from_millis(me_route_no_writer_wait_ms), + me_route_hybrid_max_wait: Duration::from_millis(me_route_hybrid_max_wait_ms), + me_route_blocking_send_timeout: Duration::from_millis( + me_route_blocking_send_timeout_ms, + ), me_route_inline_recovery_attempts, me_route_inline_recovery_wait: Duration::from_millis(me_route_inline_recovery_wait_ms), me_health_interval_ms_unhealthy: AtomicU64::new(me_health_interval_ms_unhealthy.max(1)), diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 8ce3de3..4035111 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -312,41 +312,28 @@ impl MePool { let mut p = Vec::with_capacity(12); p.extend_from_slice(&RPC_PING_U32.to_le_bytes()); p.extend_from_slice(&sent_id.to_le_bytes()); - { - let mut tracker = ping_tracker_ping.lock().await; - let now_epoch_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - let mut run_cleanup = false; - if let Some(pool) = pool_ping.upgrade() { - let last_cleanup_ms = pool + let now_epoch_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let mut run_cleanup = false; + if let Some(pool) = pool_ping.upgrade() { + let last_cleanup_ms = pool + .ping_tracker_last_cleanup_epoch_ms + .load(Ordering::Relaxed); + if now_epoch_ms.saturating_sub(last_cleanup_ms) >= 30_000 + && pool .ping_tracker_last_cleanup_epoch_ms - .load(Ordering::Relaxed); - if now_epoch_ms.saturating_sub(last_cleanup_ms) >= 30_000 - && pool - .ping_tracker_last_cleanup_epoch_ms - .compare_exchange( - last_cleanup_ms, - now_epoch_ms, - Ordering::AcqRel, - Ordering::Relaxed, - ) - .is_ok() - { - run_cleanup = true; - } + .compare_exchange( + last_cleanup_ms, + now_epoch_ms, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .is_ok() + { + run_cleanup = true; } - - if run_cleanup { - let before = tracker.len(); - tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120)); - let expired = before.saturating_sub(tracker.len()); - if expired > 0 { - stats_ping.increment_me_keepalive_timeout_by(expired as u64); - } - } - tracker.insert(sent_id, (std::time::Instant::now(), writer_id)); } ping_id = ping_id.wrapping_add(1); stats_ping.increment_me_keepalive_sent(); @@ -367,6 +354,16 @@ impl MePool { } break; } + let mut tracker = ping_tracker_ping.lock().await; + if run_cleanup { + let before = tracker.len(); + tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120)); + let expired = before.saturating_sub(tracker.len()); + if expired > 0 { + stats_ping.increment_me_keepalive_timeout_by(expired as u64); + } + } + tracker.insert(sent_id, (std::time::Instant::now(), writer_id)); } }); diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 0f9fed6..1c255ef 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -6,6 +6,7 @@ use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use bytes::Bytes; +use tokio::sync::mpsc; use tokio::sync::mpsc::error::TrySendError; use tracing::{debug, warn}; @@ -29,6 +30,29 @@ const PICK_PENALTY_DRAINING: u64 = 600; const PICK_PENALTY_STALE: u64 = 300; const PICK_PENALTY_DEGRADED: u64 = 250; +enum TimedSendError { + Closed(T), + Timeout(T), +} + +async fn send_writer_command_with_timeout( + tx: &mpsc::Sender, + cmd: WriterCommand, + timeout: Duration, +) -> std::result::Result<(), TimedSendError> { + if timeout.is_zero() { + return tx.send(cmd).await.map_err(|err| TimedSendError::Closed(err.0)); + } + match tokio::time::timeout(timeout, tx.reserve()).await { + Ok(Ok(permit)) => { + permit.send(cmd); + Ok(()) + } + Ok(Err(_)) => Err(TimedSendError::Closed(cmd)), + Err(_) => Err(TimedSendError::Timeout(cmd)), + } +} + impl MePool { /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default. pub async fn send_proxy_req( @@ -78,8 +102,18 @@ impl MePool { let mut hybrid_last_recovery_at: Option = None; let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50)); let mut hybrid_wait_current = hybrid_wait_step; + let hybrid_deadline = Instant::now() + self.me_route_hybrid_max_wait; loop { + if matches!(no_writer_mode, MeRouteNoWriterMode::HybridAsyncPersistent) + && Instant::now() >= hybrid_deadline + { + self.stats.increment_me_no_writer_failfast_total(); + return Err(ProxyError::Proxy( + "No ME writer available in hybrid wait window".into(), + )); + } + let mut skip_writer_id: Option = None; let current_meta = self .registry .get_meta(conn_id) @@ -90,12 +124,30 @@ impl MePool { match current.tx.try_send(WriterCommand::Data(current_payload.clone())) { Ok(()) => return Ok(()), Err(TrySendError::Full(cmd)) => { - if current.tx.send(cmd).await.is_ok() { - return Ok(()); + match send_writer_command_with_timeout( + ¤t.tx, + cmd, + self.me_route_blocking_send_timeout, + ) + .await + { + Ok(()) => return Ok(()), + Err(TimedSendError::Closed(_)) => { + warn!(writer_id = current.writer_id, "ME writer channel closed"); + self.remove_writer_and_close_clients(current.writer_id).await; + continue; + } + Err(TimedSendError::Timeout(_)) => { + debug!( + conn_id, + writer_id = current.writer_id, + timeout_ms = self.me_route_blocking_send_timeout.as_millis() + as u64, + "ME writer send timed out for bound writer, trying reroute" + ); + skip_writer_id = Some(current.writer_id); + } } - warn!(writer_id = current.writer_id, "ME writer channel closed"); - self.remove_writer_and_close_clients(current.writer_id).await; - continue; } Err(TrySendError::Closed(_)) => { warn!(writer_id = current.writer_id, "ME writer channel closed"); @@ -200,6 +252,9 @@ impl MePool { .candidate_indices_for_dc(&writers_snapshot, routed_dc, true) .await; } + if let Some(skip_writer_id) = skip_writer_id { + candidate_indices.retain(|idx| writers_snapshot[*idx].id != skip_writer_id); + } if candidate_indices.is_empty() { let pick_mode = self.writer_pick_mode(); match no_writer_mode { @@ -422,7 +477,13 @@ impl MePool { self.stats.increment_me_writer_pick_blocking_fallback_total(); let effective_our_addr = SocketAddr::new(w.source_ip, our_addr.port()); let (payload, meta) = build_routed_payload(effective_our_addr); - match w.tx.send(WriterCommand::Data(payload.clone())).await { + match send_writer_command_with_timeout( + &w.tx, + WriterCommand::Data(payload.clone()), + self.me_route_blocking_send_timeout, + ) + .await + { Ok(()) => { self.stats .increment_me_writer_pick_success_fallback_total(pick_mode); @@ -439,11 +500,20 @@ impl MePool { } return Ok(()); } - Err(_) => { + Err(TimedSendError::Closed(_)) => { self.stats.increment_me_writer_pick_closed_total(pick_mode); warn!(writer_id = w.id, "ME writer channel closed (blocking)"); self.remove_writer_and_close_clients(w.id).await; } + Err(TimedSendError::Timeout(_)) => { + self.stats.increment_me_writer_pick_full_total(pick_mode); + debug!( + conn_id, + writer_id = w.id, + timeout_ms = self.me_route_blocking_send_timeout.as_millis() as u64, + "ME writer blocking fallback send timed out" + ); + } } } } From 03891db0c91ad7b91bfa01da20dc1cf4de565cd1 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 18 Mar 2026 22:36:33 +0300 Subject: [PATCH 237/261] Update Cargo.toml Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index eba2e2e..b289231 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.22" +version = "3.3.23" edition = "2024" [dependencies] From c47495d6713f83d937a98c0272f2c0bbfc4ce176 Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Thu, 19 Mar 2026 00:36:02 +0300 Subject: [PATCH 238/261] Update install.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Вернул старый функционал + добавил новый: - Вернул автоматическое создание конфига с секретом - Вернул автоматическое создание службы - Добавил удаление службы и telemt через `install.sh uninstall` - Полное удаление вместе с конфигом через `install.sh --purge` - Добавил установку нужной версии `install.sh 3.3.15` --- install.sh | 568 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 489 insertions(+), 79 deletions(-) diff --git a/install.sh b/install.sh index 2dd207b..330bc3e 100644 --- a/install.sh +++ b/install.sh @@ -1,115 +1,525 @@ #!/bin/sh set -eu +# --- Global Configurations --- REPO="${REPO:-telemt/telemt}" BIN_NAME="${BIN_NAME:-telemt}" -VERSION="${1:-${VERSION:-latest}}" -INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}" +INSTALL_DIR="${INSTALL_DIR:-/bin}" +CONFIG_DIR="${CONFIG_DIR:-/etc/telemt}" +CONFIG_FILE="${CONFIG_FILE:-${CONFIG_DIR}/telemt.toml}" +WORK_DIR="${WORK_DIR:-/opt/telemt}" +SERVICE_NAME="telemt" +TEMP_DIR="" +SUDO="" -say() { - printf '%s\n' "$*" -} +# --- Argument Parsing --- +ACTION="install" +TARGET_VERSION="${VERSION:-latest}" -die() { - printf 'Error: %s\n' "$*" >&2 - exit 1 -} - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" -} - -detect_os() { - os="$(uname -s)" - case "$os" in - Linux) printf 'linux\n' ;; - OpenBSD) printf 'openbsd\n' ;; - *) printf '%s\n' "$os" ;; +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + ACTION="help" + shift + ;; + uninstall|--uninstall) + [ "$ACTION" != "purge" ] && ACTION="uninstall" + shift + ;; + --purge) + ACTION="purge" + shift + ;; + install|--install) + ACTION="install" + shift + ;; + -*) + printf '[ERROR] Unknown option: %s\n' "$1" >&2 + exit 1 + ;; + *) + if [ "$ACTION" = "install" ]; then + TARGET_VERSION="$1" + fi + shift + ;; esac +done + +# --- Core Functions --- +say() { printf '[INFO] %s\n' "$*"; } +die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; } + +cleanup() { + if [ -n "${TEMP_DIR:-}" ] && [ -d "$TEMP_DIR" ]; then + rm -rf -- "$TEMP_DIR" + fi +} + +trap cleanup EXIT INT TERM + +show_help() { + say "Usage: $0 [version | install | uninstall | --purge | --help]" + say " version Install specific version (e.g. 1.0.0, default: latest)" + say " uninstall Remove the binary and service (keeps config)" + say " --purge Remove everything including configuration" + exit 0 +} + +user_exists() { + if command -v getent >/dev/null 2>&1; then + getent passwd "$1" >/dev/null 2>&1 + else + grep -q "^${1}:" /etc/passwd 2>/dev/null + fi +} + +group_exists() { + if command -v getent >/dev/null 2>&1; then + getent group "$1" >/dev/null 2>&1 + else + grep -q "^${1}:" /etc/group 2>/dev/null + fi +} + +verify_common() { + [ -z "$BIN_NAME" ] && die "BIN_NAME cannot be empty." + [ -z "$INSTALL_DIR" ] && die "INSTALL_DIR cannot be empty." + [ -z "$CONFIG_DIR" ] && die "CONFIG_DIR cannot be empty." + + if [ "$(id -u)" -eq 0 ]; then + SUDO="" + else + if ! command -v sudo >/dev/null 2>&1; then + die "This script requires root or sudo. Neither found." + fi + SUDO="sudo" + say "sudo is available. Caching credentials..." + if ! sudo -v; then + die "Failed to cache sudo credentials" + fi + fi + + case "${INSTALL_DIR}${CONFIG_DIR}${WORK_DIR}" in + *[!a-zA-Z0-9_./-]*) + die "Invalid characters in path variables. Only alphanumeric, _, ., -, and / are allowed." + ;; + esac + + case "$BIN_NAME" in + *[!a-zA-Z0-9_-]*) die "Invalid characters in BIN_NAME: $BIN_NAME" ;; + esac + + for path in "$CONFIG_DIR" "$WORK_DIR"; do + check_path="$path" + + while [ "$check_path" != "/" ] && [ "${check_path%"/"}" != "$check_path" ]; do + check_path="${check_path%"/"}" + done + [ -z "$check_path" ] && check_path="/" + + case "$check_path" in + /|/bin|/sbin|/usr|/usr/bin|/usr/local|/etc|/opt|/var|/home|/root|/tmp) + die "Safety check failed: '$path' is a critical system directory." + ;; + esac + done + + for cmd in uname grep find rm chown chmod mv head mktemp; do + command -v "$cmd" >/dev/null 2>&1 || die "Required command not found: $cmd" + done +} + +verify_install_deps() { + if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then + die "Neither curl nor wget is installed." + fi + command -v tar >/dev/null 2>&1 || die "Required command not found: tar" + command -v gzip >/dev/null 2>&1 || die "Required command not found: gzip" + command -v cp >/dev/null 2>&1 || command -v install >/dev/null 2>&1 || die "Need cp or install" + + if ! command -v setcap >/dev/null 2>&1; then + say "setcap is missing. Installing required capability tools..." + if command -v apk >/dev/null 2>&1; then + $SUDO apk add --no-cache libcap || die "Failed to install libcap" + elif command -v apt-get >/dev/null 2>&1; then + $SUDO apt-get update -qq && $SUDO apt-get install -y -qq libcap2-bin || die "Failed to install libcap2-bin" + elif command -v dnf >/dev/null 2>&1 || command -v yum >/dev/null 2>&1; then + $SUDO ${YUM_CMD:-yum} install -y -q libcap || die "Failed to install libcap" + else + die "Cannot install 'setcap'. Package manager not found. Please install libcap manually." + fi + fi } detect_arch() { - arch="$(uname -m)" - case "$arch" in - x86_64|amd64) printf 'x86_64\n' ;; - aarch64|arm64) printf 'aarch64\n' ;; - *) die "unsupported architecture: $arch" ;; + sys_arch="$(uname -m)" + case "$sys_arch" in + x86_64|amd64) echo "x86_64" ;; + aarch64|arm64) echo "aarch64" ;; + *) die "Unsupported architecture: $sys_arch" ;; esac } detect_libc() { - case "$(ldd --version 2>&1 || true)" in - *musl*) printf 'musl\n' ;; - *) printf 'gnu\n' ;; - esac + if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -qi musl; then + echo "musl"; return 0 + fi + + if grep -q '^ID=alpine' /etc/os-release 2>/dev/null || grep -q '^ID="alpine"' /etc/os-release 2>/dev/null; then + echo "musl"; return 0 + fi + for f in /lib/ld-musl-*.so.* /lib64/ld-musl-*.so.*; do + if [ -e "$f" ]; then + echo "musl"; return 0 + fi + done + echo "gnu" } -fetch_to_stdout() { - url="$1" +fetch_file() { + fetch_url="$1" + fetch_out="$2" + if command -v curl >/dev/null 2>&1; then - curl -fsSL "$url" + curl -fsSL "$fetch_url" -o "$fetch_out" || return 1 elif command -v wget >/dev/null 2>&1; then - wget -qO- "$url" + wget -qO "$fetch_out" "$fetch_url" || return 1 else - die "neither curl nor wget is installed" + die "curl or wget required" + fi +} + +ensure_user_group() { + nologin_bin="/bin/false" + + cmd_nologin="$(command -v nologin 2>/dev/null || true)" + if [ -n "$cmd_nologin" ] && [ -x "$cmd_nologin" ]; then + nologin_bin="$cmd_nologin" + else + for bin in /sbin/nologin /usr/sbin/nologin; do + if [ -x "$bin" ]; then + nologin_bin="$bin" + break + fi + done + fi + + if ! group_exists telemt; then + if command -v groupadd >/dev/null 2>&1; then + $SUDO groupadd -r telemt || die "Failed to create group via groupadd" + elif command -v addgroup >/dev/null 2>&1; then + $SUDO addgroup -S telemt || die "Failed to create group via addgroup" + else + die "Cannot create group: neither groupadd nor addgroup found" + fi + fi + + if ! user_exists telemt; then + if command -v useradd >/dev/null 2>&1; then + $SUDO useradd -r -g telemt -d "$WORK_DIR" -s "$nologin_bin" -c "Telemt Proxy" telemt || die "Failed to create user via useradd" + elif command -v adduser >/dev/null 2>&1; then + $SUDO adduser -S -D -H -h "$WORK_DIR" -s "$nologin_bin" -G telemt telemt || die "Failed to create user via adduser" + else + die "Cannot create user: neither useradd nor adduser found" + fi + fi +} + +setup_dirs() { + say "Setting up directories..." + $SUDO mkdir -p "$WORK_DIR" "$CONFIG_DIR" || die "Failed to create directories" + $SUDO chown telemt:telemt "$WORK_DIR" || die "Failed to set owner on WORK_DIR" + $SUDO chmod 750 "$WORK_DIR" || die "Failed to set permissions on WORK_DIR" +} + +stop_service() { + say "Stopping service if running..." + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + $SUDO systemctl stop "$SERVICE_NAME" 2>/dev/null || true + elif command -v rc-service >/dev/null 2>&1; then + $SUDO rc-service "$SERVICE_NAME" stop 2>/dev/null || true fi } install_binary() { - src="$1" - dst="$2" + bin_src="$1" + bin_dst="$2" - if [ -w "$INSTALL_DIR" ] || { [ ! -e "$INSTALL_DIR" ] && [ -w "$(dirname "$INSTALL_DIR")" ]; }; then - mkdir -p "$INSTALL_DIR" - install -m 0755 "$src" "$dst" - elif command -v sudo >/dev/null 2>&1; then - sudo mkdir -p "$INSTALL_DIR" - sudo install -m 0755 "$src" "$dst" + $SUDO mkdir -p "$INSTALL_DIR" || die "Failed to create install directory" + if command -v install >/dev/null 2>&1; then + $SUDO install -m 0755 "$bin_src" "$bin_dst" || die "Failed to install binary" else - die "cannot write to $INSTALL_DIR and sudo is not available" + $SUDO rm -f "$bin_dst" + $SUDO cp "$bin_src" "$bin_dst" || die "Failed to copy binary" + $SUDO chmod 0755 "$bin_dst" || die "Failed to set permissions" + fi + + if [ ! -x "$bin_dst" ]; then + die "Failed to install binary or it is not executable: $bin_dst" + fi + + say "Granting network bind capabilities to bind port 443..." + if ! $SUDO setcap cap_net_bind_service=+ep "$bin_dst" 2>/dev/null; then + say "[WARNING] Failed to apply setcap. The service will NOT be able to open port 443!" + say "[WARNING] This usually happens inside unprivileged Docker/LXC containers." fi } -need_cmd uname -need_cmd tar -need_cmd mktemp -need_cmd grep -need_cmd install +generate_secret() { + if command -v openssl >/dev/null 2>&1; then + secret="$(openssl rand -hex 16 2>/dev/null)" && [ -n "$secret" ] && { echo "$secret"; return 0; } + fi + if command -v xxd >/dev/null 2>&1; then + secret="$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | xxd -p | tr -d '\n')" && [ -n "$secret" ] && { echo "$secret"; return 0; } + fi + secret="$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | od -An -tx1 | tr -d ' \n')" && [ -n "$secret" ] && { echo "$secret"; return 0; } + return 1 +} -ARCH="$(detect_arch)" -OS="$(detect_os)" +generate_config_content() { + cat </dev/null && config_exists=1 || true + else + [ -f "$CONFIG_FILE" ] && config_exists=1 || true + fi + + if [ "$config_exists" -eq 1 ]; then + say "Config already exists, skipping generation." + return 0 + fi + + toml_secret="$(generate_secret)" || die "Failed to generate secret" + say "Creating config at $CONFIG_FILE..." + + tmp_conf="$(mktemp "${TEMP_DIR:-/tmp}/telemt_conf.XXXXXX")" || die "Failed to create temp config" + generate_config_content "$toml_secret" > "$tmp_conf" || die "Failed to write temp config" + + $SUDO mv "$tmp_conf" "$CONFIG_FILE" || die "Failed to install config file" + $SUDO chown root:telemt "$CONFIG_FILE" || die "Failed to set owner" + $SUDO chmod 640 "$CONFIG_FILE" || die "Failed to set config permissions" + + say "Secret for user 'hello': $toml_secret" +} + +generate_systemd_content() { + cat </dev/null 2>&1 && [ -d /run/systemd/system ]; then + say "Installing systemd service..." + tmp_svc="$(mktemp "${TEMP_DIR:-/tmp}/${SERVICE_NAME}.service.XXXXXX")" || die "Failed to create temp service" + generate_systemd_content > "$tmp_svc" || die "Failed to generate service content" + + $SUDO mv "$tmp_svc" "/etc/systemd/system/${SERVICE_NAME}.service" || die "Failed to move service file" + $SUDO chown root:root "/etc/systemd/system/${SERVICE_NAME}.service" + $SUDO chmod 644 "/etc/systemd/system/${SERVICE_NAME}.service" + + $SUDO systemctl daemon-reload || die "Failed to reload systemd" + $SUDO systemctl enable "$SERVICE_NAME" || die "Failed to enable service" + $SUDO systemctl start "$SERVICE_NAME" || die "Failed to start service" + + elif command -v rc-update >/dev/null 2>&1; then + say "Installing OpenRC service..." + tmp_svc="$(mktemp "${TEMP_DIR:-/tmp}/${SERVICE_NAME}.init.XXXXXX")" || die "Failed to create temp file" + generate_openrc_content > "$tmp_svc" || die "Failed to generate init content" + + $SUDO mv "$tmp_svc" "/etc/init.d/${SERVICE_NAME}" || die "Failed to move service file" + $SUDO chown root:root "/etc/init.d/${SERVICE_NAME}" + $SUDO chmod 0755 "/etc/init.d/${SERVICE_NAME}" + + $SUDO rc-update add "$SERVICE_NAME" default 2>/dev/null || die "Failed to register service" + $SUDO rc-service "$SERVICE_NAME" start 2>/dev/null || die "Failed to start OpenRC service" + else + say "No service manager found. You can start it manually with:" + if [ -n "$SUDO" ]; then + say " sudo -u telemt ${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}" + else + say " su -s /bin/sh telemt -c '${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}'" + fi + fi +} + +kill_user_procs() { + say "Ensuring $BIN_NAME processes are killed..." + + if pkill_cmd="$(command -v pkill 2>/dev/null)"; then + $SUDO "$pkill_cmd" -u telemt "$BIN_NAME" 2>/dev/null || true + sleep 1 + $SUDO "$pkill_cmd" -9 -u telemt "$BIN_NAME" 2>/dev/null || true + elif killall_cmd="$(command -v killall 2>/dev/null)"; then + $SUDO "$killall_cmd" "$BIN_NAME" 2>/dev/null || true + sleep 1 + $SUDO "$killall_cmd" -9 "$BIN_NAME" 2>/dev/null || true + fi +} + +uninstall() { + purge_data=0 + [ "$ACTION" = "purge" ] && purge_data=1 + + say "Uninstalling $BIN_NAME..." + stop_service + + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + $SUDO systemctl disable "$SERVICE_NAME" 2>/dev/null || true + $SUDO rm -f "/etc/systemd/system/${SERVICE_NAME}.service" + $SUDO systemctl daemon-reload || true + elif command -v rc-update >/dev/null 2>&1; then + $SUDO rc-update del "$SERVICE_NAME" 2>/dev/null || true + $SUDO rm -f "/etc/init.d/${SERVICE_NAME}" + fi + + kill_user_procs + + $SUDO rm -f "${INSTALL_DIR}/${BIN_NAME}" + + $SUDO userdel telemt 2>/dev/null || $SUDO deluser telemt 2>/dev/null || true + $SUDO groupdel telemt 2>/dev/null || $SUDO delgroup telemt 2>/dev/null || true + + if [ "$purge_data" -eq 1 ]; then + say "Purging configuration and data..." + $SUDO rm -rf "$CONFIG_DIR" "$WORK_DIR" + else + say "Note: Configuration in $CONFIG_DIR was kept. Run with '--purge' to remove it." + fi + + say "Uninstallation complete." + exit 0 +} + +# ============================================================================ +# Main Entry Point +# ============================================================================ + +case "$ACTION" in + help) + show_help ;; - *) - URL="https://github.com/$REPO/releases/download/${VERSION}/${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz" + uninstall|purge) + verify_common + uninstall + ;; + install) + say "Starting installation..." + verify_common + verify_install_deps + + ARCH="$(detect_arch)" + LIBC="$(detect_libc)" + say "Detected system: $ARCH-linux-$LIBC" + + FILE_NAME="${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz" + FILE_NAME="$(printf '%s' "$FILE_NAME" | tr -d ' \t\n\r')" + + if [ "$TARGET_VERSION" = "latest" ]; then + DL_URL="https://github.com/${REPO}/releases/latest/download/${FILE_NAME}" + else + DL_URL="https://github.com/${REPO}/releases/download/${TARGET_VERSION}/${FILE_NAME}" + fi + + TEMP_DIR="$(mktemp -d)" || die "Failed to create temp directory" + if [ -z "$TEMP_DIR" ] || [ ! -d "$TEMP_DIR" ]; then + die "Temp directory creation failed" + fi + + say "Downloading from $DL_URL..." + fetch_file "$DL_URL" "${TEMP_DIR}/archive.tar.gz" || die "Download failed (check version or network)" + + gzip -dc "${TEMP_DIR}/archive.tar.gz" | tar -xf - -C "$TEMP_DIR" || die "Extraction failed" + + EXTRACTED_BIN="$(find "$TEMP_DIR" -type f -name "$BIN_NAME" -print 2>/dev/null | head -n 1)" + [ -z "$EXTRACTED_BIN" ] && die "Binary '$BIN_NAME' not found in archive" + + ensure_user_group + setup_dirs + stop_service + + say "Installing binary..." + install_binary "$EXTRACTED_BIN" "${INSTALL_DIR}/${BIN_NAME}" + + install_config + install_service + + say "" + say "=============================================" + say "Installation complete!" + say "=============================================" + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + say "To check the logs, run:" + say " journalctl -u $SERVICE_NAME -f" + say "" + fi + say "To get user connection links, run:" + if command -v jq >/dev/null 2>&1; then + say " curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | \"User: \\(.username)\\n\\(.links.tls[0] // empty)\"'" + else + say " curl -s http://127.0.0.1:9091/v1/users" + say " (Note: Install 'jq' package to see the links nicely formatted)" + fi ;; esac - -TMPDIR="$(mktemp -d)" -trap 'rm -rf "$TMPDIR"' EXIT INT TERM - -say "Installing $BIN_NAME ($VERSION) for $ARCH-linux-$LIBC..." -fetch_to_stdout "$URL" | tar -xzf - -C "$TMPDIR" - -[ -f "$TMPDIR/$BIN_NAME" ] || die "archive did not contain $BIN_NAME" - -install_binary "$TMPDIR/$BIN_NAME" "$INSTALL_DIR/$BIN_NAME" - -say "Installed: $INSTALL_DIR/$BIN_NAME" -"$INSTALL_DIR/$BIN_NAME" --version 2>/dev/null || true From bdac6e348003cb739dbb413183dc4517f00ceb60 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 00:59:37 +0300 Subject: [PATCH 239/261] Create CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 163 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..6d10cb0 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,163 @@ +# TELEMT Code of Conduct + +## 1. Purpose + +Telemt exists to solve technical problems. + +It is not a platform for ideology, politics, or personal agendas. + +All interaction here is defined by systems, constraints, and outcomes. + +Technology has consequences. +Responsibility is inherent. + +> **Zweck bestimmt die Form.** +> Purpose defines form. + +--- + +## 2. Principles + +* **Technical over emotional** + Arguments are grounded in data, logs, reproducible cases, or clear reasoning. + +* **Clarity over noise** + Communication is structured, concise, and relevant. + +* **Independence** + Telemt does not represent any state, ideology, or organization. + +* **Open participation** + Access is open. Standards are not. + +* **Responsibility over capability** + Capability does not justify careless use. + +* **Cooperation over friction** + Progress is achieved through coordination and mutual support. + +> **Fakten sind nicht verhandelbar.** +> Facts are not negotiable. + +--- + +## 3. Expected Behavior + +Participants are expected to: + +* Communicate directly and respectfully +* Support claims with evidence +* Stay within technical scope +* Accept critique and provide it constructively +* Reduce noise, duplication, and ambiguity +* Help others reach correct and reproducible outcomes +* Act in a way that improves the system as a whole + +> **Wer behauptet, belegt.** +> Whoever claims, proves. + +--- + +## 4. Unacceptable Behavior + +The following is not allowed: + +* Personal attacks, insults, harassment, intimidation +* Political discourse, propaganda, ideological conflict +* Off-topic or disruptive discussion +* Spam, flooding, or repeated low-quality input +* Misinformation presented as fact +* Attempts to degrade or destabilize Telemt +* Use of Telemt or its space to enable harm + +> **Störung ist kein Beitrag.** +> Disruption is not contribution. + +--- + +## 5. Security and Misuse + +Telemt is intended for lawful and responsible use. + +* Do not use it to plan, coordinate, or execute harm +* Do not publish vulnerabilities without responsible disclosure +* Report security issues privately where possible + +Security is both technical and behavioral. + +> **Verantwortung endet nicht am Code.** +> Responsibility does not end at the code. + +--- + +## 6. Scope + +This Code of Conduct applies to all official spaces: + +* Source repositories (issues, pull requests, discussions) +* Documentation +* Communication channels associated with Telemt + +--- + +## 7. Enforcement + +Maintainers may act to preserve the integrity of Telemt: + +* Remove content +* Lock discussions +* Reject contributions +* Restrict or ban participants + +Actions are taken to maintain function, continuity, and signal quality. + +> **Ordnung ist Voraussetzung der Funktion.** +> Order is the precondition of function. + +--- + +## 8. Maintainer Authority + +Maintainers have final authority in interpretation and enforcement. + +Authority exists to ensure continuity, consistency, and technical direction. + +--- + +## 9. Final + +Telemt is built on discipline, structure, and shared intent. + +Signal over noise. +Facts over opinion. +Systems over rhetoric. + +Work here is collective. +Outcomes are shared. +Responsibility is distributed. + +> **Ordnung ist Voraussetzung der Freiheit.** + +If you contribute — contribute with precision. +If you speak — speak with substance. +If you engage — engage constructively. + +--- + +## 10. After All + +Systems outlive intentions. + +What is built will be used. +What is released will propagate. +What is maintained will define the future state. + +There is no neutral infrastructure. + +> **Jedes System trägt Verantwortung.** +> Every system carries responsibility. + +Stability requires discipline. +Freedom requires structure. + +In the end, the system reflects its contributors. From c8ffc23cf7dd909e96d31e56aa74fe95f7c69f79 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 01:18:02 +0300 Subject: [PATCH 240/261] Update CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 113 +++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 34 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 6d10cb0..84c5f77 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,17 +1,19 @@ -# TELEMT Code of Conduct +# Code of Conduct ## 1. Purpose Telemt exists to solve technical problems. -It is not a platform for ideology, politics, or personal agendas. +Telemt is open to contributors who want to learn, improve and build meaningful systems together. -All interaction here is defined by systems, constraints, and outcomes. +It is a place for building, testing, reasoning, documenting, and improving systems. -Technology has consequences. -Responsibility is inherent. +Discussions that advance this work are in scope. Discussions that divert it are not. + +Technology has consequences. Responsibility is inherent. > **Zweck bestimmt die Form.** + > Purpose defines form. --- @@ -24,20 +26,24 @@ Responsibility is inherent. * **Clarity over noise** Communication is structured, concise, and relevant. -* **Independence** - Telemt does not represent any state, ideology, or organization. +* **Openness with standards** + Participation is open. The work remains disciplined. -* **Open participation** - Access is open. Standards are not. +* **Independence of judgment** + Claims are evaluated on technical merit, not affiliation or posture. * **Responsibility over capability** Capability does not justify careless use. * **Cooperation over friction** - Progress is achieved through coordination and mutual support. + Progress depends on coordination, mutual support, and honest review. -> **Fakten sind nicht verhandelbar.** -> Facts are not negotiable. +* **Good intent, rigorous method** + Assume good intent, but require rigor. + +> **Aussagen gelten nach ihrer Begründung.** + +> Claims are weighed by evidence. --- @@ -53,7 +59,12 @@ Participants are expected to: * Help others reach correct and reproducible outcomes * Act in a way that improves the system as a whole +Precision is learned. + +New contributors are welcome. They are expected to grow into these standards. Existing contributors are expected to make that growth possible. + > **Wer behauptet, belegt.** + > Whoever claims, proves. --- @@ -62,22 +73,25 @@ Participants are expected to: The following is not allowed: -* Personal attacks, insults, harassment, intimidation -* Political discourse, propaganda, ideological conflict -* Off-topic or disruptive discussion +* Personal attacks, insults, harassment, or intimidation +* Repeatedly derailing discussion away from Telemt’s purpose * Spam, flooding, or repeated low-quality input * Misinformation presented as fact -* Attempts to degrade or destabilize Telemt -* Use of Telemt or its space to enable harm +* Attempts to degrade, destabilize, or exhaust Telemt or its participants +* Use of Telemt or its spaces to enable harm + +Telemt is not a venue for disputes that displace technical work. +Such discussions may be closed, removed, or redirected. > **Störung ist kein Beitrag.** + > Disruption is not contribution. --- ## 5. Security and Misuse -Telemt is intended for lawful and responsible use. +Telemt is intended for responsible use. * Do not use it to plan, coordinate, or execute harm * Do not publish vulnerabilities without responsible disclosure @@ -86,11 +100,24 @@ Telemt is intended for lawful and responsible use. Security is both technical and behavioral. > **Verantwortung endet nicht am Code.** + > Responsibility does not end at the code. --- -## 6. Scope +## 6. Openness + +Telemt is open to contributors of different backgrounds, experience levels, and working styles. + +Standards are public, legible, and applied to the work itself. + +Questions are welcome. Careful disagreement is welcome. Honest correction is welcome. + +Gatekeeping by obscurity, status signaling, or hostility is not. + +--- + +## 7. Scope This Code of Conduct applies to all official spaces: @@ -100,31 +127,43 @@ This Code of Conduct applies to all official spaces: --- -## 7. Enforcement +## 8. Maintainer Stewardship -Maintainers may act to preserve the integrity of Telemt: +Maintainers are responsible for final decisions in matters of conduct, scope, and direction. -* Remove content -* Lock discussions -* Reject contributions -* Restrict or ban participants +This responsibility is stewardship: preserving continuity, protecting signal, maintaining standards, and keeping Telemt workable for others. -Actions are taken to maintain function, continuity, and signal quality. +Judgment should be exercised with restraint, consistency, and institutional responsibility. + +Not every decision requires extended debate. +Not every intervention requires public explanation. + +All decisions are expected to serve the durability, clarity, and integrity of Telemt. > **Ordnung ist Voraussetzung der Funktion.** + > Order is the precondition of function. --- -## 8. Maintainer Authority +## 9. Enforcement -Maintainers have final authority in interpretation and enforcement. +Maintainers may act to preserve the integrity of Telemt, including by: -Authority exists to ensure continuity, consistency, and technical direction. +* Removing content +* Locking discussions +* Rejecting contributions +* Restricting or banning participants + +Actions are taken to maintain function, continuity, and signal quality. + +Where possible, correction is preferred to exclusion. + +Where necessary, exclusion is preferred to decay. --- -## 9. Final +## 10. Final Telemt is built on discipline, structure, and shared intent. @@ -132,19 +171,23 @@ Signal over noise. Facts over opinion. Systems over rhetoric. -Work here is collective. +Work is collective. Outcomes are shared. Responsibility is distributed. +Precision is learned. +Rigor is expected. +Help is part of the work. + > **Ordnung ist Voraussetzung der Freiheit.** -If you contribute — contribute with precision. +If you contribute — contribute with care. If you speak — speak with substance. If you engage — engage constructively. --- -## 10. After All +## 11. After All Systems outlive intentions. @@ -152,12 +195,14 @@ What is built will be used. What is released will propagate. What is maintained will define the future state. -There is no neutral infrastructure. +There is no neutral infrastructure, only infrastructure shaped well or poorly. > **Jedes System trägt Verantwortung.** + > Every system carries responsibility. Stability requires discipline. Freedom requires structure. +Trust requires honesty. In the end, the system reflects its contributors. From 6f9aef7bb458d5447acc29d494151f183602e43f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 13:08:35 +0300 Subject: [PATCH 241/261] ME Writer stuck-up in draining-state fixes Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/metrics.rs | 58 +++++++++++++ src/stats/mod.rs | 32 +++++++ src/transport/middle_proxy/health.rs | 4 + .../middle_proxy/health_regression_tests.rs | 84 +++++++++++++++++++ src/transport/middle_proxy/pool_writer.rs | 26 +++++- 5 files changed, 203 insertions(+), 1 deletion(-) diff --git a/src/metrics.rs b/src/metrics.rs index 3de9896..4f7f4b6 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1692,6 +1692,57 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_writer_close_signal_drop_total Close-signal drops for already-removed ME writers" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_close_signal_drop_total counter"); + let _ = writeln!( + out, + "telemt_me_writer_close_signal_drop_total {}", + if me_allows_normal { + stats.get_me_writer_close_signal_drop_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_writer_close_signal_channel_full_total Close-signal drops caused by full writer command channels" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_writer_close_signal_channel_full_total counter" + ); + let _ = writeln!( + out, + "telemt_me_writer_close_signal_channel_full_total {}", + if me_allows_normal { + stats.get_me_writer_close_signal_channel_full_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_draining_writers_reap_progress_total Draining-writer removals processed by reap cleanup" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_draining_writers_reap_progress_total counter" + ); + let _ = writeln!( + out, + "telemt_me_draining_writers_reap_progress_total {}", + if me_allows_normal { + stats.get_me_draining_writers_reap_progress_total() + } else { + 0 + } + ); + let _ = writeln!(out, "# HELP telemt_me_writer_removed_total Total ME writer removals"); let _ = writeln!(out, "# TYPE telemt_me_writer_removed_total counter"); let _ = writeln!( @@ -2124,6 +2175,13 @@ mod tests { assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter")); assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter")); assert!(output.contains("# TYPE telemt_me_writer_removed_total counter")); + assert!(output.contains("# TYPE telemt_me_writer_close_signal_drop_total counter")); + assert!(output.contains( + "# TYPE telemt_me_writer_close_signal_channel_full_total counter" + )); + assert!(output.contains( + "# TYPE telemt_me_draining_writers_reap_progress_total counter" + )); assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_total counter")); assert!(output.contains("# TYPE telemt_pool_drain_soft_evict_writer_total counter")); assert!(output.contains( diff --git a/src/stats/mod.rs b/src/stats/mod.rs index 83cd03d..ad1d16b 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -123,6 +123,9 @@ pub struct Stats { pool_drain_soft_evict_total: AtomicU64, pool_drain_soft_evict_writer_total: AtomicU64, pool_stale_pick_total: AtomicU64, + me_writer_close_signal_drop_total: AtomicU64, + me_writer_close_signal_channel_full_total: AtomicU64, + me_draining_writers_reap_progress_total: AtomicU64, me_writer_removed_total: AtomicU64, me_writer_removed_unexpected_total: AtomicU64, me_refill_triggered_total: AtomicU64, @@ -734,6 +737,24 @@ impl Stats { self.pool_stale_pick_total.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_writer_close_signal_drop_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_close_signal_drop_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_close_signal_channel_full_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_close_signal_channel_full_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_draining_writers_reap_progress_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_draining_writers_reap_progress_total + .fetch_add(1, Ordering::Relaxed); + } + } pub fn increment_me_writer_removed_total(&self) { if self.telemetry_me_allows_debug() { self.me_writer_removed_total.fetch_add(1, Ordering::Relaxed); @@ -1259,6 +1280,17 @@ impl Stats { pub fn get_pool_stale_pick_total(&self) -> u64 { self.pool_stale_pick_total.load(Ordering::Relaxed) } + pub fn get_me_writer_close_signal_drop_total(&self) -> u64 { + self.me_writer_close_signal_drop_total.load(Ordering::Relaxed) + } + pub fn get_me_writer_close_signal_channel_full_total(&self) -> u64 { + self.me_writer_close_signal_channel_full_total + .load(Ordering::Relaxed) + } + pub fn get_me_draining_writers_reap_progress_total(&self) -> u64 { + self.me_draining_writers_reap_progress_total + .load(Ordering::Relaxed) + } pub fn get_me_writer_removed_total(&self) -> u64 { self.me_writer_removed_total.load(Ordering::Relaxed) } diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 0b9b749..6d0af64 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -314,6 +314,8 @@ pub(super) async fn reap_draining_writers( } pool.stats.increment_pool_force_close_total(); pool.remove_writer_and_close_clients(writer_id).await; + pool.stats + .increment_me_draining_writers_reap_progress_total(); closed_total = closed_total.saturating_add(1); } for writer_id in empty_writer_ids { @@ -324,6 +326,8 @@ pub(super) async fn reap_draining_writers( continue; } pool.remove_writer_and_close_clients(writer_id).await; + pool.stats + .increment_me_draining_writers_reap_progress_total(); closed_total = closed_total.saturating_add(1); } diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs index 606f7e5..565ac74 100644 --- a/src/transport/middle_proxy/health_regression_tests.rs +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; use std::time::{Duration, Instant}; +use bytes::Bytes; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -209,6 +210,89 @@ async fn reap_draining_writers_removes_empty_draining_writers() { assert_eq!(current_writer_ids(&pool).await, vec![3]); } +#[tokio::test] +async fn reap_draining_writers_does_not_block_on_stuck_writer_close_signal() { + let pool = make_pool(128).await; + let now_epoch_secs = MePool::now_epoch_secs(); + + let (blocked_tx, blocked_rx) = mpsc::channel::(1); + assert!( + blocked_tx + .try_send(WriterCommand::Data(Bytes::from_static(b"stuck"))) + .is_ok() + ); + let blocked_rx_guard = tokio::spawn(async move { + let _hold_rx = blocked_rx; + tokio::time::sleep(Duration::from_secs(30)).await; + }); + + let blocked_writer_id = 90u64; + let blocked_writer = MeWriter { + id: blocked_writer_id, + addr: SocketAddr::new( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 4500 + blocked_writer_id as u16, + ), + source_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + writer_dc: 2, + generation: 1, + contour: Arc::new(AtomicU8::new(WriterContour::Draining.as_u8())), + created_at: Instant::now() - Duration::from_secs(blocked_writer_id), + tx: blocked_tx.clone(), + cancel: CancellationToken::new(), + degraded: Arc::new(AtomicBool::new(false)), + rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)), + draining: Arc::new(AtomicBool::new(true)), + draining_started_at_epoch_secs: Arc::new(AtomicU64::new( + now_epoch_secs.saturating_sub(120), + )), + drain_deadline_epoch_secs: Arc::new(AtomicU64::new(0)), + allow_drain_fallback: Arc::new(AtomicBool::new(false)), + }; + pool.writers.write().await.push(blocked_writer); + pool.registry + .register_writer(blocked_writer_id, blocked_tx) + .await; + pool.conn_count.fetch_add(1, Ordering::Relaxed); + + insert_draining_writer(&pool, 91, now_epoch_secs.saturating_sub(110), 0, 0).await; + + let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); + + let reap_res = tokio::time::timeout( + Duration::from_millis(500), + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed), + ) + .await; + blocked_rx_guard.abort(); + + assert!(reap_res.is_ok(), "reap should not block on close signal"); + assert!(current_writer_ids(&pool).await.is_empty()); + assert_eq!(pool.stats.get_me_writer_close_signal_drop_total(), 2); + assert_eq!(pool.stats.get_me_writer_close_signal_channel_full_total(), 1); + assert_eq!(pool.stats.get_me_draining_writers_reap_progress_total(), 2); + let activity = pool.registry.writer_activity_snapshot().await; + assert!(!activity.bound_clients_by_writer.contains_key(&blocked_writer_id)); + assert!(!activity.bound_clients_by_writer.contains_key(&91)); + let (probe_conn_id, _rx) = pool.registry.register().await; + assert!( + !pool.registry + .bind_writer( + probe_conn_id, + blocked_writer_id, + ConnMeta { + target_dc: 2, + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6400), + our_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443), + proto_flags: 0, + }, + ) + .await + ); + let _ = pool.registry.unregister(probe_conn_id).await; +} + #[tokio::test] async fn reap_draining_writers_overflow_closes_oldest_non_empty_writers() { let pool = make_pool(2).await; diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 4035111..a6186b6 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -8,6 +8,7 @@ use bytes::Bytes; use bytes::BytesMut; use rand::Rng; use tokio::sync::mpsc; +use tokio::sync::mpsc::error::TrySendError; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; @@ -525,6 +526,11 @@ impl MePool { self.conn_count.fetch_sub(1, Ordering::Relaxed); } } + // State invariant: + // - writer is removed from `self.writers` (pool visibility), + // - writer is removed from registry routing/binding maps via `writer_lost`. + // The close command below is only a best-effort accelerator for task shutdown. + // Cleanup progress must never depend on command-channel availability. let conns = self.registry.writer_lost(writer_id).await; { let mut tracker = self.ping_tracker.lock().await; @@ -532,7 +538,25 @@ impl MePool { } self.rtt_stats.lock().await.remove(&writer_id); if let Some(tx) = close_tx { - let _ = tx.send(WriterCommand::Close).await; + match tx.try_send(WriterCommand::Close) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + self.stats.increment_me_writer_close_signal_drop_total(); + self.stats + .increment_me_writer_close_signal_channel_full_total(); + debug!( + writer_id, + "Skipping close signal for removed writer: command channel is full" + ); + } + Err(TrySendError::Closed(_)) => { + self.stats.increment_me_writer_close_signal_drop_total(); + debug!( + writer_id, + "Skipping close signal for removed writer: command channel is closed" + ); + } + } } if trigger_refill && let Some(addr) = removed_addr From 3279f6d46a5744b754741abf828e6b4d50c3a3ac Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:07:20 +0300 Subject: [PATCH 242/261] Cleanup-path as non-blocking Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/load.rs | 51 ++++++++++++++ src/transport/middle_proxy/pool_writer.rs | 8 +-- src/transport/middle_proxy/reader.rs | 32 ++++++--- src/transport/middle_proxy/registry.rs | 85 +++++++++++++++++------ src/transport/middle_proxy/send.rs | 28 +++++--- 5 files changed, 158 insertions(+), 46 deletions(-) diff --git a/src/config/load.rs b/src/config/load.rs index 0635f80..c296697 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -612,6 +612,11 @@ impl ProxyConfig { "general.me_route_backpressure_base_timeout_ms must be > 0".to_string(), )); } + if config.general.me_route_backpressure_base_timeout_ms > 5000 { + return Err(ProxyError::Config( + "general.me_route_backpressure_base_timeout_ms must be within [1, 5000]".to_string(), + )); + } if config.general.me_route_backpressure_high_timeout_ms < config.general.me_route_backpressure_base_timeout_ms @@ -620,6 +625,11 @@ impl ProxyConfig { "general.me_route_backpressure_high_timeout_ms must be >= general.me_route_backpressure_base_timeout_ms".to_string(), )); } + if config.general.me_route_backpressure_high_timeout_ms > 5000 { + return Err(ProxyError::Config( + "general.me_route_backpressure_high_timeout_ms must be within [1, 5000]".to_string(), + )); + } if !(1..=100).contains(&config.general.me_route_backpressure_high_watermark_pct) { return Err(ProxyError::Config( @@ -1624,6 +1634,47 @@ mod tests { let _ = std::fs::remove_file(path_valid); } + #[test] + fn me_route_backpressure_base_timeout_ms_out_of_range_is_rejected() { + let toml = r#" + [general] + me_route_backpressure_base_timeout_ms = 5001 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_route_backpressure_base_timeout_ms_out_of_range_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.me_route_backpressure_base_timeout_ms must be within [1, 5000]")); + let _ = std::fs::remove_file(path); + } + + #[test] + fn me_route_backpressure_high_timeout_ms_out_of_range_is_rejected() { + let toml = r#" + [general] + me_route_backpressure_base_timeout_ms = 100 + me_route_backpressure_high_timeout_ms = 5001 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_me_route_backpressure_high_timeout_ms_out_of_range_test.toml"); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + assert!(err.contains("general.me_route_backpressure_high_timeout_ms must be within [1, 5000]")); + let _ = std::fs::remove_file(path); + } + #[test] fn me_route_no_writer_wait_ms_out_of_range_is_rejected() { let toml = r#" diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index a6186b6..7d78b84 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -492,11 +492,9 @@ impl MePool { } pub(crate) async fn remove_writer_and_close_clients(self: &Arc, writer_id: u64) { - let conns = self.remove_writer_only(writer_id).await; - for bound in conns { - let _ = self.registry.route(bound.conn_id, super::MeResponse::Close).await; - let _ = self.registry.unregister(bound.conn_id).await; - } + // Full client cleanup now happens inside `registry.writer_lost` to keep + // writer reap/remove paths strictly non-blocking per connection. + let _ = self.remove_writer_only(writer_id).await; } async fn remove_writer_only(self: &Arc, writer_id: u64) -> Vec { diff --git a/src/transport/middle_proxy/reader.rs b/src/transport/middle_proxy/reader.rs index 785bc2c..8b15fc1 100644 --- a/src/transport/middle_proxy/reader.rs +++ b/src/transport/middle_proxy/reader.rs @@ -8,6 +8,7 @@ use bytes::{Bytes, BytesMut}; use tokio::io::AsyncReadExt; use tokio::net::TcpStream; use tokio::sync::{Mutex, mpsc}; +use tokio::sync::mpsc::error::TrySendError; use tokio_util::sync::CancellationToken; use tracing::{debug, trace, warn}; @@ -173,12 +174,12 @@ pub(crate) async fn reader_loop( } else if pt == RPC_CLOSE_EXT_U32 && body.len() >= 8 { let cid = u64::from_le_bytes(body[0..8].try_into().unwrap()); debug!(cid, "RPC_CLOSE_EXT from ME"); - reg.route(cid, MeResponse::Close).await; + let _ = reg.route_nowait(cid, MeResponse::Close).await; reg.unregister(cid).await; } else if pt == RPC_CLOSE_CONN_U32 && body.len() >= 8 { let cid = u64::from_le_bytes(body[0..8].try_into().unwrap()); debug!(cid, "RPC_CLOSE_CONN from ME"); - reg.route(cid, MeResponse::Close).await; + let _ = reg.route_nowait(cid, MeResponse::Close).await; reg.unregister(cid).await; } else if pt == RPC_PING_U32 && body.len() >= 8 { let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap()); @@ -186,13 +187,15 @@ pub(crate) async fn reader_loop( let mut pong = Vec::with_capacity(12); pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes()); pong.extend_from_slice(&ping_id.to_le_bytes()); - if tx - .send(WriterCommand::DataAndFlush(Bytes::from(pong))) - .await - .is_err() - { - warn!("PONG send failed"); - break; + match tx.try_send(WriterCommand::DataAndFlush(Bytes::from(pong))) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + debug!(ping_id, "PONG dropped: writer command channel is full"); + } + Err(TrySendError::Closed(_)) => { + warn!("PONG send failed: writer channel closed"); + break; + } } } else if pt == RPC_PONG_U32 && body.len() >= 8 { let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap()); @@ -232,6 +235,13 @@ async fn send_close_conn(tx: &mpsc::Sender, conn_id: u64) { let mut p = Vec::with_capacity(12); p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes()); p.extend_from_slice(&conn_id.to_le_bytes()); - - let _ = tx.send(WriterCommand::DataAndFlush(Bytes::from(p))).await; + match tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + debug!(conn_id, "ME close_conn signal skipped: writer command channel is full"); + } + Err(TrySendError::Closed(_)) => { + debug!(conn_id, "ME close_conn signal skipped: writer command channel is closed"); + } + } } diff --git a/src/transport/middle_proxy/registry.rs b/src/transport/middle_proxy/registry.rs index b8a926e..2ee55c1 100644 --- a/src/transport/middle_proxy/registry.rs +++ b/src/transport/middle_proxy/registry.rs @@ -169,6 +169,7 @@ impl ConnRegistry { None } + #[allow(dead_code)] pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult { let tx = { let inner = self.inner.read().await; @@ -445,30 +446,38 @@ impl ConnRegistry { } pub async fn writer_lost(&self, writer_id: u64) -> Vec { - let mut inner = self.inner.write().await; - inner.writers.remove(&writer_id); - inner.last_meta_for_writer.remove(&writer_id); - inner.writer_idle_since_epoch_secs.remove(&writer_id); - let conns = inner - .conns_for_writer - .remove(&writer_id) - .unwrap_or_default() - .into_iter() - .collect::>(); - + let mut close_txs = Vec::>::new(); let mut out = Vec::new(); - for conn_id in conns { - if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) { - continue; - } - inner.writer_for_conn.remove(&conn_id); - if let Some(m) = inner.meta.get(&conn_id) { - out.push(BoundConn { - conn_id, - meta: m.clone(), - }); + { + let mut inner = self.inner.write().await; + inner.writers.remove(&writer_id); + inner.last_meta_for_writer.remove(&writer_id); + inner.writer_idle_since_epoch_secs.remove(&writer_id); + let conns = inner + .conns_for_writer + .remove(&writer_id) + .unwrap_or_default() + .into_iter() + .collect::>(); + + for conn_id in conns { + if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) { + continue; + } + inner.writer_for_conn.remove(&conn_id); + if let Some(client_tx) = inner.map.remove(&conn_id) { + close_txs.push(client_tx); + } + if let Some(meta) = inner.meta.remove(&conn_id) { + out.push(BoundConn { conn_id, meta }); + } } } + + for client_tx in close_txs { + let _ = client_tx.try_send(MeResponse::Close); + } + out } @@ -491,6 +500,7 @@ impl ConnRegistry { #[cfg(test)] mod tests { use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::time::Duration; use super::ConnMeta; use super::ConnRegistry; @@ -663,6 +673,39 @@ mod tests { assert!(registry.is_writer_empty(20).await); } + #[tokio::test] + async fn writer_lost_removes_bound_conn_from_registry_and_signals_close() { + let registry = ConnRegistry::new(); + let (conn_id, mut rx) = registry.register().await; + let (writer_tx, _writer_rx) = tokio::sync::mpsc::channel(8); + registry.register_writer(10, writer_tx).await; + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443); + + assert!( + registry + .bind_writer( + conn_id, + 10, + ConnMeta { + target_dc: 2, + client_addr: addr, + our_addr: addr, + proto_flags: 0, + }, + ) + .await + ); + + let lost = registry.writer_lost(10).await; + assert_eq!(lost.len(), 1); + assert_eq!(lost[0].conn_id, conn_id); + assert!(registry.get_writer(conn_id).await.is_none()); + assert!(registry.get_meta(conn_id).await.is_none()); + assert_eq!(registry.unregister(conn_id).await, None); + let close = tokio::time::timeout(Duration::from_millis(50), rx.recv()).await; + assert!(matches!(close, Ok(Some(MeResponse::Close)))); + } + #[tokio::test] async fn bind_writer_rejects_unregistered_writer() { let registry = ConnRegistry::new(); diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 1c255ef..6791064 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -643,13 +643,19 @@ impl MePool { let mut p = Vec::with_capacity(12); p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes()); p.extend_from_slice(&conn_id.to_le_bytes()); - if w.tx - .send(WriterCommand::DataAndFlush(Bytes::from(p))) - .await - .is_err() - { - debug!("ME close write failed"); - self.remove_writer_and_close_clients(w.writer_id).await; + match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + debug!( + conn_id, + writer_id = w.writer_id, + "ME close skipped: writer command channel is full" + ); + } + Err(TrySendError::Closed(_)) => { + debug!("ME close write failed"); + self.remove_writer_and_close_clients(w.writer_id).await; + } } } else { debug!(conn_id, "ME close skipped (writer missing)"); @@ -666,8 +672,12 @@ impl MePool { p.extend_from_slice(&conn_id.to_le_bytes()); match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) { Ok(()) => {} - Err(TrySendError::Full(cmd)) => { - let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await; + Err(TrySendError::Full(_)) => { + debug!( + conn_id, + writer_id = w.writer_id, + "ME close_conn skipped: writer command channel is full" + ); } Err(TrySendError::Closed(_)) => { debug!(conn_id, "ME close_conn skipped: writer channel closed"); From ae3ced8e7c41f20ac2ed9943f275f8769372ad7e Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:42:59 +0300 Subject: [PATCH 243/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b289231..d4ef990 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.23" +version = "3.3.24" edition = "2024" [dependencies] From 05c066c676328e7d26a82f462662ce0b960dd4a3 Mon Sep 17 00:00:00 2001 From: TEMAndroid Date: Thu, 19 Mar 2026 15:54:01 +0300 Subject: [PATCH 244/261] fix(docker): expose port 9091 and allow external API access Add 9091 port mapping to compose.yml to make the REST API reachable from outside the container. Previously only port 9090 (metrics) was published, making the documented curl commands non-functional. fixes #434 --- Dockerfile | 1 + docker-compose.yml | 1 + docs/QUICK_START_GUIDE.en.md | 2 ++ docs/QUICK_START_GUIDE.ru.md | 2 ++ 4 files changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7abe548..15a4900 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ USER telemt EXPOSE 443 EXPOSE 9090 +EXPOSE 9091 ENTRYPOINT ["/app/telemt"] CMD ["config.toml"] diff --git a/docker-compose.yml b/docker-compose.yml index 5866c4d..815ba24 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: ports: - "443:443" - "127.0.0.1:9090:9090" + - "127.0.0.1:9091:9091" # Allow caching 'proxy-secret' in read-only container working_dir: /run/telemt volumes: diff --git a/docs/QUICK_START_GUIDE.en.md b/docs/QUICK_START_GUIDE.en.md index c43a2dc..ffb387f 100644 --- a/docs/QUICK_START_GUIDE.en.md +++ b/docs/QUICK_START_GUIDE.en.md @@ -181,6 +181,8 @@ docker compose down docker build -t telemt:local . docker run --name telemt --restart unless-stopped \ -p 443:443 \ + -p 9090:9090 \ + -p 9091:9091 \ -e RUST_LOG=info \ -v "$PWD/config.toml:/app/config.toml:ro" \ --read-only \ diff --git a/docs/QUICK_START_GUIDE.ru.md b/docs/QUICK_START_GUIDE.ru.md index 35fbd27..e4c5005 100644 --- a/docs/QUICK_START_GUIDE.ru.md +++ b/docs/QUICK_START_GUIDE.ru.md @@ -183,6 +183,8 @@ docker compose down docker build -t telemt:local . docker run --name telemt --restart unless-stopped \ -p 443:443 \ + -p 9090:9090 \ + -p 9091:9091 \ -e RUST_LOG=info \ -v "$PWD/config.toml:/app/config.toml:ro" \ --read-only \ From f655924323a02e337f9fdfc87c826d89145df3af Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 16:15:00 +0300 Subject: [PATCH 245/261] Update health.rs Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 35 +++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 6d0af64..21619c7 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -30,6 +30,11 @@ const HEALTH_DRAIN_CLOSE_BUDGET_MIN: usize = 16; const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256; const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN: usize = 8; const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX: usize = 256; +const HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS: u64 = 1; +#[cfg(not(test))] +const HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE: bool = true; +#[cfg(test)] +const HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE: bool = false; #[derive(Debug, Clone)] struct DcFloorPlanEntry { @@ -99,6 +104,8 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut adaptive_idle_since, &mut adaptive_recover_until, &mut floor_warn_next_allowed, + &mut drain_warn_next_allowed, + &mut drain_soft_evict_next_allowed, ) .await; let v6_degraded = check_family( @@ -116,6 +123,8 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut adaptive_idle_since, &mut adaptive_recover_until, &mut floor_warn_next_allowed, + &mut drain_warn_next_allowed, + &mut drain_soft_evict_next_allowed, ) .await; degraded_interval = v4_degraded || v6_degraded; @@ -154,6 +163,11 @@ pub(super) async fn reap_draining_writers( } draining_writers.push(writer); } + if HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE { + for writer in draining_writers.drain(..) { + force_close_writer_ids.push(writer.id); + } + } if drain_threshold > 0 && draining_writers.len() > drain_threshold as usize { draining_writers.sort_by(|left, right| { @@ -299,10 +313,14 @@ pub(super) async fn reap_draining_writers( } } - let close_budget = health_drain_close_budget(); let requested_force_close = force_close_writer_ids.len(); let requested_empty_close = empty_writer_ids.len(); let requested_close_total = requested_force_close.saturating_add(requested_empty_close); + let close_budget = if HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE { + requested_close_total + } else { + health_drain_close_budget() + }; let mut closed_writer_ids = HashSet::::new(); let mut closed_total = 0usize; for writer_id in force_close_writer_ids { @@ -396,6 +414,8 @@ async fn check_family( adaptive_idle_since: &mut HashMap<(i32, IpFamily), Instant>, adaptive_recover_until: &mut HashMap<(i32, IpFamily), Instant>, floor_warn_next_allowed: &mut HashMap<(i32, IpFamily), Instant>, + drain_warn_next_allowed: &mut HashMap, + drain_soft_evict_next_allowed: &mut HashMap, ) -> bool { let enabled = match family { IpFamily::V4 => pool.decision.ipv4_me, @@ -476,8 +496,15 @@ async fn check_family( floor_plan.active_writers_current, floor_plan.warm_writers_current, ); + let mut next_drain_reap_at = Instant::now(); for (dc, endpoints) in dc_endpoints { + if Instant::now() >= next_drain_reap_at { + reap_draining_writers(pool, drain_warn_next_allowed, drain_soft_evict_next_allowed) + .await; + next_drain_reap_at = Instant::now() + + Duration::from_secs(HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS); + } if endpoints.is_empty() { continue; } @@ -621,6 +648,12 @@ async fn check_family( let mut restored = 0usize; for _ in 0..missing { + if Instant::now() >= next_drain_reap_at { + reap_draining_writers(pool, drain_warn_next_allowed, drain_soft_evict_next_allowed) + .await; + next_drain_reap_at = Instant::now() + + Duration::from_secs(HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS); + } if reconnect_budget == 0 { break; } From dc3363aa0d05bcbf83453cf567134b144997341d Mon Sep 17 00:00:00 2001 From: Dimasssss Date: Thu, 19 Mar 2026 16:23:32 +0300 Subject: [PATCH 246/261] Update install.sh --- install.sh | 583 ++++++++++++++++++++++++++++------------------------- 1 file changed, 307 insertions(+), 276 deletions(-) diff --git a/install.sh b/install.sh index 330bc3e..90c28f4 100644 --- a/install.sh +++ b/install.sh @@ -1,154 +1,190 @@ #!/bin/sh set -eu -# --- Global Configurations --- REPO="${REPO:-telemt/telemt}" BIN_NAME="${BIN_NAME:-telemt}" INSTALL_DIR="${INSTALL_DIR:-/bin}" CONFIG_DIR="${CONFIG_DIR:-/etc/telemt}" CONFIG_FILE="${CONFIG_FILE:-${CONFIG_DIR}/telemt.toml}" WORK_DIR="${WORK_DIR:-/opt/telemt}" +TLS_DOMAIN="${TLS_DOMAIN:-petrovich.ru}" SERVICE_NAME="telemt" TEMP_DIR="" SUDO="" +CONFIG_PARENT_DIR="" +SERVICE_START_FAILED=0 -# --- Argument Parsing --- ACTION="install" TARGET_VERSION="${VERSION:-latest}" while [ $# -gt 0 ]; do case "$1" in - -h|--help) - ACTION="help" - shift - ;; + -h|--help) ACTION="help"; shift ;; uninstall|--uninstall) - [ "$ACTION" != "purge" ] && ACTION="uninstall" - shift - ;; - --purge) - ACTION="purge" - shift - ;; - install|--install) - ACTION="install" - shift - ;; - -*) - printf '[ERROR] Unknown option: %s\n' "$1" >&2 - exit 1 - ;; + if [ "$ACTION" != "purge" ]; then ACTION="uninstall"; fi + shift ;; + purge|--purge) ACTION="purge"; shift ;; + install|--install) ACTION="install"; shift ;; + -*) printf '[ERROR] Unknown option: %s\n' "$1" >&2; exit 1 ;; *) - if [ "$ACTION" = "install" ]; then - TARGET_VERSION="$1" - fi - shift - ;; + if [ "$ACTION" = "install" ]; then TARGET_VERSION="$1" + else printf '[WARNING] Ignoring extra argument: %s\n' "$1" >&2; fi + shift ;; esac done -# --- Core Functions --- -say() { printf '[INFO] %s\n' "$*"; } +say() { + if [ "$#" -eq 0 ] || [ -z "${1:-}" ]; then + printf '\n' + else + printf '[INFO] %s\n' "$*" + fi +} die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; } +write_root() { $SUDO sh -c 'cat > "$1"' _ "$1"; } + cleanup() { if [ -n "${TEMP_DIR:-}" ] && [ -d "$TEMP_DIR" ]; then rm -rf -- "$TEMP_DIR" fi } - trap cleanup EXIT INT TERM show_help() { - say "Usage: $0 [version | install | uninstall | --purge | --help]" - say " version Install specific version (e.g. 1.0.0, default: latest)" - say " uninstall Remove the binary and service (keeps config)" - say " --purge Remove everything including configuration" + say "Usage: $0 [ | install | uninstall | purge | --help ]" + say " Install specific version (e.g. 3.3.15, default: latest)" + say " install Install the latest version" + say " uninstall Remove the binary and service (keeps config and user)" + say " purge Remove everything including configuration, data, and user" exit 0 } -user_exists() { - if command -v getent >/dev/null 2>&1; then - getent passwd "$1" >/dev/null 2>&1 +check_os_entity() { + if command -v getent >/dev/null 2>&1; then getent "$1" "$2" >/dev/null 2>&1 + else grep -q "^${2}:" "/etc/$1" 2>/dev/null; fi +} + +normalize_path() { + printf '%s\n' "$1" | tr -s '/' | sed 's|/$||; s|^$|/|' +} + +get_realpath() { + path_in="$1" + case "$path_in" in /*) ;; *) path_in="$(pwd)/$path_in" ;; esac + + if command -v realpath >/dev/null 2>&1; then + if realpath_out="$(realpath -m "$path_in" 2>/dev/null)"; then + printf '%s\n' "$realpath_out" + return + fi + fi + + if command -v readlink >/dev/null 2>&1; then + resolved_path="$(readlink -f "$path_in" 2>/dev/null || true)" + if [ -n "$resolved_path" ]; then + printf '%s\n' "$resolved_path" + return + fi + fi + + d="${path_in%/*}"; b="${path_in##*/}" + if [ -z "$d" ]; then d="/"; fi + if [ "$d" = "$path_in" ]; then d="/"; b="$path_in"; fi + + if [ -d "$d" ]; then + abs_d="$(cd "$d" >/dev/null 2>&1 && pwd || true)" + if [ -n "$abs_d" ]; then + if [ "$b" = "." ] || [ -z "$b" ]; then printf '%s\n' "$abs_d" + elif [ "$abs_d" = "/" ]; then printf '/%s\n' "$b" + else printf '%s/%s\n' "$abs_d" "$b"; fi + else + normalize_path "$path_in" + fi else - grep -q "^${1}:" /etc/passwd 2>/dev/null + normalize_path "$path_in" fi } -group_exists() { - if command -v getent >/dev/null 2>&1; then - getent group "$1" >/dev/null 2>&1 - else - grep -q "^${1}:" /etc/group 2>/dev/null - fi +get_svc_mgr() { + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then echo "systemd" + elif command -v rc-service >/dev/null 2>&1; then echo "openrc" + else echo "none"; fi } verify_common() { - [ -z "$BIN_NAME" ] && die "BIN_NAME cannot be empty." - [ -z "$INSTALL_DIR" ] && die "INSTALL_DIR cannot be empty." - [ -z "$CONFIG_DIR" ] && die "CONFIG_DIR cannot be empty." + [ -n "$BIN_NAME" ] || die "BIN_NAME cannot be empty." + [ -n "$INSTALL_DIR" ] || die "INSTALL_DIR cannot be empty." + [ -n "$CONFIG_DIR" ] || die "CONFIG_DIR cannot be empty." + [ -n "$CONFIG_FILE" ] || die "CONFIG_FILE cannot be empty." + + case "${INSTALL_DIR}${CONFIG_DIR}${WORK_DIR}${CONFIG_FILE}" in + *[!a-zA-Z0-9_./-]*) die "Invalid characters in paths. Only alphanumeric, _, ., -, and / allowed." ;; + esac + + case "$TARGET_VERSION" in *[!a-zA-Z0-9_.-]*) die "Invalid characters in version." ;; esac + case "$BIN_NAME" in *[!a-zA-Z0-9_-]*) die "Invalid characters in BIN_NAME." ;; esac + + INSTALL_DIR="$(get_realpath "$INSTALL_DIR")" + CONFIG_DIR="$(get_realpath "$CONFIG_DIR")" + WORK_DIR="$(get_realpath "$WORK_DIR")" + CONFIG_FILE="$(get_realpath "$CONFIG_FILE")" + + CONFIG_PARENT_DIR="${CONFIG_FILE%/*}" + if [ -z "$CONFIG_PARENT_DIR" ]; then CONFIG_PARENT_DIR="/"; fi + if [ "$CONFIG_PARENT_DIR" = "$CONFIG_FILE" ]; then CONFIG_PARENT_DIR="."; fi if [ "$(id -u)" -eq 0 ]; then SUDO="" else - if ! command -v sudo >/dev/null 2>&1; then - die "This script requires root or sudo. Neither found." - fi + command -v sudo >/dev/null 2>&1 || die "This script requires root or sudo. Neither found." SUDO="sudo" - say "sudo is available. Caching credentials..." - if ! sudo -v; then - die "Failed to cache sudo credentials" + if ! sudo -n true 2>/dev/null; then + if ! [ -t 0 ]; then + die "sudo requires a password, but no TTY detected. Aborting to prevent hang." + fi fi fi - case "${INSTALL_DIR}${CONFIG_DIR}${WORK_DIR}" in - *[!a-zA-Z0-9_./-]*) - die "Invalid characters in path variables. Only alphanumeric, _, ., -, and / are allowed." - ;; - esac - - case "$BIN_NAME" in - *[!a-zA-Z0-9_-]*) die "Invalid characters in BIN_NAME: $BIN_NAME" ;; - esac - - for path in "$CONFIG_DIR" "$WORK_DIR"; do - check_path="$path" - - while [ "$check_path" != "/" ] && [ "${check_path%"/"}" != "$check_path" ]; do - check_path="${check_path%"/"}" - done - [ -z "$check_path" ] && check_path="/" + if [ -n "$SUDO" ]; then + if $SUDO sh -c '[ -d "$1" ]' _ "$CONFIG_FILE"; then + die "Safety check failed: CONFIG_FILE '$CONFIG_FILE' is a directory." + fi + elif [ -d "$CONFIG_FILE" ]; then + die "Safety check failed: CONFIG_FILE '$CONFIG_FILE' is a directory." + fi + for path in "$CONFIG_DIR" "$CONFIG_PARENT_DIR" "$WORK_DIR"; do + check_path="$(get_realpath "$path")" case "$check_path" in - /|/bin|/sbin|/usr|/usr/bin|/usr/local|/etc|/opt|/var|/home|/root|/tmp) - die "Safety check failed: '$path' is a critical system directory." - ;; + /|/bin|/sbin|/usr|/usr/bin|/usr/sbin|/usr/local|/usr/local/bin|/usr/local/sbin|/usr/local/etc|/usr/local/share|/etc|/var|/var/lib|/var/log|/var/run|/home|/root|/tmp|/lib|/lib64|/opt|/run|/boot|/dev|/sys|/proc) + die "Safety check failed: '$path' (resolved to '$check_path') is a critical system directory." ;; esac done - for cmd in uname grep find rm chown chmod mv head mktemp; do + check_install_dir="$(get_realpath "$INSTALL_DIR")" + case "$check_install_dir" in + /|/etc|/var|/home|/root|/tmp|/usr|/usr/local|/opt|/boot|/dev|/sys|/proc|/run) + die "Safety check failed: INSTALL_DIR '$INSTALL_DIR' is a critical system directory." ;; + esac + + for cmd in id uname grep find rm chown chmod mv mktemp mkdir tr dd sed ps head sleep cat tar gzip rmdir; do command -v "$cmd" >/dev/null 2>&1 || die "Required command not found: $cmd" done } verify_install_deps() { - if ! command -v curl >/dev/null 2>&1 && ! command -v wget >/dev/null 2>&1; then - die "Neither curl nor wget is installed." - fi - command -v tar >/dev/null 2>&1 || die "Required command not found: tar" - command -v gzip >/dev/null 2>&1 || die "Required command not found: gzip" + command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1 || die "Neither curl nor wget is installed." command -v cp >/dev/null 2>&1 || command -v install >/dev/null 2>&1 || die "Need cp or install" if ! command -v setcap >/dev/null 2>&1; then - say "setcap is missing. Installing required capability tools..." if command -v apk >/dev/null 2>&1; then - $SUDO apk add --no-cache libcap || die "Failed to install libcap" + $SUDO apk add --no-cache libcap-utils >/dev/null 2>&1 || $SUDO apk add --no-cache libcap >/dev/null 2>&1 || true elif command -v apt-get >/dev/null 2>&1; then - $SUDO apt-get update -qq && $SUDO apt-get install -y -qq libcap2-bin || die "Failed to install libcap2-bin" - elif command -v dnf >/dev/null 2>&1 || command -v yum >/dev/null 2>&1; then - $SUDO ${YUM_CMD:-yum} install -y -q libcap || die "Failed to install libcap" - else - die "Cannot install 'setcap'. Package manager not found. Please install libcap manually." + $SUDO apt-get update -q >/dev/null 2>&1 || true + $SUDO apt-get install -y -q libcap2-bin >/dev/null 2>&1 || true + elif command -v dnf >/dev/null 2>&1; then $SUDO dnf install -y -q libcap >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then $SUDO yum install -y -q libcap >/dev/null 2>&1 || true fi fi } @@ -163,122 +199,96 @@ detect_arch() { } detect_libc() { - if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -qi musl; then - echo "musl"; return 0 - fi - - if grep -q '^ID=alpine' /etc/os-release 2>/dev/null || grep -q '^ID="alpine"' /etc/os-release 2>/dev/null; then - echo "musl"; return 0 - fi for f in /lib/ld-musl-*.so.* /lib64/ld-musl-*.so.*; do - if [ -e "$f" ]; then - echo "musl"; return 0 - fi + if [ -e "$f" ]; then echo "musl"; return 0; fi done + if grep -qE '^ID="?alpine"?' /etc/os-release 2>/dev/null; then echo "musl"; return 0; fi + if command -v ldd >/dev/null 2>&1 && (ldd --version 2>&1 || true) | grep -qi musl; then echo "musl"; return 0; fi echo "gnu" } fetch_file() { - fetch_url="$1" - fetch_out="$2" - - if command -v curl >/dev/null 2>&1; then - curl -fsSL "$fetch_url" -o "$fetch_out" || return 1 - elif command -v wget >/dev/null 2>&1; then - wget -qO "$fetch_out" "$fetch_url" || return 1 - else - die "curl or wget required" - fi + if command -v curl >/dev/null 2>&1; then curl -fsSL "$1" -o "$2" + else wget -q -O "$2" "$1"; fi } ensure_user_group() { - nologin_bin="/bin/false" + nologin_bin="$(command -v nologin 2>/dev/null || command -v false 2>/dev/null || echo /bin/false)" - cmd_nologin="$(command -v nologin 2>/dev/null || true)" - if [ -n "$cmd_nologin" ] && [ -x "$cmd_nologin" ]; then - nologin_bin="$cmd_nologin" - else - for bin in /sbin/nologin /usr/sbin/nologin; do - if [ -x "$bin" ]; then - nologin_bin="$bin" - break - fi - done + if ! check_os_entity group telemt; then + if command -v groupadd >/dev/null 2>&1; then $SUDO groupadd -r telemt + elif command -v addgroup >/dev/null 2>&1; then $SUDO addgroup -S telemt + else die "Cannot create group"; fi fi - if ! group_exists telemt; then - if command -v groupadd >/dev/null 2>&1; then - $SUDO groupadd -r telemt || die "Failed to create group via groupadd" - elif command -v addgroup >/dev/null 2>&1; then - $SUDO addgroup -S telemt || die "Failed to create group via addgroup" - else - die "Cannot create group: neither groupadd nor addgroup found" - fi - fi - - if ! user_exists telemt; then + if ! check_os_entity passwd telemt; then if command -v useradd >/dev/null 2>&1; then - $SUDO useradd -r -g telemt -d "$WORK_DIR" -s "$nologin_bin" -c "Telemt Proxy" telemt || die "Failed to create user via useradd" + $SUDO useradd -r -g telemt -d "$WORK_DIR" -s "$nologin_bin" -c "Telemt Proxy" telemt elif command -v adduser >/dev/null 2>&1; then - $SUDO adduser -S -D -H -h "$WORK_DIR" -s "$nologin_bin" -G telemt telemt || die "Failed to create user via adduser" - else - die "Cannot create user: neither useradd nor adduser found" - fi + if adduser --help 2>&1 | grep -q -- '-S'; then + $SUDO adduser -S -D -H -h "$WORK_DIR" -s "$nologin_bin" -G telemt telemt + else + $SUDO adduser --system --home "$WORK_DIR" --shell "$nologin_bin" --no-create-home --ingroup telemt --disabled-password telemt + fi + else die "Cannot create user"; fi fi } setup_dirs() { - say "Setting up directories..." - $SUDO mkdir -p "$WORK_DIR" "$CONFIG_DIR" || die "Failed to create directories" - $SUDO chown telemt:telemt "$WORK_DIR" || die "Failed to set owner on WORK_DIR" - $SUDO chmod 750 "$WORK_DIR" || die "Failed to set permissions on WORK_DIR" + $SUDO mkdir -p "$WORK_DIR" "$CONFIG_DIR" "$CONFIG_PARENT_DIR" || die "Failed to create directories" + + $SUDO chown telemt:telemt "$WORK_DIR" && $SUDO chmod 750 "$WORK_DIR" + $SUDO chown root:telemt "$CONFIG_DIR" && $SUDO chmod 750 "$CONFIG_DIR" + + if [ "$CONFIG_PARENT_DIR" != "$CONFIG_DIR" ] && [ "$CONFIG_PARENT_DIR" != "." ] && [ "$CONFIG_PARENT_DIR" != "/" ]; then + $SUDO chown root:telemt "$CONFIG_PARENT_DIR" && $SUDO chmod 750 "$CONFIG_PARENT_DIR" + fi } stop_service() { - say "Stopping service if running..." - if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + svc="$(get_svc_mgr)" + if [ "$svc" = "systemd" ] && systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then $SUDO systemctl stop "$SERVICE_NAME" 2>/dev/null || true - elif command -v rc-service >/dev/null 2>&1; then + elif [ "$svc" = "openrc" ] && rc-service "$SERVICE_NAME" status >/dev/null 2>&1; then $SUDO rc-service "$SERVICE_NAME" stop 2>/dev/null || true fi } install_binary() { - bin_src="$1" - bin_dst="$2" + bin_src="$1"; bin_dst="$2" + if [ -e "$INSTALL_DIR" ] && [ ! -d "$INSTALL_DIR" ]; then + die "'$INSTALL_DIR' is not a directory." + fi $SUDO mkdir -p "$INSTALL_DIR" || die "Failed to create install directory" if command -v install >/dev/null 2>&1; then $SUDO install -m 0755 "$bin_src" "$bin_dst" || die "Failed to install binary" else - $SUDO rm -f "$bin_dst" - $SUDO cp "$bin_src" "$bin_dst" || die "Failed to copy binary" - $SUDO chmod 0755 "$bin_dst" || die "Failed to set permissions" + $SUDO rm -f "$bin_dst" 2>/dev/null || true + $SUDO cp "$bin_src" "$bin_dst" && $SUDO chmod 0755 "$bin_dst" || die "Failed to copy binary" fi - if [ ! -x "$bin_dst" ]; then - die "Failed to install binary or it is not executable: $bin_dst" - fi + $SUDO sh -c '[ -x "$1" ]' _ "$bin_dst" || die "Binary not executable: $bin_dst" - say "Granting network bind capabilities to bind port 443..." - if ! $SUDO setcap cap_net_bind_service=+ep "$bin_dst" 2>/dev/null; then - say "[WARNING] Failed to apply setcap. The service will NOT be able to open port 443!" - say "[WARNING] This usually happens inside unprivileged Docker/LXC containers." + if command -v setcap >/dev/null 2>&1; then + $SUDO setcap cap_net_bind_service=+ep "$bin_dst" 2>/dev/null || true fi } generate_secret() { - if command -v openssl >/dev/null 2>&1; then - secret="$(openssl rand -hex 16 2>/dev/null)" && [ -n "$secret" ] && { echo "$secret"; return 0; } + secret="$(command -v openssl >/dev/null 2>&1 && openssl rand -hex 16 2>/dev/null || true)" + if [ -z "$secret" ] || [ "${#secret}" -ne 32 ]; then + if command -v od >/dev/null 2>&1; then secret="$(dd if=/dev/urandom bs=16 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" + elif command -v hexdump >/dev/null 2>&1; then secret="$(dd if=/dev/urandom bs=16 count=1 2>/dev/null | hexdump -e '1/1 "%02x"')" + elif command -v xxd >/dev/null 2>&1; then secret="$(dd if=/dev/urandom bs=16 count=1 2>/dev/null | xxd -p | tr -d '\n')" + fi fi - if command -v xxd >/dev/null 2>&1; then - secret="$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | xxd -p | tr -d '\n')" && [ -n "$secret" ] && { echo "$secret"; return 0; } - fi - secret="$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | od -An -tx1 | tr -d ' \n')" && [ -n "$secret" ] && { echo "$secret"; return 0; } - return 1 + if [ "${#secret}" -eq 32 ]; then echo "$secret"; else return 1; fi } generate_config_content() { + escaped_tls_domain="$(printf '%s\n' "$TLS_DOMAIN" | tr -d '[:cntrl:]' | sed 's/\\/\\\\/g; s/"/\\"/g')" + cat </dev/null && config_exists=1 || true - else - [ -f "$CONFIG_FILE" ] && config_exists=1 || true - fi - - if [ "$config_exists" -eq 1 ]; then - say "Config already exists, skipping generation." + if $SUDO sh -c '[ -f "$1" ]' _ "$CONFIG_FILE"; then + say " -> Config already exists at $CONFIG_FILE. Skipping creation." + return 0 + fi + elif [ -f "$CONFIG_FILE" ]; then + say " -> Config already exists at $CONFIG_FILE. Skipping creation." return 0 fi - toml_secret="$(generate_secret)" || die "Failed to generate secret" - say "Creating config at $CONFIG_FILE..." + toml_secret="$(generate_secret)" || die "Failed to generate secret." - tmp_conf="$(mktemp "${TEMP_DIR:-/tmp}/telemt_conf.XXXXXX")" || die "Failed to create temp config" - generate_config_content "$toml_secret" > "$tmp_conf" || die "Failed to write temp config" + generate_config_content "$toml_secret" | write_root "$CONFIG_FILE" || die "Failed to install config" + $SUDO chown root:telemt "$CONFIG_FILE" && $SUDO chmod 640 "$CONFIG_FILE" - $SUDO mv "$tmp_conf" "$CONFIG_FILE" || die "Failed to install config file" - $SUDO chown root:telemt "$CONFIG_FILE" || die "Failed to set owner" - $SUDO chmod 640 "$CONFIG_FILE" || die "Failed to set config permissions" - - say "Secret for user 'hello': $toml_secret" + say " -> Config created successfully." + say " -> Generated secret for default user 'hello': $toml_secret" } generate_systemd_content() { cat </dev/null 2>&1 && [ -d /run/systemd/system ]; then - say "Installing systemd service..." - tmp_svc="$(mktemp "${TEMP_DIR:-/tmp}/${SERVICE_NAME}.service.XXXXXX")" || die "Failed to create temp service" - generate_systemd_content > "$tmp_svc" || die "Failed to generate service content" + svc="$(get_svc_mgr)" + if [ "$svc" = "systemd" ]; then + generate_systemd_content | write_root "/etc/systemd/system/${SERVICE_NAME}.service" + $SUDO chown root:root "/etc/systemd/system/${SERVICE_NAME}.service" && $SUDO chmod 644 "/etc/systemd/system/${SERVICE_NAME}.service" - $SUDO mv "$tmp_svc" "/etc/systemd/system/${SERVICE_NAME}.service" || die "Failed to move service file" - $SUDO chown root:root "/etc/systemd/system/${SERVICE_NAME}.service" - $SUDO chmod 644 "/etc/systemd/system/${SERVICE_NAME}.service" + $SUDO systemctl daemon-reload || true + $SUDO systemctl enable "$SERVICE_NAME" || true + + if ! $SUDO systemctl start "$SERVICE_NAME"; then + say "[WARNING] Failed to start service" + SERVICE_START_FAILED=1 + fi + elif [ "$svc" = "openrc" ]; then + generate_openrc_content | write_root "/etc/init.d/${SERVICE_NAME}" + $SUDO chown root:root "/etc/init.d/${SERVICE_NAME}" && $SUDO chmod 0755 "/etc/init.d/${SERVICE_NAME}" - $SUDO systemctl daemon-reload || die "Failed to reload systemd" - $SUDO systemctl enable "$SERVICE_NAME" || die "Failed to enable service" - $SUDO systemctl start "$SERVICE_NAME" || die "Failed to start service" - - elif command -v rc-update >/dev/null 2>&1; then - say "Installing OpenRC service..." - tmp_svc="$(mktemp "${TEMP_DIR:-/tmp}/${SERVICE_NAME}.init.XXXXXX")" || die "Failed to create temp file" - generate_openrc_content > "$tmp_svc" || die "Failed to generate init content" - - $SUDO mv "$tmp_svc" "/etc/init.d/${SERVICE_NAME}" || die "Failed to move service file" - $SUDO chown root:root "/etc/init.d/${SERVICE_NAME}" - $SUDO chmod 0755 "/etc/init.d/${SERVICE_NAME}" - - $SUDO rc-update add "$SERVICE_NAME" default 2>/dev/null || die "Failed to register service" - $SUDO rc-service "$SERVICE_NAME" start 2>/dev/null || die "Failed to start OpenRC service" + $SUDO rc-update add "$SERVICE_NAME" default 2>/dev/null || true + + if ! $SUDO rc-service "$SERVICE_NAME" start 2>/dev/null; then + say "[WARNING] Failed to start service" + SERVICE_START_FAILED=1 + fi else - say "No service manager found. You can start it manually with:" - if [ -n "$SUDO" ]; then - say " sudo -u telemt ${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}" - else - say " su -s /bin/sh telemt -c '${INSTALL_DIR}/${BIN_NAME} ${CONFIG_FILE}'" + cmd="\"${INSTALL_DIR}/${BIN_NAME}\" \"${CONFIG_FILE}\"" + if [ -n "$SUDO" ]; then + say " -> Service manager not found. Start manually: sudo -u telemt $cmd" + else + say " -> Service manager not found. Start manually: su -s /bin/sh telemt -c '$cmd'" fi fi } kill_user_procs() { - say "Ensuring $BIN_NAME processes are killed..." - - if pkill_cmd="$(command -v pkill 2>/dev/null)"; then - $SUDO "$pkill_cmd" -u telemt "$BIN_NAME" 2>/dev/null || true + if command -v pkill >/dev/null 2>&1; then + $SUDO pkill -u telemt "$BIN_NAME" 2>/dev/null || true sleep 1 - $SUDO "$pkill_cmd" -9 -u telemt "$BIN_NAME" 2>/dev/null || true - elif killall_cmd="$(command -v killall 2>/dev/null)"; then - $SUDO "$killall_cmd" "$BIN_NAME" 2>/dev/null || true - sleep 1 - $SUDO "$killall_cmd" -9 "$BIN_NAME" 2>/dev/null || true + $SUDO pkill -9 -u telemt "$BIN_NAME" 2>/dev/null || true + else + if command -v pgrep >/dev/null 2>&1; then + pids="$(pgrep -u telemt 2>/dev/null || true)" + else + pids="$(ps -u telemt -o pid= 2>/dev/null || true)" + fi + + if [ -n "$pids" ]; then + for pid in $pids; do + case "$pid" in ''|*[!0-9]*) continue ;; *) $SUDO kill "$pid" 2>/dev/null || true ;; esac + done + sleep 1 + for pid in $pids; do + case "$pid" in ''|*[!0-9]*) continue ;; *) $SUDO kill -9 "$pid" 2>/dev/null || true ;; esac + done + fi fi } uninstall() { - purge_data=0 - [ "$ACTION" = "purge" ] && purge_data=1 + say "Starting uninstallation of $BIN_NAME..." - say "Uninstalling $BIN_NAME..." + say ">>> Stage 1: Stopping services" stop_service - if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + say ">>> Stage 2: Removing service configuration" + svc="$(get_svc_mgr)" + if [ "$svc" = "systemd" ]; then $SUDO systemctl disable "$SERVICE_NAME" 2>/dev/null || true $SUDO rm -f "/etc/systemd/system/${SERVICE_NAME}.service" - $SUDO systemctl daemon-reload || true - elif command -v rc-update >/dev/null 2>&1; then + $SUDO systemctl daemon-reload 2>/dev/null || true + elif [ "$svc" = "openrc" ]; then $SUDO rc-update del "$SERVICE_NAME" 2>/dev/null || true $SUDO rm -f "/etc/init.d/${SERVICE_NAME}" fi + say ">>> Stage 3: Terminating user processes" kill_user_procs + say ">>> Stage 4: Removing binary" $SUDO rm -f "${INSTALL_DIR}/${BIN_NAME}" - $SUDO userdel telemt 2>/dev/null || $SUDO deluser telemt 2>/dev/null || true - $SUDO groupdel telemt 2>/dev/null || $SUDO delgroup telemt 2>/dev/null || true - - if [ "$purge_data" -eq 1 ]; then - say "Purging configuration and data..." + if [ "$ACTION" = "purge" ]; then + say ">>> Stage 5: Purging configuration, data, and user" $SUDO rm -rf "$CONFIG_DIR" "$WORK_DIR" + $SUDO rm -f "$CONFIG_FILE" + if [ "$CONFIG_PARENT_DIR" != "$CONFIG_DIR" ] && [ "$CONFIG_PARENT_DIR" != "." ] && [ "$CONFIG_PARENT_DIR" != "/" ]; then + $SUDO rmdir "$CONFIG_PARENT_DIR" 2>/dev/null || true + fi + $SUDO userdel telemt 2>/dev/null || $SUDO deluser telemt 2>/dev/null || true + $SUDO groupdel telemt 2>/dev/null || $SUDO delgroup telemt 2>/dev/null || true else - say "Note: Configuration in $CONFIG_DIR was kept. Run with '--purge' to remove it." + say "Note: Configuration and user kept. Run with 'purge' to remove completely." fi - - say "Uninstallation complete." + + printf '\n====================================================================\n' + printf ' UNINSTALLATION COMPLETE\n' + printf '====================================================================\n\n' exit 0 } -# ============================================================================ -# Main Entry Point -# ============================================================================ - case "$ACTION" in - help) - show_help - ;; - uninstall|purge) - verify_common - uninstall - ;; + help) show_help ;; + uninstall|purge) verify_common; uninstall ;; install) - say "Starting installation..." - verify_common - verify_install_deps + say "Starting installation of $BIN_NAME (Version: $TARGET_VERSION)" - ARCH="$(detect_arch)" - LIBC="$(detect_libc)" - say "Detected system: $ARCH-linux-$LIBC" + say ">>> Stage 1: Verifying environment and dependencies" + verify_common; verify_install_deps + if [ "$TARGET_VERSION" != "latest" ]; then + TARGET_VERSION="${TARGET_VERSION#v}" + fi + + ARCH="$(detect_arch)"; LIBC="$(detect_libc)" FILE_NAME="${BIN_NAME}-${ARCH}-linux-${LIBC}.tar.gz" - FILE_NAME="$(printf '%s' "$FILE_NAME" | tr -d ' \t\n\r')" - + if [ "$TARGET_VERSION" = "latest" ]; then DL_URL="https://github.com/${REPO}/releases/latest/download/${FILE_NAME}" - else + else DL_URL="https://github.com/${REPO}/releases/download/${TARGET_VERSION}/${FILE_NAME}" fi - TEMP_DIR="$(mktemp -d)" || die "Failed to create temp directory" + say ">>> Stage 2: Downloading archive" + TEMP_DIR="$(mktemp -d)" || die "Temp directory creation failed" if [ -z "$TEMP_DIR" ] || [ ! -d "$TEMP_DIR" ]; then - die "Temp directory creation failed" + die "Temp directory is invalid or was not created" fi - say "Downloading from $DL_URL..." - fetch_file "$DL_URL" "${TEMP_DIR}/archive.tar.gz" || die "Download failed (check version or network)" + fetch_file "$DL_URL" "${TEMP_DIR}/${FILE_NAME}" || die "Download failed" - gzip -dc "${TEMP_DIR}/archive.tar.gz" | tar -xf - -C "$TEMP_DIR" || die "Extraction failed" + say ">>> Stage 3: Extracting archive" + if ! gzip -dc "${TEMP_DIR}/${FILE_NAME}" | tar -xf - -C "$TEMP_DIR" 2>/dev/null; then + die "Extraction failed (downloaded archive might be invalid or 404)." + fi - EXTRACTED_BIN="$(find "$TEMP_DIR" -type f -name "$BIN_NAME" -print 2>/dev/null | head -n 1)" - [ -z "$EXTRACTED_BIN" ] && die "Binary '$BIN_NAME' not found in archive" + EXTRACTED_BIN="$(find "$TEMP_DIR" -type f -name "$BIN_NAME" -print 2>/dev/null | head -n 1 || true)" + [ -n "$EXTRACTED_BIN" ] || die "Binary '$BIN_NAME' not found in archive" - ensure_user_group - setup_dirs - stop_service - - say "Installing binary..." + say ">>> Stage 4: Setting up environment (User, Group, Directories)" + ensure_user_group; setup_dirs; stop_service + + say ">>> Stage 5: Installing binary" install_binary "$EXTRACTED_BIN" "${INSTALL_DIR}/${BIN_NAME}" - + + say ">>> Stage 6: Generating configuration" install_config + + say ">>> Stage 7: Installing and starting service" install_service - say "" - say "=============================================" - say "Installation complete!" - say "=============================================" - if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then - say "To check the logs, run:" - say " journalctl -u $SERVICE_NAME -f" - say "" - fi - say "To get user connection links, run:" - if command -v jq >/dev/null 2>&1; then - say " curl -s http://127.0.0.1:9091/v1/users | jq -r '.data[] | \"User: \\(.username)\\n\\(.links.tls[0] // empty)\"'" + if [ "${SERVICE_START_FAILED:-0}" -eq 1 ]; then + printf '\n====================================================================\n' + printf ' INSTALLATION COMPLETED WITH WARNINGS\n' + printf '====================================================================\n\n' + printf 'The service was installed but failed to start automatically.\n' + printf 'Please check the logs to determine the issue.\n\n' else - say " curl -s http://127.0.0.1:9091/v1/users" - say " (Note: Install 'jq' package to see the links nicely formatted)" + printf '\n====================================================================\n' + printf ' INSTALLATION SUCCESS\n' + printf '====================================================================\n\n' fi + + svc="$(get_svc_mgr)" + if [ "$svc" = "systemd" ]; then + printf 'To check the status of your proxy service, run:\n' + printf ' systemctl status %s\n\n' "$SERVICE_NAME" + elif [ "$svc" = "openrc" ]; then + printf 'To check the status of your proxy service, run:\n' + printf ' rc-service %s status\n\n' "$SERVICE_NAME" + fi + + printf 'To get your user connection links (for Telegram), run:\n' + if command -v jq >/dev/null 2>&1; then + printf ' curl -s http://127.0.0.1:9091/v1/users | jq -r '\''.data[] | "User: \\(.username)\\n\\(.links.tls[0] // empty)\\n"'\''\n' + else + printf ' curl -s http://127.0.0.1:9091/v1/users\n' + printf ' (Tip: Install '\''jq'\'' for a much cleaner output)\n' + fi + + printf '\n====================================================================\n' ;; esac From 8d1faece60a40bc2c2adbebd96cf5975d4bf8037 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 17:45:17 +0300 Subject: [PATCH 247/261] Instadrain + Hard-remove for long draining-state Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/model.rs | 1 + src/api/runtime_stats.rs | 1 + src/cli.rs | 1 + src/config/defaults.rs | 4 + src/config/hot_reload.rs | 9 ++ src/config/types.rs | 5 + src/maestro/me_startup.rs | 15 +++ src/transport/middle_proxy/config_updater.rs | 2 + src/transport/middle_proxy/health.rs | 92 ++++++++++++++----- .../middle_proxy/health_adversarial_tests.rs | 5 +- .../middle_proxy/health_integration_tests.rs | 1 + .../middle_proxy/health_regression_tests.rs | 66 ++++++++++--- src/transport/middle_proxy/mod.rs | 2 +- src/transport/middle_proxy/pool.rs | 5 + src/transport/middle_proxy/pool_status.rs | 2 + 15 files changed, 170 insertions(+), 41 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index ac4e297..e98de8b 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -364,6 +364,7 @@ pub(super) struct MinimalMeRuntimeData { pub(super) me_reconnect_backoff_cap_ms: u64, pub(super) me_reconnect_fast_retry_count: u32, pub(super) me_pool_drain_ttl_secs: u64, + pub(super) me_instadrain: bool, pub(super) me_pool_drain_soft_evict_enabled: bool, pub(super) me_pool_drain_soft_evict_grace_secs: u64, pub(super) me_pool_drain_soft_evict_per_writer: u8, diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index f8948d1..cdeacc0 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -431,6 +431,7 @@ async fn get_minimal_payload_cached( me_reconnect_backoff_cap_ms: runtime.me_reconnect_backoff_cap_ms, me_reconnect_fast_retry_count: runtime.me_reconnect_fast_retry_count, me_pool_drain_ttl_secs: runtime.me_pool_drain_ttl_secs, + me_instadrain: runtime.me_instadrain, me_pool_drain_soft_evict_enabled: runtime.me_pool_drain_soft_evict_enabled, me_pool_drain_soft_evict_grace_secs: runtime.me_pool_drain_soft_evict_grace_secs, me_pool_drain_soft_evict_per_writer: runtime.me_pool_drain_soft_evict_per_writer, diff --git a/src/cli.rs b/src/cli.rs index a1182a7..5fbd7d5 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -198,6 +198,7 @@ desync_all_full = false update_every = 43200 hardswap = false me_pool_drain_ttl_secs = 90 +me_instadrain = false me_pool_min_fresh_ratio = 0.8 me_reinit_drain_timeout_secs = 120 diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 54a53b3..6d74c93 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -613,6 +613,10 @@ pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 { 90 } +pub(crate) fn default_me_instadrain() -> bool { + false +} + pub(crate) fn default_me_pool_drain_threshold() -> u64 { 128 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 7b94999..1315f9c 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -56,6 +56,7 @@ pub struct HotFields { pub me_reinit_coalesce_window_ms: u64, pub hardswap: bool, pub me_pool_drain_ttl_secs: u64, + pub me_instadrain: bool, pub me_pool_drain_threshold: u64, pub me_pool_drain_soft_evict_enabled: bool, pub me_pool_drain_soft_evict_grace_secs: u64, @@ -143,6 +144,7 @@ impl HotFields { me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms, hardswap: cfg.general.hardswap, me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs, + me_instadrain: cfg.general.me_instadrain, me_pool_drain_threshold: cfg.general.me_pool_drain_threshold, me_pool_drain_soft_evict_enabled: cfg.general.me_pool_drain_soft_evict_enabled, me_pool_drain_soft_evict_grace_secs: cfg.general.me_pool_drain_soft_evict_grace_secs, @@ -477,6 +479,7 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig { cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms; cfg.general.hardswap = new.general.hardswap; cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs; + cfg.general.me_instadrain = new.general.me_instadrain; cfg.general.me_pool_drain_threshold = new.general.me_pool_drain_threshold; cfg.general.me_pool_drain_soft_evict_enabled = new.general.me_pool_drain_soft_evict_enabled; cfg.general.me_pool_drain_soft_evict_grace_secs = @@ -869,6 +872,12 @@ fn log_changes( old_hot.me_pool_drain_ttl_secs, new_hot.me_pool_drain_ttl_secs, ); } + if old_hot.me_instadrain != new_hot.me_instadrain { + info!( + "config reload: me_instadrain: {} → {}", + old_hot.me_instadrain, new_hot.me_instadrain, + ); + } if old_hot.me_pool_drain_threshold != new_hot.me_pool_drain_threshold { info!( diff --git a/src/config/types.rs b/src/config/types.rs index 047f3c2..ecd051d 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -812,6 +812,10 @@ pub struct GeneralConfig { #[serde(default = "default_me_pool_drain_ttl_secs")] pub me_pool_drain_ttl_secs: u64, + /// Force-remove any draining writer on the next cleanup tick, regardless of age/deadline. + #[serde(default = "default_me_instadrain")] + pub me_instadrain: bool, + /// Maximum allowed number of draining ME writers before oldest ones are force-closed in batches. /// Set to 0 to disable threshold-based draining cleanup and keep timeout-only behavior. #[serde(default = "default_me_pool_drain_threshold")] @@ -1020,6 +1024,7 @@ impl Default for GeneralConfig { me_secret_atomic_snapshot: default_me_secret_atomic_snapshot(), proxy_secret_len_max: default_proxy_secret_len_max(), me_pool_drain_ttl_secs: default_me_pool_drain_ttl_secs(), + me_instadrain: default_me_instadrain(), me_pool_drain_threshold: default_me_pool_drain_threshold(), me_pool_drain_soft_evict_enabled: default_me_pool_drain_soft_evict_enabled(), me_pool_drain_soft_evict_grace_secs: default_me_pool_drain_soft_evict_grace_secs(), diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs index 827b00c..0b1310a 100644 --- a/src/maestro/me_startup.rs +++ b/src/maestro/me_startup.rs @@ -237,6 +237,7 @@ pub(crate) async fn initialize_me_pool( config.general.me_adaptive_floor_max_warm_writers_global, config.general.hardswap, config.general.me_pool_drain_ttl_secs, + config.general.me_instadrain, config.general.me_pool_drain_threshold, config.general.me_pool_drain_soft_evict_enabled, config.general.me_pool_drain_soft_evict_grace_secs, @@ -342,6 +343,13 @@ pub(crate) async fn initialize_me_pool( ) .await; }); + let pool_drain_enforcer = pool_bg.clone(); + tokio::spawn(async move { + crate::transport::middle_proxy::me_drain_timeout_enforcer( + pool_drain_enforcer, + ) + .await; + }); break; } Err(e) => { @@ -409,6 +417,13 @@ pub(crate) async fn initialize_me_pool( ) .await; }); + let pool_drain_enforcer = pool.clone(); + tokio::spawn(async move { + crate::transport::middle_proxy::me_drain_timeout_enforcer( + pool_drain_enforcer, + ) + .await; + }); break Some(pool); } diff --git a/src/transport/middle_proxy/config_updater.rs b/src/transport/middle_proxy/config_updater.rs index 43a3569..26ec497 100644 --- a/src/transport/middle_proxy/config_updater.rs +++ b/src/transport/middle_proxy/config_updater.rs @@ -298,6 +298,7 @@ async fn run_update_cycle( pool.update_runtime_reinit_policy( cfg.general.hardswap, cfg.general.me_pool_drain_ttl_secs, + cfg.general.me_instadrain, cfg.general.me_pool_drain_threshold, cfg.general.me_pool_drain_soft_evict_enabled, cfg.general.me_pool_drain_soft_evict_grace_secs, @@ -530,6 +531,7 @@ pub async fn me_config_updater( pool.update_runtime_reinit_policy( cfg.general.hardswap, cfg.general.me_pool_drain_ttl_secs, + cfg.general.me_instadrain, cfg.general.me_pool_drain_threshold, cfg.general.me_pool_drain_soft_evict_enabled, cfg.general.me_pool_drain_soft_evict_grace_secs, diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 21619c7..8b62cff 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -12,6 +12,7 @@ use crate::crypto::SecureRandom; use crate::network::IpFamily; use super::MePool; +use super::pool::MeWriter; const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff #[allow(dead_code)] @@ -31,10 +32,7 @@ const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256; const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN: usize = 8; const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX: usize = 256; const HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS: u64 = 1; -#[cfg(not(test))] -const HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE: bool = true; -#[cfg(test)] -const HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE: bool = false; +const HEALTH_DRAIN_TIMEOUT_ENFORCER_INTERVAL_SECS: u64 = 1; #[derive(Debug, Clone)] struct DcFloorPlanEntry { @@ -131,6 +129,55 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c } } +pub async fn me_drain_timeout_enforcer(pool: Arc) { + let mut drain_warn_next_allowed: HashMap = HashMap::new(); + let mut drain_soft_evict_next_allowed: HashMap = HashMap::new(); + loop { + tokio::time::sleep(Duration::from_secs( + HEALTH_DRAIN_TIMEOUT_ENFORCER_INTERVAL_SECS, + )) + .await; + reap_draining_writers( + &pool, + &mut drain_warn_next_allowed, + &mut drain_soft_evict_next_allowed, + ) + .await; + } +} + +fn draining_writer_timeout_expired( + pool: &MePool, + writer: &MeWriter, + now_epoch_secs: u64, + drain_ttl_secs: u64, +) -> bool { + if pool + .me_instadrain + .load(std::sync::atomic::Ordering::Relaxed) + { + return true; + } + + let deadline_epoch_secs = writer + .drain_deadline_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + if deadline_epoch_secs != 0 { + return now_epoch_secs >= deadline_epoch_secs; + } + + if drain_ttl_secs == 0 { + return false; + } + let drain_started_at_epoch_secs = writer + .draining_started_at_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + if drain_started_at_epoch_secs == 0 { + return false; + } + now_epoch_secs.saturating_sub(drain_started_at_epoch_secs) > drain_ttl_secs +} + pub(super) async fn reap_draining_writers( pool: &Arc, warn_next_allowed: &mut HashMap, @@ -146,11 +193,16 @@ pub(super) async fn reap_draining_writers( let activity = pool.registry.writer_activity_snapshot().await; let mut draining_writers = Vec::new(); let mut empty_writer_ids = Vec::::new(); + let mut timeout_expired_writer_ids = Vec::::new(); let mut force_close_writer_ids = Vec::::new(); for writer in writers { if !writer.draining.load(std::sync::atomic::Ordering::Relaxed) { continue; } + if draining_writer_timeout_expired(pool, &writer, now_epoch_secs, drain_ttl_secs) { + timeout_expired_writer_ids.push(writer.id); + continue; + } if activity .bound_clients_by_writer .get(&writer.id) @@ -163,11 +215,6 @@ pub(super) async fn reap_draining_writers( } draining_writers.push(writer); } - if HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE { - for writer in draining_writers.drain(..) { - force_close_writer_ids.push(writer.id); - } - } if drain_threshold > 0 && draining_writers.len() > drain_threshold as usize { draining_writers.sort_by(|left, right| { @@ -221,14 +268,6 @@ pub(super) async fn reap_draining_writers( "ME draining writer remains non-empty past drain TTL" ); } - let deadline_epoch_secs = writer - .drain_deadline_epoch_secs - .load(std::sync::atomic::Ordering::Relaxed); - if deadline_epoch_secs != 0 && now_epoch_secs >= deadline_epoch_secs { - warn!(writer_id = writer.id, "Drain timeout, force-closing"); - force_close_writer_ids.push(writer.id); - active_draining_writer_ids.remove(&writer.id); - } } warn_next_allowed.retain(|writer_id, _| active_draining_writer_ids.contains(writer_id)); @@ -313,15 +352,21 @@ pub(super) async fn reap_draining_writers( } } + let mut closed_writer_ids = HashSet::::new(); + for writer_id in timeout_expired_writer_ids { + if !closed_writer_ids.insert(writer_id) { + continue; + } + pool.stats.increment_pool_force_close_total(); + pool.remove_writer_and_close_clients(writer_id).await; + pool.stats + .increment_me_draining_writers_reap_progress_total(); + } + let requested_force_close = force_close_writer_ids.len(); let requested_empty_close = empty_writer_ids.len(); let requested_close_total = requested_force_close.saturating_add(requested_empty_close); - let close_budget = if HEALTH_DRAIN_STRICT_IMMEDIATE_FORCE_CLOSE { - requested_close_total - } else { - health_drain_close_budget() - }; - let mut closed_writer_ids = HashSet::::new(); + let close_budget = health_drain_close_budget(); let mut closed_total = 0usize; for writer_id in force_close_writer_ids { if closed_total >= close_budget { @@ -1581,6 +1626,7 @@ mod tests { general.me_adaptive_floor_max_warm_writers_global, general.hardswap, general.me_pool_drain_ttl_secs, + general.me_instadrain, general.me_pool_drain_threshold, general.me_pool_drain_soft_evict_enabled, general.me_pool_drain_soft_evict_grace_secs, diff --git a/src/transport/middle_proxy/health_adversarial_tests.rs b/src/transport/middle_proxy/health_adversarial_tests.rs index 3f182e4..ae517b3 100644 --- a/src/transport/middle_proxy/health_adversarial_tests.rs +++ b/src/transport/middle_proxy/health_adversarial_tests.rs @@ -81,6 +81,7 @@ async fn make_pool( general.me_adaptive_floor_max_warm_writers_global, general.hardswap, general.me_pool_drain_ttl_secs, + general.me_instadrain, general.me_pool_drain_threshold, general.me_pool_drain_soft_evict_enabled, general.me_pool_drain_soft_evict_grace_secs, @@ -213,7 +214,7 @@ async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycle insert_draining_writer( &pool, writer_id, - now_epoch_secs.saturating_sub(600).saturating_add(writer_id), + now_epoch_secs.saturating_sub(20), 1, 0, ) @@ -230,7 +231,7 @@ async fn reap_draining_writers_respects_threshold_across_multiple_overflow_cycle } assert_eq!(writer_count(&pool).await, threshold as usize); - assert_eq!(sorted_writer_ids(&pool).await, vec![58, 59, 60]); + assert_eq!(sorted_writer_ids(&pool).await, vec![1, 2, 3]); } #[tokio::test] diff --git a/src/transport/middle_proxy/health_integration_tests.rs b/src/transport/middle_proxy/health_integration_tests.rs index 7f99d2a..fbbffce 100644 --- a/src/transport/middle_proxy/health_integration_tests.rs +++ b/src/transport/middle_proxy/health_integration_tests.rs @@ -80,6 +80,7 @@ async fn make_pool( general.me_adaptive_floor_max_warm_writers_global, general.hardswap, general.me_pool_drain_ttl_secs, + general.me_instadrain, general.me_pool_drain_threshold, general.me_pool_drain_soft_evict_enabled, general.me_pool_drain_soft_evict_grace_secs, diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs index 565ac74..bcdaf2e 100644 --- a/src/transport/middle_proxy/health_regression_tests.rs +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -74,6 +74,7 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc { general.me_adaptive_floor_max_warm_writers_global, general.hardswap, general.me_pool_drain_ttl_secs, + general.me_instadrain, general.me_pool_drain_threshold, general.me_pool_drain_soft_evict_enabled, general.me_pool_drain_soft_evict_grace_secs, @@ -180,8 +181,14 @@ async fn current_writer_ids(pool: &Arc) -> Vec { async fn reap_draining_writers_drops_warn_state_for_removed_writer() { let pool = make_pool(128).await; let now_epoch_secs = MePool::now_epoch_secs(); - let conn_ids = - insert_draining_writer(&pool, 7, now_epoch_secs.saturating_sub(180), 1, 0).await; + let conn_ids = insert_draining_writer( + &pool, + 7, + now_epoch_secs.saturating_sub(180), + 1, + now_epoch_secs.saturating_add(3_600), + ) + .await; let mut warn_next_allowed = HashMap::new(); let mut soft_evict_next_allowed = HashMap::new(); @@ -331,17 +338,17 @@ async fn reap_draining_writers_deadline_force_close_applies_under_threshold() { #[tokio::test] async fn reap_draining_writers_limits_closes_per_health_tick() { - let pool = make_pool(128).await; + let pool = make_pool(1).await; let now_epoch_secs = MePool::now_epoch_secs(); let close_budget = health_drain_close_budget(); - let writer_total = close_budget.saturating_add(19); + let writer_total = close_budget.saturating_add(20); for writer_id in 1..=writer_total as u64 { insert_draining_writer( &pool, writer_id, now_epoch_secs.saturating_sub(20), 1, - now_epoch_secs.saturating_sub(1), + 0, ) .await; } @@ -364,8 +371,8 @@ async fn reap_draining_writers_backlog_drains_across_ticks() { &pool, writer_id, now_epoch_secs.saturating_sub(20), - 1, - now_epoch_secs.saturating_sub(1), + 0, + 0, ) .await; } @@ -393,7 +400,7 @@ async fn reap_draining_writers_threshold_backlog_converges_to_threshold() { insert_draining_writer( &pool, writer_id, - now_epoch_secs.saturating_sub(200).saturating_add(writer_id), + now_epoch_secs.saturating_sub(20), 1, 0, ) @@ -429,27 +436,27 @@ async fn reap_draining_writers_threshold_zero_preserves_non_expired_non_empty_wr #[tokio::test] async fn reap_draining_writers_prioritizes_force_close_before_empty_cleanup() { - let pool = make_pool(128).await; + let pool = make_pool(1).await; let now_epoch_secs = MePool::now_epoch_secs(); let close_budget = health_drain_close_budget(); - for writer_id in 1..=close_budget as u64 { + for writer_id in 1..=close_budget.saturating_add(1) as u64 { insert_draining_writer( &pool, writer_id, now_epoch_secs.saturating_sub(20), 1, - now_epoch_secs.saturating_sub(1), + 0, ) .await; } - let empty_writer_id = close_budget as u64 + 1; + let empty_writer_id = close_budget.saturating_add(2) as u64; insert_draining_writer(&pool, empty_writer_id, now_epoch_secs.saturating_sub(20), 0, 0).await; let mut warn_next_allowed = HashMap::new(); let mut soft_evict_next_allowed = HashMap::new(); reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; - assert_eq!(current_writer_ids(&pool).await, vec![empty_writer_id]); + assert_eq!(current_writer_ids(&pool).await, vec![1, empty_writer_id]); } #[tokio::test] @@ -571,7 +578,14 @@ async fn reap_draining_writers_soft_evicts_stuck_writer_with_per_writer_cap() { .store(1, Ordering::Relaxed); let now_epoch_secs = MePool::now_epoch_secs(); - insert_draining_writer(&pool, 77, now_epoch_secs.saturating_sub(240), 3, 0).await; + insert_draining_writer( + &pool, + 77, + now_epoch_secs.saturating_sub(240), + 3, + now_epoch_secs.saturating_add(3_600), + ) + .await; let mut warn_next_allowed = HashMap::new(); let mut soft_evict_next_allowed = HashMap::new(); @@ -595,7 +609,14 @@ async fn reap_draining_writers_soft_evict_respects_cooldown_per_writer() { .store(60_000, Ordering::Relaxed); let now_epoch_secs = MePool::now_epoch_secs(); - insert_draining_writer(&pool, 88, now_epoch_secs.saturating_sub(240), 3, 0).await; + insert_draining_writer( + &pool, + 88, + now_epoch_secs.saturating_sub(240), + 3, + now_epoch_secs.saturating_add(3_600), + ) + .await; let mut warn_next_allowed = HashMap::new(); let mut soft_evict_next_allowed = HashMap::new(); @@ -608,6 +629,21 @@ async fn reap_draining_writers_soft_evict_respects_cooldown_per_writer() { assert_eq!(pool.stats.get_pool_drain_soft_evict_writer_total(), 1); } +#[tokio::test] +async fn reap_draining_writers_instadrain_removes_non_expired_writers_immediately() { + let pool = make_pool(0).await; + pool.me_instadrain.store(true, Ordering::Relaxed); + let now_epoch_secs = MePool::now_epoch_secs(); + insert_draining_writer(&pool, 101, now_epoch_secs.saturating_sub(5), 1, 0).await; + insert_draining_writer(&pool, 102, now_epoch_secs.saturating_sub(4), 1, 0).await; + let mut warn_next_allowed = HashMap::new(); + let mut soft_evict_next_allowed = HashMap::new(); + + reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; + + assert!(current_writer_ids(&pool).await.is_empty()); +} + #[test] fn general_config_default_drain_threshold_remains_enabled() { assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 128); diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 590c996..26ded29 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -30,7 +30,7 @@ mod health_adversarial_tests; use bytes::Bytes; -pub use health::me_health_monitor; +pub use health::{me_drain_timeout_enforcer, me_health_monitor}; #[allow(unused_imports)] pub use ping::{run_me_ping, format_sample_line, format_me_route, MePingReport, MePingSample, MePingFamily}; pub use pool::MePool; diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index d09f07c..441d41d 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -171,6 +171,7 @@ pub struct MePool { pub(super) endpoint_quarantine: Arc>>, pub(super) kdf_material_fingerprint: Arc>>, pub(super) me_pool_drain_ttl_secs: AtomicU64, + pub(super) me_instadrain: AtomicBool, pub(super) me_pool_drain_threshold: AtomicU64, pub(super) me_pool_drain_soft_evict_enabled: AtomicBool, pub(super) me_pool_drain_soft_evict_grace_secs: AtomicU64, @@ -279,6 +280,7 @@ impl MePool { me_adaptive_floor_max_warm_writers_global: u32, hardswap: bool, me_pool_drain_ttl_secs: u64, + me_instadrain: bool, me_pool_drain_threshold: u64, me_pool_drain_soft_evict_enabled: bool, me_pool_drain_soft_evict_grace_secs: u64, @@ -462,6 +464,7 @@ impl MePool { endpoint_quarantine: Arc::new(Mutex::new(HashMap::new())), kdf_material_fingerprint: Arc::new(RwLock::new(HashMap::new())), me_pool_drain_ttl_secs: AtomicU64::new(me_pool_drain_ttl_secs), + me_instadrain: AtomicBool::new(me_instadrain), me_pool_drain_threshold: AtomicU64::new(me_pool_drain_threshold), me_pool_drain_soft_evict_enabled: AtomicBool::new(me_pool_drain_soft_evict_enabled), me_pool_drain_soft_evict_grace_secs: AtomicU64::new(me_pool_drain_soft_evict_grace_secs), @@ -524,6 +527,7 @@ impl MePool { &self, hardswap: bool, drain_ttl_secs: u64, + instadrain: bool, pool_drain_threshold: u64, pool_drain_soft_evict_enabled: bool, pool_drain_soft_evict_grace_secs: u64, @@ -568,6 +572,7 @@ impl MePool { self.hardswap.store(hardswap, Ordering::Relaxed); self.me_pool_drain_ttl_secs .store(drain_ttl_secs, Ordering::Relaxed); + self.me_instadrain.store(instadrain, Ordering::Relaxed); self.me_pool_drain_threshold .store(pool_drain_threshold, Ordering::Relaxed); self.me_pool_drain_soft_evict_enabled diff --git a/src/transport/middle_proxy/pool_status.rs b/src/transport/middle_proxy/pool_status.rs index 214ee49..5fe45cb 100644 --- a/src/transport/middle_proxy/pool_status.rs +++ b/src/transport/middle_proxy/pool_status.rs @@ -126,6 +126,7 @@ pub(crate) struct MeApiRuntimeSnapshot { pub me_reconnect_backoff_cap_ms: u64, pub me_reconnect_fast_retry_count: u32, pub me_pool_drain_ttl_secs: u64, + pub me_instadrain: bool, pub me_pool_drain_soft_evict_enabled: bool, pub me_pool_drain_soft_evict_grace_secs: u64, pub me_pool_drain_soft_evict_per_writer: u8, @@ -583,6 +584,7 @@ impl MePool { me_reconnect_backoff_cap_ms: self.me_reconnect_backoff_cap.as_millis() as u64, me_reconnect_fast_retry_count: self.me_reconnect_fast_retry_count, me_pool_drain_ttl_secs: self.me_pool_drain_ttl_secs.load(Ordering::Relaxed), + me_instadrain: self.me_instadrain.load(Ordering::Relaxed), me_pool_drain_soft_evict_enabled: self .me_pool_drain_soft_evict_enabled .load(Ordering::Relaxed), From ad8ada33c9c76558bcff2756a09e4cccacd5565f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:24:01 +0300 Subject: [PATCH 248/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d4ef990..2a9cbe3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.24" +version = "3.3.25" edition = "2024" [dependencies] From 1c6c73beda08c2b26d687329d332b8252faeb105 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 00:41:40 +0300 Subject: [PATCH 249/261] ME Writers Anti-stuck and Quarantine fixes Co-Authored-By: Nook Scheel --- src/transport/middle_proxy/pool_refill.rs | 14 +++++-- src/transport/middle_proxy/pool_writer.rs | 47 +++++++++++++++-------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index fc916f4..895bdb5 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -71,11 +71,19 @@ impl MePool { } if let Some((addr, expiry)) = earliest_quarantine { + let remaining = expiry.saturating_duration_since(now); + if remaining.is_zero() { + return vec![addr]; + } + drop(guard); debug!( %addr, - wait_ms = expiry.saturating_duration_since(now).as_millis(), - "All ME endpoints are quarantined for the DC group; retrying earliest one" + wait_ms = remaining.as_millis(), + "All ME endpoints quarantined; waiting for earliest to expire" ); + // After sleeping, the quarantine entry is expired but not removed yet. + // Callers that check is_endpoint_quarantined() will lazily clean it via retain(). + tokio::time::sleep(remaining).await; return vec![addr]; } @@ -311,4 +319,4 @@ impl MePool { dc_guard.remove(&dc_key); }); } -} +} \ No newline at end of file diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index 7d78b84..fb1ba10 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -242,21 +242,27 @@ impl MePool { stats_reader_close.increment_me_idle_close_by_peer_total(); info!(writer_id, "ME socket closed by peer on idle writer"); } - if let Some(pool) = pool.upgrade() - && cleanup_for_reader - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() + if cleanup_for_reader + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() { - pool.remove_writer_and_close_clients(writer_id).await; + if let Some(pool) = pool.upgrade() { + pool.remove_writer_and_close_clients(writer_id).await; + } else { + // Pool is gone (shutdown). Remove writer from Vec directly + // as a last resort — no registry/refill side effects needed + // during shutdown. conn_count is not decremented here because + // the pool (and its counters) are already dropped. + let mut ws = writers_arc.write().await; + ws.retain(|w| w.id != writer_id); + debug!(writer_id, remaining = ws.len(), "Writer removed during pool shutdown"); + } } if let Err(e) = res { if !idle_close_by_peer { warn!(error = %e, "ME reader ended"); } } - let mut ws = writers_arc.write().await; - ws.retain(|w| w.id != writer_id); - info!(remaining = ws.len(), "Dead ME writer removed from pool"); }); let pool_ping = Arc::downgrade(self); @@ -346,12 +352,13 @@ impl MePool { stats_ping.increment_me_keepalive_failed(); debug!("ME ping failed, removing dead writer"); cancel_ping.cancel(); - if let Some(pool) = pool_ping.upgrade() - && cleanup_for_ping - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() + if cleanup_for_ping + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() { - pool.remove_writer_and_close_clients(writer_id).await; + if let Some(pool) = pool_ping.upgrade() { + pool.remove_writer_and_close_clients(writer_id).await; + } } break; } @@ -556,13 +563,19 @@ impl MePool { } } } + // Quarantine flapping endpoints regardless of draining state — + // a rapidly dying endpoint is unstable whether it was draining or not. + if let Some(addr) = removed_addr { + if let Some(uptime) = removed_uptime { + self.maybe_quarantine_flapping_endpoint(addr, uptime).await; + } + } + // Only trigger immediate refill for unexpected (non-draining) removals. + // Draining writers are intentionally being retired. if trigger_refill && let Some(addr) = removed_addr && let Some(writer_dc) = removed_dc { - if let Some(uptime) = removed_uptime { - self.maybe_quarantine_flapping_endpoint(addr, uptime).await; - } self.trigger_immediate_refill_for_dc(addr, writer_dc); } conns @@ -645,4 +658,4 @@ impl MePool { } } } -} +} \ No newline at end of file From e40361b1717d2567d5f528c503f7af18a98ca258 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 00:45:04 +0300 Subject: [PATCH 250/261] Cargo.toml + Cargo.lock Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- Cargo.lock | 302 ++++++++++++++++++++++++++++------------------------- Cargo.toml | 3 +- 2 files changed, 164 insertions(+), 141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a704404..79e302f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,15 +45,24 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] [[package]] name = "asn1-rs" @@ -135,9 +144,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "block-buffer" @@ -159,9 +168,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytes" @@ -186,9 +195,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.55" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "shlex", @@ -214,9 +223,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -265,18 +274,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.58" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.58" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstyle", "clap_lex", @@ -284,9 +293,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "core-foundation-sys" @@ -486,7 +495,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -572,9 +581,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -587,9 +596,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -597,15 +606,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -614,38 +623,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -655,7 +664,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -691,20 +699,20 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", "wasip2", "wasip3", ] @@ -894,7 +902,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -1076,9 +1084,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "ipnetwork" @@ -1127,9 +1135,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "js-sys" -version = "0.3.85" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", @@ -1169,26 +1177,27 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.181" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459427e2af2b9c839b132acb702a1c654d95e10f8c326bfc2ad11310e458b1c5" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "libc", - "redox_syscall 0.7.1", + "plain", + "redox_syscall 0.7.3", ] [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1295,7 +1304,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases 0.1.1", "libc", @@ -1318,7 +1327,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -1385,9 +1394,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oorandom" @@ -1426,9 +1435,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -1436,6 +1445,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "plotters" version = "0.3.7" @@ -1495,7 +1510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1515,7 +1530,7 @@ checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.10.0", + "bitflags 2.11.0", "num-traits", "rand", "rand_chacha", @@ -1545,7 +1560,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.2", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -1554,9 +1569,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", "getrandom 0.3.4", @@ -1582,16 +1597,16 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -1602,6 +1617,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.2" @@ -1666,16 +1687,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] name = "redox_syscall" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35985aa610addc02e24fc232012c86fd11f14111180f902b67e2d5331f8ebf2b" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] @@ -1703,9 +1724,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" @@ -1785,11 +1806,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -1798,9 +1819,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "once_cell", "ring", @@ -1903,7 +1924,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2011,12 +2032,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2044,9 +2065,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.114" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -2082,15 +2103,16 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "telemt" -version = "3.3.19" +version = "3.3.25" dependencies = [ "aes", "anyhow", + "arc-swap", "base64", "bytes", "cbc", @@ -2143,12 +2165,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.25.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2180,7 +2202,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2191,7 +2213,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2256,9 +2278,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -2271,9 +2293,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -2281,7 +2303,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.2", + "socket2 0.6.3", "tokio-macros", "tracing", "windows-sys 0.61.2", @@ -2289,13 +2311,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2409,7 +2431,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "bytes", "futures-util", "http", @@ -2452,7 +2474,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2478,9 +2500,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -2514,9 +2536,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-xid" @@ -2614,9 +2636,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", @@ -2627,9 +2649,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.58" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", "futures-util", @@ -2641,9 +2663,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2651,22 +2673,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] @@ -2699,7 +2721,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", "semver", @@ -2707,9 +2729,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.85" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", @@ -2773,7 +2795,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2784,7 +2806,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3035,9 +3057,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] @@ -3072,7 +3094,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn 2.0.114", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -3088,7 +3110,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -3100,7 +3122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.10.0", + "bitflags 2.11.0", "indexmap", "log", "serde", @@ -3172,28 +3194,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.8.39" +version = "0.8.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.39" +version = "0.8.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3213,7 +3235,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "synstructure 0.13.2", ] @@ -3234,7 +3256,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3267,11 +3289,11 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "zmij" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de98dfa5d5b7fef4ee834d0073d560c9ca7b6c46a71d058c48db7960f8cfaf7" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 2a9cbe3..fab12f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.25" +version = "3.3.26" edition = "2024" [dependencies] @@ -40,6 +40,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } parking_lot = "0.12" dashmap = "5.5" +arc-swap = "1.7" lru = "0.16" rand = "0.9" chrono = { version = "0.4", features = ["serde"] } From 4f11aa07722925272cbefa2292598fd5a302f2d6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:25:07 +0300 Subject: [PATCH 251/261] Update README.md --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2102a3a..f58a300 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,12 @@ ### 🇷🇺 RU -#### Релиз 3.3.15 Semistable +#### О релизах [3.3.15](https://github.com/telemt/telemt/releases/tag/3.3.15) по итогам работы в продакшн признан одним из самых стабильных и рекомендуется к использованию, когда cutting-edge фичи некритичны! +[3.3.24](https://github.com/telemt/telemt/releases/tag/3.3.24) даёт баланс стабильности и передового функционала, а так же последние исправления по безопасности и багам + Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **API**, **статистики**, **UX** --- @@ -40,10 +42,12 @@ ### 🇬🇧 EN -#### Release 3.3.15 Semistable +#### About releases [3.3.15](https://github.com/telemt/telemt/releases/tag/3.3.15) is, based on the results of his work in production, recognized as one of the most stable and recommended for use when cutting-edge features are not so necessary! +[3.3.24](https://github.com/telemt/telemt/releases/tag/3.3.24) provides a balance of stability and advanced functionality, as well as the latest security and bug fixes + We are looking forward to your feedback and improvement proposals — especially regarding **API**, **statistics**, **UX** --- From ed4d1167dd6c33f0307e314c9b3c1807708b7215 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:09:23 +0300 Subject: [PATCH 252/261] ME Writers Advanced Cleanup Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/transport/middle_proxy/health.rs | 92 ++++++++++++++++++++++- src/transport/middle_proxy/pool_refill.rs | 44 ++++++----- src/transport/middle_proxy/pool_writer.rs | 31 +++++--- 3 files changed, 133 insertions(+), 34 deletions(-) diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 8b62cff..5829de4 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -1327,6 +1327,33 @@ async fn recover_single_endpoint_outage( } let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms(); + let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine(); + if !bypass_quarantine { + let quarantine_remaining = { + let mut guard = pool.endpoint_quarantine.lock().await; + let quarantine_now = Instant::now(); + guard.retain(|_, expiry| *expiry > quarantine_now); + guard + .get(&endpoint) + .map(|expiry| expiry.saturating_duration_since(quarantine_now)) + }; + + if let Some(remaining) = quarantine_remaining + && !remaining.is_zero() + { + outage_next_attempt.insert(key, now + remaining); + debug!( + dc = %key.0, + family = ?key.1, + %endpoint, + required, + wait_ms = remaining.as_millis(), + "Single-endpoint outage reconnect deferred by endpoint quarantine" + ); + return; + } + } + if *reconnect_budget == 0 { outage_next_attempt.insert(key, now + Duration::from_millis(min_backoff_ms.max(250))); debug!( @@ -1342,7 +1369,6 @@ async fn recover_single_endpoint_outage( pool.stats .increment_me_single_endpoint_outage_reconnect_attempt_total(); - let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine(); let attempt_ok = if bypass_quarantine { pool.stats .increment_me_single_endpoint_quarantine_bypass_total(); @@ -1561,9 +1587,10 @@ mod tests { use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; - use super::reap_draining_writers; + use super::{reap_draining_writers, recover_single_endpoint_outage}; use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; use crate::crypto::SecureRandom; + use crate::network::IpFamily; use crate::network::probe::NetworkDecision; use crate::stats::Stats; use crate::transport::middle_proxy::codec::WriterCommand; @@ -1745,4 +1772,65 @@ mod tests { assert_eq!(pool.registry.get_writer(conn_b).await.unwrap().writer_id, 20); assert_eq!(pool.registry.get_writer(conn_c).await.unwrap().writer_id, 30); } + + #[tokio::test] + async fn removing_draining_writer_still_quarantines_flapping_endpoint() { + let pool = make_pool(1).await; + let now_epoch_secs = MePool::now_epoch_secs(); + let writer_id = 11u64; + let writer_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4000 + writer_id as u16); + let conn_id = + insert_draining_writer(&pool, writer_id, now_epoch_secs.saturating_sub(5)).await; + + assert!(pool + .registry + .evict_bound_conn_if_writer(conn_id, writer_id) + .await); + pool.remove_writer_and_close_clients(writer_id).await; + + assert!(pool.is_endpoint_quarantined(writer_addr).await); + } + + #[tokio::test] + async fn single_endpoint_outage_respects_quarantine_when_bypass_disabled() { + let pool = make_pool(1).await; + pool.me_single_endpoint_outage_disable_quarantine + .store(false, Ordering::Relaxed); + + let key = (2, IpFamily::V4); + let endpoint = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 7443); + let quarantine_ttl = Duration::from_millis(200); + { + let mut guard = pool.endpoint_quarantine.lock().await; + guard.insert(endpoint, Instant::now() + quarantine_ttl); + } + + let rng = Arc::new(SecureRandom::new()); + let mut outage_backoff = HashMap::new(); + let mut outage_next_attempt = HashMap::new(); + let mut reconnect_budget = 1usize; + let started_at = Instant::now(); + + recover_single_endpoint_outage( + &pool, + &rng, + key, + endpoint, + 1, + &mut outage_backoff, + &mut outage_next_attempt, + &mut reconnect_budget, + ) + .await; + + assert_eq!(reconnect_budget, 1); + assert_eq!( + pool.stats + .get_me_single_endpoint_outage_reconnect_attempt_total(), + 0 + ); + assert_eq!(pool.stats.get_me_single_endpoint_quarantine_bypass_total(), 0); + let next_attempt = outage_next_attempt.get(&key).copied().unwrap(); + assert!(next_attempt >= started_at + Duration::from_millis(120)); + } } diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 895bdb5..43e2e6b 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -49,28 +49,31 @@ impl MePool { return Vec::new(); } - let mut guard = self.endpoint_quarantine.lock().await; - let now = Instant::now(); - guard.retain(|_, expiry| *expiry > now); + loop { + let mut guard = self.endpoint_quarantine.lock().await; + let now = Instant::now(); + guard.retain(|_, expiry| *expiry > now); - let mut ready = Vec::::with_capacity(endpoints.len()); - let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None; - for addr in endpoints { - if let Some(expiry) = guard.get(addr).copied() { - match earliest_quarantine { - Some((_, current_expiry)) if current_expiry <= expiry => {} - _ => earliest_quarantine = Some((*addr, expiry)), + let mut ready = Vec::::with_capacity(endpoints.len()); + let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None; + for addr in endpoints { + if let Some(expiry) = guard.get(addr).copied() { + match earliest_quarantine { + Some((_, current_expiry)) if current_expiry <= expiry => {} + _ => earliest_quarantine = Some((*addr, expiry)), + } + } else { + ready.push(*addr); } - } else { - ready.push(*addr); } - } - if !ready.is_empty() { - return ready; - } + if !ready.is_empty() { + return ready; + } - if let Some((addr, expiry)) = earliest_quarantine { + let Some((addr, expiry)) = earliest_quarantine else { + return Vec::new(); + }; let remaining = expiry.saturating_duration_since(now); if remaining.is_zero() { return vec![addr]; @@ -81,13 +84,8 @@ impl MePool { wait_ms = remaining.as_millis(), "All ME endpoints quarantined; waiting for earliest to expire" ); - // After sleeping, the quarantine entry is expired but not removed yet. - // Callers that check is_endpoint_quarantined() will lazily clean it via retain(). tokio::time::sleep(remaining).await; - return vec![addr]; } - - Vec::new() } pub(super) async fn has_refill_inflight_for_dc_key(&self, key: RefillDcKey) -> bool { @@ -319,4 +317,4 @@ impl MePool { dc_guard.remove(&dc_key); }); } -} \ No newline at end of file +} diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index fb1ba10..e368ead 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -142,6 +142,9 @@ impl MePool { seq_no: 0, crc_mode: hs.crc_mode, }; + let cleanup_done = Arc::new(AtomicBool::new(false)); + let cleanup_for_writer = cleanup_done.clone(); + let pool_writer = Arc::downgrade(self); let cancel_wr = cancel.clone(); tokio::spawn(async move { loop { @@ -160,6 +163,17 @@ impl MePool { _ = cancel_wr.cancelled() => break, } } + cancel_wr.cancel(); + if cleanup_for_writer + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + if let Some(pool) = pool_writer.upgrade() { + pool.remove_writer_and_close_clients(writer_id).await; + } else { + debug!(writer_id, "ME writer cleanup skipped: pool dropped"); + } + } }); let writer = MeWriter { id: writer_id, @@ -196,7 +210,6 @@ impl MePool { let cancel_ping = cancel.clone(); let tx_ping = tx.clone(); let ping_tracker_ping = ping_tracker.clone(); - let cleanup_done = Arc::new(AtomicBool::new(false)); let cleanup_for_reader = cleanup_done.clone(); let cleanup_for_ping = cleanup_done.clone(); let keepalive_enabled = self.me_keepalive_enabled; @@ -242,6 +255,7 @@ impl MePool { stats_reader_close.increment_me_idle_close_by_peer_total(); info!(writer_id, "ME socket closed by peer on idle writer"); } + cancel_reader_token.cancel(); if cleanup_for_reader .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) .is_ok() @@ -249,13 +263,12 @@ impl MePool { if let Some(pool) = pool.upgrade() { pool.remove_writer_and_close_clients(writer_id).await; } else { - // Pool is gone (shutdown). Remove writer from Vec directly - // as a last resort — no registry/refill side effects needed - // during shutdown. conn_count is not decremented here because - // the pool (and its counters) are already dropped. - let mut ws = writers_arc.write().await; - ws.retain(|w| w.id != writer_id); - debug!(writer_id, remaining = ws.len(), "Writer removed during pool shutdown"); + let remaining = writers_arc.read().await.len(); + debug!( + writer_id, + remaining, + "ME reader cleanup skipped: pool dropped" + ); } } if let Err(e) = res { @@ -658,4 +671,4 @@ impl MePool { } } } -} \ No newline at end of file +} From f61d25ebe0b62395a226e0af70afc5d03016a2f6 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:11:47 +0300 Subject: [PATCH 253/261] Authoritative Teardown + Orphan Watchdog + Force-Close Safery Policy Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/cli.rs | 8 +- src/config/defaults.rs | 12 +- src/config/load.rs | 39 +++ src/config/types.rs | 11 +- src/maestro/me_startup.rs | 165 ++++++++--- src/transport/middle_proxy/health.rs | 256 ++++++++++++------ .../middle_proxy/health_regression_tests.rs | 23 +- src/transport/middle_proxy/mod.rs | 2 +- src/transport/middle_proxy/pool.rs | 29 +- src/transport/middle_proxy/pool_refill.rs | 49 ++-- src/transport/middle_proxy/pool_writer.rs | 94 ++++--- 11 files changed, 477 insertions(+), 211 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 5fbd7d5..b6e2d92 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -199,8 +199,14 @@ update_every = 43200 hardswap = false me_pool_drain_ttl_secs = 90 me_instadrain = false +me_pool_drain_threshold = 32 +me_pool_drain_soft_evict_grace_secs = 10 +me_pool_drain_soft_evict_per_writer = 2 +me_pool_drain_soft_evict_budget_per_core = 16 +me_pool_drain_soft_evict_cooldown_ms = 1000 +me_bind_stale_mode = "never" me_pool_min_fresh_ratio = 0.8 -me_reinit_drain_timeout_secs = 120 +me_reinit_drain_timeout_secs = 90 [network] ipv4 = true diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 6d74c93..fea8305 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -40,10 +40,10 @@ const DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS: u64 = 3000; const DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS: u64 = 250; const DEFAULT_ME_C2ME_SEND_TIMEOUT_MS: u64 = 4000; const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED: bool = true; -const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS: u64 = 30; -const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER: u8 = 1; -const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE: u16 = 8; -const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 5000; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS: u64 = 10; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER: u8 = 2; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE: u16 = 16; +const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 1000; const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30; const DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS: u64 = 250; const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2; @@ -606,7 +606,7 @@ pub(crate) fn default_proxy_secret_len_max() -> usize { } pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 { - 120 + 90 } pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 { @@ -618,7 +618,7 @@ pub(crate) fn default_me_instadrain() -> bool { } pub(crate) fn default_me_pool_drain_threshold() -> u64 { - 128 + 32 } pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool { diff --git a/src/config/load.rs b/src/config/load.rs index c296697..14799ed 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -2037,6 +2037,45 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn force_close_default_matches_drain_ttl() { + let toml = r#" + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_force_close_default_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert_eq!(cfg.general.me_reinit_drain_timeout_secs, 90); + assert_eq!(cfg.general.effective_me_pool_force_close_secs(), 90); + let _ = std::fs::remove_file(path); + } + + #[test] + fn force_close_zero_uses_runtime_safety_fallback() { + let toml = r#" + [general] + me_reinit_drain_timeout_secs = 0 + + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_force_close_zero_fallback_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert_eq!(cfg.general.me_reinit_drain_timeout_secs, 0); + assert_eq!(cfg.general.effective_me_pool_force_close_secs(), 300); + let _ = std::fs::remove_file(path); + } + #[test] fn force_close_bumped_when_below_drain_ttl() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index ecd051d..d018187 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -135,8 +135,8 @@ impl MeSocksKdfPolicy { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "lowercase")] pub enum MeBindStaleMode { - Never, #[default] + Never, Ttl, Always, } @@ -855,7 +855,7 @@ pub struct GeneralConfig { pub me_pool_min_fresh_ratio: f32, /// Drain timeout in seconds for stale ME writers after endpoint map changes. - /// Set to 0 to keep stale writers draining indefinitely (no force-close). + /// Set to 0 to use the runtime safety fallback timeout. #[serde(default = "default_me_reinit_drain_timeout_secs")] pub me_reinit_drain_timeout_secs: u64, @@ -1068,8 +1068,13 @@ impl GeneralConfig { /// Resolve force-close timeout for stale writers. /// `me_reinit_drain_timeout_secs` remains backward-compatible alias. + /// A configured `0` uses the runtime safety fallback (300s). pub fn effective_me_pool_force_close_secs(&self) -> u64 { - self.me_reinit_drain_timeout_secs + if self.me_reinit_drain_timeout_secs == 0 { + 300 + } else { + self.me_reinit_drain_timeout_secs + } } } diff --git a/src/maestro/me_startup.rs b/src/maestro/me_startup.rs index 0b1310a..eb45cc4 100644 --- a/src/maestro/me_startup.rs +++ b/src/maestro/me_startup.rs @@ -332,25 +332,76 @@ pub(crate) async fn initialize_me_pool( "Middle-End pool initialized successfully" ); - let pool_health = pool_bg.clone(); - let rng_health = rng_bg.clone(); - let min_conns = pool_size; - tokio::spawn(async move { - crate::transport::middle_proxy::me_health_monitor( - pool_health, - rng_health, - min_conns, - ) - .await; - }); - let pool_drain_enforcer = pool_bg.clone(); - tokio::spawn(async move { - crate::transport::middle_proxy::me_drain_timeout_enforcer( - pool_drain_enforcer, - ) - .await; - }); - break; + // ── Supervised background tasks ────────────────── + // Each task runs inside a nested tokio::spawn so + // that a panic is caught via JoinHandle and the + // outer loop restarts the task automatically. + let pool_health = pool_bg.clone(); + let rng_health = rng_bg.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + loop { + let p = pool_health.clone(); + let r = rng_health.clone(); + let res = tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + p, r, min_conns, + ) + .await; + }) + .await; + match res { + Ok(()) => warn!("me_health_monitor exited unexpectedly, restarting"), + Err(e) => { + error!(error = %e, "me_health_monitor panicked, restarting in 1s"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + let pool_drain_enforcer = pool_bg.clone(); + tokio::spawn(async move { + loop { + let p = pool_drain_enforcer.clone(); + let res = tokio::spawn(async move { + crate::transport::middle_proxy::me_drain_timeout_enforcer(p).await; + }) + .await; + match res { + Ok(()) => warn!("me_drain_timeout_enforcer exited unexpectedly, restarting"), + Err(e) => { + error!(error = %e, "me_drain_timeout_enforcer panicked, restarting in 1s"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + let pool_watchdog = pool_bg.clone(); + tokio::spawn(async move { + loop { + let p = pool_watchdog.clone(); + let res = tokio::spawn(async move { + crate::transport::middle_proxy::me_zombie_writer_watchdog(p).await; + }) + .await; + match res { + Ok(()) => warn!("me_zombie_writer_watchdog exited unexpectedly, restarting"), + Err(e) => { + error!(error = %e, "me_zombie_writer_watchdog panicked, restarting in 1s"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + // CRITICAL: keep the current-thread runtime + // alive. Without this, block_on() returns, + // the Runtime is dropped, and ALL spawned + // background tasks (health monitor, drain + // enforcer, zombie watchdog) are silently + // cancelled — causing the draining-writer + // leak that brought us here. + std::future::pending::<()>().await; + unreachable!(); } Err(e) => { startup_tracker_bg.set_me_last_error(Some(e.to_string())).await; @@ -408,23 +459,65 @@ pub(crate) async fn initialize_me_pool( "Middle-End pool initialized successfully" ); - let pool_clone = pool.clone(); - let rng_clone = rng.clone(); - let min_conns = pool_size; - tokio::spawn(async move { - crate::transport::middle_proxy::me_health_monitor( - pool_clone, rng_clone, min_conns, - ) - .await; - }); - let pool_drain_enforcer = pool.clone(); - tokio::spawn(async move { - crate::transport::middle_proxy::me_drain_timeout_enforcer( - pool_drain_enforcer, - ) - .await; - }); - + // ── Supervised background tasks ────────────────── + let pool_clone = pool.clone(); + let rng_clone = rng.clone(); + let min_conns = pool_size; + tokio::spawn(async move { + loop { + let p = pool_clone.clone(); + let r = rng_clone.clone(); + let res = tokio::spawn(async move { + crate::transport::middle_proxy::me_health_monitor( + p, r, min_conns, + ) + .await; + }) + .await; + match res { + Ok(()) => warn!("me_health_monitor exited unexpectedly, restarting"), + Err(e) => { + error!(error = %e, "me_health_monitor panicked, restarting in 1s"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + let pool_drain_enforcer = pool.clone(); + tokio::spawn(async move { + loop { + let p = pool_drain_enforcer.clone(); + let res = tokio::spawn(async move { + crate::transport::middle_proxy::me_drain_timeout_enforcer(p).await; + }) + .await; + match res { + Ok(()) => warn!("me_drain_timeout_enforcer exited unexpectedly, restarting"), + Err(e) => { + error!(error = %e, "me_drain_timeout_enforcer panicked, restarting in 1s"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + let pool_watchdog = pool.clone(); + tokio::spawn(async move { + loop { + let p = pool_watchdog.clone(); + let res = tokio::spawn(async move { + crate::transport::middle_proxy::me_zombie_writer_watchdog(p).await; + }) + .await; + match res { + Ok(()) => warn!("me_zombie_writer_watchdog exited unexpectedly, restarting"), + Err(e) => { + error!(error = %e, "me_zombie_writer_watchdog panicked, restarting in 1s"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + break Some(pool); } Err(e) => { diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 5829de4..9d4cc70 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -1327,33 +1327,6 @@ async fn recover_single_endpoint_outage( } let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms(); - let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine(); - if !bypass_quarantine { - let quarantine_remaining = { - let mut guard = pool.endpoint_quarantine.lock().await; - let quarantine_now = Instant::now(); - guard.retain(|_, expiry| *expiry > quarantine_now); - guard - .get(&endpoint) - .map(|expiry| expiry.saturating_duration_since(quarantine_now)) - }; - - if let Some(remaining) = quarantine_remaining - && !remaining.is_zero() - { - outage_next_attempt.insert(key, now + remaining); - debug!( - dc = %key.0, - family = ?key.1, - %endpoint, - required, - wait_ms = remaining.as_millis(), - "Single-endpoint outage reconnect deferred by endpoint quarantine" - ); - return; - } - } - if *reconnect_budget == 0 { outage_next_attempt.insert(key, now + Duration::from_millis(min_backoff_ms.max(250))); debug!( @@ -1369,6 +1342,7 @@ async fn recover_single_endpoint_outage( pool.stats .increment_me_single_endpoint_outage_reconnect_attempt_total(); + let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine(); let attempt_ok = if bypass_quarantine { pool.stats .increment_me_single_endpoint_quarantine_bypass_total(); @@ -1576,6 +1550,170 @@ async fn maybe_rotate_single_endpoint_shadow( ); } +/// Last-resort safety net for draining writers stuck past their deadline. +/// +/// Runs every `TICK_SECS` and force-closes any draining writer whose +/// `drain_deadline_epoch_secs` has been exceeded by more than a threshold. +/// +/// Two thresholds: +/// - `SOFT_THRESHOLD_SECS` (60s): writers with no bound clients +/// - `HARD_THRESHOLD_SECS` (300s): writers WITH bound clients (unconditional) +/// +/// Intentionally kept trivial and independent of pool config to minimise +/// the probability of panicking itself. Uses `SystemTime` directly +/// as a fallback clock source and timeouts on every lock acquisition +/// and writer removal so one stuck writer cannot block the rest. +pub async fn me_zombie_writer_watchdog(pool: Arc) { + use std::time::{SystemTime, UNIX_EPOCH}; + + const TICK_SECS: u64 = 30; + const SOFT_THRESHOLD_SECS: u64 = 60; + const HARD_THRESHOLD_SECS: u64 = 300; + const LOCK_TIMEOUT_SECS: u64 = 5; + const REMOVE_TIMEOUT_SECS: u64 = 10; + const HARD_DETACH_TIMEOUT_STREAK: u8 = 3; + + let mut removal_timeout_streak = HashMap::::new(); + + loop { + tokio::time::sleep(Duration::from_secs(TICK_SECS)).await; + + let now = match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(d) => d.as_secs(), + Err(_) => continue, + }; + + // Phase 1: collect zombie IDs under a short read-lock with timeout. + let zombie_ids_with_meta: Vec<(u64, bool)> = { + let Ok(ws) = tokio::time::timeout( + Duration::from_secs(LOCK_TIMEOUT_SECS), + pool.writers.read(), + ) + .await + else { + warn!("zombie_watchdog: writers read-lock timeout, skipping tick"); + continue; + }; + ws.iter() + .filter(|w| w.draining.load(std::sync::atomic::Ordering::Relaxed)) + .filter_map(|w| { + let deadline = w + .drain_deadline_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + if deadline == 0 { + return None; + } + let overdue = now.saturating_sub(deadline); + if overdue == 0 { + return None; + } + let started = w + .draining_started_at_epoch_secs + .load(std::sync::atomic::Ordering::Relaxed); + let drain_age = now.saturating_sub(started); + if drain_age > HARD_THRESHOLD_SECS { + return Some((w.id, true)); + } + if overdue > SOFT_THRESHOLD_SECS { + return Some((w.id, false)); + } + None + }) + .collect() + }; + // read lock released here + + if zombie_ids_with_meta.is_empty() { + removal_timeout_streak.clear(); + continue; + } + + let mut active_zombie_ids = HashSet::::with_capacity(zombie_ids_with_meta.len()); + for (writer_id, _) in &zombie_ids_with_meta { + active_zombie_ids.insert(*writer_id); + } + removal_timeout_streak.retain(|writer_id, _| active_zombie_ids.contains(writer_id)); + + warn!( + zombie_count = zombie_ids_with_meta.len(), + soft_threshold_secs = SOFT_THRESHOLD_SECS, + hard_threshold_secs = HARD_THRESHOLD_SECS, + "Zombie draining writers detected by watchdog, force-closing" + ); + + // Phase 2: remove each writer individually with a timeout. + // One stuck removal cannot block the rest. + for (writer_id, had_clients) in &zombie_ids_with_meta { + let result = tokio::time::timeout( + Duration::from_secs(REMOVE_TIMEOUT_SECS), + pool.remove_writer_and_close_clients(*writer_id), + ) + .await; + match result { + Ok(()) => { + removal_timeout_streak.remove(writer_id); + pool.stats.increment_pool_force_close_total(); + pool.stats + .increment_me_draining_writers_reap_progress_total(); + info!( + writer_id, + had_clients, + "Zombie writer removed by watchdog" + ); + } + Err(_) => { + let streak = removal_timeout_streak + .entry(*writer_id) + .and_modify(|value| *value = value.saturating_add(1)) + .or_insert(1); + warn!( + writer_id, + had_clients, + timeout_streak = *streak, + "Zombie writer removal timed out" + ); + if *streak < HARD_DETACH_TIMEOUT_STREAK { + continue; + } + + let hard_detach = tokio::time::timeout( + Duration::from_secs(REMOVE_TIMEOUT_SECS), + pool.remove_draining_writer_hard_detach(*writer_id), + ) + .await; + match hard_detach { + Ok(true) => { + removal_timeout_streak.remove(writer_id); + pool.stats.increment_pool_force_close_total(); + pool.stats + .increment_me_draining_writers_reap_progress_total(); + info!( + writer_id, + had_clients, + "Zombie writer hard-detached after repeated timeouts" + ); + } + Ok(false) => { + removal_timeout_streak.remove(writer_id); + debug!( + writer_id, + had_clients, + "Zombie hard-detach skipped (writer already gone or no longer draining)" + ); + } + Err(_) => { + warn!( + writer_id, + had_clients, + "Zombie hard-detach timed out, will retry next tick" + ); + } + } + } + } + } + } +} #[cfg(test)] mod tests { use std::collections::HashMap; @@ -1587,10 +1725,9 @@ mod tests { use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; - use super::{reap_draining_writers, recover_single_endpoint_outage}; + use super::reap_draining_writers; use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; use crate::crypto::SecureRandom; - use crate::network::IpFamily; use crate::network::probe::NetworkDecision; use crate::stats::Stats; use crate::transport::middle_proxy::codec::WriterCommand; @@ -1772,65 +1909,4 @@ mod tests { assert_eq!(pool.registry.get_writer(conn_b).await.unwrap().writer_id, 20); assert_eq!(pool.registry.get_writer(conn_c).await.unwrap().writer_id, 30); } - - #[tokio::test] - async fn removing_draining_writer_still_quarantines_flapping_endpoint() { - let pool = make_pool(1).await; - let now_epoch_secs = MePool::now_epoch_secs(); - let writer_id = 11u64; - let writer_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4000 + writer_id as u16); - let conn_id = - insert_draining_writer(&pool, writer_id, now_epoch_secs.saturating_sub(5)).await; - - assert!(pool - .registry - .evict_bound_conn_if_writer(conn_id, writer_id) - .await); - pool.remove_writer_and_close_clients(writer_id).await; - - assert!(pool.is_endpoint_quarantined(writer_addr).await); - } - - #[tokio::test] - async fn single_endpoint_outage_respects_quarantine_when_bypass_disabled() { - let pool = make_pool(1).await; - pool.me_single_endpoint_outage_disable_quarantine - .store(false, Ordering::Relaxed); - - let key = (2, IpFamily::V4); - let endpoint = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 7443); - let quarantine_ttl = Duration::from_millis(200); - { - let mut guard = pool.endpoint_quarantine.lock().await; - guard.insert(endpoint, Instant::now() + quarantine_ttl); - } - - let rng = Arc::new(SecureRandom::new()); - let mut outage_backoff = HashMap::new(); - let mut outage_next_attempt = HashMap::new(); - let mut reconnect_budget = 1usize; - let started_at = Instant::now(); - - recover_single_endpoint_outage( - &pool, - &rng, - key, - endpoint, - 1, - &mut outage_backoff, - &mut outage_next_attempt, - &mut reconnect_budget, - ) - .await; - - assert_eq!(reconnect_budget, 1); - assert_eq!( - pool.stats - .get_me_single_endpoint_outage_reconnect_attempt_total(), - 0 - ); - assert_eq!(pool.stats.get_me_single_endpoint_quarantine_bypass_total(), 0); - let next_attempt = outage_next_attempt.get(&key).copied().unwrap(); - assert!(next_attempt >= started_at + Duration::from_millis(120)); - } } diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs index bcdaf2e..230cd64 100644 --- a/src/transport/middle_proxy/health_regression_tests.rs +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -12,7 +12,9 @@ use super::codec::WriterCommand; use super::health::{health_drain_close_budget, reap_draining_writers}; use super::pool::{MePool, MeWriter, WriterContour}; use super::registry::ConnMeta; -use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; +use crate::config::{ + GeneralConfig, MeBindStaleMode, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode, +}; use crate::crypto::SecureRandom; use crate::network::probe::NetworkDecision; use crate::stats::Stats; @@ -646,10 +648,23 @@ async fn reap_draining_writers_instadrain_removes_non_expired_writers_immediatel #[test] fn general_config_default_drain_threshold_remains_enabled() { - assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 128); + assert_eq!(GeneralConfig::default().me_pool_drain_threshold, 32); assert!(GeneralConfig::default().me_pool_drain_soft_evict_enabled); assert_eq!( - GeneralConfig::default().me_pool_drain_soft_evict_per_writer, - 1 + GeneralConfig::default().me_pool_drain_soft_evict_grace_secs, + 10 ); + assert_eq!( + GeneralConfig::default().me_pool_drain_soft_evict_per_writer, + 2 + ); + assert_eq!( + GeneralConfig::default().me_pool_drain_soft_evict_budget_per_core, + 16 + ); + assert_eq!( + GeneralConfig::default().me_pool_drain_soft_evict_cooldown_ms, + 1000 + ); + assert_eq!(GeneralConfig::default().me_bind_stale_mode, MeBindStaleMode::Never); } diff --git a/src/transport/middle_proxy/mod.rs b/src/transport/middle_proxy/mod.rs index 26ded29..8c57717 100644 --- a/src/transport/middle_proxy/mod.rs +++ b/src/transport/middle_proxy/mod.rs @@ -30,7 +30,7 @@ mod health_adversarial_tests; use bytes::Bytes; -pub use health::{me_drain_timeout_enforcer, me_health_monitor}; +pub use health::{me_drain_timeout_enforcer, me_health_monitor, me_zombie_writer_watchdog}; #[allow(unused_imports)] pub use ping::{run_me_ping, format_sample_line, format_me_route, MePingReport, MePingSample, MePingFamily}; pub use pool::MePool; diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index 441d41d..f825058 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -18,6 +18,8 @@ use crate::transport::UpstreamManager; use super::ConnRegistry; use super::codec::WriterCommand; +const ME_FORCE_CLOSE_SAFETY_FALLBACK_SECS: u64 = 300; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub(super) struct RefillDcKey { pub dc: i32, @@ -229,6 +231,14 @@ impl MePool { .as_secs() } + fn normalize_force_close_secs(force_close_secs: u64) -> u64 { + if force_close_secs == 0 { + ME_FORCE_CLOSE_SAFETY_FALLBACK_SECS + } else { + force_close_secs + } + } + pub fn new( proxy_tag: Option>, proxy_secret: Vec, @@ -477,7 +487,9 @@ impl MePool { me_pool_drain_soft_evict_cooldown_ms: AtomicU64::new( me_pool_drain_soft_evict_cooldown_ms.max(1), ), - me_pool_force_close_secs: AtomicU64::new(me_pool_force_close_secs), + me_pool_force_close_secs: AtomicU64::new(Self::normalize_force_close_secs( + me_pool_force_close_secs, + )), me_pool_min_fresh_ratio_permille: AtomicU32::new(Self::ratio_to_permille( me_pool_min_fresh_ratio, )), @@ -587,8 +599,10 @@ impl MePool { ); self.me_pool_drain_soft_evict_cooldown_ms .store(pool_drain_soft_evict_cooldown_ms.max(1), Ordering::Relaxed); - self.me_pool_force_close_secs - .store(force_close_secs, Ordering::Relaxed); + self.me_pool_force_close_secs.store( + Self::normalize_force_close_secs(force_close_secs), + Ordering::Relaxed, + ); self.me_pool_min_fresh_ratio_permille .store(Self::ratio_to_permille(min_fresh_ratio), Ordering::Relaxed); self.me_hardswap_warmup_delay_min_ms @@ -733,12 +747,9 @@ impl MePool { } pub(super) fn force_close_timeout(&self) -> Option { - let secs = self.me_pool_force_close_secs.load(Ordering::Relaxed); - if secs == 0 { - None - } else { - Some(Duration::from_secs(secs)) - } + let secs = + Self::normalize_force_close_secs(self.me_pool_force_close_secs.load(Ordering::Relaxed)); + Some(Duration::from_secs(secs)) } pub(super) fn drain_soft_evict_enabled(&self) -> bool { diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index 43e2e6b..e4fb95f 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -49,43 +49,36 @@ impl MePool { return Vec::new(); } - loop { - let mut guard = self.endpoint_quarantine.lock().await; - let now = Instant::now(); - guard.retain(|_, expiry| *expiry > now); + let mut guard = self.endpoint_quarantine.lock().await; + let now = Instant::now(); + guard.retain(|_, expiry| *expiry > now); - let mut ready = Vec::::with_capacity(endpoints.len()); - let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None; - for addr in endpoints { - if let Some(expiry) = guard.get(addr).copied() { - match earliest_quarantine { - Some((_, current_expiry)) if current_expiry <= expiry => {} - _ => earliest_quarantine = Some((*addr, expiry)), - } - } else { - ready.push(*addr); + let mut ready = Vec::::with_capacity(endpoints.len()); + let mut earliest_quarantine: Option<(SocketAddr, Instant)> = None; + for addr in endpoints { + if let Some(expiry) = guard.get(addr).copied() { + match earliest_quarantine { + Some((_, current_expiry)) if current_expiry <= expiry => {} + _ => earliest_quarantine = Some((*addr, expiry)), } + } else { + ready.push(*addr); } + } - if !ready.is_empty() { - return ready; - } + if !ready.is_empty() { + return ready; + } - let Some((addr, expiry)) = earliest_quarantine else { - return Vec::new(); - }; - let remaining = expiry.saturating_duration_since(now); - if remaining.is_zero() { - return vec![addr]; - } - drop(guard); + if let Some((addr, expiry)) = earliest_quarantine { debug!( %addr, - wait_ms = remaining.as_millis(), - "All ME endpoints quarantined; waiting for earliest to expire" + wait_ms = expiry.saturating_duration_since(now).as_millis(), + "All ME endpoints are quarantined for the DC group; waiting for quarantine expiry" ); - tokio::time::sleep(remaining).await; } + + Vec::new() } pub(super) async fn has_refill_inflight_for_dc_key(&self, key: RefillDcKey) -> bool { diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index e368ead..e3ea44d 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -20,7 +20,6 @@ use crate::protocol::constants::{RPC_CLOSE_EXT_U32, RPC_PING_U32}; use super::codec::{RpcWriter, WriterCommand}; use super::pool::{MePool, MeWriter, WriterContour}; use super::reader::reader_loop; -use super::registry::BoundConn; use super::wire::build_proxy_req_payload; const ME_ACTIVE_PING_SECS: u64 = 25; @@ -28,6 +27,12 @@ const ME_ACTIVE_PING_JITTER_SECS: i64 = 5; const ME_IDLE_KEEPALIVE_MAX_SECS: u64 = 5; const ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS: u64 = 700; +#[derive(Clone, Copy)] +enum WriterTeardownMode { + Any, + DrainingOnly, +} + fn is_me_peer_closed_error(error: &ProxyError) -> bool { matches!(error, ProxyError::Io(ioe) if ioe.kind() == ErrorKind::UnexpectedEof) } @@ -142,10 +147,10 @@ impl MePool { seq_no: 0, crc_mode: hs.crc_mode, }; + let cancel_wr = cancel.clone(); let cleanup_done = Arc::new(AtomicBool::new(false)); let cleanup_for_writer = cleanup_done.clone(); - let pool_writer = Arc::downgrade(self); - let cancel_wr = cancel.clone(); + let pool_writer_task = Arc::downgrade(self); tokio::spawn(async move { loop { tokio::select! { @@ -163,15 +168,14 @@ impl MePool { _ = cancel_wr.cancelled() => break, } } - cancel_wr.cancel(); if cleanup_for_writer .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) .is_ok() { - if let Some(pool) = pool_writer.upgrade() { + if let Some(pool) = pool_writer_task.upgrade() { pool.remove_writer_and_close_clients(writer_id).await; } else { - debug!(writer_id, "ME writer cleanup skipped: pool dropped"); + cancel_wr.cancel(); } } }); @@ -255,7 +259,6 @@ impl MePool { stats_reader_close.increment_me_idle_close_by_peer_total(); info!(writer_id, "ME socket closed by peer on idle writer"); } - cancel_reader_token.cancel(); if cleanup_for_reader .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) .is_ok() @@ -263,12 +266,9 @@ impl MePool { if let Some(pool) = pool.upgrade() { pool.remove_writer_and_close_clients(writer_id).await; } else { - let remaining = writers_arc.read().await.len(); - debug!( - writer_id, - remaining, - "ME reader cleanup skipped: pool dropped" - ); + // Fallback for shutdown races: make writer task exit quickly so stale + // channels are observable by periodic prune. + cancel_reader_token.cancel(); } } if let Err(e) = res { @@ -276,6 +276,8 @@ impl MePool { warn!(error = %e, "ME reader ended"); } } + let remaining = writers_arc.read().await.len(); + debug!(writer_id, remaining, "ME reader task finished"); }); let pool_ping = Arc::downgrade(self); @@ -365,13 +367,12 @@ impl MePool { stats_ping.increment_me_keepalive_failed(); debug!("ME ping failed, removing dead writer"); cancel_ping.cancel(); - if cleanup_for_ping - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() + if let Some(pool) = pool_ping.upgrade() + && cleanup_for_ping + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() { - if let Some(pool) = pool_ping.upgrade() { - pool.remove_writer_and_close_clients(writer_id).await; - } + pool.remove_writer_and_close_clients(writer_id).await; } break; } @@ -514,18 +515,49 @@ impl MePool { pub(crate) async fn remove_writer_and_close_clients(self: &Arc, writer_id: u64) { // Full client cleanup now happens inside `registry.writer_lost` to keep // writer reap/remove paths strictly non-blocking per connection. - let _ = self.remove_writer_only(writer_id).await; + let _ = self + .remove_writer_with_mode(writer_id, WriterTeardownMode::Any) + .await; } - async fn remove_writer_only(self: &Arc, writer_id: u64) -> Vec { + pub(super) async fn remove_draining_writer_hard_detach( + self: &Arc, + writer_id: u64, + ) -> bool { + self.remove_writer_with_mode(writer_id, WriterTeardownMode::DrainingOnly) + .await + } + + async fn remove_writer_only(self: &Arc, writer_id: u64) -> bool { + self.remove_writer_with_mode(writer_id, WriterTeardownMode::Any) + .await + } + + // Authoritative teardown primitive shared by normal cleanup and watchdog path. + // Lock-order invariant: + // 1) mutate `writers` under pool write lock, + // 2) release pool lock, + // 3) run registry/metrics/refill side effects. + // `registry.writer_lost` must never run while `writers` lock is held. + async fn remove_writer_with_mode( + self: &Arc, + writer_id: u64, + mode: WriterTeardownMode, + ) -> bool { let mut close_tx: Option> = None; let mut removed_addr: Option = None; let mut removed_dc: Option = None; let mut removed_uptime: Option = None; let mut trigger_refill = false; + let mut removed = false; { let mut ws = self.writers.write().await; if let Some(pos) = ws.iter().position(|w| w.id == writer_id) { + if matches!(mode, WriterTeardownMode::DrainingOnly) + && !ws[pos].draining.load(Ordering::Relaxed) + { + return false; + } let w = ws.remove(pos); let was_draining = w.draining.load(Ordering::Relaxed); if was_draining { @@ -542,6 +574,7 @@ impl MePool { } close_tx = Some(w.tx.clone()); self.conn_count.fetch_sub(1, Ordering::Relaxed); + removed = true; } } // State invariant: @@ -549,7 +582,7 @@ impl MePool { // - writer is removed from registry routing/binding maps via `writer_lost`. // The close command below is only a best-effort accelerator for task shutdown. // Cleanup progress must never depend on command-channel availability. - let conns = self.registry.writer_lost(writer_id).await; + let _ = self.registry.writer_lost(writer_id).await; { let mut tracker = self.ping_tracker.lock().await; tracker.retain(|_, (_, wid)| *wid != writer_id); @@ -576,22 +609,17 @@ impl MePool { } } } - // Quarantine flapping endpoints regardless of draining state — - // a rapidly dying endpoint is unstable whether it was draining or not. if let Some(addr) = removed_addr { if let Some(uptime) = removed_uptime { self.maybe_quarantine_flapping_endpoint(addr, uptime).await; } + if trigger_refill + && let Some(writer_dc) = removed_dc + { + self.trigger_immediate_refill_for_dc(addr, writer_dc); + } } - // Only trigger immediate refill for unexpected (non-draining) removals. - // Draining writers are intentionally being retired. - if trigger_refill - && let Some(addr) = removed_addr - && let Some(writer_dc) = removed_dc - { - self.trigger_immediate_refill_for_dc(addr, writer_dc); - } - conns + removed } pub(crate) async fn mark_writer_draining_with_timeout( From ef9b7b149236030f5bf4f658f435de267d06cf4f Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:45:53 +0300 Subject: [PATCH 254/261] Teardown Monitoring in API Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/model.rs | 10 + src/api/runtime_min.rs | 124 ++++++++++++ src/api/runtime_stats.rs | 25 ++- src/metrics.rs | 178 +++++++++++++++++- src/transport/middle_proxy/health.rs | 33 +++- .../middle_proxy/health_adversarial_tests.rs | 7 +- .../middle_proxy/health_regression_tests.rs | 11 +- src/transport/middle_proxy/pool_writer.rs | 112 +++++++++-- src/transport/middle_proxy/send.rs | 31 ++- 9 files changed, 496 insertions(+), 35 deletions(-) diff --git a/src/api/model.rs b/src/api/model.rs index e98de8b..91d83b2 100644 --- a/src/api/model.rs +++ b/src/api/model.rs @@ -205,6 +205,16 @@ pub(super) struct ZeroPoolData { pub(super) refill_failed_total: u64, pub(super) writer_restored_same_endpoint_total: u64, pub(super) writer_restored_fallback_total: u64, + pub(super) teardown_attempt_total_normal: u64, + pub(super) teardown_attempt_total_hard_detach: u64, + pub(super) teardown_success_total_normal: u64, + pub(super) teardown_success_total_hard_detach: u64, + pub(super) teardown_timeout_total: u64, + pub(super) teardown_escalation_total: u64, + pub(super) teardown_noop_total: u64, + pub(super) teardown_cleanup_side_effect_failures_total: u64, + pub(super) teardown_duration_count_total: u64, + pub(super) teardown_duration_sum_seconds_total: f64, } #[derive(Serialize, Clone)] diff --git a/src/api/runtime_min.rs b/src/api/runtime_min.rs index f334dd0..ae3b23f 100644 --- a/src/api/runtime_min.rs +++ b/src/api/runtime_min.rs @@ -4,6 +4,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::Serialize; use crate::config::ProxyConfig; +use crate::stats::{ + MeWriterCleanupSideEffectStep, MeWriterTeardownMode, MeWriterTeardownReason, Stats, +}; use super::ApiShared; @@ -98,6 +101,50 @@ pub(super) struct RuntimeMeQualityCountersData { pub(super) reconnect_success_total: u64, } +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityTeardownAttemptData { + pub(super) reason: &'static str, + pub(super) mode: &'static str, + pub(super) total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityTeardownSuccessData { + pub(super) mode: &'static str, + pub(super) total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityTeardownSideEffectData { + pub(super) step: &'static str, + pub(super) total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityTeardownDurationBucketData { + pub(super) le_seconds: &'static str, + pub(super) total: u64, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityTeardownDurationData { + pub(super) mode: &'static str, + pub(super) count: u64, + pub(super) sum_seconds: f64, + pub(super) buckets: Vec, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityTeardownData { + pub(super) attempts: Vec, + pub(super) success: Vec, + pub(super) timeout_total: u64, + pub(super) escalation_total: u64, + pub(super) noop_total: u64, + pub(super) cleanup_side_effect_failures: Vec, + pub(super) duration: Vec, +} + #[derive(Serialize)] pub(super) struct RuntimeMeQualityRouteDropData { pub(super) no_conn_total: u64, @@ -120,6 +167,7 @@ pub(super) struct RuntimeMeQualityDcRttData { #[derive(Serialize)] pub(super) struct RuntimeMeQualityPayload { pub(super) counters: RuntimeMeQualityCountersData, + pub(super) teardown: RuntimeMeQualityTeardownData, pub(super) route_drops: RuntimeMeQualityRouteDropData, pub(super) dc_rtt: Vec, } @@ -374,6 +422,7 @@ pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> Runtime reconnect_attempt_total: shared.stats.get_me_reconnect_attempts(), reconnect_success_total: shared.stats.get_me_reconnect_success(), }, + teardown: build_runtime_me_teardown_data(shared), route_drops: RuntimeMeQualityRouteDropData { no_conn_total: shared.stats.get_me_route_drop_no_conn(), channel_closed_total: shared.stats.get_me_route_drop_channel_closed(), @@ -397,6 +446,81 @@ pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> Runtime } } +fn build_runtime_me_teardown_data(shared: &ApiShared) -> RuntimeMeQualityTeardownData { + let attempts = MeWriterTeardownReason::ALL + .iter() + .copied() + .flat_map(|reason| { + MeWriterTeardownMode::ALL + .iter() + .copied() + .map(move |mode| RuntimeMeQualityTeardownAttemptData { + reason: reason.as_str(), + mode: mode.as_str(), + total: shared.stats.get_me_writer_teardown_attempt_total(reason, mode), + }) + }) + .collect(); + + let success = MeWriterTeardownMode::ALL + .iter() + .copied() + .map(|mode| RuntimeMeQualityTeardownSuccessData { + mode: mode.as_str(), + total: shared.stats.get_me_writer_teardown_success_total(mode), + }) + .collect(); + + let cleanup_side_effect_failures = MeWriterCleanupSideEffectStep::ALL + .iter() + .copied() + .map(|step| RuntimeMeQualityTeardownSideEffectData { + step: step.as_str(), + total: shared + .stats + .get_me_writer_cleanup_side_effect_failures_total(step), + }) + .collect(); + + let duration = MeWriterTeardownMode::ALL + .iter() + .copied() + .map(|mode| { + let count = shared.stats.get_me_writer_teardown_duration_count(mode); + let mut buckets: Vec = Stats::me_writer_teardown_duration_bucket_labels() + .iter() + .enumerate() + .map(|(bucket_idx, label)| RuntimeMeQualityTeardownDurationBucketData { + le_seconds: label, + total: shared + .stats + .get_me_writer_teardown_duration_bucket_total(mode, bucket_idx), + }) + .collect(); + buckets.push(RuntimeMeQualityTeardownDurationBucketData { + le_seconds: "+Inf", + total: count, + }); + RuntimeMeQualityTeardownDurationData { + mode: mode.as_str(), + count, + sum_seconds: shared.stats.get_me_writer_teardown_duration_sum_seconds(mode), + buckets, + } + }) + .collect(); + + RuntimeMeQualityTeardownData { + attempts, + success, + timeout_total: shared.stats.get_me_writer_teardown_timeout_total(), + escalation_total: shared.stats.get_me_writer_teardown_escalation_total(), + noop_total: shared.stats.get_me_writer_teardown_noop_total(), + cleanup_side_effect_failures, + duration, + } +} + pub(super) async fn build_runtime_upstream_quality_data( shared: &ApiShared, ) -> RuntimeUpstreamQualityData { diff --git a/src/api/runtime_stats.rs b/src/api/runtime_stats.rs index cdeacc0..22ff82e 100644 --- a/src/api/runtime_stats.rs +++ b/src/api/runtime_stats.rs @@ -1,7 +1,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use crate::config::ApiConfig; -use crate::stats::Stats; +use crate::stats::{MeWriterTeardownMode, Stats}; use crate::transport::upstream::IpPreference; use crate::transport::UpstreamRouteKind; @@ -106,6 +106,29 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer refill_failed_total: stats.get_me_refill_failed_total(), writer_restored_same_endpoint_total: stats.get_me_writer_restored_same_endpoint_total(), writer_restored_fallback_total: stats.get_me_writer_restored_fallback_total(), + teardown_attempt_total_normal: stats + .get_me_writer_teardown_attempt_total_by_mode(MeWriterTeardownMode::Normal), + teardown_attempt_total_hard_detach: stats + .get_me_writer_teardown_attempt_total_by_mode(MeWriterTeardownMode::HardDetach), + teardown_success_total_normal: stats + .get_me_writer_teardown_success_total(MeWriterTeardownMode::Normal), + teardown_success_total_hard_detach: stats + .get_me_writer_teardown_success_total(MeWriterTeardownMode::HardDetach), + teardown_timeout_total: stats.get_me_writer_teardown_timeout_total(), + teardown_escalation_total: stats.get_me_writer_teardown_escalation_total(), + teardown_noop_total: stats.get_me_writer_teardown_noop_total(), + teardown_cleanup_side_effect_failures_total: stats + .get_me_writer_cleanup_side_effect_failures_total_all(), + teardown_duration_count_total: stats + .get_me_writer_teardown_duration_count(MeWriterTeardownMode::Normal) + .saturating_add( + stats.get_me_writer_teardown_duration_count(MeWriterTeardownMode::HardDetach), + ), + teardown_duration_sum_seconds_total: stats + .get_me_writer_teardown_duration_sum_seconds(MeWriterTeardownMode::Normal) + + stats.get_me_writer_teardown_duration_sum_seconds( + MeWriterTeardownMode::HardDetach, + ), }, desync: ZeroDesyncData { secure_padding_invalid_total: stats.get_secure_padding_invalid(), diff --git a/src/metrics.rs b/src/metrics.rs index 4f7f4b6..b7272b2 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -16,7 +16,9 @@ use tracing::{info, warn, debug}; use crate::config::ProxyConfig; use crate::ip_tracker::UserIpTracker; use crate::stats::beobachten::BeobachtenStore; -use crate::stats::Stats; +use crate::stats::{ + MeWriterCleanupSideEffectStep, MeWriterTeardownMode, MeWriterTeardownReason, Stats, +}; use crate::transport::{ListenOptions, create_listener}; pub async fn serve( @@ -1770,6 +1772,169 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp } ); + let _ = writeln!( + out, + "# HELP telemt_me_writer_teardown_attempt_total ME writer teardown attempts by reason and mode" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_teardown_attempt_total counter"); + for reason in MeWriterTeardownReason::ALL { + for mode in MeWriterTeardownMode::ALL { + let _ = writeln!( + out, + "telemt_me_writer_teardown_attempt_total{{reason=\"{}\",mode=\"{}\"}} {}", + reason.as_str(), + mode.as_str(), + if me_allows_normal { + stats.get_me_writer_teardown_attempt_total(reason, mode) + } else { + 0 + } + ); + } + } + + let _ = writeln!( + out, + "# HELP telemt_me_writer_teardown_success_total ME writer teardown successes by mode" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_teardown_success_total counter"); + for mode in MeWriterTeardownMode::ALL { + let _ = writeln!( + out, + "telemt_me_writer_teardown_success_total{{mode=\"{}\"}} {}", + mode.as_str(), + if me_allows_normal { + stats.get_me_writer_teardown_success_total(mode) + } else { + 0 + } + ); + } + + let _ = writeln!( + out, + "# HELP telemt_me_writer_teardown_timeout_total Teardown operations that timed out" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_teardown_timeout_total counter"); + let _ = writeln!( + out, + "telemt_me_writer_teardown_timeout_total {}", + if me_allows_normal { + stats.get_me_writer_teardown_timeout_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_writer_teardown_escalation_total Watchdog teardown escalations to hard detach" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_writer_teardown_escalation_total counter" + ); + let _ = writeln!( + out, + "telemt_me_writer_teardown_escalation_total {}", + if me_allows_normal { + stats.get_me_writer_teardown_escalation_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_writer_teardown_noop_total Teardown operations that became no-op" + ); + let _ = writeln!(out, "# TYPE telemt_me_writer_teardown_noop_total counter"); + let _ = writeln!( + out, + "telemt_me_writer_teardown_noop_total {}", + if me_allows_normal { + stats.get_me_writer_teardown_noop_total() + } else { + 0 + } + ); + + let _ = writeln!( + out, + "# HELP telemt_me_writer_teardown_duration_seconds ME writer teardown latency histogram by mode" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_writer_teardown_duration_seconds histogram" + ); + let bucket_labels = Stats::me_writer_teardown_duration_bucket_labels(); + for mode in MeWriterTeardownMode::ALL { + for (bucket_idx, label) in bucket_labels.iter().enumerate() { + let _ = writeln!( + out, + "telemt_me_writer_teardown_duration_seconds_bucket{{mode=\"{}\",le=\"{}\"}} {}", + mode.as_str(), + label, + if me_allows_normal { + stats.get_me_writer_teardown_duration_bucket_total(mode, bucket_idx) + } else { + 0 + } + ); + } + let _ = writeln!( + out, + "telemt_me_writer_teardown_duration_seconds_bucket{{mode=\"{}\",le=\"+Inf\"}} {}", + mode.as_str(), + if me_allows_normal { + stats.get_me_writer_teardown_duration_count(mode) + } else { + 0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_teardown_duration_seconds_sum{{mode=\"{}\"}} {:.6}", + mode.as_str(), + if me_allows_normal { + stats.get_me_writer_teardown_duration_sum_seconds(mode) + } else { + 0.0 + } + ); + let _ = writeln!( + out, + "telemt_me_writer_teardown_duration_seconds_count{{mode=\"{}\"}} {}", + mode.as_str(), + if me_allows_normal { + stats.get_me_writer_teardown_duration_count(mode) + } else { + 0 + } + ); + } + + let _ = writeln!( + out, + "# HELP telemt_me_writer_cleanup_side_effect_failures_total Failed cleanup side effects by step" + ); + let _ = writeln!( + out, + "# TYPE telemt_me_writer_cleanup_side_effect_failures_total counter" + ); + for step in MeWriterCleanupSideEffectStep::ALL { + let _ = writeln!( + out, + "telemt_me_writer_cleanup_side_effect_failures_total{{step=\"{}\"}} {}", + step.as_str(), + if me_allows_normal { + stats.get_me_writer_cleanup_side_effect_failures_total(step) + } else { + 0 + } + ); + } + let _ = writeln!(out, "# HELP telemt_me_refill_triggered_total Immediate ME refill runs started"); let _ = writeln!(out, "# TYPE telemt_me_refill_triggered_total counter"); let _ = writeln!( @@ -2175,6 +2340,17 @@ mod tests { assert!(output.contains("# TYPE telemt_me_rpc_proxy_req_signal_sent_total counter")); assert!(output.contains("# TYPE telemt_me_idle_close_by_peer_total counter")); assert!(output.contains("# TYPE telemt_me_writer_removed_total counter")); + assert!(output.contains("# TYPE telemt_me_writer_teardown_attempt_total counter")); + assert!(output.contains("# TYPE telemt_me_writer_teardown_success_total counter")); + assert!(output.contains("# TYPE telemt_me_writer_teardown_timeout_total counter")); + assert!(output.contains("# TYPE telemt_me_writer_teardown_escalation_total counter")); + assert!(output.contains("# TYPE telemt_me_writer_teardown_noop_total counter")); + assert!(output.contains( + "# TYPE telemt_me_writer_teardown_duration_seconds histogram" + )); + assert!(output.contains( + "# TYPE telemt_me_writer_cleanup_side_effect_failures_total counter" + )); assert!(output.contains("# TYPE telemt_me_writer_close_signal_drop_total counter")); assert!(output.contains( "# TYPE telemt_me_writer_close_signal_channel_full_total counter" diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 9d4cc70..30e562b 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -10,6 +10,7 @@ use tracing::{debug, info, warn}; use crate::config::MeFloorMode; use crate::crypto::SecureRandom; use crate::network::IpFamily; +use crate::stats::MeWriterTeardownReason; use super::MePool; use super::pool::MeWriter; @@ -358,7 +359,8 @@ pub(super) async fn reap_draining_writers( continue; } pool.stats.increment_pool_force_close_total(); - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients(writer_id, MeWriterTeardownReason::ReapTimeoutExpired) + .await; pool.stats .increment_me_draining_writers_reap_progress_total(); } @@ -376,7 +378,8 @@ pub(super) async fn reap_draining_writers( continue; } pool.stats.increment_pool_force_close_total(); - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients(writer_id, MeWriterTeardownReason::ReapThresholdForce) + .await; pool.stats .increment_me_draining_writers_reap_progress_total(); closed_total = closed_total.saturating_add(1); @@ -388,7 +391,8 @@ pub(super) async fn reap_draining_writers( if !closed_writer_ids.insert(writer_id) { continue; } - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients(writer_id, MeWriterTeardownReason::ReapEmpty) + .await; pool.stats .increment_me_draining_writers_reap_progress_total(); closed_total = closed_total.saturating_add(1); @@ -1646,11 +1650,14 @@ pub async fn me_zombie_writer_watchdog(pool: Arc) { for (writer_id, had_clients) in &zombie_ids_with_meta { let result = tokio::time::timeout( Duration::from_secs(REMOVE_TIMEOUT_SECS), - pool.remove_writer_and_close_clients(*writer_id), + pool.remove_writer_and_close_clients( + *writer_id, + MeWriterTeardownReason::WatchdogStuckDraining, + ), ) .await; match result { - Ok(()) => { + Ok(true) => { removal_timeout_streak.remove(writer_id); pool.stats.increment_pool_force_close_total(); pool.stats @@ -1661,7 +1668,16 @@ pub async fn me_zombie_writer_watchdog(pool: Arc) { "Zombie writer removed by watchdog" ); } + Ok(false) => { + removal_timeout_streak.remove(writer_id); + debug!( + writer_id, + had_clients, + "Zombie writer watchdog removal became no-op" + ); + } Err(_) => { + pool.stats.increment_me_writer_teardown_timeout_total(); let streak = removal_timeout_streak .entry(*writer_id) .and_modify(|value| *value = value.saturating_add(1)) @@ -1675,10 +1691,14 @@ pub async fn me_zombie_writer_watchdog(pool: Arc) { if *streak < HARD_DETACH_TIMEOUT_STREAK { continue; } + pool.stats.increment_me_writer_teardown_escalation_total(); let hard_detach = tokio::time::timeout( Duration::from_secs(REMOVE_TIMEOUT_SECS), - pool.remove_draining_writer_hard_detach(*writer_id), + pool.remove_draining_writer_hard_detach( + *writer_id, + MeWriterTeardownReason::WatchdogStuckDraining, + ), ) .await; match hard_detach { @@ -1702,6 +1722,7 @@ pub async fn me_zombie_writer_watchdog(pool: Arc) { ); } Err(_) => { + pool.stats.increment_me_writer_teardown_timeout_total(); warn!( writer_id, had_clients, diff --git a/src/transport/middle_proxy/health_adversarial_tests.rs b/src/transport/middle_proxy/health_adversarial_tests.rs index ae517b3..93b1d2b 100644 --- a/src/transport/middle_proxy/health_adversarial_tests.rs +++ b/src/transport/middle_proxy/health_adversarial_tests.rs @@ -316,7 +316,12 @@ async fn reap_draining_writers_maintains_warn_state_subset_property_under_bulk_c let ids = sorted_writer_ids(&pool).await; for writer_id in ids.into_iter().take(3) { - let _ = pool.remove_writer_and_close_clients(writer_id).await; + let _ = pool + .remove_writer_and_close_clients( + writer_id, + crate::stats::MeWriterTeardownReason::ReapEmpty, + ) + .await; } reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; diff --git a/src/transport/middle_proxy/health_regression_tests.rs b/src/transport/middle_proxy/health_regression_tests.rs index 230cd64..3c7b919 100644 --- a/src/transport/middle_proxy/health_regression_tests.rs +++ b/src/transport/middle_proxy/health_regression_tests.rs @@ -197,7 +197,9 @@ async fn reap_draining_writers_drops_warn_state_for_removed_writer() { reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.contains_key(&7)); - let _ = pool.remove_writer_and_close_clients(7).await; + let _ = pool + .remove_writer_and_close_clients(7, crate::stats::MeWriterTeardownReason::ReapEmpty) + .await; assert!(pool.registry.get_writer(conn_ids[0]).await.is_none()); reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; @@ -527,7 +529,12 @@ async fn reap_draining_writers_warn_state_never_exceeds_live_draining_population let existing_writer_ids = current_writer_ids(&pool).await; for writer_id in existing_writer_ids.into_iter().take(4) { - let _ = pool.remove_writer_and_close_clients(writer_id).await; + let _ = pool + .remove_writer_and_close_clients( + writer_id, + crate::stats::MeWriterTeardownReason::ReapEmpty, + ) + .await; } reap_draining_writers(&pool, &mut warn_next_allowed, &mut soft_evict_next_allowed).await; assert!(warn_next_allowed.len() <= pool.writers.read().await.len()); diff --git a/src/transport/middle_proxy/pool_writer.rs b/src/transport/middle_proxy/pool_writer.rs index e3ea44d..b0ba776 100644 --- a/src/transport/middle_proxy/pool_writer.rs +++ b/src/transport/middle_proxy/pool_writer.rs @@ -16,6 +16,9 @@ use crate::config::MeBindStaleMode; use crate::crypto::SecureRandom; use crate::error::{ProxyError, Result}; use crate::protocol::constants::{RPC_CLOSE_EXT_U32, RPC_PING_U32}; +use crate::stats::{ + MeWriterCleanupSideEffectStep, MeWriterTeardownMode, MeWriterTeardownReason, +}; use super::codec::{RpcWriter, WriterCommand}; use super::pool::{MePool, MeWriter, WriterContour}; @@ -28,7 +31,7 @@ const ME_IDLE_KEEPALIVE_MAX_SECS: u64 = 5; const ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS: u64 = 700; #[derive(Clone, Copy)] -enum WriterTeardownMode { +enum WriterRemoveGuardMode { Any, DrainingOnly, } @@ -49,9 +52,16 @@ impl MePool { for writer_id in closed_writer_ids { if self.registry.is_writer_empty(writer_id).await { - let _ = self.remove_writer_only(writer_id).await; + let _ = self + .remove_writer_only(writer_id, MeWriterTeardownReason::PruneClosedWriter) + .await; } else { - let _ = self.remove_writer_and_close_clients(writer_id).await; + let _ = self + .remove_writer_and_close_clients( + writer_id, + MeWriterTeardownReason::PruneClosedWriter, + ) + .await; } } } @@ -173,7 +183,11 @@ impl MePool { .is_ok() { if let Some(pool) = pool_writer_task.upgrade() { - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients( + writer_id, + MeWriterTeardownReason::WriterTaskExit, + ) + .await; } else { cancel_wr.cancel(); } @@ -264,7 +278,11 @@ impl MePool { .is_ok() { if let Some(pool) = pool.upgrade() { - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients( + writer_id, + MeWriterTeardownReason::ReaderExit, + ) + .await; } else { // Fallback for shutdown races: make writer task exit quickly so stale // channels are observable by periodic prune. @@ -372,7 +390,11 @@ impl MePool { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) .is_ok() { - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients( + writer_id, + MeWriterTeardownReason::PingSendFail, + ) + .await; } break; } @@ -465,7 +487,11 @@ impl MePool { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) .is_ok() { - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients( + writer_id, + MeWriterTeardownReason::SignalSendFail, + ) + .await; } break; } @@ -499,7 +525,11 @@ impl MePool { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) .is_ok() { - pool.remove_writer_and_close_clients(writer_id).await; + pool.remove_writer_and_close_clients( + writer_id, + MeWriterTeardownReason::SignalSendFail, + ) + .await; } break; } @@ -512,25 +542,48 @@ impl MePool { Ok(()) } - pub(crate) async fn remove_writer_and_close_clients(self: &Arc, writer_id: u64) { + pub(crate) async fn remove_writer_and_close_clients( + self: &Arc, + writer_id: u64, + reason: MeWriterTeardownReason, + ) -> bool { // Full client cleanup now happens inside `registry.writer_lost` to keep // writer reap/remove paths strictly non-blocking per connection. - let _ = self - .remove_writer_with_mode(writer_id, WriterTeardownMode::Any) - .await; + self.remove_writer_with_mode( + writer_id, + reason, + MeWriterTeardownMode::Normal, + WriterRemoveGuardMode::Any, + ) + .await } pub(super) async fn remove_draining_writer_hard_detach( self: &Arc, writer_id: u64, + reason: MeWriterTeardownReason, ) -> bool { - self.remove_writer_with_mode(writer_id, WriterTeardownMode::DrainingOnly) - .await + self.remove_writer_with_mode( + writer_id, + reason, + MeWriterTeardownMode::HardDetach, + WriterRemoveGuardMode::DrainingOnly, + ) + .await } - async fn remove_writer_only(self: &Arc, writer_id: u64) -> bool { - self.remove_writer_with_mode(writer_id, WriterTeardownMode::Any) - .await + async fn remove_writer_only( + self: &Arc, + writer_id: u64, + reason: MeWriterTeardownReason, + ) -> bool { + self.remove_writer_with_mode( + writer_id, + reason, + MeWriterTeardownMode::Normal, + WriterRemoveGuardMode::Any, + ) + .await } // Authoritative teardown primitive shared by normal cleanup and watchdog path. @@ -542,8 +595,13 @@ impl MePool { async fn remove_writer_with_mode( self: &Arc, writer_id: u64, - mode: WriterTeardownMode, + reason: MeWriterTeardownReason, + mode: MeWriterTeardownMode, + guard_mode: WriterRemoveGuardMode, ) -> bool { + let started_at = Instant::now(); + self.stats + .increment_me_writer_teardown_attempt_total(reason, mode); let mut close_tx: Option> = None; let mut removed_addr: Option = None; let mut removed_dc: Option = None; @@ -553,9 +611,12 @@ impl MePool { { let mut ws = self.writers.write().await; if let Some(pos) = ws.iter().position(|w| w.id == writer_id) { - if matches!(mode, WriterTeardownMode::DrainingOnly) + if matches!(guard_mode, WriterRemoveGuardMode::DrainingOnly) && !ws[pos].draining.load(Ordering::Relaxed) { + self.stats.increment_me_writer_teardown_noop_total(); + self.stats + .observe_me_writer_teardown_duration(mode, started_at.elapsed()); return false; } let w = ws.remove(pos); @@ -595,6 +656,9 @@ impl MePool { self.stats.increment_me_writer_close_signal_drop_total(); self.stats .increment_me_writer_close_signal_channel_full_total(); + self.stats.increment_me_writer_cleanup_side_effect_failures_total( + MeWriterCleanupSideEffectStep::CloseSignalChannelFull, + ); debug!( writer_id, "Skipping close signal for removed writer: command channel is full" @@ -602,6 +666,9 @@ impl MePool { } Err(TrySendError::Closed(_)) => { self.stats.increment_me_writer_close_signal_drop_total(); + self.stats.increment_me_writer_cleanup_side_effect_failures_total( + MeWriterCleanupSideEffectStep::CloseSignalChannelClosed, + ); debug!( writer_id, "Skipping close signal for removed writer: command channel is closed" @@ -619,6 +686,13 @@ impl MePool { self.trigger_immediate_refill_for_dc(addr, writer_dc); } } + if removed { + self.stats.increment_me_writer_teardown_success_total(mode); + } else { + self.stats.increment_me_writer_teardown_noop_total(); + } + self.stats + .observe_me_writer_teardown_duration(mode, started_at.elapsed()); removed } diff --git a/src/transport/middle_proxy/send.rs b/src/transport/middle_proxy/send.rs index 6791064..82118d8 100644 --- a/src/transport/middle_proxy/send.rs +++ b/src/transport/middle_proxy/send.rs @@ -14,6 +14,7 @@ use crate::config::{MeRouteNoWriterMode, MeWriterPickMode}; use crate::error::{ProxyError, Result}; use crate::network::IpFamily; use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32}; +use crate::stats::MeWriterTeardownReason; use super::MePool; use super::codec::WriterCommand; @@ -134,7 +135,11 @@ impl MePool { Ok(()) => return Ok(()), Err(TimedSendError::Closed(_)) => { warn!(writer_id = current.writer_id, "ME writer channel closed"); - self.remove_writer_and_close_clients(current.writer_id).await; + self.remove_writer_and_close_clients( + current.writer_id, + MeWriterTeardownReason::RouteChannelClosed, + ) + .await; continue; } Err(TimedSendError::Timeout(_)) => { @@ -151,7 +156,11 @@ impl MePool { } Err(TrySendError::Closed(_)) => { warn!(writer_id = current.writer_id, "ME writer channel closed"); - self.remove_writer_and_close_clients(current.writer_id).await; + self.remove_writer_and_close_clients( + current.writer_id, + MeWriterTeardownReason::RouteChannelClosed, + ) + .await; continue; } } @@ -458,7 +467,11 @@ impl MePool { Err(TrySendError::Closed(_)) => { self.stats.increment_me_writer_pick_closed_total(pick_mode); warn!(writer_id = w.id, "ME writer channel closed"); - self.remove_writer_and_close_clients(w.id).await; + self.remove_writer_and_close_clients( + w.id, + MeWriterTeardownReason::RouteChannelClosed, + ) + .await; continue; } } @@ -503,7 +516,11 @@ impl MePool { Err(TimedSendError::Closed(_)) => { self.stats.increment_me_writer_pick_closed_total(pick_mode); warn!(writer_id = w.id, "ME writer channel closed (blocking)"); - self.remove_writer_and_close_clients(w.id).await; + self.remove_writer_and_close_clients( + w.id, + MeWriterTeardownReason::RouteChannelClosed, + ) + .await; } Err(TimedSendError::Timeout(_)) => { self.stats.increment_me_writer_pick_full_total(pick_mode); @@ -654,7 +671,11 @@ impl MePool { } Err(TrySendError::Closed(_)) => { debug!("ME close write failed"); - self.remove_writer_and_close_clients(w.writer_id).await; + self.remove_writer_and_close_clients( + w.writer_id, + MeWriterTeardownReason::CloseRpcChannelClosed, + ) + .await; } } } else { From aba4205dccd3acb98bd5b8d403bce82dde21dd2b Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:46:35 +0300 Subject: [PATCH 255/261] Teardown Monitoring in Metrics Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/stats/mod.rs | 357 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 357 insertions(+) diff --git a/src/stats/mod.rs b/src/stats/mod.rs index ad1d16b..0df4dc0 100644 --- a/src/stats/mod.rs +++ b/src/stats/mod.rs @@ -19,6 +19,137 @@ use tracing::debug; use crate::config::{MeTelemetryLevel, MeWriterPickMode}; use self::telemetry::TelemetryPolicy; +const ME_WRITER_TEARDOWN_MODE_COUNT: usize = 2; +const ME_WRITER_TEARDOWN_REASON_COUNT: usize = 11; +const ME_WRITER_CLEANUP_SIDE_EFFECT_STEP_COUNT: usize = 2; +const ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT: usize = 12; +const ME_WRITER_TEARDOWN_DURATION_BUCKET_BOUNDS_MICROS: [u64; ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT] = [ + 1_000, + 5_000, + 10_000, + 25_000, + 50_000, + 100_000, + 250_000, + 500_000, + 1_000_000, + 2_500_000, + 5_000_000, + 10_000_000, +]; +const ME_WRITER_TEARDOWN_DURATION_BUCKET_LABELS: [&str; ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT] = [ + "0.001", + "0.005", + "0.01", + "0.025", + "0.05", + "0.1", + "0.25", + "0.5", + "1", + "2.5", + "5", + "10", +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum MeWriterTeardownMode { + Normal = 0, + HardDetach = 1, +} + +impl MeWriterTeardownMode { + pub const ALL: [Self; ME_WRITER_TEARDOWN_MODE_COUNT] = + [Self::Normal, Self::HardDetach]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::Normal => "normal", + Self::HardDetach => "hard_detach", + } + } + + const fn idx(self) -> usize { + self as usize + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum MeWriterTeardownReason { + ReaderExit = 0, + WriterTaskExit = 1, + PingSendFail = 2, + SignalSendFail = 3, + RouteChannelClosed = 4, + CloseRpcChannelClosed = 5, + PruneClosedWriter = 6, + ReapTimeoutExpired = 7, + ReapThresholdForce = 8, + ReapEmpty = 9, + WatchdogStuckDraining = 10, +} + +impl MeWriterTeardownReason { + pub const ALL: [Self; ME_WRITER_TEARDOWN_REASON_COUNT] = [ + Self::ReaderExit, + Self::WriterTaskExit, + Self::PingSendFail, + Self::SignalSendFail, + Self::RouteChannelClosed, + Self::CloseRpcChannelClosed, + Self::PruneClosedWriter, + Self::ReapTimeoutExpired, + Self::ReapThresholdForce, + Self::ReapEmpty, + Self::WatchdogStuckDraining, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::ReaderExit => "reader_exit", + Self::WriterTaskExit => "writer_task_exit", + Self::PingSendFail => "ping_send_fail", + Self::SignalSendFail => "signal_send_fail", + Self::RouteChannelClosed => "route_channel_closed", + Self::CloseRpcChannelClosed => "close_rpc_channel_closed", + Self::PruneClosedWriter => "prune_closed_writer", + Self::ReapTimeoutExpired => "reap_timeout_expired", + Self::ReapThresholdForce => "reap_threshold_force", + Self::ReapEmpty => "reap_empty", + Self::WatchdogStuckDraining => "watchdog_stuck_draining", + } + } + + const fn idx(self) -> usize { + self as usize + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum MeWriterCleanupSideEffectStep { + CloseSignalChannelFull = 0, + CloseSignalChannelClosed = 1, +} + +impl MeWriterCleanupSideEffectStep { + pub const ALL: [Self; ME_WRITER_CLEANUP_SIDE_EFFECT_STEP_COUNT] = + [Self::CloseSignalChannelFull, Self::CloseSignalChannelClosed]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::CloseSignalChannelFull => "close_signal_channel_full", + Self::CloseSignalChannelClosed => "close_signal_channel_closed", + } + } + + const fn idx(self) -> usize { + self as usize + } +} + // ============= Stats ============= #[derive(Default)] @@ -128,6 +259,18 @@ pub struct Stats { me_draining_writers_reap_progress_total: AtomicU64, me_writer_removed_total: AtomicU64, me_writer_removed_unexpected_total: AtomicU64, + me_writer_teardown_attempt_total: + [[AtomicU64; ME_WRITER_TEARDOWN_MODE_COUNT]; ME_WRITER_TEARDOWN_REASON_COUNT], + me_writer_teardown_success_total: [AtomicU64; ME_WRITER_TEARDOWN_MODE_COUNT], + me_writer_teardown_timeout_total: AtomicU64, + me_writer_teardown_escalation_total: AtomicU64, + me_writer_teardown_noop_total: AtomicU64, + me_writer_cleanup_side_effect_failures_total: + [AtomicU64; ME_WRITER_CLEANUP_SIDE_EFFECT_STEP_COUNT], + me_writer_teardown_duration_bucket_hits: + [[AtomicU64; ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT + 1]; ME_WRITER_TEARDOWN_MODE_COUNT], + me_writer_teardown_duration_sum_micros: [AtomicU64; ME_WRITER_TEARDOWN_MODE_COUNT], + me_writer_teardown_duration_count: [AtomicU64; ME_WRITER_TEARDOWN_MODE_COUNT], me_refill_triggered_total: AtomicU64, me_refill_skipped_inflight_total: AtomicU64, me_refill_failed_total: AtomicU64, @@ -765,6 +908,74 @@ impl Stats { self.me_writer_removed_unexpected_total.fetch_add(1, Ordering::Relaxed); } } + pub fn increment_me_writer_teardown_attempt_total( + &self, + reason: MeWriterTeardownReason, + mode: MeWriterTeardownMode, + ) { + if self.telemetry_me_allows_normal() { + self.me_writer_teardown_attempt_total[reason.idx()][mode.idx()] + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_teardown_success_total(&self, mode: MeWriterTeardownMode) { + if self.telemetry_me_allows_normal() { + self.me_writer_teardown_success_total[mode.idx()].fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_teardown_timeout_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_teardown_timeout_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_teardown_escalation_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_teardown_escalation_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_teardown_noop_total(&self) { + if self.telemetry_me_allows_normal() { + self.me_writer_teardown_noop_total + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn increment_me_writer_cleanup_side_effect_failures_total( + &self, + step: MeWriterCleanupSideEffectStep, + ) { + if self.telemetry_me_allows_normal() { + self.me_writer_cleanup_side_effect_failures_total[step.idx()] + .fetch_add(1, Ordering::Relaxed); + } + } + pub fn observe_me_writer_teardown_duration( + &self, + mode: MeWriterTeardownMode, + duration: Duration, + ) { + if !self.telemetry_me_allows_normal() { + return; + } + let duration_micros = duration.as_micros().min(u64::MAX as u128) as u64; + let mut bucket_idx = ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT; + for (idx, upper_bound_micros) in ME_WRITER_TEARDOWN_DURATION_BUCKET_BOUNDS_MICROS + .iter() + .copied() + .enumerate() + { + if duration_micros <= upper_bound_micros { + bucket_idx = idx; + break; + } + } + self.me_writer_teardown_duration_bucket_hits[mode.idx()][bucket_idx] + .fetch_add(1, Ordering::Relaxed); + self.me_writer_teardown_duration_sum_micros[mode.idx()] + .fetch_add(duration_micros, Ordering::Relaxed); + self.me_writer_teardown_duration_count[mode.idx()].fetch_add(1, Ordering::Relaxed); + } pub fn increment_me_refill_triggered_total(&self) { if self.telemetry_me_allows_debug() { self.me_refill_triggered_total.fetch_add(1, Ordering::Relaxed); @@ -1297,6 +1508,79 @@ impl Stats { pub fn get_me_writer_removed_unexpected_total(&self) -> u64 { self.me_writer_removed_unexpected_total.load(Ordering::Relaxed) } + pub fn get_me_writer_teardown_attempt_total( + &self, + reason: MeWriterTeardownReason, + mode: MeWriterTeardownMode, + ) -> u64 { + self.me_writer_teardown_attempt_total[reason.idx()][mode.idx()] + .load(Ordering::Relaxed) + } + pub fn get_me_writer_teardown_attempt_total_by_mode(&self, mode: MeWriterTeardownMode) -> u64 { + MeWriterTeardownReason::ALL + .iter() + .copied() + .map(|reason| self.get_me_writer_teardown_attempt_total(reason, mode)) + .sum() + } + pub fn get_me_writer_teardown_success_total(&self, mode: MeWriterTeardownMode) -> u64 { + self.me_writer_teardown_success_total[mode.idx()].load(Ordering::Relaxed) + } + pub fn get_me_writer_teardown_timeout_total(&self) -> u64 { + self.me_writer_teardown_timeout_total.load(Ordering::Relaxed) + } + pub fn get_me_writer_teardown_escalation_total(&self) -> u64 { + self.me_writer_teardown_escalation_total + .load(Ordering::Relaxed) + } + pub fn get_me_writer_teardown_noop_total(&self) -> u64 { + self.me_writer_teardown_noop_total.load(Ordering::Relaxed) + } + pub fn get_me_writer_cleanup_side_effect_failures_total( + &self, + step: MeWriterCleanupSideEffectStep, + ) -> u64 { + self.me_writer_cleanup_side_effect_failures_total[step.idx()] + .load(Ordering::Relaxed) + } + pub fn get_me_writer_cleanup_side_effect_failures_total_all(&self) -> u64 { + MeWriterCleanupSideEffectStep::ALL + .iter() + .copied() + .map(|step| self.get_me_writer_cleanup_side_effect_failures_total(step)) + .sum() + } + pub fn me_writer_teardown_duration_bucket_labels( + ) -> &'static [&'static str; ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT] { + &ME_WRITER_TEARDOWN_DURATION_BUCKET_LABELS + } + pub fn get_me_writer_teardown_duration_bucket_hits( + &self, + mode: MeWriterTeardownMode, + bucket_idx: usize, + ) -> u64 { + self.me_writer_teardown_duration_bucket_hits[mode.idx()][bucket_idx] + .load(Ordering::Relaxed) + } + pub fn get_me_writer_teardown_duration_bucket_total( + &self, + mode: MeWriterTeardownMode, + bucket_idx: usize, + ) -> u64 { + let capped_idx = bucket_idx.min(ME_WRITER_TEARDOWN_DURATION_BUCKET_COUNT); + let mut total = 0u64; + for idx in 0..=capped_idx { + total = total.saturating_add(self.get_me_writer_teardown_duration_bucket_hits(mode, idx)); + } + total + } + pub fn get_me_writer_teardown_duration_count(&self, mode: MeWriterTeardownMode) -> u64 { + self.me_writer_teardown_duration_count[mode.idx()].load(Ordering::Relaxed) + } + pub fn get_me_writer_teardown_duration_sum_seconds(&self, mode: MeWriterTeardownMode) -> f64 { + self.me_writer_teardown_duration_sum_micros[mode.idx()].load(Ordering::Relaxed) as f64 + / 1_000_000.0 + } pub fn get_me_refill_triggered_total(&self) -> u64 { self.me_refill_triggered_total.load(Ordering::Relaxed) } @@ -1800,6 +2084,79 @@ mod tests { assert_eq!(stats.get_me_keepalive_sent(), 0); assert_eq!(stats.get_me_route_drop_queue_full(), 0); } + + #[test] + fn test_teardown_counters_and_duration() { + let stats = Stats::new(); + stats.increment_me_writer_teardown_attempt_total( + MeWriterTeardownReason::ReaderExit, + MeWriterTeardownMode::Normal, + ); + stats.increment_me_writer_teardown_success_total(MeWriterTeardownMode::Normal); + stats.observe_me_writer_teardown_duration( + MeWriterTeardownMode::Normal, + Duration::from_millis(3), + ); + stats.increment_me_writer_cleanup_side_effect_failures_total( + MeWriterCleanupSideEffectStep::CloseSignalChannelFull, + ); + + assert_eq!( + stats.get_me_writer_teardown_attempt_total( + MeWriterTeardownReason::ReaderExit, + MeWriterTeardownMode::Normal + ), + 1 + ); + assert_eq!( + stats.get_me_writer_teardown_success_total(MeWriterTeardownMode::Normal), + 1 + ); + assert_eq!( + stats.get_me_writer_teardown_duration_count(MeWriterTeardownMode::Normal), + 1 + ); + assert!( + stats.get_me_writer_teardown_duration_sum_seconds(MeWriterTeardownMode::Normal) > 0.0 + ); + assert_eq!( + stats.get_me_writer_cleanup_side_effect_failures_total( + MeWriterCleanupSideEffectStep::CloseSignalChannelFull + ), + 1 + ); + } + + #[test] + fn test_teardown_counters_respect_me_silent() { + let stats = Stats::new(); + stats.apply_telemetry_policy(TelemetryPolicy { + core_enabled: true, + user_enabled: true, + me_level: MeTelemetryLevel::Silent, + }); + stats.increment_me_writer_teardown_attempt_total( + MeWriterTeardownReason::ReaderExit, + MeWriterTeardownMode::Normal, + ); + stats.increment_me_writer_teardown_timeout_total(); + stats.observe_me_writer_teardown_duration( + MeWriterTeardownMode::Normal, + Duration::from_millis(1), + ); + assert_eq!( + stats.get_me_writer_teardown_attempt_total( + MeWriterTeardownReason::ReaderExit, + MeWriterTeardownMode::Normal + ), + 0 + ); + assert_eq!(stats.get_me_writer_teardown_timeout_total(), 0); + assert_eq!( + stats.get_me_writer_teardown_duration_count(MeWriterTeardownMode::Normal), + 0 + ); + } #[test] fn test_replay_checker_basic() { From 4a610d83a36dd6489e7317b0cda791029a5ad6a2 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:56:13 +0300 Subject: [PATCH 256/261] Update Cargo.toml Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fab12f7..7c5fdc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.26" +version = "3.3.27" edition = "2024" [dependencies] From cd65fb432bbe4fd293ce9412fe36bf76c8652971 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:58:55 +0300 Subject: [PATCH 257/261] Update README.md --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f58a300..5980885 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,7 @@ #### О релизах -[3.3.15](https://github.com/telemt/telemt/releases/tag/3.3.15) по итогам работы в продакшн признан одним из самых стабильных и рекомендуется к использованию, когда cutting-edge фичи некритичны! - -[3.3.24](https://github.com/telemt/telemt/releases/tag/3.3.24) даёт баланс стабильности и передового функционала, а так же последние исправления по безопасности и багам +[3.3.27](https://github.com/telemt/telemt/releases/tag/3.3.27) даёт баланс стабильности и передового функционала, а так же последние исправления по безопасности и багам Будем рады вашему фидбеку и предложениям по улучшению — особенно в части **API**, **статистики**, **UX** @@ -44,9 +42,7 @@ #### About releases -[3.3.15](https://github.com/telemt/telemt/releases/tag/3.3.15) is, based on the results of his work in production, recognized as one of the most stable and recommended for use when cutting-edge features are not so necessary! - -[3.3.24](https://github.com/telemt/telemt/releases/tag/3.3.24) provides a balance of stability and advanced functionality, as well as the latest security and bug fixes +[3.3.27](https://github.com/telemt/telemt/releases/tag/3.3.27) provides a balance of stability and advanced functionality, as well as the latest security and bug fixes We are looking forward to your feedback and improvement proposals — especially regarding **API**, **statistics**, **UX** From db36945293a1b5c8f5aebce3988e2b104bc96332 Mon Sep 17 00:00:00 2001 From: Dmitry Dzhus Date: Fri, 20 Mar 2026 12:00:41 +0000 Subject: [PATCH 258/261] Fix typo in systemd service metadata --- src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index b6e2d92..87dcfb5 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -268,7 +268,7 @@ fn generate_systemd_unit(exe_path: &Path, config_path: &Path) -> String { format!( r#"[Unit] Description=Telemt MTProxy -Documentation=https://github.com/nicepkg/telemt +Documentation=https://github.com/telemt/telemt After=network-online.target Wants=network-online.target From 5c0eb6dbe8677bd077e5ea00e183711f5f283f34 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:05:24 +0300 Subject: [PATCH 259/261] TLS Fetcher Upstream Selection Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/config/defaults.rs | 4 +++ src/config/hot_reload.rs | 1 + src/config/load.rs | 56 ++++++++++++++++++++++++++++++++++++ src/config/types.rs | 6 ++++ src/maestro/tls_bootstrap.rs | 8 ++++++ src/tls_front/fetcher.rs | 16 ++++++++--- 6 files changed, 87 insertions(+), 4 deletions(-) diff --git a/src/config/defaults.rs b/src/config/defaults.rs index fea8305..be540b0 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -65,6 +65,10 @@ pub(crate) fn default_tls_domain() -> String { "petrovich.ru".to_string() } +pub(crate) fn default_tls_fetch_scope() -> String { + String::new() +} + pub(crate) fn default_mask_port() -> u16 { 443 } diff --git a/src/config/hot_reload.rs b/src/config/hot_reload.rs index 1315f9c..4cf7676 100644 --- a/src/config/hot_reload.rs +++ b/src/config/hot_reload.rs @@ -623,6 +623,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b } if old.censorship.tls_domain != new.censorship.tls_domain || old.censorship.tls_domains != new.censorship.tls_domains + || old.censorship.tls_fetch_scope != new.censorship.tls_fetch_scope || old.censorship.mask != new.censorship.mask || old.censorship.mask_host != new.censorship.mask_host || old.censorship.mask_port != new.censorship.mask_port diff --git a/src/config/load.rs b/src/config/load.rs index 14799ed..fbd2b33 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -779,6 +779,9 @@ impl ProxyConfig { config.censorship.mask_host = Some(config.censorship.tls_domain.clone()); } + // Normalize optional TLS fetch scope: whitespace-only values disable scoped routing. + config.censorship.tls_fetch_scope = config.censorship.tls_fetch_scope.trim().to_string(); + // Merge primary + extra TLS domains, deduplicate (primary always first). if !config.censorship.tls_domains.is_empty() { let mut all = Vec::with_capacity(1 + config.censorship.tls_domains.len()); @@ -2097,6 +2100,59 @@ mod tests { let _ = std::fs::remove_file(path); } + #[test] + fn tls_fetch_scope_default_is_empty() { + let toml = r#" + [censorship] + tls_domain = "example.com" + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_tls_fetch_scope_default_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert!(cfg.censorship.tls_fetch_scope.is_empty()); + let _ = std::fs::remove_file(path); + } + + #[test] + fn tls_fetch_scope_is_trimmed_during_load() { + let toml = r#" + [censorship] + tls_domain = "example.com" + tls_fetch_scope = " me " + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_tls_fetch_scope_trim_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert_eq!(cfg.censorship.tls_fetch_scope, "me"); + let _ = std::fs::remove_file(path); + } + + #[test] + fn tls_fetch_scope_whitespace_becomes_empty() { + let toml = r#" + [censorship] + tls_domain = "example.com" + tls_fetch_scope = " " + + [access.users] + user = "00000000000000000000000000000000" + "#; + let dir = std::env::temp_dir(); + let path = dir.join("telemt_tls_fetch_scope_blank_test.toml"); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + assert!(cfg.censorship.tls_fetch_scope.is_empty()); + let _ = std::fs::remove_file(path); + } + #[test] fn invalid_ad_tag_is_disabled_during_load() { let toml = r#" diff --git a/src/config/types.rs b/src/config/types.rs index d018187..c99000d 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1308,6 +1308,11 @@ pub struct AntiCensorshipConfig { #[serde(default)] pub tls_domains: Vec, + /// Upstream scope used for TLS front metadata fetches. + /// Empty value keeps default upstream routing behavior. + #[serde(default = "default_tls_fetch_scope")] + pub tls_fetch_scope: String, + #[serde(default = "default_true")] pub mask: bool, @@ -1365,6 +1370,7 @@ impl Default for AntiCensorshipConfig { Self { tls_domain: default_tls_domain(), tls_domains: Vec::new(), + tls_fetch_scope: default_tls_fetch_scope(), mask: default_true(), mask_host: None, mask_port: default_mask_port(), diff --git a/src/maestro/tls_bootstrap.rs b/src/maestro/tls_bootstrap.rs index a0b0b5a..73eec4c 100644 --- a/src/maestro/tls_bootstrap.rs +++ b/src/maestro/tls_bootstrap.rs @@ -38,12 +38,15 @@ pub(crate) async fn bootstrap_tls_front( .clone() .unwrap_or_else(|| config.censorship.tls_domain.clone()); let mask_unix_sock = config.censorship.mask_unix_sock.clone(); + let tls_fetch_scope = (!config.censorship.tls_fetch_scope.is_empty()) + .then(|| config.censorship.tls_fetch_scope.clone()); let fetch_timeout = Duration::from_secs(5); let cache_initial = cache.clone(); let domains_initial = tls_domains.to_vec(); let host_initial = mask_host.clone(); let unix_sock_initial = mask_unix_sock.clone(); + let scope_initial = tls_fetch_scope.clone(); let upstream_initial = upstream_manager.clone(); tokio::spawn(async move { let mut join = tokio::task::JoinSet::new(); @@ -51,6 +54,7 @@ pub(crate) async fn bootstrap_tls_front( let cache_domain = cache_initial.clone(); let host_domain = host_initial.clone(); let unix_sock_domain = unix_sock_initial.clone(); + let scope_domain = scope_initial.clone(); let upstream_domain = upstream_initial.clone(); join.spawn(async move { match crate::tls_front::fetcher::fetch_real_tls( @@ -59,6 +63,7 @@ pub(crate) async fn bootstrap_tls_front( &domain, fetch_timeout, Some(upstream_domain), + scope_domain.as_deref(), proxy_protocol, unix_sock_domain.as_deref(), ) @@ -100,6 +105,7 @@ pub(crate) async fn bootstrap_tls_front( let domains_refresh = tls_domains.to_vec(); let host_refresh = mask_host.clone(); let unix_sock_refresh = mask_unix_sock.clone(); + let scope_refresh = tls_fetch_scope.clone(); let upstream_refresh = upstream_manager.clone(); tokio::spawn(async move { loop { @@ -112,6 +118,7 @@ pub(crate) async fn bootstrap_tls_front( let cache_domain = cache_refresh.clone(); let host_domain = host_refresh.clone(); let unix_sock_domain = unix_sock_refresh.clone(); + let scope_domain = scope_refresh.clone(); let upstream_domain = upstream_refresh.clone(); join.spawn(async move { match crate::tls_front::fetcher::fetch_real_tls( @@ -120,6 +127,7 @@ pub(crate) async fn bootstrap_tls_front( &domain, fetch_timeout, Some(upstream_domain), + scope_domain.as_deref(), proxy_protocol, unix_sock_domain.as_deref(), ) diff --git a/src/tls_front/fetcher.rs b/src/tls_front/fetcher.rs index 38872af..366e5d3 100644 --- a/src/tls_front/fetcher.rs +++ b/src/tls_front/fetcher.rs @@ -394,15 +394,17 @@ async fn connect_tcp_with_upstream( port: u16, connect_timeout: Duration, upstream: Option>, + scope: Option<&str>, ) -> Result { if let Some(manager) = upstream { if let Some(addr) = resolve_socket_addr(host, port) { - match manager.connect(addr, None, None).await { + match manager.connect(addr, None, scope).await { Ok(stream) => return Ok(stream), Err(e) => { warn!( host = %host, port = port, + scope = ?scope, error = %e, "Upstream connect failed, using direct connect" ); @@ -410,12 +412,13 @@ async fn connect_tcp_with_upstream( } } else if let Ok(mut addrs) = tokio::net::lookup_host((host, port)).await { if let Some(addr) = addrs.find(|a| a.is_ipv4()) { - match manager.connect(addr, None, None).await { + match manager.connect(addr, None, scope).await { Ok(stream) => return Ok(stream), Err(e) => { warn!( host = %host, port = port, + scope = ?scope, error = %e, "Upstream connect failed, using direct connect" ); @@ -537,6 +540,7 @@ async fn fetch_via_raw_tls( sni: &str, connect_timeout: Duration, upstream: Option>, + scope: Option<&str>, proxy_protocol: u8, unix_sock: Option<&str>, ) -> Result { @@ -572,7 +576,7 @@ async fn fetch_via_raw_tls( #[cfg(not(unix))] let _ = unix_sock; - let stream = connect_tcp_with_upstream(host, port, connect_timeout, upstream).await?; + let stream = connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope).await?; fetch_via_raw_tls_stream(stream, sni, connect_timeout, proxy_protocol).await } @@ -675,6 +679,7 @@ async fn fetch_via_rustls( sni: &str, connect_timeout: Duration, upstream: Option>, + scope: Option<&str>, proxy_protocol: u8, unix_sock: Option<&str>, ) -> Result { @@ -710,7 +715,7 @@ async fn fetch_via_rustls( #[cfg(not(unix))] let _ = unix_sock; - let stream = connect_tcp_with_upstream(host, port, connect_timeout, upstream).await?; + let stream = connect_tcp_with_upstream(host, port, connect_timeout, upstream, scope).await?; fetch_via_rustls_stream(stream, host, sni, proxy_protocol).await } @@ -726,6 +731,7 @@ pub async fn fetch_real_tls( sni: &str, connect_timeout: Duration, upstream: Option>, + scope: Option<&str>, proxy_protocol: u8, unix_sock: Option<&str>, ) -> Result { @@ -735,6 +741,7 @@ pub async fn fetch_real_tls( sni, connect_timeout, upstream.clone(), + scope, proxy_protocol, unix_sock, ) @@ -753,6 +760,7 @@ pub async fn fetch_real_tls( sni, connect_timeout, upstream, + scope, proxy_protocol, unix_sock, ) From 269ba537ad893457c3ca11855a3661210b596c3a Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:07:12 +0300 Subject: [PATCH 260/261] ME Draining on Dual-Stack Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com> --- src/api/runtime_min.rs | 41 +++ src/transport/middle_proxy/health.rs | 311 ++++++++++++++++-- src/transport/middle_proxy/pool.rs | 238 +++++++++++++- src/transport/middle_proxy/pool_refill.rs | 5 +- src/transport/middle_proxy/pool_reinit.rs | 55 +++- .../middle_proxy/pool_runtime_api.rs | 51 ++- 6 files changed, 664 insertions(+), 37 deletions(-) diff --git a/src/api/runtime_min.rs b/src/api/runtime_min.rs index ae3b23f..3a107dc 100644 --- a/src/api/runtime_min.rs +++ b/src/api/runtime_min.rs @@ -154,6 +154,25 @@ pub(super) struct RuntimeMeQualityRouteDropData { pub(super) queue_full_high_total: u64, } +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityFamilyStateData { + pub(super) family: &'static str, + pub(super) state: &'static str, + pub(super) state_since_epoch_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) suppressed_until_epoch_secs: Option, + pub(super) fail_streak: u32, + pub(super) recover_success_streak: u32, +} + +#[derive(Serialize)] +pub(super) struct RuntimeMeQualityDrainGateData { + pub(super) route_quorum_ok: bool, + pub(super) redundancy_ok: bool, + pub(super) block_reason: &'static str, + pub(super) updated_at_epoch_secs: u64, +} + #[derive(Serialize)] pub(super) struct RuntimeMeQualityDcRttData { pub(super) dc: i16, @@ -169,6 +188,8 @@ pub(super) struct RuntimeMeQualityPayload { pub(super) counters: RuntimeMeQualityCountersData, pub(super) teardown: RuntimeMeQualityTeardownData, pub(super) route_drops: RuntimeMeQualityRouteDropData, + pub(super) family_states: Vec, + pub(super) drain_gate: RuntimeMeQualityDrainGateData, pub(super) dc_rtt: Vec, } @@ -409,6 +430,19 @@ pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> Runtime }; let status = pool.api_status_snapshot().await; + let family_states = pool + .api_family_state_snapshot() + .into_iter() + .map(|entry| RuntimeMeQualityFamilyStateData { + family: entry.family, + state: entry.state, + state_since_epoch_secs: entry.state_since_epoch_secs, + suppressed_until_epoch_secs: entry.suppressed_until_epoch_secs, + fail_streak: entry.fail_streak, + recover_success_streak: entry.recover_success_streak, + }) + .collect(); + let drain_gate_snapshot = pool.api_drain_gate_snapshot(); RuntimeMeQualityData { enabled: true, reason: None, @@ -430,6 +464,13 @@ pub(super) async fn build_runtime_me_quality_data(shared: &ApiShared) -> Runtime queue_full_base_total: shared.stats.get_me_route_drop_queue_full_base(), queue_full_high_total: shared.stats.get_me_route_drop_queue_full_high(), }, + family_states, + drain_gate: RuntimeMeQualityDrainGateData { + route_quorum_ok: drain_gate_snapshot.route_quorum_ok, + redundancy_ok: drain_gate_snapshot.redundancy_ok, + block_reason: drain_gate_snapshot.block_reason, + updated_at_epoch_secs: drain_gate_snapshot.updated_at_epoch_secs, + }, dc_rtt: status .dcs .into_iter() diff --git a/src/transport/middle_proxy/health.rs b/src/transport/middle_proxy/health.rs index 30e562b..d53b4ef 100644 --- a/src/transport/middle_proxy/health.rs +++ b/src/transport/middle_proxy/health.rs @@ -13,7 +13,7 @@ use crate::network::IpFamily; use crate::stats::MeWriterTeardownReason; use super::MePool; -use super::pool::MeWriter; +use super::pool::{MeFamilyRuntimeState, MeWriter}; const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff #[allow(dead_code)] @@ -34,6 +34,33 @@ const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MIN: usize = 8; const HEALTH_DRAIN_SOFT_EVICT_BUDGET_MAX: usize = 256; const HEALTH_DRAIN_REAP_OPPORTUNISTIC_INTERVAL_SECS: u64 = 1; const HEALTH_DRAIN_TIMEOUT_ENFORCER_INTERVAL_SECS: u64 = 1; +const FAMILY_SUPPRESS_FAIL_STREAK_THRESHOLD: u32 = 6; +const FAMILY_SUPPRESS_WINDOW_SECS: u64 = 120; +const FAMILY_RECOVER_PROBE_INTERVAL_SECS: u64 = 5; +const FAMILY_RECOVER_SUCCESS_STREAK_REQUIRED: u32 = 3; + +#[derive(Debug, Clone)] +struct FamilyCircuitState { + state: MeFamilyRuntimeState, + state_since_at: Instant, + suppressed_until: Option, + next_probe_at: Instant, + fail_streak: u32, + recover_success_streak: u32, +} + +impl FamilyCircuitState { + fn new(now: Instant) -> Self { + Self { + state: MeFamilyRuntimeState::Healthy, + state_since_at: now, + suppressed_until: None, + next_probe_at: now, + fail_streak: 0, + recover_success_streak: 0, + } + } +} #[derive(Debug, Clone)] struct DcFloorPlanEntry { @@ -73,6 +100,25 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c let mut floor_warn_next_allowed: HashMap<(i32, IpFamily), Instant> = HashMap::new(); let mut drain_warn_next_allowed: HashMap = HashMap::new(); let mut drain_soft_evict_next_allowed: HashMap = HashMap::new(); + let mut family_v4_circuit = FamilyCircuitState::new(Instant::now()); + let mut family_v6_circuit = FamilyCircuitState::new(Instant::now()); + let init_epoch_secs = MePool::now_epoch_secs(); + pool.set_family_runtime_state( + IpFamily::V4, + family_v4_circuit.state, + init_epoch_secs, + 0, + family_v4_circuit.fail_streak, + family_v4_circuit.recover_success_streak, + ); + pool.set_family_runtime_state( + IpFamily::V6, + family_v6_circuit.state, + init_epoch_secs, + 0, + family_v6_circuit.fail_streak, + family_v6_circuit.recover_success_streak, + ); let mut degraded_interval = true; loop { let interval = if degraded_interval { @@ -88,7 +134,9 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut drain_soft_evict_next_allowed, ) .await; - let v4_degraded = check_family( + let now = Instant::now(); + let now_epoch_secs = MePool::now_epoch_secs(); + let v4_degraded_raw = check_family( IpFamily::V4, &pool, &rng, @@ -107,25 +155,53 @@ pub async fn me_health_monitor(pool: Arc, rng: Arc, _min_c &mut drain_soft_evict_next_allowed, ) .await; - let v6_degraded = check_family( - IpFamily::V6, + let v4_degraded = apply_family_circuit_result( &pool, - &rng, - &mut backoff, - &mut next_attempt, - &mut inflight, - &mut outage_backoff, - &mut outage_next_attempt, - &mut single_endpoint_outage, - &mut shadow_rotate_deadline, - &mut idle_refresh_next_attempt, - &mut adaptive_idle_since, - &mut adaptive_recover_until, - &mut floor_warn_next_allowed, - &mut drain_warn_next_allowed, - &mut drain_soft_evict_next_allowed, - ) - .await; + IpFamily::V4, + &mut family_v4_circuit, + Some(v4_degraded_raw), + false, + now, + now_epoch_secs, + ); + + let v6_check_ran = should_run_family_check(&mut family_v6_circuit, now); + let v6_degraded_raw = if v6_check_ran { + check_family( + IpFamily::V6, + &pool, + &rng, + &mut backoff, + &mut next_attempt, + &mut inflight, + &mut outage_backoff, + &mut outage_next_attempt, + &mut single_endpoint_outage, + &mut shadow_rotate_deadline, + &mut idle_refresh_next_attempt, + &mut adaptive_idle_since, + &mut adaptive_recover_until, + &mut floor_warn_next_allowed, + &mut drain_warn_next_allowed, + &mut drain_soft_evict_next_allowed, + ) + .await + } else { + false + }; + let v6_degraded = apply_family_circuit_result( + &pool, + IpFamily::V6, + &mut family_v6_circuit, + if v6_check_ran { + Some(v6_degraded_raw) + } else { + None + }, + true, + now, + now_epoch_secs, + ); degraded_interval = v4_degraded || v6_degraded; } } @@ -147,6 +223,148 @@ pub async fn me_drain_timeout_enforcer(pool: Arc) { } } +fn should_run_family_check(circuit: &mut FamilyCircuitState, now: Instant) -> bool { + match circuit.state { + MeFamilyRuntimeState::Suppressed => { + if now < circuit.next_probe_at { + return false; + } + circuit.next_probe_at = + now + Duration::from_secs(FAMILY_RECOVER_PROBE_INTERVAL_SECS); + true + } + _ => true, + } +} + +fn apply_family_circuit_result( + pool: &Arc, + family: IpFamily, + circuit: &mut FamilyCircuitState, + degraded: Option, + allow_suppress: bool, + now: Instant, + now_epoch_secs: u64, +) -> bool { + let Some(degraded) = degraded else { + // Preserve suppression state when probe tick is intentionally skipped. + return false; + }; + + let previous_state = circuit.state; + match circuit.state { + MeFamilyRuntimeState::Suppressed => { + if degraded { + circuit.fail_streak = circuit.fail_streak.saturating_add(1); + circuit.recover_success_streak = 0; + let until = now + Duration::from_secs(FAMILY_SUPPRESS_WINDOW_SECS); + circuit.suppressed_until = Some(until); + circuit.state_since_at = now; + warn!( + ?family, + fail_streak = circuit.fail_streak, + suppress_secs = FAMILY_SUPPRESS_WINDOW_SECS, + "ME family remains suppressed due to ongoing failures" + ); + } else { + circuit.fail_streak = 0; + circuit.recover_success_streak = 1; + circuit.state = MeFamilyRuntimeState::Recovering; + } + } + MeFamilyRuntimeState::Recovering => { + if degraded { + circuit.fail_streak = circuit.fail_streak.saturating_add(1); + if allow_suppress { + circuit.state = MeFamilyRuntimeState::Suppressed; + let until = now + Duration::from_secs(FAMILY_SUPPRESS_WINDOW_SECS); + circuit.suppressed_until = Some(until); + circuit.next_probe_at = + now + Duration::from_secs(FAMILY_RECOVER_PROBE_INTERVAL_SECS); + warn!( + ?family, + fail_streak = circuit.fail_streak, + suppress_secs = FAMILY_SUPPRESS_WINDOW_SECS, + "ME family temporarily suppressed after repeated degradation" + ); + } else { + circuit.state = MeFamilyRuntimeState::Degraded; + } + } else { + circuit.recover_success_streak = circuit.recover_success_streak.saturating_add(1); + if circuit.recover_success_streak >= FAMILY_RECOVER_SUCCESS_STREAK_REQUIRED { + circuit.fail_streak = 0; + circuit.recover_success_streak = 0; + circuit.suppressed_until = None; + circuit.state = MeFamilyRuntimeState::Healthy; + info!( + ?family, + "ME family suppression lifted after stable recovery probes" + ); + } + } + } + _ => { + if degraded { + circuit.fail_streak = circuit.fail_streak.saturating_add(1); + circuit.recover_success_streak = 0; + circuit.state = MeFamilyRuntimeState::Degraded; + if allow_suppress && circuit.fail_streak >= FAMILY_SUPPRESS_FAIL_STREAK_THRESHOLD { + circuit.state = MeFamilyRuntimeState::Suppressed; + let until = now + Duration::from_secs(FAMILY_SUPPRESS_WINDOW_SECS); + circuit.suppressed_until = Some(until); + circuit.next_probe_at = + now + Duration::from_secs(FAMILY_RECOVER_PROBE_INTERVAL_SECS); + warn!( + ?family, + fail_streak = circuit.fail_streak, + suppress_secs = FAMILY_SUPPRESS_WINDOW_SECS, + "ME family temporarily suppressed after repeated degradation" + ); + } + } else { + circuit.fail_streak = 0; + circuit.recover_success_streak = 0; + circuit.suppressed_until = None; + circuit.state = MeFamilyRuntimeState::Healthy; + } + } + } + + if previous_state != circuit.state { + circuit.state_since_at = now; + } + + let suppressed_until_epoch_secs = circuit + .suppressed_until + .and_then(|until| { + if until > now { + Some( + now_epoch_secs + .saturating_add(until.saturating_duration_since(now).as_secs()), + ) + } else { + None + } + }) + .unwrap_or(0); + let state_since_epoch_secs = if previous_state == circuit.state { + pool.family_runtime_state_since_epoch_secs(family) + } else { + now_epoch_secs + }; + pool.set_family_runtime_state( + family, + circuit.state, + state_since_epoch_secs, + suppressed_until_epoch_secs, + circuit.fail_streak, + circuit.recover_success_streak, + ); + + !matches!(circuit.state, MeFamilyRuntimeState::Suppressed) && degraded +} + fn draining_writer_timeout_expired( pool: &MePool, writer: &MeWriter, @@ -1746,13 +1964,19 @@ mod tests { use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; - use super::reap_draining_writers; + use super::{ + FamilyCircuitState, apply_family_circuit_result, reap_draining_writers, + should_run_family_check, + }; use crate::config::{GeneralConfig, MeRouteNoWriterMode, MeSocksKdfPolicy, MeWriterPickMode}; use crate::crypto::SecureRandom; + use crate::network::IpFamily; use crate::network::probe::NetworkDecision; use crate::stats::Stats; use crate::transport::middle_proxy::codec::WriterCommand; - use crate::transport::middle_proxy::pool::{MePool, MeWriter, WriterContour}; + use crate::transport::middle_proxy::pool::{ + MeFamilyRuntimeState, MePool, MeWriter, WriterContour, + }; use crate::transport::middle_proxy::registry::ConnMeta; async fn make_pool(me_pool_drain_threshold: u64) -> Arc { @@ -1930,4 +2154,47 @@ mod tests { assert_eq!(pool.registry.get_writer(conn_b).await.unwrap().writer_id, 20); assert_eq!(pool.registry.get_writer(conn_c).await.unwrap().writer_id, 30); } + + #[tokio::test] + async fn suppressed_family_probe_skip_preserves_suppressed_state() { + let pool = make_pool(0).await; + let now = Instant::now(); + let now_epoch_secs = MePool::now_epoch_secs(); + let suppressed_until_epoch_secs = now_epoch_secs.saturating_add(60); + pool.set_family_runtime_state( + IpFamily::V6, + MeFamilyRuntimeState::Suppressed, + now_epoch_secs, + suppressed_until_epoch_secs, + 7, + 0, + ); + + let mut circuit = FamilyCircuitState { + state: MeFamilyRuntimeState::Suppressed, + state_since_at: now, + suppressed_until: Some(now + Duration::from_secs(60)), + next_probe_at: now + Duration::from_secs(5), + fail_streak: 7, + recover_success_streak: 0, + }; + + assert!(!should_run_family_check(&mut circuit, now)); + assert!(!apply_family_circuit_result( + &pool, + IpFamily::V6, + &mut circuit, + None, + true, + now, + now_epoch_secs, + )); + assert_eq!(circuit.state, MeFamilyRuntimeState::Suppressed); + assert_eq!(circuit.fail_streak, 7); + assert_eq!(circuit.recover_success_streak, 0); + assert_eq!( + pool.family_runtime_state(IpFamily::V6), + MeFamilyRuntimeState::Suppressed, + ); + } } diff --git a/src/transport/middle_proxy/pool.rs b/src/transport/middle_proxy/pool.rs index f825058..27bcb07 100644 --- a/src/transport/middle_proxy/pool.rs +++ b/src/transport/middle_proxy/pool.rs @@ -74,6 +74,64 @@ impl WriterContour { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum MeFamilyRuntimeState { + Healthy = 0, + Degraded = 1, + Suppressed = 2, + Recovering = 3, +} + +impl MeFamilyRuntimeState { + pub(crate) fn from_u8(value: u8) -> Self { + match value { + 1 => Self::Degraded, + 2 => Self::Suppressed, + 3 => Self::Recovering, + _ => Self::Healthy, + } + } + + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Healthy => "healthy", + Self::Degraded => "degraded", + Self::Suppressed => "suppressed", + Self::Recovering => "recovering", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum MeDrainGateReason { + Open = 0, + CoverageQuorum = 1, + Redundancy = 2, + SuppressionActive = 3, +} + +impl MeDrainGateReason { + pub(crate) fn from_u8(value: u8) -> Self { + match value { + 1 => Self::CoverageQuorum, + 2 => Self::Redundancy, + 3 => Self::SuppressionActive, + _ => Self::Open, + } + } + + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Open => "open", + Self::CoverageQuorum => "coverage_quorum", + Self::Redundancy => "redundancy", + Self::SuppressionActive => "suppression_active", + } + } +} + #[derive(Debug, Clone)] pub struct SecretSnapshot { pub epoch: u64, @@ -203,6 +261,20 @@ pub struct MePool { pub(super) me_health_interval_ms_unhealthy: AtomicU64, pub(super) me_health_interval_ms_healthy: AtomicU64, pub(super) me_warn_rate_limit_ms: AtomicU64, + pub(super) me_family_v4_runtime_state: AtomicU8, + pub(super) me_family_v6_runtime_state: AtomicU8, + pub(super) me_family_v4_state_since_epoch_secs: AtomicU64, + pub(super) me_family_v6_state_since_epoch_secs: AtomicU64, + pub(super) me_family_v4_suppressed_until_epoch_secs: AtomicU64, + pub(super) me_family_v6_suppressed_until_epoch_secs: AtomicU64, + pub(super) me_family_v4_fail_streak: AtomicU32, + pub(super) me_family_v6_fail_streak: AtomicU32, + pub(super) me_family_v4_recover_success_streak: AtomicU32, + pub(super) me_family_v6_recover_success_streak: AtomicU32, + pub(super) me_last_drain_gate_route_quorum_ok: AtomicBool, + pub(super) me_last_drain_gate_redundancy_ok: AtomicBool, + pub(super) me_last_drain_gate_block_reason: AtomicU8, + pub(super) me_last_drain_gate_updated_at_epoch_secs: AtomicU64, pub(super) runtime_ready: AtomicBool, pool_size: usize, pub(super) preferred_endpoints_by_dc: Arc>>>, @@ -518,6 +590,20 @@ impl MePool { me_health_interval_ms_unhealthy: AtomicU64::new(me_health_interval_ms_unhealthy.max(1)), me_health_interval_ms_healthy: AtomicU64::new(me_health_interval_ms_healthy.max(1)), me_warn_rate_limit_ms: AtomicU64::new(me_warn_rate_limit_ms.max(1)), + me_family_v4_runtime_state: AtomicU8::new(MeFamilyRuntimeState::Healthy as u8), + me_family_v6_runtime_state: AtomicU8::new(MeFamilyRuntimeState::Healthy as u8), + me_family_v4_state_since_epoch_secs: AtomicU64::new(Self::now_epoch_secs()), + me_family_v6_state_since_epoch_secs: AtomicU64::new(Self::now_epoch_secs()), + me_family_v4_suppressed_until_epoch_secs: AtomicU64::new(0), + me_family_v6_suppressed_until_epoch_secs: AtomicU64::new(0), + me_family_v4_fail_streak: AtomicU32::new(0), + me_family_v6_fail_streak: AtomicU32::new(0), + me_family_v4_recover_success_streak: AtomicU32::new(0), + me_family_v6_recover_success_streak: AtomicU32::new(0), + me_last_drain_gate_route_quorum_ok: AtomicBool::new(false), + me_last_drain_gate_redundancy_ok: AtomicBool::new(false), + me_last_drain_gate_block_reason: AtomicU8::new(MeDrainGateReason::Open as u8), + me_last_drain_gate_updated_at_epoch_secs: AtomicU64::new(Self::now_epoch_secs()), runtime_ready: AtomicBool::new(false), preferred_endpoints_by_dc: Arc::new(RwLock::new(preferred_endpoints_by_dc)), }) @@ -535,6 +621,153 @@ impl MePool { self.runtime_ready.load(Ordering::Relaxed) } + pub(super) fn set_family_runtime_state( + &self, + family: IpFamily, + state: MeFamilyRuntimeState, + state_since_epoch_secs: u64, + suppressed_until_epoch_secs: u64, + fail_streak: u32, + recover_success_streak: u32, + ) { + match family { + IpFamily::V4 => { + self.me_family_v4_runtime_state + .store(state as u8, Ordering::Relaxed); + self.me_family_v4_state_since_epoch_secs + .store(state_since_epoch_secs, Ordering::Relaxed); + self.me_family_v4_suppressed_until_epoch_secs + .store(suppressed_until_epoch_secs, Ordering::Relaxed); + self.me_family_v4_fail_streak + .store(fail_streak, Ordering::Relaxed); + self.me_family_v4_recover_success_streak + .store(recover_success_streak, Ordering::Relaxed); + } + IpFamily::V6 => { + self.me_family_v6_runtime_state + .store(state as u8, Ordering::Relaxed); + self.me_family_v6_state_since_epoch_secs + .store(state_since_epoch_secs, Ordering::Relaxed); + self.me_family_v6_suppressed_until_epoch_secs + .store(suppressed_until_epoch_secs, Ordering::Relaxed); + self.me_family_v6_fail_streak + .store(fail_streak, Ordering::Relaxed); + self.me_family_v6_recover_success_streak + .store(recover_success_streak, Ordering::Relaxed); + } + } + } + + pub(crate) fn family_runtime_state(&self, family: IpFamily) -> MeFamilyRuntimeState { + match family { + IpFamily::V4 => MeFamilyRuntimeState::from_u8( + self.me_family_v4_runtime_state.load(Ordering::Relaxed), + ), + IpFamily::V6 => MeFamilyRuntimeState::from_u8( + self.me_family_v6_runtime_state.load(Ordering::Relaxed), + ), + } + } + + pub(crate) fn family_runtime_state_since_epoch_secs(&self, family: IpFamily) -> u64 { + match family { + IpFamily::V4 => self + .me_family_v4_state_since_epoch_secs + .load(Ordering::Relaxed), + IpFamily::V6 => self + .me_family_v6_state_since_epoch_secs + .load(Ordering::Relaxed), + } + } + + pub(crate) fn family_suppressed_until_epoch_secs(&self, family: IpFamily) -> u64 { + match family { + IpFamily::V4 => self + .me_family_v4_suppressed_until_epoch_secs + .load(Ordering::Relaxed), + IpFamily::V6 => self + .me_family_v6_suppressed_until_epoch_secs + .load(Ordering::Relaxed), + } + } + + pub(crate) fn family_fail_streak(&self, family: IpFamily) -> u32 { + match family { + IpFamily::V4 => self.me_family_v4_fail_streak.load(Ordering::Relaxed), + IpFamily::V6 => self.me_family_v6_fail_streak.load(Ordering::Relaxed), + } + } + + pub(crate) fn family_recover_success_streak(&self, family: IpFamily) -> u32 { + match family { + IpFamily::V4 => self + .me_family_v4_recover_success_streak + .load(Ordering::Relaxed), + IpFamily::V6 => self + .me_family_v6_recover_success_streak + .load(Ordering::Relaxed), + } + } + + pub(crate) fn is_family_temporarily_suppressed( + &self, + family: IpFamily, + now_epoch_secs: u64, + ) -> bool { + self.family_suppressed_until_epoch_secs(family) > now_epoch_secs + } + + pub(super) fn family_enabled_for_drain_coverage( + &self, + family: IpFamily, + now_epoch_secs: u64, + ) -> bool { + let configured = match family { + IpFamily::V4 => self.decision.ipv4_me, + IpFamily::V6 => self.decision.ipv6_me, + }; + configured && !self.is_family_temporarily_suppressed(family, now_epoch_secs) + } + + pub(super) fn set_last_drain_gate( + &self, + route_quorum_ok: bool, + redundancy_ok: bool, + block_reason: MeDrainGateReason, + updated_at_epoch_secs: u64, + ) { + self.me_last_drain_gate_route_quorum_ok + .store(route_quorum_ok, Ordering::Relaxed); + self.me_last_drain_gate_redundancy_ok + .store(redundancy_ok, Ordering::Relaxed); + self.me_last_drain_gate_block_reason + .store(block_reason as u8, Ordering::Relaxed); + self.me_last_drain_gate_updated_at_epoch_secs + .store(updated_at_epoch_secs, Ordering::Relaxed); + } + + pub(crate) fn last_drain_gate_route_quorum_ok(&self) -> bool { + self.me_last_drain_gate_route_quorum_ok + .load(Ordering::Relaxed) + } + + pub(crate) fn last_drain_gate_redundancy_ok(&self) -> bool { + self.me_last_drain_gate_redundancy_ok + .load(Ordering::Relaxed) + } + + pub(crate) fn last_drain_gate_block_reason(&self) -> MeDrainGateReason { + MeDrainGateReason::from_u8( + self.me_last_drain_gate_block_reason + .load(Ordering::Relaxed), + ) + } + + pub(crate) fn last_drain_gate_updated_at_epoch_secs(&self) -> u64 { + self.me_last_drain_gate_updated_at_epoch_secs + .load(Ordering::Relaxed) + } + pub fn update_runtime_reinit_policy( &self, hardswap: bool, @@ -1021,9 +1254,10 @@ impl MePool { } pub(super) async fn active_coverage_required_total(&self) -> usize { + let now_epoch_secs = Self::now_epoch_secs(); let mut endpoints_by_dc = HashMap::>::new(); - if self.decision.ipv4_me { + if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch_secs) { let map = self.proxy_map_v4.read().await; for (dc, addrs) in map.iter() { let entry = endpoints_by_dc.entry(*dc).or_default(); @@ -1033,7 +1267,7 @@ impl MePool { } } - if self.decision.ipv6_me { + if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch_secs) { let map = self.proxy_map_v6.read().await; for (dc, addrs) in map.iter() { let entry = endpoints_by_dc.entry(*dc).or_default(); diff --git a/src/transport/middle_proxy/pool_refill.rs b/src/transport/middle_proxy/pool_refill.rs index e4fb95f..3c5d4b3 100644 --- a/src/transport/middle_proxy/pool_refill.rs +++ b/src/transport/middle_proxy/pool_refill.rs @@ -164,9 +164,10 @@ impl MePool { } async fn endpoints_for_dc(&self, target_dc: i32) -> Vec { + let now_epoch_secs = Self::now_epoch_secs(); let mut endpoints = HashSet::::new(); - if self.decision.ipv4_me { + if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch_secs) { let map = self.proxy_map_v4.read().await; if let Some(addrs) = map.get(&target_dc) { for (ip, port) in addrs { @@ -175,7 +176,7 @@ impl MePool { } } - if self.decision.ipv6_me { + if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch_secs) { let map = self.proxy_map_v6.read().await; if let Some(addrs) = map.get(&target_dc) { for (ip, port) in addrs { diff --git a/src/transport/middle_proxy/pool_reinit.rs b/src/transport/middle_proxy/pool_reinit.rs index 0d5c6f4..bfd56c6 100644 --- a/src/transport/middle_proxy/pool_reinit.rs +++ b/src/transport/middle_proxy/pool_reinit.rs @@ -11,8 +11,9 @@ use tracing::{debug, info, warn}; use std::collections::hash_map::DefaultHasher; use crate::crypto::SecureRandom; +use crate::network::IpFamily; -use super::pool::{MePool, WriterContour}; +use super::pool::{MeDrainGateReason, MePool, WriterContour}; const ME_HARDSWAP_PENDING_TTL_SECS: u64 = 1800; @@ -120,9 +121,10 @@ impl MePool { } async fn desired_dc_endpoints(&self) -> HashMap> { + let now_epoch_secs = Self::now_epoch_secs(); let mut out: HashMap> = HashMap::new(); - if self.decision.ipv4_me { + if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch_secs) { let map_v4 = self.proxy_map_v4.read().await.clone(); for (dc, addrs) in map_v4 { let entry = out.entry(dc).or_default(); @@ -132,7 +134,7 @@ impl MePool { } } - if self.decision.ipv6_me { + if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch_secs) { let map_v6 = self.proxy_map_v6.read().await.clone(); for (dc, addrs) in map_v6 { let entry = out.entry(dc).or_default(); @@ -313,13 +315,23 @@ impl MePool { pub async fn zero_downtime_reinit_after_map_change(self: &Arc, rng: &SecureRandom) { let desired_by_dc = self.desired_dc_endpoints().await; + let now_epoch_secs = Self::now_epoch_secs(); + let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs); + let v6_suppressed = self.is_family_temporarily_suppressed(IpFamily::V6, now_epoch_secs); if desired_by_dc.is_empty() { warn!("ME endpoint map is empty; skipping stale writer drain"); + let reason = if (self.decision.ipv4_me && v4_suppressed) + || (self.decision.ipv6_me && v6_suppressed) + { + MeDrainGateReason::SuppressionActive + } else { + MeDrainGateReason::CoverageQuorum + }; + self.set_last_drain_gate(false, false, reason, now_epoch_secs); return; } let desired_map_hash = Self::desired_map_hash(&desired_by_dc); - let now_epoch_secs = Self::now_epoch_secs(); let previous_generation = self.current_generation(); let hardswap = self.hardswap.load(Ordering::Relaxed); let generation = if hardswap { @@ -390,7 +402,17 @@ impl MePool { .load(Ordering::Relaxed), ); let (coverage_ratio, missing_dc) = Self::coverage_ratio(&desired_by_dc, &active_writer_addrs); + let mut route_quorum_ok = coverage_ratio >= min_ratio; + let mut redundancy_ok = missing_dc.is_empty(); + let mut redundancy_missing_dc = missing_dc.clone(); + let mut gate_coverage_ratio = coverage_ratio; if !hardswap && coverage_ratio < min_ratio { + self.set_last_drain_gate( + false, + redundancy_ok, + MeDrainGateReason::CoverageQuorum, + now_epoch_secs, + ); warn!( previous_generation, generation, @@ -411,7 +433,17 @@ impl MePool { .collect(); let (fresh_coverage_ratio, fresh_missing_dc) = Self::coverage_ratio(&desired_by_dc, &fresh_writer_addrs); - if !fresh_missing_dc.is_empty() { + route_quorum_ok = fresh_coverage_ratio >= min_ratio; + redundancy_ok = fresh_missing_dc.is_empty(); + redundancy_missing_dc = fresh_missing_dc.clone(); + gate_coverage_ratio = fresh_coverage_ratio; + if fresh_coverage_ratio < min_ratio { + self.set_last_drain_gate( + false, + redundancy_ok, + MeDrainGateReason::CoverageQuorum, + now_epoch_secs, + ); warn!( previous_generation, generation, @@ -421,13 +453,16 @@ impl MePool { ); return; } - } else if !missing_dc.is_empty() { + } + + self.set_last_drain_gate(route_quorum_ok, redundancy_ok, MeDrainGateReason::Open, now_epoch_secs); + if !redundancy_ok { warn!( - missing_dc = ?missing_dc, - // Keep stale writers alive when fresh coverage is incomplete. - "ME reinit coverage incomplete; keeping stale writers" + missing_dc = ?redundancy_missing_dc, + coverage_ratio = format_args!("{gate_coverage_ratio:.3}"), + min_ratio = format_args!("{min_ratio:.3}"), + "ME reinit proceeds with weighted quorum while some DC groups remain uncovered" ); - return; } if hardswap { diff --git a/src/transport/middle_proxy/pool_runtime_api.rs b/src/transport/middle_proxy/pool_runtime_api.rs index 37ef298..adacd4e 100644 --- a/src/transport/middle_proxy/pool_runtime_api.rs +++ b/src/transport/middle_proxy/pool_runtime_api.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::time::Instant; -use super::pool::{MePool, RefillDcKey}; +use super::pool::{MeDrainGateReason, MePool, RefillDcKey}; use crate::network::IpFamily; #[derive(Clone, Debug)] @@ -36,6 +36,24 @@ pub(crate) struct MeApiNatStunSnapshot { pub stun_backoff_remaining_ms: Option, } +#[derive(Clone, Debug)] +pub(crate) struct MeApiFamilyStateSnapshot { + pub family: &'static str, + pub state: &'static str, + pub state_since_epoch_secs: u64, + pub suppressed_until_epoch_secs: Option, + pub fail_streak: u32, + pub recover_success_streak: u32, +} + +#[derive(Clone, Debug)] +pub(crate) struct MeApiDrainGateSnapshot { + pub route_quorum_ok: bool, + pub redundancy_ok: bool, + pub block_reason: &'static str, + pub updated_at_epoch_secs: u64, +} + impl MePool { pub(crate) async fn api_refill_snapshot(&self) -> MeApiRefillSnapshot { let inflight_endpoints_total = self.refill_inflight.lock().await.len(); @@ -125,4 +143,35 @@ impl MePool { stun_backoff_remaining_ms, } } + + pub(crate) fn api_family_state_snapshot(&self) -> Vec { + [IpFamily::V4, IpFamily::V6] + .into_iter() + .map(|family| { + let state = self.family_runtime_state(family); + let suppressed_until = self.family_suppressed_until_epoch_secs(family); + MeApiFamilyStateSnapshot { + family: match family { + IpFamily::V4 => "v4", + IpFamily::V6 => "v6", + }, + state: state.as_str(), + state_since_epoch_secs: self.family_runtime_state_since_epoch_secs(family), + suppressed_until_epoch_secs: (suppressed_until != 0).then_some(suppressed_until), + fail_streak: self.family_fail_streak(family), + recover_success_streak: self.family_recover_success_streak(family), + } + }) + .collect() + } + + pub(crate) fn api_drain_gate_snapshot(&self) -> MeApiDrainGateSnapshot { + let reason: MeDrainGateReason = self.last_drain_gate_block_reason(); + MeApiDrainGateSnapshot { + route_quorum_ok: self.last_drain_gate_route_quorum_ok(), + redundancy_ok: self.last_drain_gate_redundancy_ok(), + block_reason: reason.as_str(), + updated_at_epoch_secs: self.last_drain_gate_updated_at_epoch_secs(), + } + } } From 2605929b932aa8fac725ed72e30752583fe5b69c Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:26:57 +0300 Subject: [PATCH 261/261] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7c5fdc2..886c061 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telemt" -version = "3.3.27" +version = "3.3.28" edition = "2024" [dependencies]