mirror of
https://github.com/telemt/telemt.git
synced 2026-07-04 08:41:11 +03:00
Compare commits
5 Commits
3.3.34
..
a1e401b00d
| Author | SHA1 | Date | |
|---|---|---|---|
| a1e401b00d | |||
| 4d83d02a8f | |||
| fea8bc63fd | |||
| d8f7173f15 | |||
| b23d433e19 |
Generated
+1
-14
@@ -2793,7 +2793,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.32"
|
version = "3.3.31"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
@@ -2844,7 +2844,6 @@ dependencies = [
|
|||||||
"tokio-util",
|
"tokio-util",
|
||||||
"toml",
|
"toml",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-appender",
|
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"url",
|
"url",
|
||||||
"webpki-roots",
|
"webpki-roots",
|
||||||
@@ -3171,18 +3170,6 @@ dependencies = [
|
|||||||
"tracing-core",
|
"tracing-core",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tracing-appender"
|
|
||||||
version = "0.2.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf"
|
|
||||||
dependencies = [
|
|
||||||
"crossbeam-channel",
|
|
||||||
"thiserror 2.0.18",
|
|
||||||
"time",
|
|
||||||
"tracing-subscriber",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tracing-attributes"
|
name = "tracing-attributes"
|
||||||
version = "0.1.31"
|
version = "0.1.31"
|
||||||
|
|||||||
+4
-17
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.3.34"
|
version = "3.3.31"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
@@ -30,13 +30,7 @@ static_assertions = "1.1"
|
|||||||
|
|
||||||
# Network
|
# Network
|
||||||
socket2 = { version = "0.6", features = ["all"] }
|
socket2 = { version = "0.6", features = ["all"] }
|
||||||
nix = { version = "0.31", default-features = false, features = [
|
nix = { version = "0.31", default-features = false, features = ["net", "fs"] }
|
||||||
"net",
|
|
||||||
"user",
|
|
||||||
"process",
|
|
||||||
"fs",
|
|
||||||
"signal",
|
|
||||||
] }
|
|
||||||
shadowsocks = { version = "1.24", features = ["aead-cipher-2022"] }
|
shadowsocks = { version = "1.24", features = ["aead-cipher-2022"] }
|
||||||
|
|
||||||
# Serialization
|
# Serialization
|
||||||
@@ -50,7 +44,6 @@ bytes = "1.9"
|
|||||||
thiserror = "2.0"
|
thiserror = "2.0"
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
tracing-appender = "0.2"
|
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
dashmap = "6.1"
|
dashmap = "6.1"
|
||||||
arc-swap = "1.7"
|
arc-swap = "1.7"
|
||||||
@@ -75,14 +68,8 @@ hyper = { version = "1", features = ["server", "http1"] }
|
|||||||
hyper-util = { version = "0.1", features = ["tokio", "server-auto"] }
|
hyper-util = { version = "0.1", features = ["tokio", "server-auto"] }
|
||||||
http-body-util = "0.1"
|
http-body-util = "0.1"
|
||||||
httpdate = "1.0"
|
httpdate = "1.0"
|
||||||
tokio-rustls = { version = "0.26", default-features = false, features = [
|
tokio-rustls = { version = "0.26", default-features = false, features = ["tls12"] }
|
||||||
"tls12",
|
rustls = { version = "0.23", default-features = false, features = ["std", "tls12", "ring"] }
|
||||||
] }
|
|
||||||
rustls = { version = "0.23", default-features = false, features = [
|
|
||||||
"std",
|
|
||||||
"tls12",
|
|
||||||
"ring",
|
|
||||||
] }
|
|
||||||
webpki-roots = "1.0"
|
webpki-roots = "1.0"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|||||||
@@ -91,7 +91,6 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
| upstream_connect_retry_attempts | `u32` | `2` | Must be `> 0`. | Connect attempts for selected upstream before error/fallback. |
|
| upstream_connect_retry_attempts | `u32` | `2` | Must be `> 0`. | Connect attempts for selected upstream before error/fallback. |
|
||||||
| upstream_connect_retry_backoff_ms | `u64` | `100` | — | Delay between upstream connect attempts (ms). |
|
| upstream_connect_retry_backoff_ms | `u64` | `100` | — | Delay between upstream connect attempts (ms). |
|
||||||
| upstream_connect_budget_ms | `u64` | `3000` | Must be `> 0`. | Total wall-clock budget for one upstream connect request (ms). |
|
| upstream_connect_budget_ms | `u64` | `3000` | Must be `> 0`. | Total wall-clock budget for one upstream connect request (ms). |
|
||||||
| tg_connect | `u64` | `10` | Must be `> 0`. | Per-attempt upstream TCP connect timeout to Telegram DC (seconds). |
|
|
||||||
| upstream_unhealthy_fail_threshold | `u32` | `5` | Must be `> 0`. | Consecutive failed requests before upstream is marked unhealthy. |
|
| upstream_unhealthy_fail_threshold | `u32` | `5` | Must be `> 0`. | Consecutive failed requests before upstream is marked unhealthy. |
|
||||||
| upstream_connect_failfast_hard_errors | `bool` | `false` | — | Skips additional retries for hard non-transient connect errors. |
|
| upstream_connect_failfast_hard_errors | `bool` | `false` | — | Skips additional retries for hard non-transient connect errors. |
|
||||||
| stun_iface_mismatch_ignore | `bool` | `false` | none | Reserved compatibility flag in current runtime revision. |
|
| stun_iface_mismatch_ignore | `bool` | `false` | none | Reserved compatibility flag in current runtime revision. |
|
||||||
@@ -250,6 +249,7 @@ Note: When `server.proxy_protocol` is enabled, incoming PROXY protocol headers a
|
|||||||
| relay_client_idle_soft_secs | `u64` | `120` | Must be `> 0`; must be `<= relay_client_idle_hard_secs`. | Soft idle threshold for middle-relay client uplink inactivity (seconds). |
|
| relay_client_idle_soft_secs | `u64` | `120` | Must be `> 0`; must be `<= relay_client_idle_hard_secs`. | Soft idle threshold for middle-relay client uplink inactivity (seconds). |
|
||||||
| relay_client_idle_hard_secs | `u64` | `360` | Must be `> 0`; must be `>= relay_client_idle_soft_secs`. | Hard idle threshold for middle-relay client uplink inactivity (seconds). |
|
| relay_client_idle_hard_secs | `u64` | `360` | Must be `> 0`; must be `>= relay_client_idle_soft_secs`. | Hard idle threshold for middle-relay client uplink inactivity (seconds). |
|
||||||
| relay_idle_grace_after_downstream_activity_secs | `u64` | `30` | Must be `<= relay_client_idle_hard_secs`. | Extra hard-idle grace after recent downstream activity (seconds). |
|
| relay_idle_grace_after_downstream_activity_secs | `u64` | `30` | Must be `<= relay_client_idle_hard_secs`. | Extra hard-idle grace after recent downstream activity (seconds). |
|
||||||
|
| tg_connect | `u64` | `10` | — | Upstream Telegram connect timeout. |
|
||||||
| client_keepalive | `u64` | `15` | — | Client keepalive timeout. |
|
| client_keepalive | `u64` | `15` | — | Client keepalive timeout. |
|
||||||
| client_ack | `u64` | `90` | — | Client ACK timeout. |
|
| client_ack | `u64` | `90` | — | Client ACK timeout. |
|
||||||
| me_one_retry | `u8` | `12` | none | Fast reconnect attempts budget for single-endpoint DC scenarios. |
|
| me_one_retry | `u8` | `12` | none | Fast reconnect attempts budget for single-endpoint DC scenarios. |
|
||||||
|
|||||||
@@ -42,7 +42,6 @@ use events::ApiEventStore;
|
|||||||
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||||
use model::{
|
use model::{
|
||||||
ApiFailure, CreateUserRequest, HealthData, PatchUserRequest, RotateSecretRequest, SummaryData,
|
ApiFailure, CreateUserRequest, HealthData, PatchUserRequest, RotateSecretRequest, SummaryData,
|
||||||
UserActiveIps,
|
|
||||||
};
|
};
|
||||||
use runtime_edge::{
|
use runtime_edge::{
|
||||||
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
||||||
@@ -363,21 +362,6 @@ async fn handle(
|
|||||||
);
|
);
|
||||||
Ok(success_response(StatusCode::OK, data, revision))
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
}
|
}
|
||||||
("GET", "/v1/stats/users/active-ips") => {
|
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
|
||||||
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
|
||||||
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
|
|
||||||
let mut data: Vec<UserActiveIps> = active_ips_map
|
|
||||||
.into_iter()
|
|
||||||
.filter(|(_, ips)| !ips.is_empty())
|
|
||||||
.map(|(username, active_ips)| UserActiveIps {
|
|
||||||
username,
|
|
||||||
active_ips,
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
data.sort_by(|a, b| a.username.cmp(&b.username));
|
|
||||||
Ok(success_response(StatusCode::OK, data, revision))
|
|
||||||
}
|
|
||||||
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||||
|
|||||||
@@ -442,12 +442,6 @@ pub(super) struct UserInfo {
|
|||||||
pub(super) links: UserLinks,
|
pub(super) links: UserLinks,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct UserActiveIps {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) active_ips: Vec<IpAddr>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub(super) struct CreateUserResponse {
|
pub(super) struct CreateUserResponse {
|
||||||
pub(super) user: UserInfo,
|
pub(super) user: UserInfo,
|
||||||
|
|||||||
+1
-30
@@ -35,14 +35,11 @@ pub(super) struct RuntimeGatesData {
|
|||||||
pub(super) conditional_cast_enabled: bool,
|
pub(super) conditional_cast_enabled: bool,
|
||||||
pub(super) me_runtime_ready: bool,
|
pub(super) me_runtime_ready: bool,
|
||||||
pub(super) me2dc_fallback_enabled: bool,
|
pub(super) me2dc_fallback_enabled: bool,
|
||||||
pub(super) me2dc_fast_enabled: bool,
|
|
||||||
pub(super) use_middle_proxy: bool,
|
pub(super) use_middle_proxy: bool,
|
||||||
pub(super) route_mode: &'static str,
|
pub(super) route_mode: &'static str,
|
||||||
pub(super) reroute_active: bool,
|
pub(super) reroute_active: bool,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub(super) reroute_to_direct_at_epoch_secs: Option<u64>,
|
pub(super) reroute_to_direct_at_epoch_secs: Option<u64>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub(super) reroute_reason: Option<&'static str>,
|
|
||||||
pub(super) startup_status: &'static str,
|
pub(super) startup_status: &'static str,
|
||||||
pub(super) startup_stage: String,
|
pub(super) startup_stage: String,
|
||||||
pub(super) startup_progress_pct: f64,
|
pub(super) startup_progress_pct: f64,
|
||||||
@@ -50,7 +47,6 @@ pub(super) struct RuntimeGatesData {
|
|||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub(super) struct EffectiveTimeoutLimits {
|
pub(super) struct EffectiveTimeoutLimits {
|
||||||
pub(super) client_first_byte_idle_secs: u64,
|
|
||||||
pub(super) client_handshake_secs: u64,
|
pub(super) client_handshake_secs: u64,
|
||||||
pub(super) tg_connect_secs: u64,
|
pub(super) tg_connect_secs: u64,
|
||||||
pub(super) client_keepalive_secs: u64,
|
pub(super) client_keepalive_secs: u64,
|
||||||
@@ -90,7 +86,6 @@ pub(super) struct EffectiveMiddleProxyLimits {
|
|||||||
pub(super) writer_pick_mode: &'static str,
|
pub(super) writer_pick_mode: &'static str,
|
||||||
pub(super) writer_pick_sample_size: u8,
|
pub(super) writer_pick_sample_size: u8,
|
||||||
pub(super) me2dc_fallback: bool,
|
pub(super) me2dc_fallback: bool,
|
||||||
pub(super) me2dc_fast: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -100,11 +95,6 @@ pub(super) struct EffectiveUserIpPolicyLimits {
|
|||||||
pub(super) window_secs: u64,
|
pub(super) window_secs: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct EffectiveUserTcpPolicyLimits {
|
|
||||||
pub(super) global_each: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub(super) struct EffectiveLimitsData {
|
pub(super) struct EffectiveLimitsData {
|
||||||
pub(super) update_every_secs: u64,
|
pub(super) update_every_secs: u64,
|
||||||
@@ -114,7 +104,6 @@ pub(super) struct EffectiveLimitsData {
|
|||||||
pub(super) upstream: EffectiveUpstreamLimits,
|
pub(super) upstream: EffectiveUpstreamLimits,
|
||||||
pub(super) middle_proxy: EffectiveMiddleProxyLimits,
|
pub(super) middle_proxy: EffectiveMiddleProxyLimits,
|
||||||
pub(super) user_ip_policy: EffectiveUserIpPolicyLimits,
|
pub(super) user_ip_policy: EffectiveUserIpPolicyLimits,
|
||||||
pub(super) user_tcp_policy: EffectiveUserTcpPolicyLimits,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -180,8 +169,6 @@ pub(super) async fn build_runtime_gates_data(
|
|||||||
let startup_summary = build_runtime_startup_summary(shared).await;
|
let startup_summary = build_runtime_startup_summary(shared).await;
|
||||||
let route_state = shared.route_runtime.snapshot();
|
let route_state = shared.route_runtime.snapshot();
|
||||||
let route_mode = route_state.mode.as_str();
|
let route_mode = route_state.mode.as_str();
|
||||||
let fast_fallback_enabled =
|
|
||||||
cfg.general.use_middle_proxy && cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
|
||||||
let reroute_active = cfg.general.use_middle_proxy
|
let reroute_active = cfg.general.use_middle_proxy
|
||||||
&& cfg.general.me2dc_fallback
|
&& cfg.general.me2dc_fallback
|
||||||
&& matches!(route_state.mode, RelayRouteMode::Direct);
|
&& matches!(route_state.mode, RelayRouteMode::Direct);
|
||||||
@@ -190,15 +177,6 @@ pub(super) async fn build_runtime_gates_data(
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let reroute_reason = if reroute_active {
|
|
||||||
if fast_fallback_enabled {
|
|
||||||
Some("fast_not_ready_fallback")
|
|
||||||
} else {
|
|
||||||
Some("strict_grace_fallback")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
let me_runtime_ready = if !cfg.general.use_middle_proxy {
|
let me_runtime_ready = if !cfg.general.use_middle_proxy {
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
@@ -216,12 +194,10 @@ pub(super) async fn build_runtime_gates_data(
|
|||||||
conditional_cast_enabled: cfg.general.use_middle_proxy,
|
conditional_cast_enabled: cfg.general.use_middle_proxy,
|
||||||
me_runtime_ready,
|
me_runtime_ready,
|
||||||
me2dc_fallback_enabled: cfg.general.me2dc_fallback,
|
me2dc_fallback_enabled: cfg.general.me2dc_fallback,
|
||||||
me2dc_fast_enabled: fast_fallback_enabled,
|
|
||||||
use_middle_proxy: cfg.general.use_middle_proxy,
|
use_middle_proxy: cfg.general.use_middle_proxy,
|
||||||
route_mode,
|
route_mode,
|
||||||
reroute_active,
|
reroute_active,
|
||||||
reroute_to_direct_at_epoch_secs,
|
reroute_to_direct_at_epoch_secs,
|
||||||
reroute_reason,
|
|
||||||
startup_status: startup_summary.status,
|
startup_status: startup_summary.status,
|
||||||
startup_stage: startup_summary.stage,
|
startup_stage: startup_summary.stage,
|
||||||
startup_progress_pct: startup_summary.progress_pct,
|
startup_progress_pct: startup_summary.progress_pct,
|
||||||
@@ -234,9 +210,8 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD
|
|||||||
me_reinit_every_secs: cfg.general.effective_me_reinit_every_secs(),
|
me_reinit_every_secs: cfg.general.effective_me_reinit_every_secs(),
|
||||||
me_pool_force_close_secs: cfg.general.effective_me_pool_force_close_secs(),
|
me_pool_force_close_secs: cfg.general.effective_me_pool_force_close_secs(),
|
||||||
timeouts: EffectiveTimeoutLimits {
|
timeouts: EffectiveTimeoutLimits {
|
||||||
client_first_byte_idle_secs: cfg.timeouts.client_first_byte_idle_secs,
|
|
||||||
client_handshake_secs: cfg.timeouts.client_handshake,
|
client_handshake_secs: cfg.timeouts.client_handshake,
|
||||||
tg_connect_secs: cfg.general.tg_connect,
|
tg_connect_secs: cfg.timeouts.tg_connect,
|
||||||
client_keepalive_secs: cfg.timeouts.client_keepalive,
|
client_keepalive_secs: cfg.timeouts.client_keepalive,
|
||||||
client_ack_secs: cfg.timeouts.client_ack,
|
client_ack_secs: cfg.timeouts.client_ack,
|
||||||
me_one_retry: cfg.timeouts.me_one_retry,
|
me_one_retry: cfg.timeouts.me_one_retry,
|
||||||
@@ -288,16 +263,12 @@ pub(super) fn build_limits_effective_data(cfg: &ProxyConfig) -> EffectiveLimitsD
|
|||||||
writer_pick_mode: me_writer_pick_mode_label(cfg.general.me_writer_pick_mode),
|
writer_pick_mode: me_writer_pick_mode_label(cfg.general.me_writer_pick_mode),
|
||||||
writer_pick_sample_size: cfg.general.me_writer_pick_sample_size,
|
writer_pick_sample_size: cfg.general.me_writer_pick_sample_size,
|
||||||
me2dc_fallback: cfg.general.me2dc_fallback,
|
me2dc_fallback: cfg.general.me2dc_fallback,
|
||||||
me2dc_fast: cfg.general.me2dc_fast,
|
|
||||||
},
|
},
|
||||||
user_ip_policy: EffectiveUserIpPolicyLimits {
|
user_ip_policy: EffectiveUserIpPolicyLimits {
|
||||||
global_each: cfg.access.user_max_unique_ips_global_each,
|
global_each: cfg.access.user_max_unique_ips_global_each,
|
||||||
mode: user_max_unique_ips_mode_label(cfg.access.user_max_unique_ips_mode),
|
mode: user_max_unique_ips_mode_label(cfg.access.user_max_unique_ips_mode),
|
||||||
window_secs: cfg.access.user_max_unique_ips_window_secs,
|
window_secs: cfg.access.user_max_unique_ips_window_secs,
|
||||||
},
|
},
|
||||||
user_tcp_policy: EffectiveUserTcpPolicyLimits {
|
|
||||||
global_each: cfg.access.user_max_tcp_conns_global_each,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-67
@@ -144,14 +144,7 @@ pub(super) async fn create_user(
|
|||||||
.unwrap_or(UserInfo {
|
.unwrap_or(UserInfo {
|
||||||
username: body.username.clone(),
|
username: body.username.clone(),
|
||||||
user_ad_tag: None,
|
user_ad_tag: None,
|
||||||
max_tcp_conns: cfg
|
max_tcp_conns: None,
|
||||||
.access
|
|
||||||
.user_max_tcp_conns
|
|
||||||
.get(&body.username)
|
|
||||||
.copied()
|
|
||||||
.filter(|limit| *limit > 0)
|
|
||||||
.or((cfg.access.user_max_tcp_conns_global_each > 0)
|
|
||||||
.then_some(cfg.access.user_max_tcp_conns_global_each)),
|
|
||||||
expiration_rfc3339: None,
|
expiration_rfc3339: None,
|
||||||
data_quota_bytes: None,
|
data_quota_bytes: None,
|
||||||
max_unique_ips: updated_limit,
|
max_unique_ips: updated_limit,
|
||||||
@@ -402,14 +395,7 @@ pub(super) async fn users_from_config(
|
|||||||
});
|
});
|
||||||
users.push(UserInfo {
|
users.push(UserInfo {
|
||||||
user_ad_tag: cfg.access.user_ad_tags.get(&username).cloned(),
|
user_ad_tag: cfg.access.user_ad_tags.get(&username).cloned(),
|
||||||
max_tcp_conns: cfg
|
max_tcp_conns: cfg.access.user_max_tcp_conns.get(&username).copied(),
|
||||||
.access
|
|
||||||
.user_max_tcp_conns
|
|
||||||
.get(&username)
|
|
||||||
.copied()
|
|
||||||
.filter(|limit| *limit > 0)
|
|
||||||
.or((cfg.access.user_max_tcp_conns_global_each > 0)
|
|
||||||
.then_some(cfg.access.user_max_tcp_conns_global_each)),
|
|
||||||
expiration_rfc3339: cfg
|
expiration_rfc3339: cfg
|
||||||
.access
|
.access
|
||||||
.user_expirations
|
.user_expirations
|
||||||
@@ -586,54 +572,3 @@ fn resolve_tls_domains(cfg: &ProxyConfig) -> Vec<&str> {
|
|||||||
}
|
}
|
||||||
domains
|
domains
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use crate::ip_tracker::UserIpTracker;
|
|
||||||
use crate::stats::Stats;
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn users_from_config_reports_effective_tcp_limit_with_global_fallback() {
|
|
||||||
let mut cfg = ProxyConfig::default();
|
|
||||||
cfg.access.users.insert(
|
|
||||||
"alice".to_string(),
|
|
||||||
"0123456789abcdef0123456789abcdef".to_string(),
|
|
||||||
);
|
|
||||||
cfg.access.user_max_tcp_conns_global_each = 7;
|
|
||||||
|
|
||||||
let stats = Stats::new();
|
|
||||||
let tracker = UserIpTracker::new();
|
|
||||||
|
|
||||||
let users = users_from_config(&cfg, &stats, &tracker, None, None).await;
|
|
||||||
let alice = users
|
|
||||||
.iter()
|
|
||||||
.find(|entry| entry.username == "alice")
|
|
||||||
.expect("alice must be present");
|
|
||||||
assert_eq!(alice.max_tcp_conns, Some(7));
|
|
||||||
|
|
||||||
cfg.access.user_max_tcp_conns.insert("alice".to_string(), 5);
|
|
||||||
let users = users_from_config(&cfg, &stats, &tracker, None, None).await;
|
|
||||||
let alice = users
|
|
||||||
.iter()
|
|
||||||
.find(|entry| entry.username == "alice")
|
|
||||||
.expect("alice must be present");
|
|
||||||
assert_eq!(alice.max_tcp_conns, Some(5));
|
|
||||||
|
|
||||||
cfg.access.user_max_tcp_conns.insert("alice".to_string(), 0);
|
|
||||||
let users = users_from_config(&cfg, &stats, &tracker, None, None).await;
|
|
||||||
let alice = users
|
|
||||||
.iter()
|
|
||||||
.find(|entry| entry.username == "alice")
|
|
||||||
.expect("alice must be present");
|
|
||||||
assert_eq!(alice.max_tcp_conns, Some(7));
|
|
||||||
|
|
||||||
cfg.access.user_max_tcp_conns_global_each = 0;
|
|
||||||
let users = users_from_config(&cfg, &stats, &tracker, None, None).await;
|
|
||||||
let alice = users
|
|
||||||
.iter()
|
|
||||||
.find(|entry| entry.username == "alice")
|
|
||||||
.expect("alice must be present");
|
|
||||||
assert_eq!(alice.max_tcp_conns, None);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+70
-420
@@ -1,270 +1,11 @@
|
|||||||
//! CLI commands: --init (fire-and-forget setup), daemon options, subcommands
|
//! CLI commands: --init (fire-and-forget setup)
|
||||||
//!
|
|
||||||
//! Subcommands:
|
|
||||||
//! - `start [OPTIONS] [config.toml]` - Start the daemon
|
|
||||||
//! - `stop [--pid-file PATH]` - Stop a running daemon
|
|
||||||
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
|
|
||||||
//! - `status [--pid-file PATH]` - Check daemon status
|
|
||||||
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
|
|
||||||
|
|
||||||
use rand::RngExt;
|
use rand::RngExt;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
use crate::daemon::{self, DEFAULT_PID_FILE, DaemonOptions};
|
|
||||||
|
|
||||||
/// CLI subcommand to execute.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum Subcommand {
|
|
||||||
/// Run the proxy (default, or explicit `run` subcommand).
|
|
||||||
Run,
|
|
||||||
/// Start as daemon (`start` subcommand).
|
|
||||||
Start,
|
|
||||||
/// Stop a running daemon (`stop` subcommand).
|
|
||||||
Stop,
|
|
||||||
/// Reload configuration (`reload` subcommand).
|
|
||||||
Reload,
|
|
||||||
/// Check daemon status (`status` subcommand).
|
|
||||||
Status,
|
|
||||||
/// Fire-and-forget setup (`--init`).
|
|
||||||
Init,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parsed subcommand with its options.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct ParsedCommand {
|
|
||||||
pub subcommand: Subcommand,
|
|
||||||
pub pid_file: PathBuf,
|
|
||||||
pub config_path: String,
|
|
||||||
#[cfg(unix)]
|
|
||||||
pub daemon_opts: DaemonOptions,
|
|
||||||
pub init_opts: Option<InitOptions>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for ParsedCommand {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
subcommand: Subcommand::Run,
|
|
||||||
#[cfg(unix)]
|
|
||||||
pid_file: PathBuf::from(DEFAULT_PID_FILE),
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
pid_file: PathBuf::from("/var/run/telemt.pid"),
|
|
||||||
config_path: "config.toml".to_string(),
|
|
||||||
#[cfg(unix)]
|
|
||||||
daemon_opts: DaemonOptions::default(),
|
|
||||||
init_opts: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse CLI arguments into a command structure.
|
|
||||||
pub fn parse_command(args: &[String]) -> ParsedCommand {
|
|
||||||
let mut cmd = ParsedCommand::default();
|
|
||||||
|
|
||||||
// Check for --init first (legacy form)
|
|
||||||
if args.iter().any(|a| a == "--init") {
|
|
||||||
cmd.subcommand = Subcommand::Init;
|
|
||||||
cmd.init_opts = parse_init_args(args);
|
|
||||||
return cmd;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for subcommand as first argument
|
|
||||||
if let Some(first) = args.first() {
|
|
||||||
match first.as_str() {
|
|
||||||
"start" => {
|
|
||||||
cmd.subcommand = Subcommand::Start;
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
cmd.daemon_opts = parse_daemon_args(args);
|
|
||||||
// Force daemonize for start command
|
|
||||||
cmd.daemon_opts.daemonize = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
"stop" => {
|
|
||||||
cmd.subcommand = Subcommand::Stop;
|
|
||||||
}
|
|
||||||
"reload" => {
|
|
||||||
cmd.subcommand = Subcommand::Reload;
|
|
||||||
}
|
|
||||||
"status" => {
|
|
||||||
cmd.subcommand = Subcommand::Status;
|
|
||||||
}
|
|
||||||
"run" => {
|
|
||||||
cmd.subcommand = Subcommand::Run;
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
cmd.daemon_opts = parse_daemon_args(args);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
// No subcommand, default to Run
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
cmd.daemon_opts = parse_daemon_args(args);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse remaining options
|
|
||||||
let mut i = 0;
|
|
||||||
while i < args.len() {
|
|
||||||
match args[i].as_str() {
|
|
||||||
// Skip subcommand names
|
|
||||||
"start" | "stop" | "reload" | "status" | "run" => {}
|
|
||||||
// PID file option (for stop/reload/status)
|
|
||||||
"--pid-file" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
cmd.pid_file = PathBuf::from(&args[i]);
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
cmd.daemon_opts.pid_file = Some(cmd.pid_file.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--pid-file=") => {
|
|
||||||
cmd.pid_file = PathBuf::from(s.trim_start_matches("--pid-file="));
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
cmd.daemon_opts.pid_file = Some(cmd.pid_file.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Config path (positional, non-flag argument)
|
|
||||||
s if !s.starts_with('-') => {
|
|
||||||
cmd.config_path = s.to_string();
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Execute a subcommand that doesn't require starting the server.
|
|
||||||
/// Returns `Some(exit_code)` if the command was handled, `None` if server should start.
|
|
||||||
#[cfg(unix)]
|
|
||||||
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
|
||||||
match cmd.subcommand {
|
|
||||||
Subcommand::Stop => Some(cmd_stop(&cmd.pid_file)),
|
|
||||||
Subcommand::Reload => Some(cmd_reload(&cmd.pid_file)),
|
|
||||||
Subcommand::Status => Some(cmd_status(&cmd.pid_file)),
|
|
||||||
Subcommand::Init => {
|
|
||||||
if let Some(opts) = cmd.init_opts.clone() {
|
|
||||||
match run_init(opts) {
|
|
||||||
Ok(()) => Some(0),
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("[telemt] Init failed: {}", e);
|
|
||||||
Some(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Some(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Run and Start need the server
|
|
||||||
Subcommand::Run | Subcommand::Start => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
|
||||||
match cmd.subcommand {
|
|
||||||
Subcommand::Stop | Subcommand::Reload | Subcommand::Status => {
|
|
||||||
eprintln!("[telemt] Subcommand not supported on this platform");
|
|
||||||
Some(1)
|
|
||||||
}
|
|
||||||
Subcommand::Init => {
|
|
||||||
if let Some(opts) = cmd.init_opts.clone() {
|
|
||||||
match run_init(opts) {
|
|
||||||
Ok(()) => Some(0),
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("[telemt] Init failed: {}", e);
|
|
||||||
Some(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Some(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Subcommand::Run | Subcommand::Start => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Stop command: send SIGTERM to the running daemon.
|
|
||||||
#[cfg(unix)]
|
|
||||||
fn cmd_stop(pid_file: &Path) -> i32 {
|
|
||||||
use nix::sys::signal::Signal;
|
|
||||||
|
|
||||||
println!("Stopping telemt daemon...");
|
|
||||||
|
|
||||||
match daemon::signal_pid_file(pid_file, Signal::SIGTERM) {
|
|
||||||
Ok(()) => {
|
|
||||||
println!("Stop signal sent successfully");
|
|
||||||
|
|
||||||
// Wait for process to exit (up to 10 seconds)
|
|
||||||
for _ in 0..20 {
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
|
||||||
if let daemon::DaemonStatus::NotRunning = daemon::check_status(pid_file) {
|
|
||||||
println!("Daemon stopped");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
println!("Daemon may still be shutting down");
|
|
||||||
0
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("Failed to stop daemon: {}", e);
|
|
||||||
1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reload command: send SIGHUP to trigger config reload.
|
|
||||||
#[cfg(unix)]
|
|
||||||
fn cmd_reload(pid_file: &Path) -> i32 {
|
|
||||||
use nix::sys::signal::Signal;
|
|
||||||
|
|
||||||
println!("Reloading telemt configuration...");
|
|
||||||
|
|
||||||
match daemon::signal_pid_file(pid_file, Signal::SIGHUP) {
|
|
||||||
Ok(()) => {
|
|
||||||
println!("Reload signal sent successfully");
|
|
||||||
0
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("Failed to reload daemon: {}", e);
|
|
||||||
1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Status command: check if daemon is running.
|
|
||||||
#[cfg(unix)]
|
|
||||||
fn cmd_status(pid_file: &Path) -> i32 {
|
|
||||||
match daemon::check_status(pid_file) {
|
|
||||||
daemon::DaemonStatus::Running(pid) => {
|
|
||||||
println!("telemt is running (pid {})", pid);
|
|
||||||
0
|
|
||||||
}
|
|
||||||
daemon::DaemonStatus::Stale(pid) => {
|
|
||||||
println!("telemt is not running (stale pid file, was pid {})", pid);
|
|
||||||
// Clean up stale PID file
|
|
||||||
let _ = std::fs::remove_file(pid_file);
|
|
||||||
1
|
|
||||||
}
|
|
||||||
daemon::DaemonStatus::NotRunning => {
|
|
||||||
println!("telemt is not running");
|
|
||||||
1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Options for the init command
|
/// Options for the init command
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub struct InitOptions {
|
pub struct InitOptions {
|
||||||
pub port: u16,
|
pub port: u16,
|
||||||
pub domain: String,
|
pub domain: String,
|
||||||
@@ -274,64 +15,6 @@ pub struct InitOptions {
|
|||||||
pub no_start: bool,
|
pub no_start: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse daemon-related options from CLI args.
|
|
||||||
#[cfg(unix)]
|
|
||||||
pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
|
|
||||||
let mut opts = DaemonOptions::default();
|
|
||||||
let mut i = 0;
|
|
||||||
|
|
||||||
while i < args.len() {
|
|
||||||
match args[i].as_str() {
|
|
||||||
"--daemon" | "-d" => {
|
|
||||||
opts.daemonize = true;
|
|
||||||
}
|
|
||||||
"--foreground" | "-f" => {
|
|
||||||
opts.foreground = true;
|
|
||||||
}
|
|
||||||
"--pid-file" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
opts.pid_file = Some(PathBuf::from(&args[i]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--pid-file=") => {
|
|
||||||
opts.pid_file = Some(PathBuf::from(s.trim_start_matches("--pid-file=")));
|
|
||||||
}
|
|
||||||
"--run-as-user" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
opts.user = Some(args[i].clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--run-as-user=") => {
|
|
||||||
opts.user = Some(s.trim_start_matches("--run-as-user=").to_string());
|
|
||||||
}
|
|
||||||
"--run-as-group" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
opts.group = Some(args[i].clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--run-as-group=") => {
|
|
||||||
opts.group = Some(s.trim_start_matches("--run-as-group=").to_string());
|
|
||||||
}
|
|
||||||
"--working-dir" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
opts.working_dir = Some(PathBuf::from(&args[i]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--working-dir=") => {
|
|
||||||
opts.working_dir = Some(PathBuf::from(s.trim_start_matches("--working-dir=")));
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
opts
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for InitOptions {
|
impl Default for InitOptions {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -401,16 +84,10 @@ pub fn parse_init_args(args: &[String]) -> Option<InitOptions> {
|
|||||||
|
|
||||||
/// Run the fire-and-forget setup.
|
/// Run the fire-and-forget setup.
|
||||||
pub fn run_init(opts: InitOptions) -> Result<(), Box<dyn std::error::Error>> {
|
pub fn run_init(opts: InitOptions) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
use crate::service::{self, InitSystem, ServiceOptions};
|
|
||||||
|
|
||||||
eprintln!("[telemt] Fire-and-forget setup");
|
eprintln!("[telemt] Fire-and-forget setup");
|
||||||
eprintln!();
|
eprintln!();
|
||||||
|
|
||||||
// 1. Detect init system
|
// 1. Generate or validate secret
|
||||||
let init_system = service::detect_init_system();
|
|
||||||
eprintln!("[+] Detected init system: {}", init_system);
|
|
||||||
|
|
||||||
// 2. Generate or validate secret
|
|
||||||
let secret = match opts.secret {
|
let secret = match opts.secret {
|
||||||
Some(s) => {
|
Some(s) => {
|
||||||
if s.len() != 32 || !s.chars().all(|c| c.is_ascii_hexdigit()) {
|
if s.len() != 32 || !s.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||||
@@ -427,126 +104,72 @@ pub fn run_init(opts: InitOptions) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
eprintln!("[+] Port: {}", opts.port);
|
eprintln!("[+] Port: {}", opts.port);
|
||||||
eprintln!("[+] Domain: {}", opts.domain);
|
eprintln!("[+] Domain: {}", opts.domain);
|
||||||
|
|
||||||
// 3. Create config directory
|
// 2. Create config directory
|
||||||
fs::create_dir_all(&opts.config_dir)?;
|
fs::create_dir_all(&opts.config_dir)?;
|
||||||
let config_path = opts.config_dir.join("config.toml");
|
let config_path = opts.config_dir.join("config.toml");
|
||||||
|
|
||||||
// 4. Write config
|
// 3. Write config
|
||||||
let config_content = generate_config(&opts.username, &secret, opts.port, &opts.domain);
|
let config_content = generate_config(&opts.username, &secret, opts.port, &opts.domain);
|
||||||
fs::write(&config_path, &config_content)?;
|
fs::write(&config_path, &config_content)?;
|
||||||
eprintln!("[+] Config written to {}", config_path.display());
|
eprintln!("[+] Config written to {}", config_path.display());
|
||||||
|
|
||||||
// 5. Generate and write service file
|
// 4. Write systemd unit
|
||||||
let exe_path =
|
let exe_path =
|
||||||
std::env::current_exe().unwrap_or_else(|_| PathBuf::from("/usr/local/bin/telemt"));
|
std::env::current_exe().unwrap_or_else(|_| PathBuf::from("/usr/local/bin/telemt"));
|
||||||
|
|
||||||
let service_opts = ServiceOptions {
|
let unit_path = Path::new("/etc/systemd/system/telemt.service");
|
||||||
exe_path: &exe_path,
|
let unit_content = generate_systemd_unit(&exe_path, &config_path);
|
||||||
config_path: &config_path,
|
|
||||||
user: None, // Let systemd/init handle user
|
|
||||||
group: None,
|
|
||||||
pid_file: "/var/run/telemt.pid",
|
|
||||||
working_dir: Some("/var/lib/telemt"),
|
|
||||||
description: "Telemt MTProxy - Telegram MTProto Proxy",
|
|
||||||
};
|
|
||||||
|
|
||||||
let service_path = service::service_file_path(init_system);
|
match fs::write(unit_path, &unit_content) {
|
||||||
let service_content = service::generate_service_file(init_system, &service_opts);
|
|
||||||
|
|
||||||
// Ensure parent directory exists
|
|
||||||
if let Some(parent) = Path::new(service_path).parent() {
|
|
||||||
let _ = fs::create_dir_all(parent);
|
|
||||||
}
|
|
||||||
|
|
||||||
match fs::write(service_path, &service_content) {
|
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
eprintln!("[+] Service file written to {}", service_path);
|
eprintln!("[+] Systemd unit written to {}", unit_path.display());
|
||||||
|
|
||||||
// Make script executable for OpenRC/FreeBSD
|
|
||||||
#[cfg(unix)]
|
|
||||||
if init_system == InitSystem::OpenRC || init_system == InitSystem::FreeBSDRc {
|
|
||||||
use std::os::unix::fs::PermissionsExt;
|
|
||||||
let mut perms = fs::metadata(service_path)?.permissions();
|
|
||||||
perms.set_mode(0o755);
|
|
||||||
fs::set_permissions(service_path, perms)?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("[!] Cannot write service file (run as root?): {}", e);
|
eprintln!("[!] Cannot write systemd unit (run as root?): {}", e);
|
||||||
eprintln!("[!] Manual service file content:");
|
eprintln!("[!] Manual unit file content:");
|
||||||
eprintln!("{}", service_content);
|
eprintln!("{}", unit_content);
|
||||||
|
|
||||||
// Still print links and installation instructions
|
// Still print links and config
|
||||||
eprintln!();
|
|
||||||
eprintln!("{}", service::installation_instructions(init_system));
|
|
||||||
print_links(&opts.username, &secret, opts.port, &opts.domain);
|
print_links(&opts.username, &secret, opts.port, &opts.domain);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. Install and enable service based on init system
|
// 5. Reload systemd
|
||||||
match init_system {
|
run_cmd("systemctl", &["daemon-reload"]);
|
||||||
InitSystem::Systemd => {
|
|
||||||
run_cmd("systemctl", &["daemon-reload"]);
|
|
||||||
run_cmd("systemctl", &["enable", "telemt.service"]);
|
|
||||||
eprintln!("[+] Service enabled");
|
|
||||||
|
|
||||||
if !opts.no_start {
|
// 6. Enable service
|
||||||
run_cmd("systemctl", &["start", "telemt.service"]);
|
run_cmd("systemctl", &["enable", "telemt.service"]);
|
||||||
eprintln!("[+] Service started");
|
eprintln!("[+] Service enabled");
|
||||||
|
|
||||||
std::thread::sleep(std::time::Duration::from_secs(1));
|
// 7. Start service (unless --no-start)
|
||||||
let status = Command::new("systemctl")
|
if !opts.no_start {
|
||||||
.args(["is-active", "telemt.service"])
|
run_cmd("systemctl", &["start", "telemt.service"]);
|
||||||
.output();
|
eprintln!("[+] Service started");
|
||||||
|
|
||||||
match status {
|
// Brief delay then check status
|
||||||
Ok(out) if out.status.success() => {
|
std::thread::sleep(std::time::Duration::from_secs(1));
|
||||||
eprintln!("[+] Service is running");
|
let status = Command::new("systemctl")
|
||||||
}
|
.args(["is-active", "telemt.service"])
|
||||||
_ => {
|
.output();
|
||||||
eprintln!("[!] Service may not have started correctly");
|
|
||||||
eprintln!("[!] Check: journalctl -u telemt.service -n 20");
|
match status {
|
||||||
}
|
Ok(out) if out.status.success() => {
|
||||||
}
|
eprintln!("[+] Service is running");
|
||||||
} else {
|
}
|
||||||
eprintln!("[+] Service not started (--no-start)");
|
_ => {
|
||||||
eprintln!("[+] Start manually: systemctl start telemt.service");
|
eprintln!("[!] Service may not have started correctly");
|
||||||
|
eprintln!("[!] Check: journalctl -u telemt.service -n 20");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
InitSystem::OpenRC => {
|
} else {
|
||||||
run_cmd("rc-update", &["add", "telemt", "default"]);
|
eprintln!("[+] Service not started (--no-start)");
|
||||||
eprintln!("[+] Service enabled");
|
eprintln!("[+] Start manually: systemctl start telemt.service");
|
||||||
|
|
||||||
if !opts.no_start {
|
|
||||||
run_cmd("rc-service", &["telemt", "start"]);
|
|
||||||
eprintln!("[+] Service started");
|
|
||||||
} else {
|
|
||||||
eprintln!("[+] Service not started (--no-start)");
|
|
||||||
eprintln!("[+] Start manually: rc-service telemt start");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
InitSystem::FreeBSDRc => {
|
|
||||||
run_cmd("sysrc", &["telemt_enable=YES"]);
|
|
||||||
eprintln!("[+] Service enabled");
|
|
||||||
|
|
||||||
if !opts.no_start {
|
|
||||||
run_cmd("service", &["telemt", "start"]);
|
|
||||||
eprintln!("[+] Service started");
|
|
||||||
} else {
|
|
||||||
eprintln!("[+] Service not started (--no-start)");
|
|
||||||
eprintln!("[+] Start manually: service telemt start");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
InitSystem::Unknown => {
|
|
||||||
eprintln!("[!] Unknown init system - service file written but not installed");
|
|
||||||
eprintln!("[!] You may need to install it manually");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
eprintln!();
|
eprintln!();
|
||||||
|
|
||||||
// 7. Print links
|
// 8. Print links
|
||||||
print_links(&opts.username, &secret, opts.port, &opts.domain);
|
print_links(&opts.username, &secret, opts.port, &opts.domain);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -584,7 +207,6 @@ me_pool_drain_soft_evict_cooldown_ms = 1000
|
|||||||
me_bind_stale_mode = "never"
|
me_bind_stale_mode = "never"
|
||||||
me_pool_min_fresh_ratio = 0.8
|
me_pool_min_fresh_ratio = 0.8
|
||||||
me_reinit_drain_timeout_secs = 90
|
me_reinit_drain_timeout_secs = 90
|
||||||
tg_connect = 10
|
|
||||||
|
|
||||||
[network]
|
[network]
|
||||||
ipv4 = true
|
ipv4 = true
|
||||||
@@ -610,8 +232,8 @@ ip = "0.0.0.0"
|
|||||||
ip = "::"
|
ip = "::"
|
||||||
|
|
||||||
[timeouts]
|
[timeouts]
|
||||||
client_first_byte_idle_secs = 300
|
client_handshake = 15
|
||||||
client_handshake = 60
|
tg_connect = 10
|
||||||
client_keepalive = 60
|
client_keepalive = 60
|
||||||
client_ack = 300
|
client_ack = 300
|
||||||
|
|
||||||
@@ -623,7 +245,6 @@ fake_cert_len = 2048
|
|||||||
tls_full_cert_ttl_secs = 90
|
tls_full_cert_ttl_secs = 90
|
||||||
|
|
||||||
[access]
|
[access]
|
||||||
user_max_tcp_conns_global_each = 0
|
|
||||||
replay_check_len = 65536
|
replay_check_len = 65536
|
||||||
replay_window_secs = 120
|
replay_window_secs = 120
|
||||||
ignore_time_skew = false
|
ignore_time_skew = false
|
||||||
@@ -643,6 +264,35 @@ weight = 10
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn generate_systemd_unit(exe_path: &Path, config_path: &Path) -> String {
|
||||||
|
format!(
|
||||||
|
r#"[Unit]
|
||||||
|
Description=Telemt MTProxy
|
||||||
|
Documentation=https://github.com/telemt/telemt
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart={exe} {config}
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
LimitNOFILE=65535
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ReadWritePaths=/etc/telemt
|
||||||
|
PrivateTmp=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
"#,
|
||||||
|
exe = exe_path.display(),
|
||||||
|
config = config_path.display(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn run_cmd(cmd: &str, args: &[&str]) {
|
fn run_cmd(cmd: &str, args: &[&str]) {
|
||||||
match Command::new(cmd).args(args).output() {
|
match Command::new(cmd).args(args).output() {
|
||||||
Ok(output) => {
|
Ok(output) => {
|
||||||
|
|||||||
+1
-17
@@ -110,11 +110,7 @@ pub(crate) fn default_replay_window_secs() -> u64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_handshake_timeout() -> u64 {
|
pub(crate) fn default_handshake_timeout() -> u64 {
|
||||||
60
|
30
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_client_first_byte_idle_secs() -> u64 {
|
|
||||||
300
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_relay_idle_policy_v2_enabled() -> bool {
|
pub(crate) fn default_relay_idle_policy_v2_enabled() -> bool {
|
||||||
@@ -213,10 +209,6 @@ pub(crate) fn default_server_max_connections() -> u32 {
|
|||||||
10_000
|
10_000
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_listen_backlog() -> u32 {
|
|
||||||
1024
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_accept_permit_timeout_ms() -> u64 {
|
pub(crate) fn default_accept_permit_timeout_ms() -> u64 {
|
||||||
DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS
|
DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS
|
||||||
}
|
}
|
||||||
@@ -281,10 +273,6 @@ pub(crate) fn default_me2dc_fallback() -> bool {
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_me2dc_fast() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_keepalive_interval() -> u64 {
|
pub(crate) fn default_keepalive_interval() -> u64 {
|
||||||
8
|
8
|
||||||
}
|
}
|
||||||
@@ -811,10 +799,6 @@ pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
|||||||
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
||||||
0
|
0
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,7 +117,6 @@ pub struct HotFields {
|
|||||||
pub users: std::collections::HashMap<String, String>,
|
pub users: std::collections::HashMap<String, String>,
|
||||||
pub user_ad_tags: std::collections::HashMap<String, String>,
|
pub user_ad_tags: std::collections::HashMap<String, String>,
|
||||||
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
|
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
|
||||||
pub user_max_tcp_conns_global_each: usize,
|
|
||||||
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
|
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
|
||||||
pub user_data_quota: std::collections::HashMap<String, u64>,
|
pub user_data_quota: std::collections::HashMap<String, u64>,
|
||||||
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
|
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
|
||||||
@@ -241,7 +240,6 @@ impl HotFields {
|
|||||||
users: cfg.access.users.clone(),
|
users: cfg.access.users.clone(),
|
||||||
user_ad_tags: cfg.access.user_ad_tags.clone(),
|
user_ad_tags: cfg.access.user_ad_tags.clone(),
|
||||||
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
|
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
|
||||||
user_max_tcp_conns_global_each: cfg.access.user_max_tcp_conns_global_each,
|
|
||||||
user_expirations: cfg.access.user_expirations.clone(),
|
user_expirations: cfg.access.user_expirations.clone(),
|
||||||
user_data_quota: cfg.access.user_data_quota.clone(),
|
user_data_quota: cfg.access.user_data_quota.clone(),
|
||||||
user_max_unique_ips: cfg.access.user_max_unique_ips.clone(),
|
user_max_unique_ips: cfg.access.user_max_unique_ips.clone(),
|
||||||
@@ -532,7 +530,6 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
|||||||
cfg.access.users = new.access.users.clone();
|
cfg.access.users = new.access.users.clone();
|
||||||
cfg.access.user_ad_tags = new.access.user_ad_tags.clone();
|
cfg.access.user_ad_tags = new.access.user_ad_tags.clone();
|
||||||
cfg.access.user_max_tcp_conns = new.access.user_max_tcp_conns.clone();
|
cfg.access.user_max_tcp_conns = new.access.user_max_tcp_conns.clone();
|
||||||
cfg.access.user_max_tcp_conns_global_each = new.access.user_max_tcp_conns_global_each;
|
|
||||||
cfg.access.user_expirations = new.access.user_expirations.clone();
|
cfg.access.user_expirations = new.access.user_expirations.clone();
|
||||||
cfg.access.user_data_quota = new.access.user_data_quota.clone();
|
cfg.access.user_data_quota = new.access.user_data_quota.clone();
|
||||||
cfg.access.user_max_unique_ips = new.access.user_max_unique_ips.clone();
|
cfg.access.user_max_unique_ips = new.access.user_max_unique_ips.clone();
|
||||||
@@ -573,7 +570,6 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
}
|
}
|
||||||
if old.server.proxy_protocol != new.server.proxy_protocol
|
if old.server.proxy_protocol != new.server.proxy_protocol
|
||||||
|| !listeners_equal(&old.server.listeners, &new.server.listeners)
|
|| !listeners_equal(&old.server.listeners, &new.server.listeners)
|
||||||
|| old.server.listen_backlog != new.server.listen_backlog
|
|
||||||
|| old.server.listen_addr_ipv4 != new.server.listen_addr_ipv4
|
|| old.server.listen_addr_ipv4 != new.server.listen_addr_ipv4
|
||||||
|| old.server.listen_addr_ipv6 != new.server.listen_addr_ipv6
|
|| old.server.listen_addr_ipv6 != new.server.listen_addr_ipv6
|
||||||
|| old.server.listen_tcp != new.server.listen_tcp
|
|| old.server.listen_tcp != new.server.listen_tcp
|
||||||
@@ -655,9 +651,6 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
}
|
}
|
||||||
if old.general.me_route_no_writer_mode != new.general.me_route_no_writer_mode
|
if old.general.me_route_no_writer_mode != new.general.me_route_no_writer_mode
|
||||||
|| old.general.me_route_no_writer_wait_ms != new.general.me_route_no_writer_wait_ms
|
|| old.general.me_route_no_writer_wait_ms != new.general.me_route_no_writer_wait_ms
|
||||||
|| old.general.me_route_hybrid_max_wait_ms != new.general.me_route_hybrid_max_wait_ms
|
|
||||||
|| old.general.me_route_blocking_send_timeout_ms
|
|
||||||
!= new.general.me_route_blocking_send_timeout_ms
|
|
||||||
|| old.general.me_route_inline_recovery_attempts
|
|| old.general.me_route_inline_recovery_attempts
|
||||||
!= new.general.me_route_inline_recovery_attempts
|
!= new.general.me_route_inline_recovery_attempts
|
||||||
|| old.general.me_route_inline_recovery_wait_ms
|
|| old.general.me_route_inline_recovery_wait_ms
|
||||||
@@ -676,11 +669,9 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
warned = true;
|
warned = true;
|
||||||
warn!("config reload: general.me_init_retry_attempts changed; restart required");
|
warn!("config reload: general.me_init_retry_attempts changed; restart required");
|
||||||
}
|
}
|
||||||
if old.general.me2dc_fallback != new.general.me2dc_fallback
|
if old.general.me2dc_fallback != new.general.me2dc_fallback {
|
||||||
|| old.general.me2dc_fast != new.general.me2dc_fast
|
|
||||||
{
|
|
||||||
warned = true;
|
warned = true;
|
||||||
warn!("config reload: general.me2dc_fallback/me2dc_fast changed; restart required");
|
warn!("config reload: general.me2dc_fallback changed; restart required");
|
||||||
}
|
}
|
||||||
if old.general.proxy_config_v4_cache_path != new.general.proxy_config_v4_cache_path
|
if old.general.proxy_config_v4_cache_path != new.general.proxy_config_v4_cache_path
|
||||||
|| old.general.proxy_config_v6_cache_path != new.general.proxy_config_v6_cache_path
|
|| old.general.proxy_config_v6_cache_path != new.general.proxy_config_v6_cache_path
|
||||||
@@ -699,7 +690,6 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
if old.general.upstream_connect_retry_attempts != new.general.upstream_connect_retry_attempts
|
if old.general.upstream_connect_retry_attempts != new.general.upstream_connect_retry_attempts
|
||||||
|| old.general.upstream_connect_retry_backoff_ms
|
|| old.general.upstream_connect_retry_backoff_ms
|
||||||
!= new.general.upstream_connect_retry_backoff_ms
|
!= new.general.upstream_connect_retry_backoff_ms
|
||||||
|| old.general.tg_connect != new.general.tg_connect
|
|
||||||
|| old.general.upstream_unhealthy_fail_threshold
|
|| old.general.upstream_unhealthy_fail_threshold
|
||||||
!= new.general.upstream_unhealthy_fail_threshold
|
!= new.general.upstream_unhealthy_fail_threshold
|
||||||
|| old.general.upstream_connect_failfast_hard_errors
|
|| old.general.upstream_connect_failfast_hard_errors
|
||||||
@@ -1148,12 +1138,6 @@ fn log_changes(
|
|||||||
new_hot.user_max_tcp_conns.len()
|
new_hot.user_max_tcp_conns.len()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if old_hot.user_max_tcp_conns_global_each != new_hot.user_max_tcp_conns_global_each {
|
|
||||||
info!(
|
|
||||||
"config reload: user_max_tcp_conns policy global_each={}",
|
|
||||||
new_hot.user_max_tcp_conns_global_each
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if old_hot.user_expirations != new_hot.user_expirations {
|
if old_hot.user_expirations != new_hot.user_expirations {
|
||||||
info!(
|
info!(
|
||||||
"config reload: user_expirations updated ({} entries)",
|
"config reload: user_expirations updated ({} entries)",
|
||||||
|
|||||||
+4
-14
@@ -346,9 +346,9 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.general.tg_connect == 0 {
|
if config.timeouts.tg_connect == 0 {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"general.tg_connect must be > 0".to_string(),
|
"timeouts.tg_connect must be > 0".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1223,7 +1223,6 @@ mod tests {
|
|||||||
default_me_init_retry_attempts()
|
default_me_init_retry_attempts()
|
||||||
);
|
);
|
||||||
assert_eq!(cfg.general.me2dc_fallback, default_me2dc_fallback());
|
assert_eq!(cfg.general.me2dc_fallback, default_me2dc_fallback());
|
||||||
assert_eq!(cfg.general.me2dc_fast, default_me2dc_fast());
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
cfg.general.proxy_config_v4_cache_path,
|
cfg.general.proxy_config_v4_cache_path,
|
||||||
default_proxy_config_v4_cache_path()
|
default_proxy_config_v4_cache_path()
|
||||||
@@ -1328,10 +1327,6 @@ mod tests {
|
|||||||
default_api_runtime_edge_events_capacity()
|
default_api_runtime_edge_events_capacity()
|
||||||
);
|
);
|
||||||
assert_eq!(cfg.access.users, default_access_users());
|
assert_eq!(cfg.access.users, default_access_users());
|
||||||
assert_eq!(
|
|
||||||
cfg.access.user_max_tcp_conns_global_each,
|
|
||||||
default_user_max_tcp_conns_global_each()
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
cfg.access.user_max_unique_ips_mode,
|
cfg.access.user_max_unique_ips_mode,
|
||||||
UserMaxUniqueIpsMode::default()
|
UserMaxUniqueIpsMode::default()
|
||||||
@@ -1367,7 +1362,6 @@ mod tests {
|
|||||||
default_me_init_retry_attempts()
|
default_me_init_retry_attempts()
|
||||||
);
|
);
|
||||||
assert_eq!(general.me2dc_fallback, default_me2dc_fallback());
|
assert_eq!(general.me2dc_fallback, default_me2dc_fallback());
|
||||||
assert_eq!(general.me2dc_fast, default_me2dc_fast());
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
general.proxy_config_v4_cache_path,
|
general.proxy_config_v4_cache_path,
|
||||||
default_proxy_config_v4_cache_path()
|
default_proxy_config_v4_cache_path()
|
||||||
@@ -1475,10 +1469,6 @@ mod tests {
|
|||||||
|
|
||||||
let access = AccessConfig::default();
|
let access = AccessConfig::default();
|
||||||
assert_eq!(access.users, default_access_users());
|
assert_eq!(access.users, default_access_users());
|
||||||
assert_eq!(
|
|
||||||
access.user_max_tcp_conns_global_each,
|
|
||||||
default_user_max_tcp_conns_global_each()
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1924,7 +1914,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn tg_connect_zero_is_rejected() {
|
fn tg_connect_zero_is_rejected() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
[general]
|
[timeouts]
|
||||||
tg_connect = 0
|
tg_connect = 0
|
||||||
|
|
||||||
[censorship]
|
[censorship]
|
||||||
@@ -1937,7 +1927,7 @@ mod tests {
|
|||||||
let path = dir.join("telemt_tg_connect_zero_test.toml");
|
let path = dir.join("telemt_tg_connect_zero_test.toml");
|
||||||
std::fs::write(&path, toml).unwrap();
|
std::fs::write(&path, toml).unwrap();
|
||||||
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
let err = ProxyConfig::load(&path).unwrap_err().to_string();
|
||||||
assert!(err.contains("general.tg_connect must be > 0"));
|
assert!(err.contains("timeouts.tg_connect must be > 0"));
|
||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,28 +17,6 @@ fn remove_temp_config(path: &PathBuf) {
|
|||||||
let _ = fs::remove_file(path);
|
let _ = fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn default_timeouts_enable_apple_compatible_handshake_profile() {
|
|
||||||
let cfg = ProxyConfig::default();
|
|
||||||
assert_eq!(cfg.timeouts.client_first_byte_idle_secs, 300);
|
|
||||||
assert_eq!(cfg.timeouts.client_handshake, 60);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn load_accepts_zero_first_byte_idle_timeout_as_legacy_opt_out() {
|
|
||||||
let path = write_temp_config(
|
|
||||||
r#"
|
|
||||||
[timeouts]
|
|
||||||
client_first_byte_idle_secs = 0
|
|
||||||
"#,
|
|
||||||
);
|
|
||||||
|
|
||||||
let cfg = ProxyConfig::load(&path).expect("config with zero first-byte idle timeout must load");
|
|
||||||
assert_eq!(cfg.timeouts.client_first_byte_idle_secs, 0);
|
|
||||||
|
|
||||||
remove_temp_config(&path);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn load_rejects_relay_hard_idle_smaller_than_soft_idle_with_clear_error() {
|
fn load_rejects_relay_hard_idle_smaller_than_soft_idle_with_clear_error() {
|
||||||
let path = write_temp_config(
|
let path = write_temp_config(
|
||||||
|
|||||||
+4
-31
@@ -429,11 +429,6 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_me2dc_fallback")]
|
#[serde(default = "default_me2dc_fallback")]
|
||||||
pub me2dc_fallback: bool,
|
pub me2dc_fallback: bool,
|
||||||
|
|
||||||
/// Fast ME->Direct fallback mode for new sessions.
|
|
||||||
/// Active only when both `use_middle_proxy=true` and `me2dc_fallback=true`.
|
|
||||||
#[serde(default = "default_me2dc_fast")]
|
|
||||||
pub me2dc_fast: bool,
|
|
||||||
|
|
||||||
/// Enable ME keepalive padding frames.
|
/// Enable ME keepalive padding frames.
|
||||||
#[serde(default = "default_true")]
|
#[serde(default = "default_true")]
|
||||||
pub me_keepalive_enabled: bool,
|
pub me_keepalive_enabled: bool,
|
||||||
@@ -663,10 +658,6 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default = "default_upstream_connect_budget_ms")]
|
#[serde(default = "default_upstream_connect_budget_ms")]
|
||||||
pub upstream_connect_budget_ms: u64,
|
pub upstream_connect_budget_ms: u64,
|
||||||
|
|
||||||
/// Per-attempt TCP connect timeout to Telegram DC (seconds).
|
|
||||||
#[serde(default = "default_connect_timeout")]
|
|
||||||
pub tg_connect: u64,
|
|
||||||
|
|
||||||
/// Consecutive failed requests before upstream is marked unhealthy.
|
/// Consecutive failed requests before upstream is marked unhealthy.
|
||||||
#[serde(default = "default_upstream_unhealthy_fail_threshold")]
|
#[serde(default = "default_upstream_unhealthy_fail_threshold")]
|
||||||
pub upstream_unhealthy_fail_threshold: u32,
|
pub upstream_unhealthy_fail_threshold: u32,
|
||||||
@@ -948,7 +939,6 @@ impl Default for GeneralConfig {
|
|||||||
middle_proxy_warm_standby: default_middle_proxy_warm_standby(),
|
middle_proxy_warm_standby: default_middle_proxy_warm_standby(),
|
||||||
me_init_retry_attempts: default_me_init_retry_attempts(),
|
me_init_retry_attempts: default_me_init_retry_attempts(),
|
||||||
me2dc_fallback: default_me2dc_fallback(),
|
me2dc_fallback: default_me2dc_fallback(),
|
||||||
me2dc_fast: default_me2dc_fast(),
|
|
||||||
me_keepalive_enabled: default_true(),
|
me_keepalive_enabled: default_true(),
|
||||||
me_keepalive_interval_secs: default_keepalive_interval(),
|
me_keepalive_interval_secs: default_keepalive_interval(),
|
||||||
me_keepalive_jitter_secs: default_keepalive_jitter(),
|
me_keepalive_jitter_secs: default_keepalive_jitter(),
|
||||||
@@ -1011,7 +1001,6 @@ impl Default for GeneralConfig {
|
|||||||
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
upstream_connect_retry_attempts: default_upstream_connect_retry_attempts(),
|
||||||
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
upstream_connect_retry_backoff_ms: default_upstream_connect_retry_backoff_ms(),
|
||||||
upstream_connect_budget_ms: default_upstream_connect_budget_ms(),
|
upstream_connect_budget_ms: default_upstream_connect_budget_ms(),
|
||||||
tg_connect: default_connect_timeout(),
|
|
||||||
upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
|
upstream_unhealthy_fail_threshold: default_upstream_unhealthy_fail_threshold(),
|
||||||
upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(),
|
upstream_connect_failfast_hard_errors: default_upstream_connect_failfast_hard_errors(),
|
||||||
stun_iface_mismatch_ignore: false,
|
stun_iface_mismatch_ignore: false,
|
||||||
@@ -1277,11 +1266,6 @@ pub struct ServerConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub listeners: Vec<ListenerConfig>,
|
pub listeners: Vec<ListenerConfig>,
|
||||||
|
|
||||||
/// TCP `listen(2)` backlog for client-facing sockets (also used for the metrics HTTP listener).
|
|
||||||
/// The effective queue is capped by the kernel (for example `somaxconn` on Linux).
|
|
||||||
#[serde(default = "default_listen_backlog")]
|
|
||||||
pub listen_backlog: u32,
|
|
||||||
|
|
||||||
/// Maximum number of concurrent client connections.
|
/// Maximum number of concurrent client connections.
|
||||||
/// 0 means unlimited.
|
/// 0 means unlimited.
|
||||||
#[serde(default = "default_server_max_connections")]
|
#[serde(default = "default_server_max_connections")]
|
||||||
@@ -1310,7 +1294,6 @@ impl Default for ServerConfig {
|
|||||||
metrics_whitelist: default_metrics_whitelist(),
|
metrics_whitelist: default_metrics_whitelist(),
|
||||||
api: ApiConfig::default(),
|
api: ApiConfig::default(),
|
||||||
listeners: Vec::new(),
|
listeners: Vec::new(),
|
||||||
listen_backlog: default_listen_backlog(),
|
|
||||||
max_connections: default_server_max_connections(),
|
max_connections: default_server_max_connections(),
|
||||||
accept_permit_timeout_ms: default_accept_permit_timeout_ms(),
|
accept_permit_timeout_ms: default_accept_permit_timeout_ms(),
|
||||||
}
|
}
|
||||||
@@ -1319,12 +1302,6 @@ impl Default for ServerConfig {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct TimeoutsConfig {
|
pub struct TimeoutsConfig {
|
||||||
/// Maximum idle wait in seconds for the first client byte before handshake parsing starts.
|
|
||||||
/// `0` disables the separate idle phase and keeps legacy timeout behavior.
|
|
||||||
#[serde(default = "default_client_first_byte_idle_secs")]
|
|
||||||
pub client_first_byte_idle_secs: u64,
|
|
||||||
|
|
||||||
/// Maximum active handshake duration in seconds after the first client byte is received.
|
|
||||||
#[serde(default = "default_handshake_timeout")]
|
#[serde(default = "default_handshake_timeout")]
|
||||||
pub client_handshake: u64,
|
pub client_handshake: u64,
|
||||||
|
|
||||||
@@ -1346,6 +1323,9 @@ pub struct TimeoutsConfig {
|
|||||||
#[serde(default = "default_relay_idle_grace_after_downstream_activity_secs")]
|
#[serde(default = "default_relay_idle_grace_after_downstream_activity_secs")]
|
||||||
pub relay_idle_grace_after_downstream_activity_secs: u64,
|
pub relay_idle_grace_after_downstream_activity_secs: u64,
|
||||||
|
|
||||||
|
#[serde(default = "default_connect_timeout")]
|
||||||
|
pub tg_connect: u64,
|
||||||
|
|
||||||
#[serde(default = "default_keepalive")]
|
#[serde(default = "default_keepalive")]
|
||||||
pub client_keepalive: u64,
|
pub client_keepalive: u64,
|
||||||
|
|
||||||
@@ -1364,13 +1344,13 @@ pub struct TimeoutsConfig {
|
|||||||
impl Default for TimeoutsConfig {
|
impl Default for TimeoutsConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
client_first_byte_idle_secs: default_client_first_byte_idle_secs(),
|
|
||||||
client_handshake: default_handshake_timeout(),
|
client_handshake: default_handshake_timeout(),
|
||||||
relay_idle_policy_v2_enabled: default_relay_idle_policy_v2_enabled(),
|
relay_idle_policy_v2_enabled: default_relay_idle_policy_v2_enabled(),
|
||||||
relay_client_idle_soft_secs: default_relay_client_idle_soft_secs(),
|
relay_client_idle_soft_secs: default_relay_client_idle_soft_secs(),
|
||||||
relay_client_idle_hard_secs: default_relay_client_idle_hard_secs(),
|
relay_client_idle_hard_secs: default_relay_client_idle_hard_secs(),
|
||||||
relay_idle_grace_after_downstream_activity_secs:
|
relay_idle_grace_after_downstream_activity_secs:
|
||||||
default_relay_idle_grace_after_downstream_activity_secs(),
|
default_relay_idle_grace_after_downstream_activity_secs(),
|
||||||
|
tg_connect: default_connect_timeout(),
|
||||||
client_keepalive: default_keepalive(),
|
client_keepalive: default_keepalive(),
|
||||||
client_ack: default_ack_timeout(),
|
client_ack: default_ack_timeout(),
|
||||||
me_one_retry: default_me_one_retry(),
|
me_one_retry: default_me_one_retry(),
|
||||||
@@ -1633,12 +1613,6 @@ pub struct AccessConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub user_max_tcp_conns: HashMap<String, usize>,
|
pub user_max_tcp_conns: HashMap<String, usize>,
|
||||||
|
|
||||||
/// Global per-user TCP connection limit applied when a user has no
|
|
||||||
/// positive individual override.
|
|
||||||
/// `0` disables the inherited limit.
|
|
||||||
#[serde(default = "default_user_max_tcp_conns_global_each")]
|
|
||||||
pub user_max_tcp_conns_global_each: usize,
|
|
||||||
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub user_expirations: HashMap<String, DateTime<Utc>>,
|
pub user_expirations: HashMap<String, DateTime<Utc>>,
|
||||||
|
|
||||||
@@ -1675,7 +1649,6 @@ impl Default for AccessConfig {
|
|||||||
users: default_access_users(),
|
users: default_access_users(),
|
||||||
user_ad_tags: HashMap::new(),
|
user_ad_tags: HashMap::new(),
|
||||||
user_max_tcp_conns: HashMap::new(),
|
user_max_tcp_conns: HashMap::new(),
|
||||||
user_max_tcp_conns_global_each: default_user_max_tcp_conns_global_each(),
|
|
||||||
user_expirations: HashMap::new(),
|
user_expirations: HashMap::new(),
|
||||||
user_data_quota: HashMap::new(),
|
user_data_quota: HashMap::new(),
|
||||||
user_max_unique_ips: HashMap::new(),
|
user_max_unique_ips: HashMap::new(),
|
||||||
|
|||||||
@@ -1,541 +0,0 @@
|
|||||||
//! Unix daemon support for telemt.
|
|
||||||
//!
|
|
||||||
//! Provides classic Unix daemonization (double-fork), PID file management,
|
|
||||||
//! and privilege dropping for running telemt as a background service.
|
|
||||||
|
|
||||||
use std::fs::{self, File, OpenOptions};
|
|
||||||
use std::io::{self, Read, Write};
|
|
||||||
use std::os::unix::fs::OpenOptionsExt;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
use nix::fcntl::{Flock, FlockArg};
|
|
||||||
use nix::unistd::{self, ForkResult, Gid, Pid, Uid, chdir, close, fork, getpid, setsid};
|
|
||||||
use tracing::{debug, info, warn};
|
|
||||||
|
|
||||||
/// Default PID file location.
|
|
||||||
pub const DEFAULT_PID_FILE: &str = "/var/run/telemt.pid";
|
|
||||||
|
|
||||||
/// Daemon configuration options parsed from CLI.
|
|
||||||
#[derive(Debug, Clone, Default)]
|
|
||||||
pub struct DaemonOptions {
|
|
||||||
/// Run as daemon (fork to background).
|
|
||||||
pub daemonize: bool,
|
|
||||||
/// Path to PID file.
|
|
||||||
pub pid_file: Option<PathBuf>,
|
|
||||||
/// User to run as after binding sockets.
|
|
||||||
pub user: Option<String>,
|
|
||||||
/// Group to run as after binding sockets.
|
|
||||||
pub group: Option<String>,
|
|
||||||
/// Working directory for the daemon.
|
|
||||||
pub working_dir: Option<PathBuf>,
|
|
||||||
/// Explicit foreground mode (for systemd Type=simple).
|
|
||||||
pub foreground: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl DaemonOptions {
|
|
||||||
/// Returns the effective PID file path.
|
|
||||||
pub fn pid_file_path(&self) -> &Path {
|
|
||||||
self.pid_file
|
|
||||||
.as_deref()
|
|
||||||
.unwrap_or(Path::new(DEFAULT_PID_FILE))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns true if we should actually daemonize.
|
|
||||||
/// Foreground flag takes precedence.
|
|
||||||
pub fn should_daemonize(&self) -> bool {
|
|
||||||
self.daemonize && !self.foreground
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Error types for daemon operations.
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
|
||||||
pub enum DaemonError {
|
|
||||||
#[error("fork failed: {0}")]
|
|
||||||
ForkFailed(#[source] nix::Error),
|
|
||||||
|
|
||||||
#[error("setsid failed: {0}")]
|
|
||||||
SetsidFailed(#[source] nix::Error),
|
|
||||||
|
|
||||||
#[error("chdir failed: {0}")]
|
|
||||||
ChdirFailed(#[source] nix::Error),
|
|
||||||
|
|
||||||
#[error("failed to open /dev/null: {0}")]
|
|
||||||
DevNullFailed(#[source] io::Error),
|
|
||||||
|
|
||||||
#[error("failed to redirect stdio: {0}")]
|
|
||||||
RedirectFailed(#[source] nix::Error),
|
|
||||||
|
|
||||||
#[error("PID file error: {0}")]
|
|
||||||
PidFile(String),
|
|
||||||
|
|
||||||
#[error("another instance is already running (pid {0})")]
|
|
||||||
AlreadyRunning(i32),
|
|
||||||
|
|
||||||
#[error("user '{0}' not found")]
|
|
||||||
UserNotFound(String),
|
|
||||||
|
|
||||||
#[error("group '{0}' not found")]
|
|
||||||
GroupNotFound(String),
|
|
||||||
|
|
||||||
#[error("failed to set uid/gid: {0}")]
|
|
||||||
PrivilegeDrop(#[source] nix::Error),
|
|
||||||
|
|
||||||
#[error("io error: {0}")]
|
|
||||||
Io(#[from] io::Error),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Result of a successful daemonize() call.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub enum DaemonizeResult {
|
|
||||||
/// We are the parent process and should exit.
|
|
||||||
Parent,
|
|
||||||
/// We are the daemon child process and should continue.
|
|
||||||
Child,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Performs classic Unix double-fork daemonization.
|
|
||||||
///
|
|
||||||
/// This detaches the process from the controlling terminal:
|
|
||||||
/// 1. First fork - parent exits, child continues
|
|
||||||
/// 2. setsid() - become session leader
|
|
||||||
/// 3. Second fork - ensure we can never acquire a controlling terminal
|
|
||||||
/// 4. chdir("/") - don't hold any directory open
|
|
||||||
/// 5. Redirect stdin/stdout/stderr to /dev/null
|
|
||||||
///
|
|
||||||
/// Returns `DaemonizeResult::Parent` in the original parent (which should exit),
|
|
||||||
/// or `DaemonizeResult::Child` in the final daemon child.
|
|
||||||
pub fn daemonize(working_dir: Option<&Path>) -> Result<DaemonizeResult, DaemonError> {
|
|
||||||
// First fork
|
|
||||||
match unsafe { fork() } {
|
|
||||||
Ok(ForkResult::Parent { .. }) => {
|
|
||||||
// Parent exits
|
|
||||||
return Ok(DaemonizeResult::Parent);
|
|
||||||
}
|
|
||||||
Ok(ForkResult::Child) => {
|
|
||||||
// Child continues
|
|
||||||
}
|
|
||||||
Err(e) => return Err(DaemonError::ForkFailed(e)),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create new session, become session leader
|
|
||||||
setsid().map_err(DaemonError::SetsidFailed)?;
|
|
||||||
|
|
||||||
// Second fork to ensure we can never acquire a controlling terminal
|
|
||||||
match unsafe { fork() } {
|
|
||||||
Ok(ForkResult::Parent { .. }) => {
|
|
||||||
// Intermediate parent exits
|
|
||||||
std::process::exit(0);
|
|
||||||
}
|
|
||||||
Ok(ForkResult::Child) => {
|
|
||||||
// Final daemon child continues
|
|
||||||
}
|
|
||||||
Err(e) => return Err(DaemonError::ForkFailed(e)),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Change working directory
|
|
||||||
let target_dir = working_dir.unwrap_or(Path::new("/"));
|
|
||||||
chdir(target_dir).map_err(DaemonError::ChdirFailed)?;
|
|
||||||
|
|
||||||
// Redirect stdin, stdout, stderr to /dev/null
|
|
||||||
redirect_stdio_to_devnull()?;
|
|
||||||
|
|
||||||
Ok(DaemonizeResult::Child)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Redirects stdin, stdout, and stderr to /dev/null.
|
|
||||||
fn redirect_stdio_to_devnull() -> Result<(), DaemonError> {
|
|
||||||
let devnull = File::options()
|
|
||||||
.read(true)
|
|
||||||
.write(true)
|
|
||||||
.open("/dev/null")
|
|
||||||
.map_err(DaemonError::DevNullFailed)?;
|
|
||||||
|
|
||||||
let devnull_fd = std::os::unix::io::AsRawFd::as_raw_fd(&devnull);
|
|
||||||
|
|
||||||
// Use libc::dup2 directly for redirecting standard file descriptors
|
|
||||||
// nix 0.31's dup2 requires OwnedFd which doesn't work well with stdio fds
|
|
||||||
unsafe {
|
|
||||||
// Redirect stdin (fd 0)
|
|
||||||
if libc::dup2(devnull_fd, 0) < 0 {
|
|
||||||
return Err(DaemonError::RedirectFailed(nix::errno::Errno::last()));
|
|
||||||
}
|
|
||||||
// Redirect stdout (fd 1)
|
|
||||||
if libc::dup2(devnull_fd, 1) < 0 {
|
|
||||||
return Err(DaemonError::RedirectFailed(nix::errno::Errno::last()));
|
|
||||||
}
|
|
||||||
// Redirect stderr (fd 2)
|
|
||||||
if libc::dup2(devnull_fd, 2) < 0 {
|
|
||||||
return Err(DaemonError::RedirectFailed(nix::errno::Errno::last()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close original devnull fd if it's not one of the standard fds
|
|
||||||
if devnull_fd > 2 {
|
|
||||||
let _ = close(devnull_fd);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// PID file manager with flock-based locking.
|
|
||||||
pub struct PidFile {
|
|
||||||
path: PathBuf,
|
|
||||||
file: Option<File>,
|
|
||||||
locked: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PidFile {
|
|
||||||
/// Creates a new PID file manager for the given path.
|
|
||||||
pub fn new<P: AsRef<Path>>(path: P) -> Self {
|
|
||||||
Self {
|
|
||||||
path: path.as_ref().to_path_buf(),
|
|
||||||
file: None,
|
|
||||||
locked: false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Checks if another instance is already running.
|
|
||||||
///
|
|
||||||
/// Returns the PID of the running instance if one exists.
|
|
||||||
pub fn check_running(&self) -> Result<Option<i32>, DaemonError> {
|
|
||||||
if !self.path.exists() {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to read existing PID
|
|
||||||
let mut contents = String::new();
|
|
||||||
File::open(&self.path)
|
|
||||||
.and_then(|mut f| f.read_to_string(&mut contents))
|
|
||||||
.map_err(|e| {
|
|
||||||
DaemonError::PidFile(format!("cannot read {}: {}", self.path.display(), e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let pid: i32 = contents
|
|
||||||
.trim()
|
|
||||||
.parse()
|
|
||||||
.map_err(|_| DaemonError::PidFile(format!("invalid PID in {}", self.path.display())))?;
|
|
||||||
|
|
||||||
// Check if process is still running
|
|
||||||
if is_process_running(pid) {
|
|
||||||
Ok(Some(pid))
|
|
||||||
} else {
|
|
||||||
// Stale PID file
|
|
||||||
debug!(pid, path = %self.path.display(), "Removing stale PID file");
|
|
||||||
let _ = fs::remove_file(&self.path);
|
|
||||||
Ok(None)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Acquires the PID file lock and writes the current PID.
|
|
||||||
///
|
|
||||||
/// Fails if another instance is already running.
|
|
||||||
pub fn acquire(&mut self) -> Result<(), DaemonError> {
|
|
||||||
// Check for running instance first
|
|
||||||
if let Some(pid) = self.check_running()? {
|
|
||||||
return Err(DaemonError::AlreadyRunning(pid));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure parent directory exists
|
|
||||||
if let Some(parent) = self.path.parent() {
|
|
||||||
if !parent.exists() {
|
|
||||||
fs::create_dir_all(parent).map_err(|e| {
|
|
||||||
DaemonError::PidFile(format!(
|
|
||||||
"cannot create directory {}: {}",
|
|
||||||
parent.display(),
|
|
||||||
e
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open/create PID file with exclusive lock
|
|
||||||
let file = OpenOptions::new()
|
|
||||||
.write(true)
|
|
||||||
.create(true)
|
|
||||||
.truncate(true)
|
|
||||||
.mode(0o644)
|
|
||||||
.open(&self.path)
|
|
||||||
.map_err(|e| {
|
|
||||||
DaemonError::PidFile(format!("cannot open {}: {}", self.path.display(), e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// Try to acquire exclusive lock (non-blocking)
|
|
||||||
let flock = Flock::lock(file, FlockArg::LockExclusiveNonblock).map_err(|(_, errno)| {
|
|
||||||
// Check if another instance grabbed the lock
|
|
||||||
if let Some(pid) = self.check_running().ok().flatten() {
|
|
||||||
DaemonError::AlreadyRunning(pid)
|
|
||||||
} else {
|
|
||||||
DaemonError::PidFile(format!("cannot lock {}: {}", self.path.display(), errno))
|
|
||||||
}
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// Write our PID
|
|
||||||
let pid = getpid();
|
|
||||||
let mut file = flock
|
|
||||||
.unlock()
|
|
||||||
.map_err(|(_, errno)| DaemonError::PidFile(format!("unlock failed: {}", errno)))?;
|
|
||||||
|
|
||||||
writeln!(file, "{}", pid).map_err(|e| {
|
|
||||||
DaemonError::PidFile(format!(
|
|
||||||
"cannot write PID to {}: {}",
|
|
||||||
self.path.display(),
|
|
||||||
e
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// Re-acquire lock and keep it
|
|
||||||
let flock = Flock::lock(file, FlockArg::LockExclusiveNonblock).map_err(|(_, errno)| {
|
|
||||||
DaemonError::PidFile(format!("cannot re-lock {}: {}", self.path.display(), errno))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
self.file = Some(flock.unlock().map_err(|(_, errno)| {
|
|
||||||
DaemonError::PidFile(format!("unlock for storage failed: {}", errno))
|
|
||||||
})?);
|
|
||||||
self.locked = true;
|
|
||||||
|
|
||||||
info!(pid = pid.as_raw(), path = %self.path.display(), "PID file created");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Releases the PID file lock and removes the file.
|
|
||||||
pub fn release(&mut self) -> Result<(), DaemonError> {
|
|
||||||
if let Some(file) = self.file.take() {
|
|
||||||
drop(file);
|
|
||||||
}
|
|
||||||
self.locked = false;
|
|
||||||
|
|
||||||
if self.path.exists() {
|
|
||||||
fs::remove_file(&self.path).map_err(|e| {
|
|
||||||
DaemonError::PidFile(format!("cannot remove {}: {}", self.path.display(), e))
|
|
||||||
})?;
|
|
||||||
debug!(path = %self.path.display(), "PID file removed");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns the path to this PID file.
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn path(&self) -> &Path {
|
|
||||||
&self.path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for PidFile {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if self.locked {
|
|
||||||
if let Err(e) = self.release() {
|
|
||||||
warn!(error = %e, "Failed to clean up PID file on drop");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Checks if a process with the given PID is running.
|
|
||||||
fn is_process_running(pid: i32) -> bool {
|
|
||||||
// kill(pid, 0) checks if process exists without sending a signal
|
|
||||||
nix::sys::signal::kill(Pid::from_raw(pid), None).is_ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Drops privileges to the specified user and group.
|
|
||||||
///
|
|
||||||
/// This should be called after binding privileged ports but before
|
|
||||||
/// entering the main event loop.
|
|
||||||
pub fn drop_privileges(user: Option<&str>, group: Option<&str>) -> Result<(), DaemonError> {
|
|
||||||
// Look up group first (need to do this while still root)
|
|
||||||
let target_gid = if let Some(group_name) = group {
|
|
||||||
Some(lookup_group(group_name)?)
|
|
||||||
} else if let Some(user_name) = user {
|
|
||||||
// If no group specified but user is, use user's primary group
|
|
||||||
Some(lookup_user_primary_gid(user_name)?)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
// Look up user
|
|
||||||
let target_uid = if let Some(user_name) = user {
|
|
||||||
Some(lookup_user(user_name)?)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
// Drop privileges: set GID first, then UID
|
|
||||||
// (Setting UID first would prevent us from setting GID)
|
|
||||||
if let Some(gid) = target_gid {
|
|
||||||
unistd::setgid(gid).map_err(DaemonError::PrivilegeDrop)?;
|
|
||||||
// Also set supplementary groups to just this one
|
|
||||||
unistd::setgroups(&[gid]).map_err(DaemonError::PrivilegeDrop)?;
|
|
||||||
info!(gid = gid.as_raw(), "Dropped group privileges");
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(uid) = target_uid {
|
|
||||||
unistd::setuid(uid).map_err(DaemonError::PrivilegeDrop)?;
|
|
||||||
info!(uid = uid.as_raw(), "Dropped user privileges");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Looks up a user by name and returns their UID.
|
|
||||||
fn lookup_user(name: &str) -> Result<Uid, DaemonError> {
|
|
||||||
// Use libc getpwnam
|
|
||||||
let c_name =
|
|
||||||
std::ffi::CString::new(name).map_err(|_| DaemonError::UserNotFound(name.to_string()))?;
|
|
||||||
|
|
||||||
unsafe {
|
|
||||||
let pwd = libc::getpwnam(c_name.as_ptr());
|
|
||||||
if pwd.is_null() {
|
|
||||||
Err(DaemonError::UserNotFound(name.to_string()))
|
|
||||||
} else {
|
|
||||||
Ok(Uid::from_raw((*pwd).pw_uid))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Looks up a user's primary GID by username.
|
|
||||||
fn lookup_user_primary_gid(name: &str) -> Result<Gid, DaemonError> {
|
|
||||||
let c_name =
|
|
||||||
std::ffi::CString::new(name).map_err(|_| DaemonError::UserNotFound(name.to_string()))?;
|
|
||||||
|
|
||||||
unsafe {
|
|
||||||
let pwd = libc::getpwnam(c_name.as_ptr());
|
|
||||||
if pwd.is_null() {
|
|
||||||
Err(DaemonError::UserNotFound(name.to_string()))
|
|
||||||
} else {
|
|
||||||
Ok(Gid::from_raw((*pwd).pw_gid))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Looks up a group by name and returns its GID.
|
|
||||||
fn lookup_group(name: &str) -> Result<Gid, DaemonError> {
|
|
||||||
let c_name =
|
|
||||||
std::ffi::CString::new(name).map_err(|_| DaemonError::GroupNotFound(name.to_string()))?;
|
|
||||||
|
|
||||||
unsafe {
|
|
||||||
let grp = libc::getgrnam(c_name.as_ptr());
|
|
||||||
if grp.is_null() {
|
|
||||||
Err(DaemonError::GroupNotFound(name.to_string()))
|
|
||||||
} else {
|
|
||||||
Ok(Gid::from_raw((*grp).gr_gid))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reads PID from a PID file.
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
|
|
||||||
let path = path.as_ref();
|
|
||||||
let mut contents = String::new();
|
|
||||||
File::open(path)
|
|
||||||
.and_then(|mut f| f.read_to_string(&mut contents))
|
|
||||||
.map_err(|e| DaemonError::PidFile(format!("cannot read {}: {}", path.display(), e)))?;
|
|
||||||
|
|
||||||
contents
|
|
||||||
.trim()
|
|
||||||
.parse()
|
|
||||||
.map_err(|_| DaemonError::PidFile(format!("invalid PID in {}", path.display())))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Sends a signal to the process specified in a PID file.
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn signal_pid_file<P: AsRef<Path>>(
|
|
||||||
path: P,
|
|
||||||
signal: nix::sys::signal::Signal,
|
|
||||||
) -> Result<(), DaemonError> {
|
|
||||||
let pid = read_pid_file(&path)?;
|
|
||||||
|
|
||||||
if !is_process_running(pid) {
|
|
||||||
return Err(DaemonError::PidFile(format!(
|
|
||||||
"process {} from {} is not running",
|
|
||||||
pid,
|
|
||||||
path.as_ref().display()
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
nix::sys::signal::kill(Pid::from_raw(pid), signal)
|
|
||||||
.map_err(|e| DaemonError::PidFile(format!("cannot signal process {}: {}", pid, e)))?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns the status of the daemon based on PID file.
|
|
||||||
#[allow(dead_code)]
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum DaemonStatus {
|
|
||||||
/// Daemon is running with the given PID.
|
|
||||||
Running(i32),
|
|
||||||
/// PID file exists but process is not running.
|
|
||||||
Stale(i32),
|
|
||||||
/// No PID file exists.
|
|
||||||
NotRunning,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Checks the daemon status from a PID file.
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
|
|
||||||
let path = path.as_ref();
|
|
||||||
|
|
||||||
if !path.exists() {
|
|
||||||
return DaemonStatus::NotRunning;
|
|
||||||
}
|
|
||||||
|
|
||||||
match read_pid_file(path) {
|
|
||||||
Ok(pid) => {
|
|
||||||
if is_process_running(pid) {
|
|
||||||
DaemonStatus::Running(pid)
|
|
||||||
} else {
|
|
||||||
DaemonStatus::Stale(pid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(_) => DaemonStatus::NotRunning,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_daemon_options_default() {
|
|
||||||
let opts = DaemonOptions::default();
|
|
||||||
assert!(!opts.daemonize);
|
|
||||||
assert!(!opts.should_daemonize());
|
|
||||||
assert_eq!(opts.pid_file_path(), Path::new(DEFAULT_PID_FILE));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_daemon_options_foreground_overrides() {
|
|
||||||
let opts = DaemonOptions {
|
|
||||||
daemonize: true,
|
|
||||||
foreground: true,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
assert!(!opts.should_daemonize());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_check_status_not_running() {
|
|
||||||
let path = "/tmp/telemt_test_nonexistent.pid";
|
|
||||||
assert_eq!(check_status(path), DaemonStatus::NotRunning);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_pid_file_basic() {
|
|
||||||
let path = "/tmp/telemt_test_pidfile.pid";
|
|
||||||
let _ = fs::remove_file(path);
|
|
||||||
|
|
||||||
let mut pf = PidFile::new(path);
|
|
||||||
assert!(pf.check_running().unwrap().is_none());
|
|
||||||
|
|
||||||
pf.acquire().unwrap();
|
|
||||||
assert!(Path::new(path).exists());
|
|
||||||
|
|
||||||
// Read it back
|
|
||||||
let pid = read_pid_file(path).unwrap();
|
|
||||||
assert_eq!(pid, std::process::id() as i32);
|
|
||||||
|
|
||||||
pf.release().unwrap();
|
|
||||||
assert!(!Path::new(path).exists());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-305
@@ -1,305 +0,0 @@
|
|||||||
//! Logging configuration for telemt.
|
|
||||||
//!
|
|
||||||
//! Supports multiple log destinations:
|
|
||||||
//! - stderr (default, works with systemd journald)
|
|
||||||
//! - syslog (Unix only, for traditional init systems)
|
|
||||||
//! - file (with optional rotation)
|
|
||||||
|
|
||||||
#![allow(dead_code)] // Infrastructure module - used via CLI flags
|
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use tracing_subscriber::layer::SubscriberExt;
|
|
||||||
use tracing_subscriber::util::SubscriberInitExt;
|
|
||||||
use tracing_subscriber::{EnvFilter, fmt, reload};
|
|
||||||
|
|
||||||
/// Log destination configuration.
|
|
||||||
#[derive(Debug, Clone, Default)]
|
|
||||||
pub enum LogDestination {
|
|
||||||
/// Log to stderr (default, captured by systemd journald).
|
|
||||||
#[default]
|
|
||||||
Stderr,
|
|
||||||
/// Log to syslog (Unix only).
|
|
||||||
#[cfg(unix)]
|
|
||||||
Syslog,
|
|
||||||
/// Log to a file with optional rotation.
|
|
||||||
File {
|
|
||||||
path: String,
|
|
||||||
/// Rotate daily if true.
|
|
||||||
rotate_daily: bool,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Logging options parsed from CLI/config.
|
|
||||||
#[derive(Debug, Clone, Default)]
|
|
||||||
pub struct LoggingOptions {
|
|
||||||
/// Where to send logs.
|
|
||||||
pub destination: LogDestination,
|
|
||||||
/// Disable ANSI colors.
|
|
||||||
pub disable_colors: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Guard that must be held to keep file logging active.
|
|
||||||
/// When dropped, flushes and closes log files.
|
|
||||||
pub struct LoggingGuard {
|
|
||||||
_guard: Option<tracing_appender::non_blocking::WorkerGuard>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LoggingGuard {
|
|
||||||
fn new(guard: Option<tracing_appender::non_blocking::WorkerGuard>) -> Self {
|
|
||||||
Self { _guard: guard }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Creates a no-op guard for stderr/syslog logging.
|
|
||||||
pub fn noop() -> Self {
|
|
||||||
Self { _guard: None }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Initialize the tracing subscriber with the specified options.
|
|
||||||
///
|
|
||||||
/// Returns a reload handle for dynamic log level changes and a guard
|
|
||||||
/// that must be kept alive for file logging.
|
|
||||||
pub fn init_logging(
|
|
||||||
opts: &LoggingOptions,
|
|
||||||
initial_filter: &str,
|
|
||||||
) -> (
|
|
||||||
reload::Handle<EnvFilter, impl tracing::Subscriber + Send + Sync>,
|
|
||||||
LoggingGuard,
|
|
||||||
) {
|
|
||||||
let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new(initial_filter));
|
|
||||||
|
|
||||||
match &opts.destination {
|
|
||||||
LogDestination::Stderr => {
|
|
||||||
let fmt_layer = fmt::Layer::default()
|
|
||||||
.with_ansi(!opts.disable_colors)
|
|
||||||
.with_target(true);
|
|
||||||
|
|
||||||
tracing_subscriber::registry()
|
|
||||||
.with(filter_layer)
|
|
||||||
.with(fmt_layer)
|
|
||||||
.init();
|
|
||||||
|
|
||||||
(filter_handle, LoggingGuard::noop())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
LogDestination::Syslog => {
|
|
||||||
// Use a custom fmt layer that writes to syslog
|
|
||||||
let fmt_layer = fmt::Layer::default()
|
|
||||||
.with_ansi(false)
|
|
||||||
.with_target(true)
|
|
||||||
.with_writer(SyslogWriter::new);
|
|
||||||
|
|
||||||
tracing_subscriber::registry()
|
|
||||||
.with(filter_layer)
|
|
||||||
.with(fmt_layer)
|
|
||||||
.init();
|
|
||||||
|
|
||||||
(filter_handle, LoggingGuard::noop())
|
|
||||||
}
|
|
||||||
|
|
||||||
LogDestination::File { path, rotate_daily } => {
|
|
||||||
let (non_blocking, guard) = if *rotate_daily {
|
|
||||||
// Extract directory and filename prefix
|
|
||||||
let path = Path::new(path);
|
|
||||||
let dir = path.parent().unwrap_or(Path::new("/var/log"));
|
|
||||||
let prefix = path
|
|
||||||
.file_name()
|
|
||||||
.and_then(|s| s.to_str())
|
|
||||||
.unwrap_or("telemt");
|
|
||||||
|
|
||||||
let file_appender = tracing_appender::rolling::daily(dir, prefix);
|
|
||||||
tracing_appender::non_blocking(file_appender)
|
|
||||||
} else {
|
|
||||||
let file = std::fs::OpenOptions::new()
|
|
||||||
.create(true)
|
|
||||||
.append(true)
|
|
||||||
.open(path)
|
|
||||||
.expect("Failed to open log file");
|
|
||||||
tracing_appender::non_blocking(file)
|
|
||||||
};
|
|
||||||
|
|
||||||
let fmt_layer = fmt::Layer::default()
|
|
||||||
.with_ansi(false)
|
|
||||||
.with_target(true)
|
|
||||||
.with_writer(non_blocking);
|
|
||||||
|
|
||||||
tracing_subscriber::registry()
|
|
||||||
.with(filter_layer)
|
|
||||||
.with(fmt_layer)
|
|
||||||
.init();
|
|
||||||
|
|
||||||
(filter_handle, LoggingGuard::new(Some(guard)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Syslog writer for tracing.
|
|
||||||
#[cfg(unix)]
|
|
||||||
struct SyslogWriter {
|
|
||||||
_private: (),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
impl SyslogWriter {
|
|
||||||
fn new() -> Self {
|
|
||||||
// Open syslog connection on first use
|
|
||||||
static INIT: std::sync::Once = std::sync::Once::new();
|
|
||||||
INIT.call_once(|| {
|
|
||||||
unsafe {
|
|
||||||
// Open syslog with ident "telemt", LOG_PID, LOG_DAEMON facility
|
|
||||||
let ident = b"telemt\0".as_ptr() as *const libc::c_char;
|
|
||||||
libc::openlog(ident, libc::LOG_PID | libc::LOG_NDELAY, libc::LOG_DAEMON);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Self { _private: () }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
impl std::io::Write for SyslogWriter {
|
|
||||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
|
||||||
// Convert to C string, stripping newlines
|
|
||||||
let msg = String::from_utf8_lossy(buf);
|
|
||||||
let msg = msg.trim_end();
|
|
||||||
|
|
||||||
if msg.is_empty() {
|
|
||||||
return Ok(buf.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine priority based on log level in the message
|
|
||||||
let priority = if msg.contains(" ERROR ") || msg.contains(" error ") {
|
|
||||||
libc::LOG_ERR
|
|
||||||
} else if msg.contains(" WARN ") || msg.contains(" warn ") {
|
|
||||||
libc::LOG_WARNING
|
|
||||||
} else if msg.contains(" INFO ") || msg.contains(" info ") {
|
|
||||||
libc::LOG_INFO
|
|
||||||
} else if msg.contains(" DEBUG ") || msg.contains(" debug ") {
|
|
||||||
libc::LOG_DEBUG
|
|
||||||
} else {
|
|
||||||
libc::LOG_INFO
|
|
||||||
};
|
|
||||||
|
|
||||||
// Write to syslog
|
|
||||||
let c_msg = std::ffi::CString::new(msg.as_bytes())
|
|
||||||
.unwrap_or_else(|_| std::ffi::CString::new("(invalid utf8)").unwrap());
|
|
||||||
|
|
||||||
unsafe {
|
|
||||||
libc::syslog(
|
|
||||||
priority,
|
|
||||||
b"%s\0".as_ptr() as *const libc::c_char,
|
|
||||||
c_msg.as_ptr(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(buf.len())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn flush(&mut self) -> std::io::Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for SyslogWriter {
|
|
||||||
type Writer = SyslogWriter;
|
|
||||||
|
|
||||||
fn make_writer(&'a self) -> Self::Writer {
|
|
||||||
SyslogWriter::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse log destination from CLI arguments.
|
|
||||||
pub fn parse_log_destination(args: &[String]) -> LogDestination {
|
|
||||||
let mut i = 0;
|
|
||||||
while i < args.len() {
|
|
||||||
match args[i].as_str() {
|
|
||||||
#[cfg(unix)]
|
|
||||||
"--syslog" => {
|
|
||||||
return LogDestination::Syslog;
|
|
||||||
}
|
|
||||||
"--log-file" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
return LogDestination::File {
|
|
||||||
path: args[i].clone(),
|
|
||||||
rotate_daily: false,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--log-file=") => {
|
|
||||||
return LogDestination::File {
|
|
||||||
path: s.trim_start_matches("--log-file=").to_string(),
|
|
||||||
rotate_daily: false,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
"--log-file-daily" => {
|
|
||||||
i += 1;
|
|
||||||
if i < args.len() {
|
|
||||||
return LogDestination::File {
|
|
||||||
path: args[i].clone(),
|
|
||||||
rotate_daily: true,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--log-file-daily=") => {
|
|
||||||
return LogDestination::File {
|
|
||||||
path: s.trim_start_matches("--log-file-daily=").to_string(),
|
|
||||||
rotate_daily: true,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
LogDestination::Stderr
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_log_destination_default() {
|
|
||||||
let args: Vec<String> = vec![];
|
|
||||||
assert!(matches!(
|
|
||||||
parse_log_destination(&args),
|
|
||||||
LogDestination::Stderr
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_log_destination_file() {
|
|
||||||
let args = vec!["--log-file".to_string(), "/var/log/telemt.log".to_string()];
|
|
||||||
match parse_log_destination(&args) {
|
|
||||||
LogDestination::File { path, rotate_daily } => {
|
|
||||||
assert_eq!(path, "/var/log/telemt.log");
|
|
||||||
assert!(!rotate_daily);
|
|
||||||
}
|
|
||||||
_ => panic!("Expected File destination"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_parse_log_destination_file_daily() {
|
|
||||||
let args = vec!["--log-file-daily=/var/log/telemt".to_string()];
|
|
||||||
match parse_log_destination(&args) {
|
|
||||||
LogDestination::File { path, rotate_daily } => {
|
|
||||||
assert_eq!(path, "/var/log/telemt");
|
|
||||||
assert!(rotate_daily);
|
|
||||||
}
|
|
||||||
_ => panic!("Expected File destination"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
#[test]
|
|
||||||
fn test_parse_log_destination_syslog() {
|
|
||||||
let args = vec!["--syslog".to_string()];
|
|
||||||
assert!(matches!(
|
|
||||||
parse_log_destination(&args),
|
|
||||||
LogDestination::Syslog
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+20
-60
@@ -21,29 +21,10 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
if config.general.use_middle_proxy {
|
if config.general.use_middle_proxy {
|
||||||
if let Some(pool) = me_pool.as_ref() {
|
if let Some(pool) = me_pool.as_ref() {
|
||||||
let initial_ready = pool.admission_ready_conditional_cast().await;
|
let initial_ready = pool.admission_ready_conditional_cast().await;
|
||||||
let mut fallback_enabled = config.general.me2dc_fallback;
|
admission_tx.send_replace(initial_ready);
|
||||||
let mut fast_fallback_enabled = fallback_enabled && config.general.me2dc_fast;
|
let _ = route_runtime.set_mode(RelayRouteMode::Middle);
|
||||||
let (initial_gate_open, initial_route_mode, initial_fallback_reason) = if initial_ready
|
|
||||||
{
|
|
||||||
(true, RelayRouteMode::Middle, None)
|
|
||||||
} else if fast_fallback_enabled {
|
|
||||||
(
|
|
||||||
true,
|
|
||||||
RelayRouteMode::Direct,
|
|
||||||
Some("fast_not_ready_fallback"),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
(false, RelayRouteMode::Middle, None)
|
|
||||||
};
|
|
||||||
admission_tx.send_replace(initial_gate_open);
|
|
||||||
let _ = route_runtime.set_mode(initial_route_mode);
|
|
||||||
if initial_ready {
|
if initial_ready {
|
||||||
info!("Conditional-admission gate: open / ME pool READY");
|
info!("Conditional-admission gate: open / ME pool READY");
|
||||||
} else if let Some(reason) = initial_fallback_reason {
|
|
||||||
warn!(
|
|
||||||
fallback_reason = reason,
|
|
||||||
"Conditional-admission gate opened in ME fast fallback mode"
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
warn!("Conditional-admission gate: closed / ME pool is NOT ready)");
|
warn!("Conditional-admission gate: closed / ME pool is NOT ready)");
|
||||||
}
|
}
|
||||||
@@ -53,9 +34,10 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
let route_runtime_gate = route_runtime.clone();
|
let route_runtime_gate = route_runtime.clone();
|
||||||
let mut config_rx_gate = config_rx.clone();
|
let mut config_rx_gate = config_rx.clone();
|
||||||
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
|
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
|
||||||
|
let mut fallback_enabled = config.general.me2dc_fallback;
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut gate_open = initial_gate_open;
|
let mut gate_open = initial_ready;
|
||||||
let mut route_mode = initial_route_mode;
|
let mut route_mode = RelayRouteMode::Middle;
|
||||||
let mut ready_observed = initial_ready;
|
let mut ready_observed = initial_ready;
|
||||||
let mut not_ready_since = if initial_ready {
|
let mut not_ready_since = if initial_ready {
|
||||||
None
|
None
|
||||||
@@ -71,23 +53,16 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
let cfg = config_rx_gate.borrow_and_update().clone();
|
let cfg = config_rx_gate.borrow_and_update().clone();
|
||||||
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
|
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
|
||||||
fallback_enabled = cfg.general.me2dc_fallback;
|
fallback_enabled = cfg.general.me2dc_fallback;
|
||||||
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
||||||
}
|
}
|
||||||
let ready = pool_for_gate.admission_ready_conditional_cast().await;
|
let ready = pool_for_gate.admission_ready_conditional_cast().await;
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let (next_gate_open, next_route_mode, next_fallback_reason) = if ready {
|
let (next_gate_open, next_route_mode, next_fallback_active) = if ready {
|
||||||
ready_observed = true;
|
ready_observed = true;
|
||||||
not_ready_since = None;
|
not_ready_since = None;
|
||||||
(true, RelayRouteMode::Middle, None)
|
(true, RelayRouteMode::Middle, false)
|
||||||
} else if fast_fallback_enabled {
|
|
||||||
(
|
|
||||||
true,
|
|
||||||
RelayRouteMode::Direct,
|
|
||||||
Some("fast_not_ready_fallback"),
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
let not_ready_started_at = *not_ready_since.get_or_insert(now);
|
let not_ready_started_at = *not_ready_since.get_or_insert(now);
|
||||||
let not_ready_for = now.saturating_duration_since(not_ready_started_at);
|
let not_ready_for = now.saturating_duration_since(not_ready_started_at);
|
||||||
@@ -97,12 +72,11 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
STARTUP_FALLBACK_AFTER
|
STARTUP_FALLBACK_AFTER
|
||||||
};
|
};
|
||||||
if fallback_enabled && not_ready_for > fallback_after {
|
if fallback_enabled && not_ready_for > fallback_after {
|
||||||
(true, RelayRouteMode::Direct, Some("strict_grace_fallback"))
|
(true, RelayRouteMode::Direct, true)
|
||||||
} else {
|
} else {
|
||||||
(false, RelayRouteMode::Middle, None)
|
(false, RelayRouteMode::Middle, false)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let next_fallback_active = next_fallback_reason.is_some();
|
|
||||||
|
|
||||||
if next_route_mode != route_mode {
|
if next_route_mode != route_mode {
|
||||||
route_mode = next_route_mode;
|
route_mode = next_route_mode;
|
||||||
@@ -114,28 +88,17 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
"Middle-End routing restored for new sessions"
|
"Middle-End routing restored for new sessions"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
let fallback_reason = next_fallback_reason.unwrap_or("unknown");
|
let fallback_after = if ready_observed {
|
||||||
if fallback_reason == "strict_grace_fallback" {
|
RUNTIME_FALLBACK_AFTER
|
||||||
let fallback_after = if ready_observed {
|
|
||||||
RUNTIME_FALLBACK_AFTER
|
|
||||||
} else {
|
|
||||||
STARTUP_FALLBACK_AFTER
|
|
||||||
};
|
|
||||||
warn!(
|
|
||||||
target_mode = route_mode.as_str(),
|
|
||||||
cutover_generation = snapshot.generation,
|
|
||||||
grace_secs = fallback_after.as_secs(),
|
|
||||||
fallback_reason,
|
|
||||||
"ME pool stayed not-ready beyond grace; routing new sessions via Direct-DC"
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
warn!(
|
STARTUP_FALLBACK_AFTER
|
||||||
target_mode = route_mode.as_str(),
|
};
|
||||||
cutover_generation = snapshot.generation,
|
warn!(
|
||||||
fallback_reason,
|
target_mode = route_mode.as_str(),
|
||||||
"ME pool not-ready; routing new sessions via Direct-DC (fast mode)"
|
cutover_generation = snapshot.generation,
|
||||||
);
|
grace_secs = fallback_after.as_secs(),
|
||||||
}
|
"ME pool stayed not-ready beyond grace; routing new sessions via Direct-DC"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -145,10 +108,7 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
admission_tx_gate.send_replace(gate_open);
|
admission_tx_gate.send_replace(gate_open);
|
||||||
if gate_open {
|
if gate_open {
|
||||||
if next_fallback_active {
|
if next_fallback_active {
|
||||||
warn!(
|
warn!("Conditional-admission gate opened in ME fallback mode");
|
||||||
fallback_reason = next_fallback_reason.unwrap_or("unknown"),
|
|
||||||
"Conditional-admission gate opened in ME fallback mode"
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
info!("Conditional-admission gate opened / ME pool READY");
|
info!("Conditional-admission gate opened / ME pool READY");
|
||||||
}
|
}
|
||||||
|
|||||||
+26
-104
@@ -8,7 +8,6 @@ use tracing::{debug, error, info, warn};
|
|||||||
|
|
||||||
use crate::cli;
|
use crate::cli;
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::logging::LogDestination;
|
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::{
|
use crate::transport::middle_proxy::{
|
||||||
ProxyConfigData, fetch_proxy_config_with_raw_via_upstream, load_proxy_config_cache,
|
ProxyConfigData, fetch_proxy_config_with_raw_via_upstream, load_proxy_config_cache,
|
||||||
@@ -28,16 +27,7 @@ pub(crate) fn resolve_runtime_config_path(
|
|||||||
absolute.canonicalize().unwrap_or(absolute)
|
absolute.canonicalize().unwrap_or(absolute)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parsed CLI arguments.
|
pub(crate) fn parse_cli() -> (String, Option<PathBuf>, bool, Option<String>) {
|
||||||
pub(crate) struct CliArgs {
|
|
||||||
pub config_path: String,
|
|
||||||
pub data_path: Option<PathBuf>,
|
|
||||||
pub silent: bool,
|
|
||||||
pub log_level: Option<String>,
|
|
||||||
pub log_destination: LogDestination,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn parse_cli() -> CliArgs {
|
|
||||||
let mut config_path = "config.toml".to_string();
|
let mut config_path = "config.toml".to_string();
|
||||||
let mut data_path: Option<PathBuf> = None;
|
let mut data_path: Option<PathBuf> = None;
|
||||||
let mut silent = false;
|
let mut silent = false;
|
||||||
@@ -45,9 +35,6 @@ pub(crate) fn parse_cli() -> CliArgs {
|
|||||||
|
|
||||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||||
|
|
||||||
// Parse log destination
|
|
||||||
let log_destination = crate::logging::parse_log_destination(&args);
|
|
||||||
|
|
||||||
// Check for --init first (handled before tokio)
|
// Check for --init first (handled before tokio)
|
||||||
if let Some(init_opts) = cli::parse_init_args(&args) {
|
if let Some(init_opts) = cli::parse_init_args(&args) {
|
||||||
if let Err(e) = cli::run_init(init_opts) {
|
if let Err(e) = cli::run_init(init_opts) {
|
||||||
@@ -87,35 +74,36 @@ pub(crate) fn parse_cli() -> CliArgs {
|
|||||||
log_level = Some(s.trim_start_matches("--log-level=").to_string());
|
log_level = Some(s.trim_start_matches("--log-level=").to_string());
|
||||||
}
|
}
|
||||||
"--help" | "-h" => {
|
"--help" | "-h" => {
|
||||||
print_help();
|
eprintln!("Usage: telemt [config.toml] [OPTIONS]");
|
||||||
|
eprintln!();
|
||||||
|
eprintln!("Options:");
|
||||||
|
eprintln!(
|
||||||
|
" --data-path <DIR> Set data directory (absolute path; overrides config value)"
|
||||||
|
);
|
||||||
|
eprintln!(" --silent, -s Suppress info logs");
|
||||||
|
eprintln!(" --log-level <LEVEL> debug|verbose|normal|silent");
|
||||||
|
eprintln!(" --help, -h Show this help");
|
||||||
|
eprintln!();
|
||||||
|
eprintln!("Setup (fire-and-forget):");
|
||||||
|
eprintln!(
|
||||||
|
" --init Generate config, install systemd service, start"
|
||||||
|
);
|
||||||
|
eprintln!(" --port <PORT> Listen port (default: 443)");
|
||||||
|
eprintln!(
|
||||||
|
" --domain <DOMAIN> TLS domain for masking (default: www.google.com)"
|
||||||
|
);
|
||||||
|
eprintln!(
|
||||||
|
" --secret <HEX> 32-char hex secret (auto-generated if omitted)"
|
||||||
|
);
|
||||||
|
eprintln!(" --user <NAME> Username (default: user)");
|
||||||
|
eprintln!(" --config-dir <DIR> Config directory (default: /etc/telemt)");
|
||||||
|
eprintln!(" --no-start Don't start the service after install");
|
||||||
std::process::exit(0);
|
std::process::exit(0);
|
||||||
}
|
}
|
||||||
"--version" | "-V" => {
|
"--version" | "-V" => {
|
||||||
println!("telemt {}", env!("CARGO_PKG_VERSION"));
|
println!("telemt {}", env!("CARGO_PKG_VERSION"));
|
||||||
std::process::exit(0);
|
std::process::exit(0);
|
||||||
}
|
}
|
||||||
// Skip daemon-related flags (already parsed)
|
|
||||||
"--daemon" | "-d" | "--foreground" | "-f" => {}
|
|
||||||
s if s.starts_with("--pid-file") => {
|
|
||||||
if !s.contains('=') {
|
|
||||||
i += 1; // skip value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--run-as-user") => {
|
|
||||||
if !s.contains('=') {
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--run-as-group") => {
|
|
||||||
if !s.contains('=') {
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if s.starts_with("--working-dir") => {
|
|
||||||
if !s.contains('=') {
|
|
||||||
i += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s if !s.starts_with('-') => {
|
s if !s.starts_with('-') => {
|
||||||
config_path = s.to_string();
|
config_path = s.to_string();
|
||||||
}
|
}
|
||||||
@@ -126,73 +114,7 @@ pub(crate) fn parse_cli() -> CliArgs {
|
|||||||
i += 1;
|
i += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
CliArgs {
|
(config_path, data_path, silent, log_level)
|
||||||
config_path,
|
|
||||||
data_path,
|
|
||||||
silent,
|
|
||||||
log_level,
|
|
||||||
log_destination,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_help() {
|
|
||||||
eprintln!("Usage: telemt [COMMAND] [OPTIONS] [config.toml]");
|
|
||||||
eprintln!();
|
|
||||||
eprintln!("Commands:");
|
|
||||||
eprintln!(" run Run in foreground (default if no command given)");
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
eprintln!(" start Start as background daemon");
|
|
||||||
eprintln!(" stop Stop a running daemon");
|
|
||||||
eprintln!(" reload Reload configuration (send SIGHUP)");
|
|
||||||
eprintln!(" status Check if daemon is running");
|
|
||||||
}
|
|
||||||
eprintln!();
|
|
||||||
eprintln!("Options:");
|
|
||||||
eprintln!(
|
|
||||||
" --data-path <DIR> Set data directory (absolute path; overrides config value)"
|
|
||||||
);
|
|
||||||
eprintln!(" --silent, -s Suppress info logs");
|
|
||||||
eprintln!(" --log-level <LEVEL> debug|verbose|normal|silent");
|
|
||||||
eprintln!(" --help, -h Show this help");
|
|
||||||
eprintln!(" --version, -V Show version");
|
|
||||||
eprintln!();
|
|
||||||
eprintln!("Logging options:");
|
|
||||||
eprintln!(" --log-file <PATH> Log to file (default: stderr)");
|
|
||||||
eprintln!(" --log-file-daily <PATH> Log to file with daily rotation");
|
|
||||||
#[cfg(unix)]
|
|
||||||
eprintln!(" --syslog Log to syslog (Unix only)");
|
|
||||||
eprintln!();
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
eprintln!("Daemon options (Unix only):");
|
|
||||||
eprintln!(" --daemon, -d Fork to background (daemonize)");
|
|
||||||
eprintln!(" --foreground, -f Explicit foreground mode (for systemd)");
|
|
||||||
eprintln!(" --pid-file <PATH> PID file path (default: /var/run/telemt.pid)");
|
|
||||||
eprintln!(" --run-as-user <USER> Drop privileges to this user after binding");
|
|
||||||
eprintln!(" --run-as-group <GROUP> Drop privileges to this group after binding");
|
|
||||||
eprintln!(" --working-dir <DIR> Working directory for daemon mode");
|
|
||||||
eprintln!();
|
|
||||||
}
|
|
||||||
eprintln!("Setup (fire-and-forget):");
|
|
||||||
eprintln!(" --init Generate config, install systemd service, start");
|
|
||||||
eprintln!(" --port <PORT> Listen port (default: 443)");
|
|
||||||
eprintln!(" --domain <DOMAIN> TLS domain for masking (default: www.google.com)");
|
|
||||||
eprintln!(" --secret <HEX> 32-char hex secret (auto-generated if omitted)");
|
|
||||||
eprintln!(" --user <NAME> Username (default: user)");
|
|
||||||
eprintln!(" --config-dir <DIR> Config directory (default: /etc/telemt)");
|
|
||||||
eprintln!(" --no-start Don't start the service after install");
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
eprintln!();
|
|
||||||
eprintln!("Examples:");
|
|
||||||
eprintln!(" telemt config.toml Run in foreground");
|
|
||||||
eprintln!(" telemt start config.toml Start as daemon");
|
|
||||||
eprintln!(" telemt start --pid-file /tmp/t.pid Start with custom PID file");
|
|
||||||
eprintln!(" telemt stop Stop daemon");
|
|
||||||
eprintln!(" telemt reload Reload configuration");
|
|
||||||
eprintln!(" telemt status Check daemon status");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -72,7 +72,6 @@ pub(crate) async fn bind_listeners(
|
|||||||
let options = ListenOptions {
|
let options = ListenOptions {
|
||||||
reuse_port: listener_conf.reuse_allow,
|
reuse_port: listener_conf.reuse_allow,
|
||||||
ipv6_only: listener_conf.ip.is_ipv6(),
|
ipv6_only: listener_conf.ip.is_ipv6(),
|
||||||
backlog: config.server.listen_backlog,
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -277,8 +277,6 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
config.general.me_warn_rate_limit_ms,
|
config.general.me_warn_rate_limit_ms,
|
||||||
config.general.me_route_no_writer_mode,
|
config.general.me_route_no_writer_mode,
|
||||||
config.general.me_route_no_writer_wait_ms,
|
config.general.me_route_no_writer_wait_ms,
|
||||||
config.general.me_route_hybrid_max_wait_ms,
|
|
||||||
config.general.me_route_blocking_send_timeout_ms,
|
|
||||||
config.general.me_route_inline_recovery_attempts,
|
config.general.me_route_inline_recovery_attempts,
|
||||||
config.general.me_route_inline_recovery_wait_ms,
|
config.general.me_route_inline_recovery_wait_ms,
|
||||||
);
|
);
|
||||||
|
|||||||
+22
-109
@@ -47,55 +47,8 @@ use crate::transport::UpstreamManager;
|
|||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
use helpers::{parse_cli, resolve_runtime_config_path};
|
use helpers::{parse_cli, resolve_runtime_config_path};
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
use crate::daemon::{DaemonOptions, PidFile, drop_privileges};
|
|
||||||
|
|
||||||
/// Runs the full telemt runtime startup pipeline and blocks until shutdown.
|
/// Runs the full telemt runtime startup pipeline and blocks until shutdown.
|
||||||
///
|
|
||||||
/// On Unix, daemon options should be handled before calling this function
|
|
||||||
/// (daemonization must happen before tokio runtime starts).
|
|
||||||
#[cfg(unix)]
|
|
||||||
pub async fn run_with_daemon(
|
|
||||||
daemon_opts: DaemonOptions,
|
|
||||||
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|
||||||
run_inner(daemon_opts).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Runs the full telemt runtime startup pipeline and blocks until shutdown.
|
|
||||||
///
|
|
||||||
/// This is the main entry point for non-daemon mode or when called as a library.
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub async fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
pub async fn run() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
// Parse CLI to get daemon options even in simple run() path
|
|
||||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
|
||||||
let daemon_opts = crate::cli::parse_daemon_args(&args);
|
|
||||||
run_inner(daemon_opts).await
|
|
||||||
}
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
{
|
|
||||||
run_inner().await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
async fn run_inner(
|
|
||||||
daemon_opts: DaemonOptions,
|
|
||||||
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
|
||||||
// Acquire PID file if daemonizing or if explicitly requested
|
|
||||||
// Keep it alive until shutdown (underscore prefix = intentionally kept for RAII cleanup)
|
|
||||||
let _pid_file = if daemon_opts.daemonize || daemon_opts.pid_file.is_some() {
|
|
||||||
let mut pf = PidFile::new(daemon_opts.pid_file_path());
|
|
||||||
if let Err(e) = pf.acquire() {
|
|
||||||
eprintln!("[telemt] {}", e);
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
Some(pf)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let process_started_at = Instant::now();
|
let process_started_at = Instant::now();
|
||||||
let process_started_at_epoch_secs = SystemTime::now()
|
let process_started_at_epoch_secs = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
@@ -108,12 +61,7 @@ async fn run_inner(
|
|||||||
Some("load and validate config".to_string()),
|
Some("load and validate config".to_string()),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let cli_args = parse_cli();
|
let (config_path_cli, data_path, cli_silent, cli_log_level) = parse_cli();
|
||||||
let config_path_cli = cli_args.config_path;
|
|
||||||
let data_path = cli_args.data_path;
|
|
||||||
let cli_silent = cli_args.silent;
|
|
||||||
let cli_log_level = cli_args.log_level;
|
|
||||||
let log_destination = cli_args.log_destination;
|
|
||||||
let startup_cwd = match std::env::current_dir() {
|
let startup_cwd = match std::env::current_dir() {
|
||||||
Ok(cwd) => cwd,
|
Ok(cwd) => cwd,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -167,13 +115,15 @@ async fn run_inner(
|
|||||||
);
|
);
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
} else if let Err(e) = std::fs::create_dir_all(data_path) {
|
} else {
|
||||||
eprintln!(
|
if let Err(e) = std::fs::create_dir_all(data_path) {
|
||||||
"[telemt] Can't create data_path {}: {}",
|
eprintln!(
|
||||||
data_path.display(),
|
"[telemt] Can't create data_path {}: {}",
|
||||||
e
|
data_path.display(),
|
||||||
);
|
e
|
||||||
std::process::exit(1);
|
);
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = std::env::set_current_dir(data_path) {
|
if let Err(e) = std::env::set_current_dir(data_path) {
|
||||||
@@ -211,43 +161,17 @@ async fn run_inner(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
// Initialize logging based on destination
|
// Configure color output based on config
|
||||||
let _logging_guard: Option<crate::logging::LoggingGuard>;
|
let fmt_layer = if config.general.disable_colors {
|
||||||
match log_destination {
|
fmt::Layer::default().with_ansi(false)
|
||||||
crate::logging::LogDestination::Stderr => {
|
} else {
|
||||||
// Default: log to stderr (works with systemd journald)
|
fmt::Layer::default().with_ansi(true)
|
||||||
let fmt_layer = if config.general.disable_colors {
|
};
|
||||||
fmt::Layer::default().with_ansi(false)
|
|
||||||
} else {
|
|
||||||
fmt::Layer::default().with_ansi(true)
|
|
||||||
};
|
|
||||||
tracing_subscriber::registry()
|
|
||||||
.with(filter_layer)
|
|
||||||
.with(fmt_layer)
|
|
||||||
.init();
|
|
||||||
_logging_guard = None;
|
|
||||||
}
|
|
||||||
#[cfg(unix)]
|
|
||||||
crate::logging::LogDestination::Syslog => {
|
|
||||||
// Syslog: for OpenRC/FreeBSD
|
|
||||||
let logging_opts = crate::logging::LoggingOptions {
|
|
||||||
destination: log_destination,
|
|
||||||
disable_colors: true,
|
|
||||||
};
|
|
||||||
let (_, guard) = crate::logging::init_logging(&logging_opts, "info");
|
|
||||||
_logging_guard = Some(guard);
|
|
||||||
}
|
|
||||||
crate::logging::LogDestination::File { .. } => {
|
|
||||||
// File logging with optional rotation
|
|
||||||
let logging_opts = crate::logging::LoggingOptions {
|
|
||||||
destination: log_destination,
|
|
||||||
disable_colors: true,
|
|
||||||
};
|
|
||||||
let (_, guard) = crate::logging::init_logging(&logging_opts, "info");
|
|
||||||
_logging_guard = Some(guard);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
tracing_subscriber::registry()
|
||||||
|
.with(filter_layer)
|
||||||
|
.with(fmt_layer)
|
||||||
|
.init();
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(
|
.complete_component(
|
||||||
COMPONENT_TRACING_INIT,
|
COMPONENT_TRACING_INIT,
|
||||||
@@ -301,7 +225,7 @@ async fn run_inner(
|
|||||||
config.general.upstream_connect_retry_attempts,
|
config.general.upstream_connect_retry_attempts,
|
||||||
config.general.upstream_connect_retry_backoff_ms,
|
config.general.upstream_connect_retry_backoff_ms,
|
||||||
config.general.upstream_connect_budget_ms,
|
config.general.upstream_connect_budget_ms,
|
||||||
config.general.tg_connect,
|
config.timeouts.tg_connect,
|
||||||
config.general.upstream_unhealthy_fail_threshold,
|
config.general.upstream_unhealthy_fail_threshold,
|
||||||
config.general.upstream_connect_failfast_hard_errors,
|
config.general.upstream_connect_failfast_hard_errors,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -662,14 +586,6 @@ async fn run_inner(
|
|||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop privileges after binding sockets (which may require root for port < 1024)
|
|
||||||
if daemon_opts.user.is_some() || daemon_opts.group.is_some() {
|
|
||||||
if let Err(e) = drop_privileges(daemon_opts.user.as_deref(), daemon_opts.group.as_deref()) {
|
|
||||||
error!(error = %e, "Failed to drop privileges");
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
runtime_tasks::apply_runtime_log_filter(
|
runtime_tasks::apply_runtime_log_filter(
|
||||||
has_rust_log,
|
has_rust_log,
|
||||||
&effective_log_level,
|
&effective_log_level,
|
||||||
@@ -690,9 +606,6 @@ async fn run_inner(
|
|||||||
|
|
||||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||||
|
|
||||||
// Spawn signal handlers for SIGUSR1/SIGUSR2 (non-shutdown signals)
|
|
||||||
shutdown::spawn_signal_handlers(stats.clone(), process_started_at);
|
|
||||||
|
|
||||||
listeners::spawn_tcp_accept_loops(
|
listeners::spawn_tcp_accept_loops(
|
||||||
listeners,
|
listeners,
|
||||||
config_rx.clone(),
|
config_rx.clone(),
|
||||||
@@ -710,7 +623,7 @@ async fn run_inner(
|
|||||||
max_connections.clone(),
|
max_connections.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
shutdown::wait_for_shutdown(process_started_at, me_pool, stats).await;
|
shutdown::wait_for_shutdown(process_started_at, me_pool).await;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -323,12 +323,10 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
let config_rx_metrics = config_rx.clone();
|
let config_rx_metrics = config_rx.clone();
|
||||||
let ip_tracker_metrics = ip_tracker.clone();
|
let ip_tracker_metrics = ip_tracker.clone();
|
||||||
let whitelist = config.server.metrics_whitelist.clone();
|
let whitelist = config.server.metrics_whitelist.clone();
|
||||||
let listen_backlog = config.server.listen_backlog;
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
metrics::serve(
|
metrics::serve(
|
||||||
port,
|
port,
|
||||||
listen,
|
listen,
|
||||||
listen_backlog,
|
|
||||||
stats,
|
stats,
|
||||||
beobachten,
|
beobachten,
|
||||||
ip_tracker_metrics,
|
ip_tracker_metrics,
|
||||||
|
|||||||
+32
-193
@@ -1,206 +1,45 @@
|
|||||||
//! Shutdown and signal handling for telemt.
|
|
||||||
//!
|
|
||||||
//! Handles graceful shutdown on various signals:
|
|
||||||
//! - SIGINT (Ctrl+C) / SIGTERM: Graceful shutdown
|
|
||||||
//! - SIGQUIT: Graceful shutdown with stats dump
|
|
||||||
//! - SIGUSR1: Reserved for log rotation (logs acknowledgment)
|
|
||||||
//! - SIGUSR2: Dump runtime status to log
|
|
||||||
//!
|
|
||||||
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
use tokio::signal;
|
use tokio::signal;
|
||||||
#[cfg(unix)]
|
use tracing::{error, info, warn};
|
||||||
use tokio::signal::unix::{SignalKind, signal};
|
|
||||||
use tracing::{info, warn};
|
|
||||||
|
|
||||||
use crate::stats::Stats;
|
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
|
||||||
use super::helpers::{format_uptime, unit_label};
|
use super::helpers::{format_uptime, unit_label};
|
||||||
|
|
||||||
/// Signal that triggered shutdown.
|
pub(crate) async fn wait_for_shutdown(process_started_at: Instant, me_pool: Option<Arc<MePool>>) {
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
match signal::ctrl_c().await {
|
||||||
pub enum ShutdownSignal {
|
Ok(()) => {
|
||||||
/// SIGINT (Ctrl+C)
|
let shutdown_started_at = Instant::now();
|
||||||
Interrupt,
|
info!("Shutting down...");
|
||||||
/// SIGTERM
|
let uptime_secs = process_started_at.elapsed().as_secs();
|
||||||
Terminate,
|
info!("Uptime: {}", format_uptime(uptime_secs));
|
||||||
/// SIGQUIT (with stats dump)
|
if let Some(pool) = &me_pool {
|
||||||
Quit,
|
match tokio::time::timeout(
|
||||||
}
|
Duration::from_secs(2),
|
||||||
|
pool.shutdown_send_close_conn_all(),
|
||||||
impl std::fmt::Display for ShutdownSignal {
|
)
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
.await
|
||||||
match self {
|
{
|
||||||
ShutdownSignal::Interrupt => write!(f, "SIGINT"),
|
Ok(total) => {
|
||||||
ShutdownSignal::Terminate => write!(f, "SIGTERM"),
|
info!(
|
||||||
ShutdownSignal::Quit => write!(f, "SIGQUIT"),
|
close_conn_sent = total,
|
||||||
}
|
"ME shutdown: RPC_CLOSE_CONN broadcast completed"
|
||||||
}
|
);
|
||||||
}
|
}
|
||||||
|
Err(_) => {
|
||||||
/// Waits for a shutdown signal and performs graceful shutdown.
|
warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out");
|
||||||
pub(crate) async fn wait_for_shutdown(
|
}
|
||||||
process_started_at: Instant,
|
|
||||||
me_pool: Option<Arc<MePool>>,
|
|
||||||
stats: Arc<Stats>,
|
|
||||||
) {
|
|
||||||
let signal = wait_for_shutdown_signal().await;
|
|
||||||
perform_shutdown(signal, process_started_at, me_pool, &stats).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for any shutdown signal (SIGINT, SIGTERM, SIGQUIT).
|
|
||||||
#[cfg(unix)]
|
|
||||||
async fn wait_for_shutdown_signal() -> ShutdownSignal {
|
|
||||||
let mut sigint = signal(SignalKind::interrupt()).expect("Failed to register SIGINT handler");
|
|
||||||
let mut sigterm = signal(SignalKind::terminate()).expect("Failed to register SIGTERM handler");
|
|
||||||
let mut sigquit = signal(SignalKind::quit()).expect("Failed to register SIGQUIT handler");
|
|
||||||
|
|
||||||
tokio::select! {
|
|
||||||
_ = sigint.recv() => ShutdownSignal::Interrupt,
|
|
||||||
_ = sigterm.recv() => ShutdownSignal::Terminate,
|
|
||||||
_ = sigquit.recv() => ShutdownSignal::Quit,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
async fn wait_for_shutdown_signal() -> ShutdownSignal {
|
|
||||||
signal::ctrl_c().await.expect("Failed to listen for Ctrl+C");
|
|
||||||
ShutdownSignal::Interrupt
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Performs graceful shutdown sequence.
|
|
||||||
async fn perform_shutdown(
|
|
||||||
signal: ShutdownSignal,
|
|
||||||
process_started_at: Instant,
|
|
||||||
me_pool: Option<Arc<MePool>>,
|
|
||||||
stats: &Stats,
|
|
||||||
) {
|
|
||||||
let shutdown_started_at = Instant::now();
|
|
||||||
info!(signal = %signal, "Received shutdown signal");
|
|
||||||
|
|
||||||
// Dump stats if SIGQUIT
|
|
||||||
if signal == ShutdownSignal::Quit {
|
|
||||||
dump_stats(stats, process_started_at);
|
|
||||||
}
|
|
||||||
|
|
||||||
info!("Shutting down...");
|
|
||||||
let uptime_secs = process_started_at.elapsed().as_secs();
|
|
||||||
info!("Uptime: {}", format_uptime(uptime_secs));
|
|
||||||
|
|
||||||
// Graceful ME pool shutdown
|
|
||||||
if let Some(pool) = &me_pool {
|
|
||||||
match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(total) => {
|
|
||||||
info!(
|
|
||||||
close_conn_sent = total,
|
|
||||||
"ME shutdown: RPC_CLOSE_CONN broadcast completed"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let shutdown_secs = shutdown_started_at.elapsed().as_secs();
|
|
||||||
info!(
|
|
||||||
"Shutdown completed successfully in {} {}.",
|
|
||||||
shutdown_secs,
|
|
||||||
unit_label(shutdown_secs, "second", "seconds")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Dumps runtime statistics to the log.
|
|
||||||
fn dump_stats(stats: &Stats, process_started_at: Instant) {
|
|
||||||
let uptime_secs = process_started_at.elapsed().as_secs();
|
|
||||||
|
|
||||||
info!("=== Runtime Statistics Dump ===");
|
|
||||||
info!("Uptime: {}", format_uptime(uptime_secs));
|
|
||||||
|
|
||||||
// Connection stats
|
|
||||||
info!(
|
|
||||||
"Connections: total={}, current={} (direct={}, me={}), bad={}",
|
|
||||||
stats.get_connects_all(),
|
|
||||||
stats.get_current_connections_total(),
|
|
||||||
stats.get_current_connections_direct(),
|
|
||||||
stats.get_current_connections_me(),
|
|
||||||
stats.get_connects_bad(),
|
|
||||||
);
|
|
||||||
|
|
||||||
// ME pool stats
|
|
||||||
info!(
|
|
||||||
"ME keepalive: sent={}, pong={}, failed={}, timeout={}",
|
|
||||||
stats.get_me_keepalive_sent(),
|
|
||||||
stats.get_me_keepalive_pong(),
|
|
||||||
stats.get_me_keepalive_failed(),
|
|
||||||
stats.get_me_keepalive_timeout(),
|
|
||||||
);
|
|
||||||
|
|
||||||
// Relay stats
|
|
||||||
info!(
|
|
||||||
"Relay idle: soft_mark={}, hard_close={}, pressure_evict={}",
|
|
||||||
stats.get_relay_idle_soft_mark_total(),
|
|
||||||
stats.get_relay_idle_hard_close_total(),
|
|
||||||
stats.get_relay_pressure_evict_total(),
|
|
||||||
);
|
|
||||||
|
|
||||||
info!("=== End Statistics Dump ===");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Spawns a background task to handle operational signals (SIGUSR1, SIGUSR2).
|
|
||||||
///
|
|
||||||
/// These signals don't trigger shutdown but perform specific actions:
|
|
||||||
/// - SIGUSR1: Log rotation acknowledgment (for external log rotation tools)
|
|
||||||
/// - SIGUSR2: Dump runtime status to log
|
|
||||||
#[cfg(unix)]
|
|
||||||
pub(crate) fn spawn_signal_handlers(stats: Arc<Stats>, process_started_at: Instant) {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut sigusr1 =
|
|
||||||
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
|
|
||||||
let mut sigusr2 =
|
|
||||||
signal(SignalKind::user_defined2()).expect("Failed to register SIGUSR2 handler");
|
|
||||||
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
_ = sigusr1.recv() => {
|
|
||||||
handle_sigusr1();
|
|
||||||
}
|
|
||||||
_ = sigusr2.recv() => {
|
|
||||||
handle_sigusr2(&stats, process_started_at);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let shutdown_secs = shutdown_started_at.elapsed().as_secs();
|
||||||
|
info!(
|
||||||
|
"Shutdown completed successfully in {} {}.",
|
||||||
|
shutdown_secs,
|
||||||
|
unit_label(shutdown_secs, "second", "seconds")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
Err(e) => error!("Signal error: {}", e),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// No-op on non-Unix platforms.
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
pub(crate) fn spawn_signal_handlers(_stats: Arc<Stats>, _process_started_at: Instant) {
|
|
||||||
// No SIGUSR1/SIGUSR2 on non-Unix
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Handles SIGUSR1 - log rotation signal.
|
|
||||||
///
|
|
||||||
/// This signal is typically sent by logrotate or similar tools after
|
|
||||||
/// rotating log files. Since tracing-subscriber doesn't natively support
|
|
||||||
/// reopening files, we just acknowledge the signal. If file logging is
|
|
||||||
/// added in the future, this would reopen log file handles.
|
|
||||||
#[cfg(unix)]
|
|
||||||
fn handle_sigusr1() {
|
|
||||||
info!("SIGUSR1 received - log rotation acknowledged");
|
|
||||||
// Future: If using file-based logging, reopen file handles here
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Handles SIGUSR2 - dump runtime status.
|
|
||||||
#[cfg(unix)]
|
|
||||||
fn handle_sigusr2(stats: &Stats, process_started_at: Instant) {
|
|
||||||
info!("SIGUSR2 received - dumping runtime status");
|
|
||||||
dump_stats(stats, process_started_at);
|
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-48
@@ -4,8 +4,6 @@ mod api;
|
|||||||
mod cli;
|
mod cli;
|
||||||
mod config;
|
mod config;
|
||||||
mod crypto;
|
mod crypto;
|
||||||
#[cfg(unix)]
|
|
||||||
mod daemon;
|
|
||||||
mod error;
|
mod error;
|
||||||
mod ip_tracker;
|
mod ip_tracker;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -17,13 +15,11 @@ mod ip_tracker_hotpath_adversarial_tests;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "tests/ip_tracker_regression_tests.rs"]
|
#[path = "tests/ip_tracker_regression_tests.rs"]
|
||||||
mod ip_tracker_regression_tests;
|
mod ip_tracker_regression_tests;
|
||||||
mod logging;
|
|
||||||
mod maestro;
|
mod maestro;
|
||||||
mod metrics;
|
mod metrics;
|
||||||
mod network;
|
mod network;
|
||||||
mod protocol;
|
mod protocol;
|
||||||
mod proxy;
|
mod proxy;
|
||||||
mod service;
|
|
||||||
mod startup;
|
mod startup;
|
||||||
mod stats;
|
mod stats;
|
||||||
mod stream;
|
mod stream;
|
||||||
@@ -31,49 +27,8 @@ mod tls_front;
|
|||||||
mod transport;
|
mod transport;
|
||||||
mod util;
|
mod util;
|
||||||
|
|
||||||
fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
#[tokio::main]
|
||||||
// Install rustls crypto provider early
|
async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||||
|
maestro::run().await
|
||||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
|
||||||
let cmd = cli::parse_command(&args);
|
|
||||||
|
|
||||||
// Handle subcommands that don't need the server (stop, reload, status, init)
|
|
||||||
if let Some(exit_code) = cli::execute_subcommand(&cmd) {
|
|
||||||
std::process::exit(exit_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(unix)]
|
|
||||||
{
|
|
||||||
let daemon_opts = cmd.daemon_opts;
|
|
||||||
|
|
||||||
// Daemonize BEFORE runtime
|
|
||||||
if daemon_opts.should_daemonize() {
|
|
||||||
match daemon::daemonize(daemon_opts.working_dir.as_deref()) {
|
|
||||||
Ok(daemon::DaemonizeResult::Parent) => {
|
|
||||||
std::process::exit(0);
|
|
||||||
}
|
|
||||||
Ok(daemon::DaemonizeResult::Child) => {
|
|
||||||
// continue
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("[telemt] Daemonization failed: {}", e);
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tokio::runtime::Builder::new_multi_thread()
|
|
||||||
.enable_all()
|
|
||||||
.build()?
|
|
||||||
.block_on(maestro::run_with_daemon(daemon_opts))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(unix))]
|
|
||||||
{
|
|
||||||
tokio::runtime::Builder::new_multi_thread()
|
|
||||||
.enable_all()
|
|
||||||
.build()?
|
|
||||||
.block_on(maestro::run())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-70
@@ -22,7 +22,6 @@ use crate::transport::{ListenOptions, create_listener};
|
|||||||
pub async fn serve(
|
pub async fn serve(
|
||||||
port: u16,
|
port: u16,
|
||||||
listen: Option<String>,
|
listen: Option<String>,
|
||||||
listen_backlog: u32,
|
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
beobachten: Arc<BeobachtenStore>,
|
beobachten: Arc<BeobachtenStore>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
@@ -41,7 +40,7 @@ pub async fn serve(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
let is_ipv6 = addr.is_ipv6();
|
let is_ipv6 = addr.is_ipv6();
|
||||||
match bind_metrics_listener(addr, is_ipv6, listen_backlog) {
|
match bind_metrics_listener(addr, is_ipv6) {
|
||||||
Ok(listener) => {
|
Ok(listener) => {
|
||||||
info!("Metrics endpoint: http://{}/metrics and /beobachten", addr);
|
info!("Metrics endpoint: http://{}/metrics and /beobachten", addr);
|
||||||
serve_listener(
|
serve_listener(
|
||||||
@@ -61,7 +60,7 @@ pub async fn serve(
|
|||||||
let mut listener_v6 = None;
|
let mut listener_v6 = None;
|
||||||
|
|
||||||
let addr_v4 = SocketAddr::from(([0, 0, 0, 0], port));
|
let addr_v4 = SocketAddr::from(([0, 0, 0, 0], port));
|
||||||
match bind_metrics_listener(addr_v4, false, listen_backlog) {
|
match bind_metrics_listener(addr_v4, false) {
|
||||||
Ok(listener) => {
|
Ok(listener) => {
|
||||||
info!(
|
info!(
|
||||||
"Metrics endpoint: http://{}/metrics and /beobachten",
|
"Metrics endpoint: http://{}/metrics and /beobachten",
|
||||||
@@ -75,7 +74,7 @@ pub async fn serve(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], port));
|
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], port));
|
||||||
match bind_metrics_listener(addr_v6, true, listen_backlog) {
|
match bind_metrics_listener(addr_v6, true) {
|
||||||
Ok(listener) => {
|
Ok(listener) => {
|
||||||
info!(
|
info!(
|
||||||
"Metrics endpoint: http://[::]:{}/metrics and /beobachten",
|
"Metrics endpoint: http://[::]:{}/metrics and /beobachten",
|
||||||
@@ -123,15 +122,10 @@ pub async fn serve(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn bind_metrics_listener(
|
fn bind_metrics_listener(addr: SocketAddr, ipv6_only: bool) -> std::io::Result<TcpListener> {
|
||||||
addr: SocketAddr,
|
|
||||||
ipv6_only: bool,
|
|
||||||
listen_backlog: u32,
|
|
||||||
) -> std::io::Result<TcpListener> {
|
|
||||||
let options = ListenOptions {
|
let options = ListenOptions {
|
||||||
reuse_port: false,
|
reuse_port: false,
|
||||||
ipv6_only,
|
ipv6_only,
|
||||||
backlog: listen_backlog,
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
let socket = create_listener(addr, &options)?;
|
let socket = create_listener(addr, &options)?;
|
||||||
@@ -1564,40 +1558,6 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# HELP telemt_me_endpoint_quarantine_unexpected_total ME endpoint quarantines caused by unexpected writer removals"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# TYPE telemt_me_endpoint_quarantine_unexpected_total counter"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"telemt_me_endpoint_quarantine_unexpected_total {}",
|
|
||||||
if me_allows_normal {
|
|
||||||
stats.get_me_endpoint_quarantine_unexpected_total()
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# HELP telemt_me_endpoint_quarantine_draining_suppressed_total Draining writer removals that skipped endpoint quarantine"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# TYPE telemt_me_endpoint_quarantine_draining_suppressed_total counter"
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"telemt_me_endpoint_quarantine_draining_suppressed_total {}",
|
|
||||||
if me_allows_normal {
|
|
||||||
stats.get_me_endpoint_quarantine_draining_suppressed_total()
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
@@ -2358,20 +2318,6 @@ async fn render_metrics(stats: &Stats, config: &ProxyConfig, ip_tracker: &UserIp
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"# HELP telemt_me_hybrid_timeout_total ME hybrid route timeouts after bounded retry window"
|
|
||||||
);
|
|
||||||
let _ = writeln!(out, "# TYPE telemt_me_hybrid_timeout_total counter");
|
|
||||||
let _ = writeln!(
|
|
||||||
out,
|
|
||||||
"telemt_me_hybrid_timeout_total {}",
|
|
||||||
if me_allows_normal {
|
|
||||||
stats.get_me_hybrid_timeout_total()
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
);
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_me_async_recovery_trigger_total Async ME recovery trigger attempts from route path"
|
"# HELP telemt_me_async_recovery_trigger_total Async ME recovery trigger attempts from route path"
|
||||||
@@ -2662,9 +2608,6 @@ mod tests {
|
|||||||
stats.increment_me_d2c_write_mode(crate::stats::MeD2cWriteMode::Coalesced);
|
stats.increment_me_d2c_write_mode(crate::stats::MeD2cWriteMode::Coalesced);
|
||||||
stats.increment_me_d2c_quota_reject_total(crate::stats::MeD2cQuotaRejectStage::PostWrite);
|
stats.increment_me_d2c_quota_reject_total(crate::stats::MeD2cQuotaRejectStage::PostWrite);
|
||||||
stats.observe_me_d2c_frame_buf_shrink(4096);
|
stats.observe_me_d2c_frame_buf_shrink(4096);
|
||||||
stats.increment_me_endpoint_quarantine_total();
|
|
||||||
stats.increment_me_endpoint_quarantine_unexpected_total();
|
|
||||||
stats.increment_me_endpoint_quarantine_draining_suppressed_total();
|
|
||||||
stats.increment_user_connects("alice");
|
stats.increment_user_connects("alice");
|
||||||
stats.increment_user_curr_connects("alice");
|
stats.increment_user_curr_connects("alice");
|
||||||
stats.add_user_octets_from("alice", 1024);
|
stats.add_user_octets_from("alice", 1024);
|
||||||
@@ -2715,9 +2658,6 @@ mod tests {
|
|||||||
assert!(output.contains("telemt_me_d2c_quota_reject_total{stage=\"post_write\"} 1"));
|
assert!(output.contains("telemt_me_d2c_quota_reject_total{stage=\"post_write\"} 1"));
|
||||||
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_total 1"));
|
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_total 1"));
|
||||||
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_bytes_total 4096"));
|
assert!(output.contains("telemt_me_d2c_frame_buf_shrink_bytes_total 4096"));
|
||||||
assert!(output.contains("telemt_me_endpoint_quarantine_total 1"));
|
|
||||||
assert!(output.contains("telemt_me_endpoint_quarantine_unexpected_total 1"));
|
|
||||||
assert!(output.contains("telemt_me_endpoint_quarantine_draining_suppressed_total 1"));
|
|
||||||
assert!(output.contains("telemt_user_connections_total{user=\"alice\"} 1"));
|
assert!(output.contains("telemt_user_connections_total{user=\"alice\"} 1"));
|
||||||
assert!(output.contains("telemt_user_connections_current{user=\"alice\"} 1"));
|
assert!(output.contains("telemt_user_connections_current{user=\"alice\"} 1"));
|
||||||
assert!(output.contains("telemt_user_octets_from_client{user=\"alice\"} 1024"));
|
assert!(output.contains("telemt_user_octets_from_client{user=\"alice\"} 1024"));
|
||||||
@@ -2784,12 +2724,6 @@ mod tests {
|
|||||||
assert!(output.contains("# TYPE telemt_me_d2c_write_mode_total counter"));
|
assert!(output.contains("# TYPE telemt_me_d2c_write_mode_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_me_d2c_batch_frames_bucket_total counter"));
|
assert!(output.contains("# TYPE telemt_me_d2c_batch_frames_bucket_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_me_d2c_flush_duration_us_bucket_total counter"));
|
assert!(output.contains("# TYPE telemt_me_d2c_flush_duration_us_bucket_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_me_endpoint_quarantine_total counter"));
|
|
||||||
assert!(output.contains("# TYPE telemt_me_endpoint_quarantine_unexpected_total counter"));
|
|
||||||
assert!(
|
|
||||||
output
|
|
||||||
.contains("# TYPE telemt_me_endpoint_quarantine_draining_suppressed_total counter")
|
|
||||||
);
|
|
||||||
assert!(output.contains("# TYPE telemt_me_writer_removed_total counter"));
|
assert!(output.contains("# TYPE telemt_me_writer_removed_total counter"));
|
||||||
assert!(
|
assert!(
|
||||||
output
|
output
|
||||||
|
|||||||
+47
-169
@@ -416,68 +416,16 @@ where
|
|||||||
|
|
||||||
debug!(peer = %real_peer, "New connection (generic stream)");
|
debug!(peer = %real_peer, "New connection (generic stream)");
|
||||||
|
|
||||||
let first_byte = if config.timeouts.client_first_byte_idle_secs == 0 {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
let idle_timeout = Duration::from_secs(config.timeouts.client_first_byte_idle_secs);
|
|
||||||
let mut first_byte = [0u8; 1];
|
|
||||||
match timeout(idle_timeout, stream.read(&mut first_byte)).await {
|
|
||||||
Ok(Ok(0)) => {
|
|
||||||
debug!(peer = %real_peer, "Connection closed before first client byte");
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
Ok(Ok(_)) => Some(first_byte[0]),
|
|
||||||
Ok(Err(e))
|
|
||||||
if matches!(
|
|
||||||
e.kind(),
|
|
||||||
std::io::ErrorKind::UnexpectedEof
|
|
||||||
| std::io::ErrorKind::ConnectionReset
|
|
||||||
| std::io::ErrorKind::ConnectionAborted
|
|
||||||
| std::io::ErrorKind::BrokenPipe
|
|
||||||
| std::io::ErrorKind::NotConnected
|
|
||||||
) =>
|
|
||||||
{
|
|
||||||
debug!(
|
|
||||||
peer = %real_peer,
|
|
||||||
error = %e,
|
|
||||||
"Connection closed before first client byte"
|
|
||||||
);
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
Ok(Err(e)) => {
|
|
||||||
debug!(
|
|
||||||
peer = %real_peer,
|
|
||||||
error = %e,
|
|
||||||
"Failed while waiting for first client byte"
|
|
||||||
);
|
|
||||||
return Err(ProxyError::Io(e));
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
debug!(
|
|
||||||
peer = %real_peer,
|
|
||||||
idle_secs = config.timeouts.client_first_byte_idle_secs,
|
|
||||||
"Closing idle pooled connection before first client byte"
|
|
||||||
);
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let handshake_timeout = handshake_timeout_with_mask_grace(&config);
|
let handshake_timeout = handshake_timeout_with_mask_grace(&config);
|
||||||
let stats_for_timeout = stats.clone();
|
let stats_for_timeout = stats.clone();
|
||||||
let config_for_timeout = config.clone();
|
let config_for_timeout = config.clone();
|
||||||
let beobachten_for_timeout = beobachten.clone();
|
let beobachten_for_timeout = beobachten.clone();
|
||||||
let peer_for_timeout = real_peer.ip();
|
let peer_for_timeout = real_peer.ip();
|
||||||
|
|
||||||
// Phase 2: active handshake (with timeout after the first client byte)
|
// Phase 1: handshake (with timeout)
|
||||||
let outcome = match timeout(handshake_timeout, async {
|
let outcome = match timeout(handshake_timeout, async {
|
||||||
let mut first_bytes = [0u8; 5];
|
let mut first_bytes = [0u8; 5];
|
||||||
if let Some(first_byte) = first_byte {
|
stream.read_exact(&mut first_bytes).await?;
|
||||||
first_bytes[0] = first_byte;
|
|
||||||
stream.read_exact(&mut first_bytes[1..]).await?;
|
|
||||||
} else {
|
|
||||||
stream.read_exact(&mut first_bytes).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
||||||
debug!(peer = %real_peer, is_tls = is_tls, "Handshake type detected");
|
debug!(peer = %real_peer, is_tls = is_tls, "Handshake type detected");
|
||||||
@@ -788,9 +736,36 @@ impl RunningClientHandler {
|
|||||||
debug!(peer = %peer, error = %e, "Failed to configure client socket");
|
debug!(peer = %peer, error = %e, "Failed to configure client socket");
|
||||||
}
|
}
|
||||||
|
|
||||||
let outcome = match self.do_handshake().await? {
|
let handshake_timeout = handshake_timeout_with_mask_grace(&self.config);
|
||||||
Some(outcome) => outcome,
|
let stats = self.stats.clone();
|
||||||
None => return Ok(()),
|
let config_for_timeout = self.config.clone();
|
||||||
|
let beobachten_for_timeout = self.beobachten.clone();
|
||||||
|
let peer_for_timeout = peer.ip();
|
||||||
|
|
||||||
|
// Phase 1: handshake (with timeout)
|
||||||
|
let outcome = match timeout(handshake_timeout, self.do_handshake()).await {
|
||||||
|
Ok(Ok(outcome)) => outcome,
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
debug!(peer = %peer, error = %e, "Handshake failed");
|
||||||
|
record_handshake_failure_class(
|
||||||
|
&beobachten_for_timeout,
|
||||||
|
&config_for_timeout,
|
||||||
|
peer_for_timeout,
|
||||||
|
&e,
|
||||||
|
);
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
stats.increment_handshake_timeouts();
|
||||||
|
debug!(peer = %peer, "Handshake timeout");
|
||||||
|
record_beobachten_class(
|
||||||
|
&beobachten_for_timeout,
|
||||||
|
&config_for_timeout,
|
||||||
|
peer_for_timeout,
|
||||||
|
"other",
|
||||||
|
);
|
||||||
|
return Err(ProxyError::TgHandshakeTimeout);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
// Phase 2: relay (WITHOUT handshake timeout — relay has its own activity timeouts)
|
||||||
@@ -799,7 +774,7 @@ impl RunningClientHandler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn do_handshake(mut self) -> Result<Option<HandshakeOutcome>> {
|
async fn do_handshake(mut self) -> Result<HandshakeOutcome> {
|
||||||
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||||
|
|
||||||
if self.proxy_protocol_enabled {
|
if self.proxy_protocol_enabled {
|
||||||
@@ -874,108 +849,19 @@ impl RunningClientHandler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let first_byte = if self.config.timeouts.client_first_byte_idle_secs == 0 {
|
let mut first_bytes = [0u8; 5];
|
||||||
None
|
self.stream.read_exact(&mut first_bytes).await?;
|
||||||
|
|
||||||
|
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
||||||
|
let peer = self.peer;
|
||||||
|
|
||||||
|
debug!(peer = %peer, is_tls = is_tls, "Handshake type detected");
|
||||||
|
|
||||||
|
if is_tls {
|
||||||
|
self.handle_tls_client(first_bytes, local_addr).await
|
||||||
} else {
|
} else {
|
||||||
let idle_timeout =
|
self.handle_direct_client(first_bytes, local_addr).await
|
||||||
Duration::from_secs(self.config.timeouts.client_first_byte_idle_secs);
|
}
|
||||||
let mut first_byte = [0u8; 1];
|
|
||||||
match timeout(idle_timeout, self.stream.read(&mut first_byte)).await {
|
|
||||||
Ok(Ok(0)) => {
|
|
||||||
debug!(peer = %self.peer, "Connection closed before first client byte");
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
Ok(Ok(_)) => Some(first_byte[0]),
|
|
||||||
Ok(Err(e))
|
|
||||||
if matches!(
|
|
||||||
e.kind(),
|
|
||||||
std::io::ErrorKind::UnexpectedEof
|
|
||||||
| std::io::ErrorKind::ConnectionReset
|
|
||||||
| std::io::ErrorKind::ConnectionAborted
|
|
||||||
| std::io::ErrorKind::BrokenPipe
|
|
||||||
| std::io::ErrorKind::NotConnected
|
|
||||||
) =>
|
|
||||||
{
|
|
||||||
debug!(
|
|
||||||
peer = %self.peer,
|
|
||||||
error = %e,
|
|
||||||
"Connection closed before first client byte"
|
|
||||||
);
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
Ok(Err(e)) => {
|
|
||||||
debug!(
|
|
||||||
peer = %self.peer,
|
|
||||||
error = %e,
|
|
||||||
"Failed while waiting for first client byte"
|
|
||||||
);
|
|
||||||
return Err(ProxyError::Io(e));
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
debug!(
|
|
||||||
peer = %self.peer,
|
|
||||||
idle_secs = self.config.timeouts.client_first_byte_idle_secs,
|
|
||||||
"Closing idle pooled connection before first client byte"
|
|
||||||
);
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let handshake_timeout = handshake_timeout_with_mask_grace(&self.config);
|
|
||||||
let stats = self.stats.clone();
|
|
||||||
let config_for_timeout = self.config.clone();
|
|
||||||
let beobachten_for_timeout = self.beobachten.clone();
|
|
||||||
let peer_for_timeout = self.peer.ip();
|
|
||||||
let peer_for_log = self.peer;
|
|
||||||
|
|
||||||
let outcome = match timeout(handshake_timeout, async {
|
|
||||||
let mut first_bytes = [0u8; 5];
|
|
||||||
if let Some(first_byte) = first_byte {
|
|
||||||
first_bytes[0] = first_byte;
|
|
||||||
self.stream.read_exact(&mut first_bytes[1..]).await?;
|
|
||||||
} else {
|
|
||||||
self.stream.read_exact(&mut first_bytes).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let is_tls = tls::is_tls_handshake(&first_bytes[..3]);
|
|
||||||
let peer = self.peer;
|
|
||||||
|
|
||||||
debug!(peer = %peer, is_tls = is_tls, "Handshake type detected");
|
|
||||||
|
|
||||||
if is_tls {
|
|
||||||
self.handle_tls_client(first_bytes, local_addr).await
|
|
||||||
} else {
|
|
||||||
self.handle_direct_client(first_bytes, local_addr).await
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(Ok(outcome)) => outcome,
|
|
||||||
Ok(Err(e)) => {
|
|
||||||
debug!(peer = %peer_for_log, error = %e, "Handshake failed");
|
|
||||||
record_handshake_failure_class(
|
|
||||||
&beobachten_for_timeout,
|
|
||||||
&config_for_timeout,
|
|
||||||
peer_for_timeout,
|
|
||||||
&e,
|
|
||||||
);
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
stats.increment_handshake_timeouts();
|
|
||||||
debug!(peer = %peer_for_log, "Handshake timeout");
|
|
||||||
record_beobachten_class(
|
|
||||||
&beobachten_for_timeout,
|
|
||||||
&config_for_timeout,
|
|
||||||
peer_for_timeout,
|
|
||||||
"other",
|
|
||||||
);
|
|
||||||
return Err(ProxyError::TgHandshakeTimeout);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(Some(outcome))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_tls_client(
|
async fn handle_tls_client(
|
||||||
@@ -1366,11 +1252,7 @@ impl RunningClientHandler {
|
|||||||
.access
|
.access
|
||||||
.user_max_tcp_conns
|
.user_max_tcp_conns
|
||||||
.get(user)
|
.get(user)
|
||||||
.copied()
|
.map(|v| *v as u64);
|
||||||
.filter(|limit| *limit > 0)
|
|
||||||
.or((config.access.user_max_tcp_conns_global_each > 0)
|
|
||||||
.then_some(config.access.user_max_tcp_conns_global_each))
|
|
||||||
.map(|v| v as u64);
|
|
||||||
if !stats.try_acquire_user_curr_connects(user, limit) {
|
if !stats.try_acquire_user_curr_connects(user, limit) {
|
||||||
return Err(ProxyError::ConnectionLimitExceeded {
|
return Err(ProxyError::ConnectionLimitExceeded {
|
||||||
user: user.to_string(),
|
user: user.to_string(),
|
||||||
@@ -1429,11 +1311,7 @@ impl RunningClientHandler {
|
|||||||
.access
|
.access
|
||||||
.user_max_tcp_conns
|
.user_max_tcp_conns
|
||||||
.get(user)
|
.get(user)
|
||||||
.copied()
|
.map(|v| *v as u64);
|
||||||
.filter(|limit| *limit > 0)
|
|
||||||
.or((config.access.user_max_tcp_conns_global_each > 0)
|
|
||||||
.then_some(config.access.user_max_tcp_conns_global_each))
|
|
||||||
.map(|v| v as u64);
|
|
||||||
if !stats.try_acquire_user_curr_connects(user, limit) {
|
if !stats.try_acquire_user_curr_connects(user, limit) {
|
||||||
return Err(ProxyError::ConnectionLimitExceeded {
|
return Err(ProxyError::ConnectionLimitExceeded {
|
||||||
user: user.to_string(),
|
user: user.to_string(),
|
||||||
|
|||||||
+69
-115
@@ -4,7 +4,7 @@ use std::collections::{BTreeSet, HashMap};
|
|||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::hash::{BuildHasher, Hash};
|
use std::hash::{BuildHasher, Hash};
|
||||||
use std::net::{IpAddr, SocketAddr};
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Mutex, OnceLock};
|
use std::sync::{Arc, Mutex, OnceLock};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -36,6 +36,7 @@ enum C2MeCommand {
|
|||||||
|
|
||||||
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
|
const DESYNC_DEDUP_WINDOW: Duration = Duration::from_secs(60);
|
||||||
const DESYNC_DEDUP_MAX_ENTRIES: usize = 65_536;
|
const DESYNC_DEDUP_MAX_ENTRIES: usize = 65_536;
|
||||||
|
const DESYNC_DEDUP_PRUNE_SCAN_LIMIT: usize = 1024;
|
||||||
const DESYNC_FULL_CACHE_EMIT_MIN_INTERVAL: Duration = Duration::from_millis(1000);
|
const DESYNC_FULL_CACHE_EMIT_MIN_INTERVAL: Duration = Duration::from_millis(1000);
|
||||||
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
|
const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
|
||||||
const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128;
|
const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128;
|
||||||
@@ -45,6 +46,10 @@ const RELAY_IDLE_IO_POLL_MAX: Duration = Duration::from_secs(1);
|
|||||||
const TINY_FRAME_DEBT_PER_TINY: u32 = 8;
|
const TINY_FRAME_DEBT_PER_TINY: u32 = 8;
|
||||||
const TINY_FRAME_DEBT_LIMIT: u32 = 512;
|
const TINY_FRAME_DEBT_LIMIT: u32 = 512;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
const C2ME_SEND_TIMEOUT: Duration = Duration::from_millis(50);
|
||||||
|
#[cfg(not(test))]
|
||||||
|
const C2ME_SEND_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
|
#[cfg(test)]
|
||||||
const RELAY_TEST_STEP_TIMEOUT: Duration = Duration::from_secs(1);
|
const RELAY_TEST_STEP_TIMEOUT: Duration = Duration::from_secs(1);
|
||||||
const ME_D2C_FLUSH_BATCH_MAX_FRAMES_MIN: usize = 1;
|
const ME_D2C_FLUSH_BATCH_MAX_FRAMES_MIN: usize = 1;
|
||||||
const ME_D2C_FLUSH_BATCH_MAX_BYTES_MIN: usize = 4096;
|
const ME_D2C_FLUSH_BATCH_MAX_BYTES_MIN: usize = 4096;
|
||||||
@@ -52,21 +57,12 @@ const ME_D2C_FRAME_BUF_SHRINK_HYSTERESIS_FACTOR: usize = 2;
|
|||||||
const ME_D2C_SINGLE_WRITE_COALESCE_MAX_BYTES: usize = 128 * 1024;
|
const ME_D2C_SINGLE_WRITE_COALESCE_MAX_BYTES: usize = 128 * 1024;
|
||||||
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
|
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
|
||||||
static DESYNC_DEDUP: OnceLock<DashMap<u64, Instant>> = OnceLock::new();
|
static DESYNC_DEDUP: OnceLock<DashMap<u64, Instant>> = OnceLock::new();
|
||||||
static DESYNC_DEDUP_PREVIOUS: OnceLock<DashMap<u64, Instant>> = OnceLock::new();
|
|
||||||
static DESYNC_HASHER: OnceLock<RandomState> = OnceLock::new();
|
static DESYNC_HASHER: OnceLock<RandomState> = OnceLock::new();
|
||||||
static DESYNC_FULL_CACHE_LAST_EMIT_AT: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
|
static DESYNC_FULL_CACHE_LAST_EMIT_AT: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
|
||||||
static DESYNC_DEDUP_ROTATION_STATE: OnceLock<Mutex<DesyncDedupRotationState>> = OnceLock::new();
|
static DESYNC_DEDUP_EVER_SATURATED: OnceLock<AtomicBool> = OnceLock::new();
|
||||||
// Invariant for async callers:
|
|
||||||
// this std::sync::Mutex is allowed only because critical sections are short,
|
|
||||||
// synchronous, and MUST never cross an `.await`.
|
|
||||||
static RELAY_IDLE_CANDIDATE_REGISTRY: OnceLock<Mutex<RelayIdleCandidateRegistry>> = OnceLock::new();
|
static RELAY_IDLE_CANDIDATE_REGISTRY: OnceLock<Mutex<RelayIdleCandidateRegistry>> = OnceLock::new();
|
||||||
static RELAY_IDLE_MARK_SEQ: AtomicU64 = AtomicU64::new(0);
|
static RELAY_IDLE_MARK_SEQ: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
struct DesyncDedupRotationState {
|
|
||||||
current_started_at: Option<Instant>,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct RelayForensicsState {
|
struct RelayForensicsState {
|
||||||
trace_id: u64,
|
trace_id: u64,
|
||||||
conn_id: u64,
|
conn_id: u64,
|
||||||
@@ -99,7 +95,6 @@ fn relay_idle_candidate_registry() -> &'static Mutex<RelayIdleCandidateRegistry>
|
|||||||
|
|
||||||
fn relay_idle_candidate_registry_lock() -> std::sync::MutexGuard<'static, RelayIdleCandidateRegistry>
|
fn relay_idle_candidate_registry_lock() -> std::sync::MutexGuard<'static, RelayIdleCandidateRegistry>
|
||||||
{
|
{
|
||||||
// Keep lock scope narrow and synchronous: callers must drop guard before any `.await`.
|
|
||||||
let registry = relay_idle_candidate_registry();
|
let registry = relay_idle_candidate_registry();
|
||||||
match registry.lock() {
|
match registry.lock() {
|
||||||
Ok(guard) => guard,
|
Ok(guard) => guard,
|
||||||
@@ -317,76 +312,64 @@ fn should_emit_full_desync(key: u64, all_full: bool, now: Instant) -> bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
let dedup_current = DESYNC_DEDUP.get_or_init(DashMap::new);
|
let dedup = DESYNC_DEDUP.get_or_init(DashMap::new);
|
||||||
let dedup_previous = DESYNC_DEDUP_PREVIOUS.get_or_init(DashMap::new);
|
let saturated_before = dedup.len() >= DESYNC_DEDUP_MAX_ENTRIES;
|
||||||
let rotation_state =
|
let ever_saturated = DESYNC_DEDUP_EVER_SATURATED.get_or_init(|| AtomicBool::new(false));
|
||||||
DESYNC_DEDUP_ROTATION_STATE.get_or_init(|| Mutex::new(DesyncDedupRotationState::default()));
|
if saturated_before {
|
||||||
|
ever_saturated.store(true, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
let mut state = match rotation_state.lock() {
|
if let Some(mut seen_at) = dedup.get_mut(&key) {
|
||||||
Ok(guard) => guard,
|
if now.duration_since(*seen_at) >= DESYNC_DEDUP_WINDOW {
|
||||||
Err(poisoned) => {
|
*seen_at = now;
|
||||||
let mut guard = poisoned.into_inner();
|
return true;
|
||||||
*guard = DesyncDedupRotationState::default();
|
|
||||||
rotation_state.clear_poison();
|
|
||||||
guard
|
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if dedup.len() >= DESYNC_DEDUP_MAX_ENTRIES {
|
||||||
|
let mut stale_keys = Vec::new();
|
||||||
|
let mut oldest_candidate: Option<(u64, Instant)> = None;
|
||||||
|
for entry in dedup.iter().take(DESYNC_DEDUP_PRUNE_SCAN_LIMIT) {
|
||||||
|
let key = *entry.key();
|
||||||
|
let seen_at = *entry.value();
|
||||||
|
|
||||||
|
match oldest_candidate {
|
||||||
|
Some((_, oldest_seen)) if seen_at >= oldest_seen => {}
|
||||||
|
_ => oldest_candidate = Some((key, seen_at)),
|
||||||
|
}
|
||||||
|
|
||||||
|
if now.duration_since(seen_at) >= DESYNC_DEDUP_WINDOW {
|
||||||
|
stale_keys.push(*entry.key());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for stale_key in stale_keys {
|
||||||
|
dedup.remove(&stale_key);
|
||||||
|
}
|
||||||
|
if dedup.len() >= DESYNC_DEDUP_MAX_ENTRIES {
|
||||||
|
let Some((evict_key, _)) = oldest_candidate else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
dedup.remove(&evict_key);
|
||||||
|
dedup.insert(key, now);
|
||||||
|
return should_emit_full_desync_full_cache(now);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dedup.insert(key, now);
|
||||||
|
let saturated_after = dedup.len() >= DESYNC_DEDUP_MAX_ENTRIES;
|
||||||
|
// Preserve the first sequential insert that reaches capacity as a normal
|
||||||
|
// emit, while still gating concurrent newcomer churn after the cache has
|
||||||
|
// ever been observed at saturation.
|
||||||
|
let was_ever_saturated = if saturated_after {
|
||||||
|
ever_saturated.swap(true, Ordering::Relaxed)
|
||||||
|
} else {
|
||||||
|
ever_saturated.load(Ordering::Relaxed)
|
||||||
};
|
};
|
||||||
|
|
||||||
let rotate_now = match state.current_started_at {
|
if saturated_before || (saturated_after && was_ever_saturated) {
|
||||||
Some(current_started_at) => match now.checked_duration_since(current_started_at) {
|
|
||||||
Some(elapsed) => elapsed >= DESYNC_DEDUP_WINDOW,
|
|
||||||
None => true,
|
|
||||||
},
|
|
||||||
None => true,
|
|
||||||
};
|
|
||||||
if rotate_now {
|
|
||||||
dedup_previous.clear();
|
|
||||||
for entry in dedup_current.iter() {
|
|
||||||
dedup_previous.insert(*entry.key(), *entry.value());
|
|
||||||
}
|
|
||||||
dedup_current.clear();
|
|
||||||
state.current_started_at = Some(now);
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(seen_at) = dedup_current.get(&key).map(|entry| *entry.value()) {
|
|
||||||
let within_window = match now.checked_duration_since(seen_at) {
|
|
||||||
Some(elapsed) => elapsed < DESYNC_DEDUP_WINDOW,
|
|
||||||
None => true,
|
|
||||||
};
|
|
||||||
if within_window {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
dedup_current.insert(key, now);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(seen_at) = dedup_previous.get(&key).map(|entry| *entry.value()) {
|
|
||||||
let within_window = match now.checked_duration_since(seen_at) {
|
|
||||||
Some(elapsed) => elapsed < DESYNC_DEDUP_WINDOW,
|
|
||||||
None => true,
|
|
||||||
};
|
|
||||||
if within_window {
|
|
||||||
// Keep the original timestamp when promoting from previous bucket,
|
|
||||||
// so dedup expiry remains tied to first-seen time.
|
|
||||||
dedup_current.insert(key, seen_at);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
dedup_previous.remove(&key);
|
|
||||||
}
|
|
||||||
|
|
||||||
if dedup_current.len() >= DESYNC_DEDUP_MAX_ENTRIES {
|
|
||||||
// Bounded eviction path: rotate buckets instead of scanning/evicting
|
|
||||||
// arbitrary entries from a saturated single map.
|
|
||||||
dedup_previous.clear();
|
|
||||||
for entry in dedup_current.iter() {
|
|
||||||
dedup_previous.insert(*entry.key(), *entry.value());
|
|
||||||
}
|
|
||||||
dedup_current.clear();
|
|
||||||
state.current_started_at = Some(now);
|
|
||||||
dedup_current.insert(key, now);
|
|
||||||
should_emit_full_desync_full_cache(now)
|
should_emit_full_desync_full_cache(now)
|
||||||
} else {
|
} else {
|
||||||
dedup_current.insert(key, now);
|
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -422,20 +405,8 @@ fn clear_desync_dedup_for_testing() {
|
|||||||
if let Some(dedup) = DESYNC_DEDUP.get() {
|
if let Some(dedup) = DESYNC_DEDUP.get() {
|
||||||
dedup.clear();
|
dedup.clear();
|
||||||
}
|
}
|
||||||
if let Some(dedup_previous) = DESYNC_DEDUP_PREVIOUS.get() {
|
if let Some(ever_saturated) = DESYNC_DEDUP_EVER_SATURATED.get() {
|
||||||
dedup_previous.clear();
|
ever_saturated.store(false, Ordering::Relaxed);
|
||||||
}
|
|
||||||
if let Some(rotation_state) = DESYNC_DEDUP_ROTATION_STATE.get() {
|
|
||||||
match rotation_state.lock() {
|
|
||||||
Ok(mut guard) => {
|
|
||||||
*guard = DesyncDedupRotationState::default();
|
|
||||||
}
|
|
||||||
Err(poisoned) => {
|
|
||||||
let mut guard = poisoned.into_inner();
|
|
||||||
*guard = DesyncDedupRotationState::default();
|
|
||||||
rotation_state.clear_poison();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if let Some(last_emit_at) = DESYNC_FULL_CACHE_LAST_EMIT_AT.get() {
|
if let Some(last_emit_at) = DESYNC_FULL_CACHE_LAST_EMIT_AT.get() {
|
||||||
match last_emit_at.lock() {
|
match last_emit_at.lock() {
|
||||||
@@ -644,7 +615,6 @@ pub(crate) fn relay_idle_pressure_test_scope() -> std::sync::MutexGuard<'static,
|
|||||||
async fn enqueue_c2me_command(
|
async fn enqueue_c2me_command(
|
||||||
tx: &mpsc::Sender<C2MeCommand>,
|
tx: &mpsc::Sender<C2MeCommand>,
|
||||||
cmd: C2MeCommand,
|
cmd: C2MeCommand,
|
||||||
send_timeout: Option<Duration>,
|
|
||||||
) -> std::result::Result<(), mpsc::error::SendError<C2MeCommand>> {
|
) -> std::result::Result<(), mpsc::error::SendError<C2MeCommand>> {
|
||||||
match tx.try_send(cmd) {
|
match tx.try_send(cmd) {
|
||||||
Ok(()) => Ok(()),
|
Ok(()) => Ok(()),
|
||||||
@@ -655,18 +625,12 @@ async fn enqueue_c2me_command(
|
|||||||
if tx.capacity() <= C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS {
|
if tx.capacity() <= C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS {
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
}
|
}
|
||||||
let reserve_result = match send_timeout {
|
match timeout(C2ME_SEND_TIMEOUT, tx.reserve()).await {
|
||||||
Some(send_timeout) => match timeout(send_timeout, tx.reserve()).await {
|
Ok(Ok(permit)) => {
|
||||||
Ok(result) => result,
|
|
||||||
Err(_) => return Err(mpsc::error::SendError(cmd)),
|
|
||||||
},
|
|
||||||
None => tx.reserve().await,
|
|
||||||
};
|
|
||||||
match reserve_result {
|
|
||||||
Ok(permit) => {
|
|
||||||
permit.send(cmd);
|
permit.send(cmd);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
Ok(Err(_)) => Err(mpsc::error::SendError(cmd)),
|
||||||
Err(_) => Err(mpsc::error::SendError(cmd)),
|
Err(_) => Err(mpsc::error::SendError(cmd)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -792,10 +756,6 @@ where
|
|||||||
.general
|
.general
|
||||||
.me_c2me_channel_capacity
|
.me_c2me_channel_capacity
|
||||||
.max(C2ME_CHANNEL_CAPACITY_FALLBACK);
|
.max(C2ME_CHANNEL_CAPACITY_FALLBACK);
|
||||||
let c2me_send_timeout = match config.general.me_c2me_send_timeout_ms {
|
|
||||||
0 => None,
|
|
||||||
timeout_ms => Some(Duration::from_millis(timeout_ms)),
|
|
||||||
};
|
|
||||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||||
let me_pool_c2me = me_pool.clone();
|
let me_pool_c2me = me_pool.clone();
|
||||||
let c2me_sender = tokio::spawn(async move {
|
let c2me_sender = tokio::spawn(async move {
|
||||||
@@ -1172,7 +1132,7 @@ where
|
|||||||
user = %user,
|
user = %user,
|
||||||
"Middle-relay pressure eviction for idle-candidate session"
|
"Middle-relay pressure eviction for idle-candidate session"
|
||||||
);
|
);
|
||||||
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close, c2me_send_timeout).await;
|
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await;
|
||||||
main_result = Err(ProxyError::Proxy(
|
main_result = Err(ProxyError::Proxy(
|
||||||
"middle-relay session evicted under pressure (idle-candidate)".to_string(),
|
"middle-relay session evicted under pressure (idle-candidate)".to_string(),
|
||||||
));
|
));
|
||||||
@@ -1191,7 +1151,7 @@ where
|
|||||||
"Cutover affected middle session, closing client connection"
|
"Cutover affected middle session, closing client connection"
|
||||||
);
|
);
|
||||||
tokio::time::sleep(delay).await;
|
tokio::time::sleep(delay).await;
|
||||||
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close, c2me_send_timeout).await;
|
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await;
|
||||||
main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
|
main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1249,12 +1209,8 @@ where
|
|||||||
flags |= RPC_FLAG_NOT_ENCRYPTED;
|
flags |= RPC_FLAG_NOT_ENCRYPTED;
|
||||||
}
|
}
|
||||||
// Keep client read loop lightweight: route heavy ME send path via a dedicated task.
|
// Keep client read loop lightweight: route heavy ME send path via a dedicated task.
|
||||||
if enqueue_c2me_command(
|
if enqueue_c2me_command(&c2me_tx, C2MeCommand::Data { payload, flags })
|
||||||
&c2me_tx,
|
.await
|
||||||
C2MeCommand::Data { payload, flags },
|
|
||||||
c2me_send_timeout,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
main_result = Err(ProxyError::Proxy("ME sender channel closed".into()));
|
main_result = Err(ProxyError::Proxy("ME sender channel closed".into()));
|
||||||
@@ -1264,9 +1220,7 @@ where
|
|||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
debug!(conn_id, "Client EOF");
|
debug!(conn_id, "Client EOF");
|
||||||
client_closed = true;
|
client_closed = true;
|
||||||
let _ =
|
let _ = enqueue_c2me_command(&c2me_tx, C2MeCommand::Close).await;
|
||||||
enqueue_c2me_command(&c2me_tx, C2MeCommand::Close, c2me_send_timeout)
|
|
||||||
.await;
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|||||||
@@ -94,7 +94,6 @@ async fn adversarial_tls_handshake_timeout_during_masking_delay() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -142,7 +141,6 @@ async fn blackhat_proxy_protocol_slowloris_timeout() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -195,7 +193,6 @@ async fn negative_proxy_protocol_enabled_but_client_sends_tls_hello() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -242,7 +239,6 @@ async fn edge_client_stream_exactly_4_bytes_eof() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -286,7 +282,6 @@ async fn edge_client_stream_tls_header_valid_but_body_1_byte_short_eof() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -333,7 +328,6 @@ async fn integration_non_tls_modes_disabled_immediately_masks() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -47,7 +47,6 @@ async fn invariant_tls_clienthello_truncation_exact_boundary_triggers_masking()
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -178,7 +177,6 @@ async fn invariant_direct_mode_partial_header_eof_is_error_not_bad_connect() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ fn build_harness(config: ProxyConfig) -> PipelineHarness {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -34,7 +34,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -47,7 +47,6 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -48,7 +48,6 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> RedTeamHarness {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -238,7 +237,6 @@ async fn redteam_03_masking_duration_must_be_less_than_1ms_when_backend_down() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
Arc::new(Stats::new()),
|
Arc::new(Stats::new()),
|
||||||
@@ -479,7 +477,6 @@ async fn measure_invalid_probe_duration_ms(delay_ms: u64, tls_len: u16, body_sen
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
Arc::new(Stats::new()),
|
Arc::new(Stats::new()),
|
||||||
@@ -553,7 +550,6 @@ async fn capture_forwarded_probe_len(tls_len: u16, body_sent: usize) -> usize {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
Arc::new(Stats::new()),
|
Arc::new(Stats::new()),
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -34,7 +34,6 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -100,7 +100,6 @@ async fn blackhat_proxy_protocol_massive_garbage_rejected_quickly() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -147,7 +146,6 @@ async fn edge_tls_body_immediate_eof_triggers_masking_and_bad_connect() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -197,7 +195,6 @@ async fn security_classic_mode_disabled_masks_valid_length_payload() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::{UpstreamConfig, UpstreamType};
|
use crate::config::{UpstreamConfig, UpstreamType};
|
||||||
use crate::crypto::{AesCtr, sha256, sha256_hmac};
|
use crate::crypto::AesCtr;
|
||||||
use crate::protocol::constants::{
|
use crate::crypto::sha256_hmac;
|
||||||
DC_IDX_POS, HANDSHAKE_LEN, IV_LEN, PREKEY_LEN, PROTO_TAG_POS, ProtoTag, SKIP_LEN,
|
use crate::protocol::constants::ProtoTag;
|
||||||
TLS_RECORD_CHANGE_CIPHER,
|
|
||||||
};
|
|
||||||
use crate::protocol::tls;
|
use crate::protocol::tls;
|
||||||
use crate::proxy::handshake::HandshakeSuccess;
|
use crate::proxy::handshake::HandshakeSuccess;
|
||||||
use crate::stream::{CryptoReader, CryptoWriter};
|
use crate::stream::{CryptoReader, CryptoWriter};
|
||||||
@@ -341,7 +339,6 @@ async fn relay_task_abort_releases_user_gate_and_ip_reservation() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -455,7 +452,6 @@ async fn relay_cutover_releases_user_gate_and_ip_reservation() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -579,7 +575,6 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -749,7 +744,6 @@ async fn proxy_protocol_header_is_rejected_when_trust_list_is_empty() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -826,7 +820,6 @@ async fn proxy_protocol_header_from_untrusted_peer_range_is_rejected_under_load(
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -986,7 +979,6 @@ async fn short_tls_probe_is_masked_through_client_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1074,7 +1066,6 @@ async fn tls12_record_probe_is_masked_through_client_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1160,7 +1151,6 @@ async fn handle_client_stream_increments_connects_all_exactly_once() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1253,7 +1243,6 @@ async fn running_client_handler_increments_connects_all_exactly_once() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1321,163 +1310,6 @@ async fn running_client_handler_increments_connects_all_exactly_once() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
|
||||||
async fn idle_pooled_connection_closes_cleanly_in_generic_stream_path() {
|
|
||||||
let mut cfg = ProxyConfig::default();
|
|
||||||
cfg.general.beobachten = false;
|
|
||||||
cfg.timeouts.client_first_byte_idle_secs = 1;
|
|
||||||
|
|
||||||
let config = Arc::new(cfg);
|
|
||||||
let stats = Arc::new(Stats::new());
|
|
||||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
|
||||||
vec![UpstreamConfig {
|
|
||||||
upstream_type: UpstreamType::Direct {
|
|
||||||
interface: None,
|
|
||||||
bind_addresses: None,
|
|
||||||
},
|
|
||||||
weight: 1,
|
|
||||||
enabled: true,
|
|
||||||
scopes: String::new(),
|
|
||||||
selected_scope: String::new(),
|
|
||||||
}],
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
10,
|
|
||||||
1,
|
|
||||||
false,
|
|
||||||
stats.clone(),
|
|
||||||
));
|
|
||||||
let replay_checker = Arc::new(ReplayChecker::new(128, Duration::from_secs(60)));
|
|
||||||
let buffer_pool = Arc::new(BufferPool::new());
|
|
||||||
let rng = Arc::new(SecureRandom::new());
|
|
||||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
|
||||||
let ip_tracker = Arc::new(UserIpTracker::new());
|
|
||||||
let beobachten = Arc::new(BeobachtenStore::new());
|
|
||||||
|
|
||||||
let (server_side, _client_side) = duplex(4096);
|
|
||||||
let peer: SocketAddr = "198.51.100.169:55200".parse().unwrap();
|
|
||||||
|
|
||||||
let handler = tokio::spawn(handle_client_stream(
|
|
||||||
server_side,
|
|
||||||
peer,
|
|
||||||
config,
|
|
||||||
stats.clone(),
|
|
||||||
upstream_manager,
|
|
||||||
replay_checker,
|
|
||||||
buffer_pool,
|
|
||||||
rng,
|
|
||||||
None,
|
|
||||||
route_runtime,
|
|
||||||
None,
|
|
||||||
ip_tracker,
|
|
||||||
beobachten,
|
|
||||||
false,
|
|
||||||
));
|
|
||||||
|
|
||||||
// Let the spawned handler arm the idle-phase timeout before advancing paused time.
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
tokio::time::advance(Duration::from_secs(2)).await;
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
|
|
||||||
let result = tokio::time::timeout(Duration::from_secs(1), handler)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
assert!(result.is_ok());
|
|
||||||
assert_eq!(stats.get_handshake_timeouts(), 0);
|
|
||||||
assert_eq!(stats.get_connects_bad(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
|
||||||
async fn idle_pooled_connection_closes_cleanly_in_client_handler_path() {
|
|
||||||
let front_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let front_addr = front_listener.local_addr().unwrap();
|
|
||||||
|
|
||||||
let mut cfg = ProxyConfig::default();
|
|
||||||
cfg.general.beobachten = false;
|
|
||||||
cfg.timeouts.client_first_byte_idle_secs = 1;
|
|
||||||
|
|
||||||
let config = Arc::new(cfg);
|
|
||||||
let stats = Arc::new(Stats::new());
|
|
||||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
|
||||||
vec![UpstreamConfig {
|
|
||||||
upstream_type: UpstreamType::Direct {
|
|
||||||
interface: None,
|
|
||||||
bind_addresses: None,
|
|
||||||
},
|
|
||||||
weight: 1,
|
|
||||||
enabled: true,
|
|
||||||
scopes: String::new(),
|
|
||||||
selected_scope: String::new(),
|
|
||||||
}],
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
10,
|
|
||||||
1,
|
|
||||||
false,
|
|
||||||
stats.clone(),
|
|
||||||
));
|
|
||||||
let replay_checker = Arc::new(ReplayChecker::new(128, Duration::from_secs(60)));
|
|
||||||
let buffer_pool = Arc::new(BufferPool::new());
|
|
||||||
let rng = Arc::new(SecureRandom::new());
|
|
||||||
let route_runtime = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
|
||||||
let ip_tracker = Arc::new(UserIpTracker::new());
|
|
||||||
let beobachten = Arc::new(BeobachtenStore::new());
|
|
||||||
|
|
||||||
let server_task = {
|
|
||||||
let config = config.clone();
|
|
||||||
let stats = stats.clone();
|
|
||||||
let upstream_manager = upstream_manager.clone();
|
|
||||||
let replay_checker = replay_checker.clone();
|
|
||||||
let buffer_pool = buffer_pool.clone();
|
|
||||||
let rng = rng.clone();
|
|
||||||
let route_runtime = route_runtime.clone();
|
|
||||||
let ip_tracker = ip_tracker.clone();
|
|
||||||
let beobachten = beobachten.clone();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let (stream, peer) = front_listener.accept().await.unwrap();
|
|
||||||
let real_peer_report = Arc::new(std::sync::Mutex::new(None));
|
|
||||||
ClientHandler::new(
|
|
||||||
stream,
|
|
||||||
peer,
|
|
||||||
config,
|
|
||||||
stats,
|
|
||||||
upstream_manager,
|
|
||||||
replay_checker,
|
|
||||||
buffer_pool,
|
|
||||||
rng,
|
|
||||||
None,
|
|
||||||
route_runtime,
|
|
||||||
None,
|
|
||||||
ip_tracker,
|
|
||||||
beobachten,
|
|
||||||
false,
|
|
||||||
real_peer_report,
|
|
||||||
)
|
|
||||||
.run()
|
|
||||||
.await
|
|
||||||
})
|
|
||||||
};
|
|
||||||
|
|
||||||
let _client = TcpStream::connect(front_addr).await.unwrap();
|
|
||||||
|
|
||||||
// Let the accepted connection reach the idle wait before advancing paused time.
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
tokio::time::advance(Duration::from_secs(2)).await;
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
|
|
||||||
let result = tokio::time::timeout(Duration::from_secs(1), server_task)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
assert!(result.is_ok());
|
|
||||||
assert_eq!(stats.get_handshake_timeouts(), 0);
|
|
||||||
assert_eq!(stats.get_connects_bad(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn partial_tls_header_stall_triggers_handshake_timeout() {
|
async fn partial_tls_header_stall_triggers_handshake_timeout() {
|
||||||
let mut cfg = ProxyConfig::default();
|
let mut cfg = ProxyConfig::default();
|
||||||
@@ -1500,7 +1332,6 @@ async fn partial_tls_header_stall_triggers_handshake_timeout() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1646,148 +1477,6 @@ fn wrap_tls_application_data(payload: &[u8]) -> Vec<u8> {
|
|||||||
record
|
record
|
||||||
}
|
}
|
||||||
|
|
||||||
fn wrap_tls_ccs_record() -> Vec<u8> {
|
|
||||||
let mut record = Vec::with_capacity(6);
|
|
||||||
record.push(TLS_RECORD_CHANGE_CIPHER);
|
|
||||||
record.extend_from_slice(&[0x03, 0x03]);
|
|
||||||
record.extend_from_slice(&1u16.to_be_bytes());
|
|
||||||
record.push(0x01);
|
|
||||||
record
|
|
||||||
}
|
|
||||||
|
|
||||||
fn make_valid_mtproto_handshake(
|
|
||||||
secret_hex: &str,
|
|
||||||
proto_tag: ProtoTag,
|
|
||||||
dc_idx: i16,
|
|
||||||
) -> [u8; HANDSHAKE_LEN] {
|
|
||||||
let secret = hex::decode(secret_hex).expect("secret hex must decode for mtproto test helper");
|
|
||||||
|
|
||||||
let mut handshake = [0x5Au8; HANDSHAKE_LEN];
|
|
||||||
for (idx, b) in handshake[SKIP_LEN..SKIP_LEN + PREKEY_LEN + IV_LEN]
|
|
||||||
.iter_mut()
|
|
||||||
.enumerate()
|
|
||||||
{
|
|
||||||
*b = (idx as u8).wrapping_add(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
let dec_prekey = &handshake[SKIP_LEN..SKIP_LEN + PREKEY_LEN];
|
|
||||||
let dec_iv_bytes = &handshake[SKIP_LEN + PREKEY_LEN..SKIP_LEN + PREKEY_LEN + IV_LEN];
|
|
||||||
|
|
||||||
let mut dec_key_input = Vec::with_capacity(PREKEY_LEN + secret.len());
|
|
||||||
dec_key_input.extend_from_slice(dec_prekey);
|
|
||||||
dec_key_input.extend_from_slice(&secret);
|
|
||||||
let dec_key = sha256(&dec_key_input);
|
|
||||||
|
|
||||||
let mut dec_iv_arr = [0u8; IV_LEN];
|
|
||||||
dec_iv_arr.copy_from_slice(dec_iv_bytes);
|
|
||||||
let dec_iv = u128::from_be_bytes(dec_iv_arr);
|
|
||||||
|
|
||||||
let mut stream = AesCtr::new(&dec_key, dec_iv);
|
|
||||||
let keystream = stream.encrypt(&[0u8; HANDSHAKE_LEN]);
|
|
||||||
|
|
||||||
let mut target_plain = [0u8; HANDSHAKE_LEN];
|
|
||||||
target_plain[PROTO_TAG_POS..PROTO_TAG_POS + 4].copy_from_slice(&proto_tag.to_bytes());
|
|
||||||
target_plain[DC_IDX_POS..DC_IDX_POS + 2].copy_from_slice(&dc_idx.to_le_bytes());
|
|
||||||
|
|
||||||
for idx in PROTO_TAG_POS..HANDSHAKE_LEN {
|
|
||||||
handshake[idx] = target_plain[idx] ^ keystream[idx];
|
|
||||||
}
|
|
||||||
|
|
||||||
handshake
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn fragmented_tls_mtproto_with_interleaved_ccs_is_accepted() {
|
|
||||||
let secret_hex = "55555555555555555555555555555555";
|
|
||||||
let secret = [0x55u8; 16];
|
|
||||||
let client_hello = make_valid_tls_client_hello(&secret, 0);
|
|
||||||
let mtproto_handshake = make_valid_mtproto_handshake(secret_hex, ProtoTag::Secure, 2);
|
|
||||||
|
|
||||||
let mut cfg = ProxyConfig::default();
|
|
||||||
cfg.general.beobachten = false;
|
|
||||||
cfg.access.ignore_time_skew = true;
|
|
||||||
cfg.access
|
|
||||||
.users
|
|
||||||
.insert("user".to_string(), secret_hex.to_string());
|
|
||||||
|
|
||||||
let config = Arc::new(cfg);
|
|
||||||
let replay_checker = Arc::new(ReplayChecker::new(128, Duration::from_secs(60)));
|
|
||||||
let rng = SecureRandom::new();
|
|
||||||
|
|
||||||
let (server_side, mut client_side) = duplex(131072);
|
|
||||||
let peer: SocketAddr = "198.51.100.85:55007".parse().unwrap();
|
|
||||||
let (read_half, write_half) = tokio::io::split(server_side);
|
|
||||||
|
|
||||||
let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake(
|
|
||||||
&client_hello,
|
|
||||||
read_half,
|
|
||||||
write_half,
|
|
||||||
peer,
|
|
||||||
&config,
|
|
||||||
&replay_checker,
|
|
||||||
&rng,
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
HandshakeResult::Success(result) => result,
|
|
||||||
_ => panic!("expected successful TLS handshake"),
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut tls_response_head = [0u8; 5];
|
|
||||||
client_side
|
|
||||||
.read_exact(&mut tls_response_head)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(tls_response_head[0], 0x16);
|
|
||||||
let tls_response_len =
|
|
||||||
u16::from_be_bytes([tls_response_head[3], tls_response_head[4]]) as usize;
|
|
||||||
let mut tls_response_body = vec![0u8; tls_response_len];
|
|
||||||
client_side
|
|
||||||
.read_exact(&mut tls_response_body)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
client_side
|
|
||||||
.write_all(&wrap_tls_application_data(&mtproto_handshake[..13]))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
client_side.write_all(&wrap_tls_ccs_record()).await.unwrap();
|
|
||||||
client_side
|
|
||||||
.write_all(&wrap_tls_application_data(&mtproto_handshake[13..37]))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
client_side.write_all(&wrap_tls_ccs_record()).await.unwrap();
|
|
||||||
client_side
|
|
||||||
.write_all(&wrap_tls_application_data(&mtproto_handshake[37..]))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await.unwrap();
|
|
||||||
assert_eq!(&mtproto_data[..], &mtproto_handshake);
|
|
||||||
|
|
||||||
let mtproto_handshake: [u8; HANDSHAKE_LEN] = mtproto_data[..].try_into().unwrap();
|
|
||||||
let (_, _, success) = match handle_mtproto_handshake(
|
|
||||||
&mtproto_handshake,
|
|
||||||
tls_reader,
|
|
||||||
tls_writer,
|
|
||||||
peer,
|
|
||||||
&config,
|
|
||||||
&replay_checker,
|
|
||||||
true,
|
|
||||||
Some(tls_user.as_str()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
HandshakeResult::Success(result) => result,
|
|
||||||
_ => panic!("expected successful MTProto handshake"),
|
|
||||||
};
|
|
||||||
|
|
||||||
assert_eq!(success.user, "user");
|
|
||||||
assert_eq!(success.proto_tag, ProtoTag::Secure);
|
|
||||||
assert_eq!(success.dc_idx, 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn valid_tls_path_does_not_fall_back_to_mask_backend() {
|
async fn valid_tls_path_does_not_fall_back_to_mask_backend() {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
@@ -1825,7 +1514,6 @@ async fn valid_tls_path_does_not_fall_back_to_mask_backend() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1934,7 +1622,6 @@ async fn valid_tls_with_invalid_mtproto_falls_back_to_mask_backend() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -2041,7 +1728,6 @@ async fn client_handler_tls_bad_mtproto_is_forwarded_to_mask_backend() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -2163,7 +1849,6 @@ async fn alpn_mismatch_tls_probe_is_masked_through_client_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -2256,7 +1941,6 @@ async fn invalid_hmac_tls_probe_is_masked_through_client_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -2355,7 +2039,6 @@ async fn burst_invalid_tls_probes_are_masked_verbatim() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -2534,16 +2217,14 @@ async fn tcp_limit_rejection_does_not_reserve_ip_or_trigger_rollback() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn zero_tcp_limit_uses_global_fallback_and_rejects_without_side_effects() {
|
async fn zero_tcp_limit_rejects_without_ip_or_counter_side_effects() {
|
||||||
let mut config = ProxyConfig::default();
|
let mut config = ProxyConfig::default();
|
||||||
config
|
config
|
||||||
.access
|
.access
|
||||||
.user_max_tcp_conns
|
.user_max_tcp_conns
|
||||||
.insert("user".to_string(), 0);
|
.insert("user".to_string(), 0);
|
||||||
config.access.user_max_tcp_conns_global_each = 1;
|
|
||||||
|
|
||||||
let stats = Stats::new();
|
let stats = Stats::new();
|
||||||
stats.increment_user_curr_connects("user");
|
|
||||||
let ip_tracker = UserIpTracker::new();
|
let ip_tracker = UserIpTracker::new();
|
||||||
let peer_addr: SocketAddr = "198.51.100.211:50001".parse().unwrap();
|
let peer_addr: SocketAddr = "198.51.100.211:50001".parse().unwrap();
|
||||||
|
|
||||||
@@ -2560,75 +2241,10 @@ async fn zero_tcp_limit_uses_global_fallback_and_rejects_without_side_effects()
|
|||||||
result,
|
result,
|
||||||
Err(ProxyError::ConnectionLimitExceeded { user }) if user == "user"
|
Err(ProxyError::ConnectionLimitExceeded { user }) if user == "user"
|
||||||
));
|
));
|
||||||
assert_eq!(
|
|
||||||
stats.get_user_curr_connects("user"),
|
|
||||||
1,
|
|
||||||
"TCP-limit rejection must keep pre-existing in-flight connection count unchanged"
|
|
||||||
);
|
|
||||||
assert_eq!(ip_tracker.get_active_ip_count("user").await, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn zero_tcp_limit_with_disabled_global_fallback_is_unlimited() {
|
|
||||||
let mut config = ProxyConfig::default();
|
|
||||||
config
|
|
||||||
.access
|
|
||||||
.user_max_tcp_conns
|
|
||||||
.insert("user".to_string(), 0);
|
|
||||||
config.access.user_max_tcp_conns_global_each = 0;
|
|
||||||
|
|
||||||
let stats = Stats::new();
|
|
||||||
let ip_tracker = UserIpTracker::new();
|
|
||||||
let peer_addr: SocketAddr = "198.51.100.212:50002".parse().unwrap();
|
|
||||||
|
|
||||||
let result = RunningClientHandler::check_user_limits_static(
|
|
||||||
"user",
|
|
||||||
&config,
|
|
||||||
&stats,
|
|
||||||
peer_addr,
|
|
||||||
&ip_tracker,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
result.is_ok(),
|
|
||||||
"per-user zero with global fallback disabled must not enforce a TCP limit"
|
|
||||||
);
|
|
||||||
assert_eq!(stats.get_user_curr_connects("user"), 0);
|
assert_eq!(stats.get_user_curr_connects("user"), 0);
|
||||||
assert_eq!(ip_tracker.get_active_ip_count("user").await, 0);
|
assert_eq!(ip_tracker.get_active_ip_count("user").await, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn global_tcp_fallback_applies_when_per_user_limit_is_missing() {
|
|
||||||
let mut config = ProxyConfig::default();
|
|
||||||
config.access.user_max_tcp_conns_global_each = 1;
|
|
||||||
|
|
||||||
let stats = Stats::new();
|
|
||||||
stats.increment_user_curr_connects("user");
|
|
||||||
let ip_tracker = UserIpTracker::new();
|
|
||||||
let peer_addr: SocketAddr = "198.51.100.213:50003".parse().unwrap();
|
|
||||||
|
|
||||||
let result = RunningClientHandler::check_user_limits_static(
|
|
||||||
"user",
|
|
||||||
&config,
|
|
||||||
&stats,
|
|
||||||
peer_addr,
|
|
||||||
&ip_tracker,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
assert!(matches!(
|
|
||||||
result,
|
|
||||||
Err(ProxyError::ConnectionLimitExceeded { user }) if user == "user"
|
|
||||||
));
|
|
||||||
assert_eq!(
|
|
||||||
stats.get_user_curr_connects("user"),
|
|
||||||
1,
|
|
||||||
"Global fallback TCP-limit rejection must keep pre-existing counter unchanged"
|
|
||||||
);
|
|
||||||
assert_eq!(ip_tracker.get_active_ip_count("user").await, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn check_user_limits_static_success_does_not_leak_counter_or_ip_reservation() {
|
async fn check_user_limits_static_success_does_not_leak_counter_or_ip_reservation() {
|
||||||
let user = "check-helper-user";
|
let user = "check-helper-user";
|
||||||
@@ -3260,7 +2876,6 @@ async fn relay_connect_error_releases_user_and_ip_before_return() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -3821,7 +3436,6 @@ async fn untrusted_proxy_header_source_is_rejected() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -3891,7 +3505,6 @@ async fn empty_proxy_trusted_cidrs_rejects_proxy_header_by_default() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -3988,7 +3601,6 @@ async fn oversized_tls_record_is_masked_in_generic_stream_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -4091,7 +3703,6 @@ async fn oversized_tls_record_is_masked_in_client_handler_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -4208,7 +3819,6 @@ async fn tls_record_len_min_minus_1_is_rejected_in_generic_stream_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -4311,7 +3921,6 @@ async fn tls_record_len_min_minus_1_is_rejected_in_client_handler_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -4417,7 +4026,6 @@ async fn tls_record_len_16384_is_accepted_in_generic_stream_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -4518,7 +4126,6 @@ async fn tls_record_len_16384_is_accepted_in_client_handler_pipeline() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -35,7 +35,6 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ fn make_test_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats,
|
stats,
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ fn build_harness(secret_hex: &str, mask_port: u16) -> PipelineHarness {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -1302,7 +1302,6 @@ async fn direct_relay_abort_midflight_releases_route_gauge() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1409,7 +1408,6 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() {
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1531,7 +1529,6 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea
|
|||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
1,
|
1,
|
||||||
10,
|
|
||||||
1,
|
1,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1764,7 +1761,6 @@ async fn negative_direct_relay_dc_connection_refused_fails_fast() {
|
|||||||
1,
|
1,
|
||||||
100,
|
100,
|
||||||
5000,
|
5000,
|
||||||
10,
|
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
@@ -1855,7 +1851,6 @@ async fn adversarial_direct_relay_cutover_integrity() {
|
|||||||
1,
|
1,
|
||||||
100,
|
100,
|
||||||
5000,
|
5000,
|
||||||
10,
|
|
||||||
3,
|
3,
|
||||||
false,
|
false,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
|
|||||||
@@ -126,7 +126,6 @@ async fn c2me_channel_full_path_yields_then_sends() {
|
|||||||
payload: make_pooled_payload(&[0xBB, 0xCC]),
|
payload: make_pooled_payload(&[0xBB, 0xCC]),
|
||||||
flags: 2,
|
flags: 2,
|
||||||
},
|
},
|
||||||
None,
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,388 +0,0 @@
|
|||||||
//! Service manager integration for telemt.
|
|
||||||
//!
|
|
||||||
//! Supports generating service files for:
|
|
||||||
//! - systemd (Linux)
|
|
||||||
//! - OpenRC (Alpine, Gentoo)
|
|
||||||
//! - rc.d (FreeBSD)
|
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
/// Detected init/service system.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum InitSystem {
|
|
||||||
/// systemd (most modern Linux distributions)
|
|
||||||
Systemd,
|
|
||||||
/// OpenRC (Alpine, Gentoo, some BSDs)
|
|
||||||
OpenRC,
|
|
||||||
/// FreeBSD rc.d
|
|
||||||
FreeBSDRc,
|
|
||||||
/// No known init system detected
|
|
||||||
Unknown,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for InitSystem {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
match self {
|
|
||||||
InitSystem::Systemd => write!(f, "systemd"),
|
|
||||||
InitSystem::OpenRC => write!(f, "OpenRC"),
|
|
||||||
InitSystem::FreeBSDRc => write!(f, "FreeBSD rc.d"),
|
|
||||||
InitSystem::Unknown => write!(f, "unknown"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Detects the init system in use on the current host.
|
|
||||||
pub fn detect_init_system() -> InitSystem {
|
|
||||||
// Check for systemd first (most common on Linux)
|
|
||||||
if Path::new("/run/systemd/system").exists() {
|
|
||||||
return InitSystem::Systemd;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for OpenRC
|
|
||||||
if Path::new("/sbin/openrc-run").exists() || Path::new("/sbin/openrc").exists() {
|
|
||||||
return InitSystem::OpenRC;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for FreeBSD rc.d
|
|
||||||
if Path::new("/etc/rc.subr").exists() && Path::new("/etc/rc.d").exists() {
|
|
||||||
return InitSystem::FreeBSDRc;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: check if systemctl exists even without /run/systemd
|
|
||||||
if Path::new("/usr/bin/systemctl").exists() || Path::new("/bin/systemctl").exists() {
|
|
||||||
return InitSystem::Systemd;
|
|
||||||
}
|
|
||||||
|
|
||||||
InitSystem::Unknown
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns the default service file path for the given init system.
|
|
||||||
pub fn service_file_path(init_system: InitSystem) -> &'static str {
|
|
||||||
match init_system {
|
|
||||||
InitSystem::Systemd => "/etc/systemd/system/telemt.service",
|
|
||||||
InitSystem::OpenRC => "/etc/init.d/telemt",
|
|
||||||
InitSystem::FreeBSDRc => "/usr/local/etc/rc.d/telemt",
|
|
||||||
InitSystem::Unknown => "/etc/init.d/telemt",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Options for generating service files.
|
|
||||||
pub struct ServiceOptions<'a> {
|
|
||||||
/// Path to the telemt executable
|
|
||||||
pub exe_path: &'a Path,
|
|
||||||
/// Path to the configuration file
|
|
||||||
pub config_path: &'a Path,
|
|
||||||
/// User to run as (optional)
|
|
||||||
pub user: Option<&'a str>,
|
|
||||||
/// Group to run as (optional)
|
|
||||||
pub group: Option<&'a str>,
|
|
||||||
/// PID file path
|
|
||||||
pub pid_file: &'a str,
|
|
||||||
/// Working directory
|
|
||||||
pub working_dir: Option<&'a str>,
|
|
||||||
/// Description
|
|
||||||
pub description: &'a str,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> Default for ServiceOptions<'a> {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
exe_path: Path::new("/usr/local/bin/telemt"),
|
|
||||||
config_path: Path::new("/etc/telemt/config.toml"),
|
|
||||||
user: Some("telemt"),
|
|
||||||
group: Some("telemt"),
|
|
||||||
pid_file: "/var/run/telemt.pid",
|
|
||||||
working_dir: Some("/var/lib/telemt"),
|
|
||||||
description: "Telemt MTProxy - Telegram MTProto Proxy",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generates a service file for the given init system.
|
|
||||||
pub fn generate_service_file(init_system: InitSystem, opts: &ServiceOptions) -> String {
|
|
||||||
match init_system {
|
|
||||||
InitSystem::Systemd => generate_systemd_unit(opts),
|
|
||||||
InitSystem::OpenRC => generate_openrc_script(opts),
|
|
||||||
InitSystem::FreeBSDRc => generate_freebsd_rc_script(opts),
|
|
||||||
InitSystem::Unknown => generate_systemd_unit(opts), // Default to systemd format
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generates an enhanced systemd unit file.
|
|
||||||
fn generate_systemd_unit(opts: &ServiceOptions) -> String {
|
|
||||||
let user_line = opts.user.map(|u| format!("User={}", u)).unwrap_or_default();
|
|
||||||
let group_line = opts
|
|
||||||
.group
|
|
||||||
.map(|g| format!("Group={}", g))
|
|
||||||
.unwrap_or_default();
|
|
||||||
let working_dir = opts
|
|
||||||
.working_dir
|
|
||||||
.map(|d| format!("WorkingDirectory={}", d))
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
format!(
|
|
||||||
r#"[Unit]
|
|
||||||
Description={description}
|
|
||||||
Documentation=https://github.com/telemt/telemt
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
ExecStart={exe} --foreground --pid-file {pid_file} {config}
|
|
||||||
ExecReload=/bin/kill -HUP $MAINPID
|
|
||||||
PIDFile={pid_file}
|
|
||||||
Restart=always
|
|
||||||
RestartSec=5
|
|
||||||
{user}
|
|
||||||
{group}
|
|
||||||
{working_dir}
|
|
||||||
|
|
||||||
# Resource limits
|
|
||||||
LimitNOFILE=65535
|
|
||||||
LimitNPROC=4096
|
|
||||||
|
|
||||||
# Security hardening
|
|
||||||
NoNewPrivileges=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
PrivateTmp=true
|
|
||||||
PrivateDevices=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
|
||||||
RestrictNamespaces=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
LockPersonality=true
|
|
||||||
|
|
||||||
# Allow binding to privileged ports and writing to specific paths
|
|
||||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
|
||||||
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
|
|
||||||
ReadWritePaths=/etc/telemt /var/run /var/lib/telemt
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
"#,
|
|
||||||
description = opts.description,
|
|
||||||
exe = opts.exe_path.display(),
|
|
||||||
config = opts.config_path.display(),
|
|
||||||
pid_file = opts.pid_file,
|
|
||||||
user = user_line,
|
|
||||||
group = group_line,
|
|
||||||
working_dir = working_dir,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generates an OpenRC init script.
|
|
||||||
fn generate_openrc_script(opts: &ServiceOptions) -> String {
|
|
||||||
let user = opts.user.unwrap_or("root");
|
|
||||||
let group = opts.group.unwrap_or("root");
|
|
||||||
|
|
||||||
format!(
|
|
||||||
r#"#!/sbin/openrc-run
|
|
||||||
# OpenRC init script for telemt
|
|
||||||
|
|
||||||
description="{description}"
|
|
||||||
command="{exe}"
|
|
||||||
command_args="--daemon --syslog --pid-file {pid_file} {config}"
|
|
||||||
command_user="{user}:{group}"
|
|
||||||
pidfile="{pid_file}"
|
|
||||||
|
|
||||||
depend() {{
|
|
||||||
need net
|
|
||||||
use logger
|
|
||||||
after firewall
|
|
||||||
}}
|
|
||||||
|
|
||||||
start_pre() {{
|
|
||||||
checkpath --directory --owner {user}:{group} --mode 0755 /var/run
|
|
||||||
checkpath --directory --owner {user}:{group} --mode 0755 /var/lib/telemt
|
|
||||||
checkpath --directory --owner {user}:{group} --mode 0755 /var/log/telemt
|
|
||||||
}}
|
|
||||||
|
|
||||||
reload() {{
|
|
||||||
ebegin "Reloading ${{RC_SVCNAME}}"
|
|
||||||
start-stop-daemon --signal HUP --pidfile "${{pidfile}}"
|
|
||||||
eend $?
|
|
||||||
}}
|
|
||||||
"#,
|
|
||||||
description = opts.description,
|
|
||||||
exe = opts.exe_path.display(),
|
|
||||||
config = opts.config_path.display(),
|
|
||||||
pid_file = opts.pid_file,
|
|
||||||
user = user,
|
|
||||||
group = group,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Generates a FreeBSD rc.d script.
|
|
||||||
fn generate_freebsd_rc_script(opts: &ServiceOptions) -> String {
|
|
||||||
let user = opts.user.unwrap_or("root");
|
|
||||||
let group = opts.group.unwrap_or("wheel");
|
|
||||||
|
|
||||||
format!(
|
|
||||||
r#"#!/bin/sh
|
|
||||||
#
|
|
||||||
# PROVIDE: telemt
|
|
||||||
# REQUIRE: LOGIN NETWORKING
|
|
||||||
# KEYWORD: shutdown
|
|
||||||
#
|
|
||||||
# Add the following lines to /etc/rc.conf to enable telemt:
|
|
||||||
#
|
|
||||||
# telemt_enable="YES"
|
|
||||||
# telemt_config="/etc/telemt/config.toml" # optional
|
|
||||||
# telemt_user="telemt" # optional
|
|
||||||
# telemt_group="telemt" # optional
|
|
||||||
#
|
|
||||||
|
|
||||||
. /etc/rc.subr
|
|
||||||
|
|
||||||
name="telemt"
|
|
||||||
rcvar="telemt_enable"
|
|
||||||
desc="{description}"
|
|
||||||
|
|
||||||
load_rc_config $name
|
|
||||||
|
|
||||||
: ${{telemt_enable:="NO"}}
|
|
||||||
: ${{telemt_config:="{config}"}}
|
|
||||||
: ${{telemt_user:="{user}"}}
|
|
||||||
: ${{telemt_group:="{group}"}}
|
|
||||||
: ${{telemt_pidfile:="{pid_file}"}}
|
|
||||||
|
|
||||||
pidfile="${{telemt_pidfile}}"
|
|
||||||
command="{exe}"
|
|
||||||
command_args="--daemon --syslog --pid-file ${{telemt_pidfile}} ${{telemt_config}}"
|
|
||||||
|
|
||||||
start_precmd="telemt_prestart"
|
|
||||||
reload_cmd="telemt_reload"
|
|
||||||
extra_commands="reload"
|
|
||||||
|
|
||||||
telemt_prestart() {{
|
|
||||||
install -d -o ${{telemt_user}} -g ${{telemt_group}} -m 755 /var/run
|
|
||||||
install -d -o ${{telemt_user}} -g ${{telemt_group}} -m 755 /var/lib/telemt
|
|
||||||
}}
|
|
||||||
|
|
||||||
telemt_reload() {{
|
|
||||||
if [ -f "${{pidfile}}" ]; then
|
|
||||||
echo "Reloading ${{name}} configuration."
|
|
||||||
kill -HUP $(cat ${{pidfile}})
|
|
||||||
else
|
|
||||||
echo "${{name}} is not running."
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}}
|
|
||||||
|
|
||||||
run_rc_command "$1"
|
|
||||||
"#,
|
|
||||||
description = opts.description,
|
|
||||||
exe = opts.exe_path.display(),
|
|
||||||
config = opts.config_path.display(),
|
|
||||||
pid_file = opts.pid_file,
|
|
||||||
user = user,
|
|
||||||
group = group,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Installation instructions for each init system.
|
|
||||||
pub fn installation_instructions(init_system: InitSystem) -> &'static str {
|
|
||||||
match init_system {
|
|
||||||
InitSystem::Systemd => {
|
|
||||||
r#"To install and enable the service:
|
|
||||||
sudo systemctl daemon-reload
|
|
||||||
sudo systemctl enable telemt
|
|
||||||
sudo systemctl start telemt
|
|
||||||
|
|
||||||
To check status:
|
|
||||||
sudo systemctl status telemt
|
|
||||||
|
|
||||||
To view logs:
|
|
||||||
journalctl -u telemt -f
|
|
||||||
|
|
||||||
To reload configuration:
|
|
||||||
sudo systemctl reload telemt
|
|
||||||
"#
|
|
||||||
}
|
|
||||||
InitSystem::OpenRC => {
|
|
||||||
r#"To install and enable the service:
|
|
||||||
sudo chmod +x /etc/init.d/telemt
|
|
||||||
sudo rc-update add telemt default
|
|
||||||
sudo rc-service telemt start
|
|
||||||
|
|
||||||
To check status:
|
|
||||||
sudo rc-service telemt status
|
|
||||||
|
|
||||||
To reload configuration:
|
|
||||||
sudo rc-service telemt reload
|
|
||||||
"#
|
|
||||||
}
|
|
||||||
InitSystem::FreeBSDRc => {
|
|
||||||
r#"To install and enable the service:
|
|
||||||
sudo chmod +x /usr/local/etc/rc.d/telemt
|
|
||||||
sudo sysrc telemt_enable="YES"
|
|
||||||
sudo service telemt start
|
|
||||||
|
|
||||||
To check status:
|
|
||||||
sudo service telemt status
|
|
||||||
|
|
||||||
To reload configuration:
|
|
||||||
sudo service telemt reload
|
|
||||||
"#
|
|
||||||
}
|
|
||||||
InitSystem::Unknown => {
|
|
||||||
r#"No supported init system detected.
|
|
||||||
You may need to create a service file manually or run telemt directly:
|
|
||||||
telemt start /etc/telemt/config.toml
|
|
||||||
"#
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_systemd_unit_generation() {
|
|
||||||
let opts = ServiceOptions::default();
|
|
||||||
let unit = generate_systemd_unit(&opts);
|
|
||||||
assert!(unit.contains("[Unit]"));
|
|
||||||
assert!(unit.contains("[Service]"));
|
|
||||||
assert!(unit.contains("[Install]"));
|
|
||||||
assert!(unit.contains("ExecReload="));
|
|
||||||
assert!(unit.contains("PIDFile="));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_openrc_script_generation() {
|
|
||||||
let opts = ServiceOptions::default();
|
|
||||||
let script = generate_openrc_script(&opts);
|
|
||||||
assert!(script.contains("#!/sbin/openrc-run"));
|
|
||||||
assert!(script.contains("depend()"));
|
|
||||||
assert!(script.contains("reload()"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_freebsd_rc_script_generation() {
|
|
||||||
let opts = ServiceOptions::default();
|
|
||||||
let script = generate_freebsd_rc_script(&opts);
|
|
||||||
assert!(script.contains("#!/bin/sh"));
|
|
||||||
assert!(script.contains("PROVIDE: telemt"));
|
|
||||||
assert!(script.contains("run_rc_command"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_service_file_paths() {
|
|
||||||
assert_eq!(
|
|
||||||
service_file_path(InitSystem::Systemd),
|
|
||||||
"/etc/systemd/system/telemt.service"
|
|
||||||
);
|
|
||||||
assert_eq!(service_file_path(InitSystem::OpenRC), "/etc/init.d/telemt");
|
|
||||||
assert_eq!(
|
|
||||||
service_file_path(InitSystem::FreeBSDRc),
|
|
||||||
"/usr/local/etc/rc.d/telemt"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -128,8 +128,6 @@ pub struct Stats {
|
|||||||
me_crc_mismatch: AtomicU64,
|
me_crc_mismatch: AtomicU64,
|
||||||
me_seq_mismatch: AtomicU64,
|
me_seq_mismatch: AtomicU64,
|
||||||
me_endpoint_quarantine_total: AtomicU64,
|
me_endpoint_quarantine_total: AtomicU64,
|
||||||
me_endpoint_quarantine_unexpected_total: AtomicU64,
|
|
||||||
me_endpoint_quarantine_draining_suppressed_total: AtomicU64,
|
|
||||||
me_kdf_drift_total: AtomicU64,
|
me_kdf_drift_total: AtomicU64,
|
||||||
me_kdf_port_only_drift_total: AtomicU64,
|
me_kdf_port_only_drift_total: AtomicU64,
|
||||||
me_hardswap_pending_reuse_total: AtomicU64,
|
me_hardswap_pending_reuse_total: AtomicU64,
|
||||||
@@ -236,7 +234,6 @@ pub struct Stats {
|
|||||||
me_writer_restored_same_endpoint_total: AtomicU64,
|
me_writer_restored_same_endpoint_total: AtomicU64,
|
||||||
me_writer_restored_fallback_total: AtomicU64,
|
me_writer_restored_fallback_total: AtomicU64,
|
||||||
me_no_writer_failfast_total: AtomicU64,
|
me_no_writer_failfast_total: AtomicU64,
|
||||||
me_hybrid_timeout_total: AtomicU64,
|
|
||||||
me_async_recovery_trigger_total: AtomicU64,
|
me_async_recovery_trigger_total: AtomicU64,
|
||||||
me_inline_recovery_total: AtomicU64,
|
me_inline_recovery_total: AtomicU64,
|
||||||
ip_reservation_rollback_tcp_limit_total: AtomicU64,
|
ip_reservation_rollback_tcp_limit_total: AtomicU64,
|
||||||
@@ -1206,11 +1203,6 @@ impl Stats {
|
|||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn increment_me_hybrid_timeout_total(&self) {
|
|
||||||
if self.telemetry_me_allows_normal() {
|
|
||||||
self.me_hybrid_timeout_total.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn increment_me_async_recovery_trigger_total(&self) {
|
pub fn increment_me_async_recovery_trigger_total(&self) {
|
||||||
if self.telemetry_me_allows_normal() {
|
if self.telemetry_me_allows_normal() {
|
||||||
self.me_async_recovery_trigger_total
|
self.me_async_recovery_trigger_total
|
||||||
@@ -1253,18 +1245,6 @@ impl Stats {
|
|||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn increment_me_endpoint_quarantine_unexpected_total(&self) {
|
|
||||||
if self.telemetry_me_allows_normal() {
|
|
||||||
self.me_endpoint_quarantine_unexpected_total
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn increment_me_endpoint_quarantine_draining_suppressed_total(&self) {
|
|
||||||
if self.telemetry_me_allows_normal() {
|
|
||||||
self.me_endpoint_quarantine_draining_suppressed_total
|
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn increment_me_kdf_drift_total(&self) {
|
pub fn increment_me_kdf_drift_total(&self) {
|
||||||
if self.telemetry_me_allows_normal() {
|
if self.telemetry_me_allows_normal() {
|
||||||
self.me_kdf_drift_total.fetch_add(1, Ordering::Relaxed);
|
self.me_kdf_drift_total.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -1517,14 +1497,6 @@ impl Stats {
|
|||||||
pub fn get_me_endpoint_quarantine_total(&self) -> u64 {
|
pub fn get_me_endpoint_quarantine_total(&self) -> u64 {
|
||||||
self.me_endpoint_quarantine_total.load(Ordering::Relaxed)
|
self.me_endpoint_quarantine_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
pub fn get_me_endpoint_quarantine_unexpected_total(&self) -> u64 {
|
|
||||||
self.me_endpoint_quarantine_unexpected_total
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
pub fn get_me_endpoint_quarantine_draining_suppressed_total(&self) -> u64 {
|
|
||||||
self.me_endpoint_quarantine_draining_suppressed_total
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
pub fn get_me_kdf_drift_total(&self) -> u64 {
|
pub fn get_me_kdf_drift_total(&self) -> u64 {
|
||||||
self.me_kdf_drift_total.load(Ordering::Relaxed)
|
self.me_kdf_drift_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
@@ -1904,9 +1876,6 @@ impl Stats {
|
|||||||
pub fn get_me_no_writer_failfast_total(&self) -> u64 {
|
pub fn get_me_no_writer_failfast_total(&self) -> u64 {
|
||||||
self.me_no_writer_failfast_total.load(Ordering::Relaxed)
|
self.me_no_writer_failfast_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
pub fn get_me_hybrid_timeout_total(&self) -> u64 {
|
|
||||||
self.me_hybrid_timeout_total.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
pub fn get_me_async_recovery_trigger_total(&self) -> u64 {
|
pub fn get_me_async_recovery_trigger_total(&self) -> u64 {
|
||||||
self.me_async_recovery_trigger_total.load(Ordering::Relaxed)
|
self.me_async_recovery_trigger_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -244,9 +244,10 @@ fn order_profiles(
|
|||||||
if let Some(pos) = ordered
|
if let Some(pos) = ordered
|
||||||
.iter()
|
.iter()
|
||||||
.position(|profile| *profile == cached.profile)
|
.position(|profile| *profile == cached.profile)
|
||||||
&& pos != 0
|
|
||||||
{
|
{
|
||||||
ordered.swap(0, pos);
|
if pos != 0 {
|
||||||
|
ordered.swap(0, pos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -314,6 +314,53 @@ async fn run_update_cycle(
|
|||||||
reinit_tx: &mpsc::Sender<MeReinitTrigger>,
|
reinit_tx: &mpsc::Sender<MeReinitTrigger>,
|
||||||
) {
|
) {
|
||||||
let upstream = pool.upstream.clone();
|
let upstream = pool.upstream.clone();
|
||||||
|
pool.update_runtime_reinit_policy(
|
||||||
|
cfg.general.hardswap,
|
||||||
|
cfg.general.me_pool_drain_ttl_secs,
|
||||||
|
cfg.general.me_instadrain,
|
||||||
|
cfg.general.me_pool_drain_threshold,
|
||||||
|
cfg.general.me_pool_drain_soft_evict_enabled,
|
||||||
|
cfg.general.me_pool_drain_soft_evict_grace_secs,
|
||||||
|
cfg.general.me_pool_drain_soft_evict_per_writer,
|
||||||
|
cfg.general.me_pool_drain_soft_evict_budget_per_core,
|
||||||
|
cfg.general.me_pool_drain_soft_evict_cooldown_ms,
|
||||||
|
cfg.general.effective_me_pool_force_close_secs(),
|
||||||
|
cfg.general.me_pool_min_fresh_ratio,
|
||||||
|
cfg.general.me_hardswap_warmup_delay_min_ms,
|
||||||
|
cfg.general.me_hardswap_warmup_delay_max_ms,
|
||||||
|
cfg.general.me_hardswap_warmup_extra_passes,
|
||||||
|
cfg.general.me_hardswap_warmup_pass_backoff_base_ms,
|
||||||
|
cfg.general.me_bind_stale_mode,
|
||||||
|
cfg.general.me_bind_stale_ttl_secs,
|
||||||
|
cfg.general.me_secret_atomic_snapshot,
|
||||||
|
cfg.general.me_deterministic_writer_sort,
|
||||||
|
cfg.general.me_writer_pick_mode,
|
||||||
|
cfg.general.me_writer_pick_sample_size,
|
||||||
|
cfg.general.me_single_endpoint_shadow_writers,
|
||||||
|
cfg.general.me_single_endpoint_outage_mode_enabled,
|
||||||
|
cfg.general.me_single_endpoint_outage_disable_quarantine,
|
||||||
|
cfg.general.me_single_endpoint_outage_backoff_min_ms,
|
||||||
|
cfg.general.me_single_endpoint_outage_backoff_max_ms,
|
||||||
|
cfg.general.me_single_endpoint_shadow_rotate_every_secs,
|
||||||
|
cfg.general.me_floor_mode,
|
||||||
|
cfg.general.me_adaptive_floor_idle_secs,
|
||||||
|
cfg.general.me_adaptive_floor_min_writers_single_endpoint,
|
||||||
|
cfg.general.me_adaptive_floor_min_writers_multi_endpoint,
|
||||||
|
cfg.general.me_adaptive_floor_recover_grace_secs,
|
||||||
|
cfg.general.me_adaptive_floor_writers_per_core_total,
|
||||||
|
cfg.general.me_adaptive_floor_cpu_cores_override,
|
||||||
|
cfg.general
|
||||||
|
.me_adaptive_floor_max_extra_writers_single_per_core,
|
||||||
|
cfg.general
|
||||||
|
.me_adaptive_floor_max_extra_writers_multi_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_per_core,
|
||||||
|
cfg.general.me_adaptive_floor_max_active_writers_global,
|
||||||
|
cfg.general.me_adaptive_floor_max_warm_writers_global,
|
||||||
|
cfg.general.me_health_interval_ms_unhealthy,
|
||||||
|
cfg.general.me_health_interval_ms_healthy,
|
||||||
|
cfg.general.me_warn_rate_limit_ms,
|
||||||
|
);
|
||||||
|
|
||||||
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
|
let required_cfg_snapshots = cfg.general.me_config_stable_snapshots.max(1);
|
||||||
let required_secret_snapshots = cfg.general.proxy_secret_stable_snapshots.max(1);
|
let required_secret_snapshots = cfg.general.proxy_secret_stable_snapshots.max(1);
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ impl MePool {
|
|||||||
} else {
|
} else {
|
||||||
let connect_fut = async {
|
let connect_fut = async {
|
||||||
if addr.is_ipv6()
|
if addr.is_ipv6()
|
||||||
&& let Some(v6) = self.nat_runtime.detected_ipv6
|
&& let Some(v6) = self.detected_ipv6
|
||||||
{
|
{
|
||||||
match TcpSocket::new_v6() {
|
match TcpSocket::new_v6() {
|
||||||
Ok(sock) => {
|
Ok(sock) => {
|
||||||
@@ -305,7 +305,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
MeSocksKdfPolicy::Compat => {
|
MeSocksKdfPolicy::Compat => {
|
||||||
self.stats.increment_me_socks_kdf_compat_fallback();
|
self.stats.increment_me_socks_kdf_compat_fallback();
|
||||||
if self.nat_runtime.nat_probe {
|
if self.nat_probe {
|
||||||
let bind_ip = Self::direct_bind_ip_for_stun(family, upstream_egress);
|
let bind_ip = Self::direct_bind_ip_for_stun(family, upstream_egress);
|
||||||
self.maybe_reflect_public_addr(family, bind_ip).await
|
self.maybe_reflect_public_addr(family, bind_ip).await
|
||||||
} else {
|
} else {
|
||||||
@@ -313,7 +313,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if self.nat_runtime.nat_probe {
|
} else if self.nat_probe {
|
||||||
let bind_ip = Self::direct_bind_ip_for_stun(family, upstream_egress);
|
let bind_ip = Self::direct_bind_ip_for_stun(family, upstream_egress);
|
||||||
self.maybe_reflect_public_addr(family, bind_ip).await
|
self.maybe_reflect_public_addr(family, bind_ip).await
|
||||||
} else {
|
} else {
|
||||||
@@ -343,10 +343,7 @@ impl MePool {
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.as_secs() as u32;
|
.as_secs() as u32;
|
||||||
|
|
||||||
let secret_atomic_snapshot = self
|
let secret_atomic_snapshot = self.secret_atomic_snapshot.load(Ordering::Relaxed);
|
||||||
.writer_selection_policy
|
|
||||||
.secret_atomic_snapshot
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
let (ks, secret) = if secret_atomic_snapshot {
|
let (ks, secret) = if secret_atomic_snapshot {
|
||||||
let snapshot = self.secret_snapshot().await;
|
let snapshot = self.secret_snapshot().await;
|
||||||
(snapshot.key_selector, snapshot.secret)
|
(snapshot.key_selector, snapshot.secret)
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ use std::sync::Arc;
|
|||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use rand::RngExt;
|
use rand::RngExt;
|
||||||
use tokio::sync::Semaphore;
|
|
||||||
use tokio::task::JoinSet;
|
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::config::MeFloorMode;
|
use crate::config::MeFloorMode;
|
||||||
@@ -16,7 +14,6 @@ use crate::crypto::SecureRandom;
|
|||||||
use crate::network::IpFamily;
|
use crate::network::IpFamily;
|
||||||
|
|
||||||
use super::MePool;
|
use super::MePool;
|
||||||
use super::pool::MeFamilyRuntimeState;
|
|
||||||
|
|
||||||
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
const JITTER_FRAC_NUM: u64 = 2; // jitter up to 50% of backoff
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
@@ -30,9 +27,6 @@ const HEALTH_RECONNECT_BUDGET_PER_CORE: usize = 2;
|
|||||||
const HEALTH_RECONNECT_BUDGET_PER_DC: usize = 1;
|
const HEALTH_RECONNECT_BUDGET_PER_DC: usize = 1;
|
||||||
const HEALTH_RECONNECT_BUDGET_MIN: usize = 4;
|
const HEALTH_RECONNECT_BUDGET_MIN: usize = 4;
|
||||||
const HEALTH_RECONNECT_BUDGET_MAX: usize = 128;
|
const HEALTH_RECONNECT_BUDGET_MAX: usize = 128;
|
||||||
const FAMILY_SUPPRESS_FAIL_STREAK_THRESHOLD: u32 = 5;
|
|
||||||
const FAMILY_SUPPRESS_DURATION_SECS: u64 = 60;
|
|
||||||
const FAMILY_RECOVER_SUCCESS_STREAK_TARGET: u32 = 2;
|
|
||||||
const HEALTH_DRAIN_CLOSE_BUDGET_PER_CORE: usize = 16;
|
const HEALTH_DRAIN_CLOSE_BUDGET_PER_CORE: usize = 16;
|
||||||
const HEALTH_DRAIN_CLOSE_BUDGET_MIN: usize = 16;
|
const HEALTH_DRAIN_CLOSE_BUDGET_MIN: usize = 16;
|
||||||
const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256;
|
const HEALTH_DRAIN_CLOSE_BUDGET_MAX: usize = 256;
|
||||||
@@ -62,17 +56,6 @@ struct FamilyFloorPlan {
|
|||||||
target_writers_total: usize,
|
target_writers_total: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct FamilyReconnectOutcome {
|
|
||||||
key: (i32, IpFamily),
|
|
||||||
dc: i32,
|
|
||||||
family: IpFamily,
|
|
||||||
alive: usize,
|
|
||||||
required: usize,
|
|
||||||
endpoint_count: usize,
|
|
||||||
restored: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_connections: usize) {
|
||||||
let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
|
let mut backoff: HashMap<(i32, IpFamily), u64> = HashMap::new();
|
||||||
let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
let mut next_attempt: HashMap<(i32, IpFamily), Instant> = HashMap::new();
|
||||||
@@ -95,7 +78,6 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
};
|
};
|
||||||
tokio::time::sleep(interval).await;
|
tokio::time::sleep(interval).await;
|
||||||
pool.prune_closed_writers().await;
|
pool.prune_closed_writers().await;
|
||||||
pool.sweep_endpoint_quarantine().await;
|
|
||||||
reap_draining_writers(&pool, &mut drain_warn_next_allowed).await;
|
reap_draining_writers(&pool, &mut drain_warn_next_allowed).await;
|
||||||
let v4_degraded = check_family(
|
let v4_degraded = check_family(
|
||||||
IpFamily::V4,
|
IpFamily::V4,
|
||||||
@@ -131,8 +113,6 @@ pub async fn me_health_monitor(pool: Arc<MePool>, rng: Arc<SecureRandom>, _min_c
|
|||||||
&mut floor_warn_next_allowed,
|
&mut floor_warn_next_allowed,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
update_family_runtime_state(&pool, IpFamily::V4, v4_degraded);
|
|
||||||
update_family_runtime_state(&pool, IpFamily::V6, v6_degraded);
|
|
||||||
degraded_interval = v4_degraded || v6_degraded;
|
degraded_interval = v4_degraded || v6_degraded;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -155,11 +135,9 @@ pub(super) async fn reap_draining_writers(
|
|||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
let now_epoch_secs = MePool::now_epoch_secs();
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let drain_ttl_secs = pool
|
let drain_ttl_secs = pool
|
||||||
.drain_runtime
|
|
||||||
.me_pool_drain_ttl_secs
|
.me_pool_drain_ttl_secs
|
||||||
.load(std::sync::atomic::Ordering::Relaxed);
|
.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
let drain_threshold = pool
|
let drain_threshold = pool
|
||||||
.drain_runtime
|
|
||||||
.me_pool_drain_threshold
|
.me_pool_drain_threshold
|
||||||
.load(std::sync::atomic::Ordering::Relaxed);
|
.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
let activity = pool.registry.writer_activity_snapshot().await;
|
let activity = pool.registry.writer_activity_snapshot().await;
|
||||||
@@ -243,10 +221,7 @@ pub(super) async fn reap_draining_writers(
|
|||||||
endpoint = %writer.addr,
|
endpoint = %writer.addr,
|
||||||
generation = writer.generation,
|
generation = writer.generation,
|
||||||
drain_ttl_secs,
|
drain_ttl_secs,
|
||||||
force_close_secs = pool
|
force_close_secs = pool.me_pool_force_close_secs.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
.drain_runtime
|
|
||||||
.me_pool_force_close_secs
|
|
||||||
.load(std::sync::atomic::Ordering::Relaxed),
|
|
||||||
allow_drain_fallback = writer.allow_drain_fallback,
|
allow_drain_fallback = writer.allow_drain_fallback,
|
||||||
"ME draining writer remains non-empty past drain TTL"
|
"ME draining writer remains non-empty past drain TTL"
|
||||||
);
|
);
|
||||||
@@ -390,8 +365,7 @@ async fn check_family(
|
|||||||
endpoints.sort_unstable();
|
endpoints.sort_unstable();
|
||||||
endpoints.dedup();
|
endpoints.dedup();
|
||||||
}
|
}
|
||||||
let reconnect_budget = health_reconnect_budget(pool, dc_endpoints.len());
|
let mut reconnect_budget = health_reconnect_budget(pool, dc_endpoints.len());
|
||||||
let reconnect_sem = Arc::new(Semaphore::new(reconnect_budget));
|
|
||||||
|
|
||||||
if pool.floor_mode() == MeFloorMode::Static {
|
if pool.floor_mode() == MeFloorMode::Static {
|
||||||
adaptive_idle_since.clear();
|
adaptive_idle_since.clear();
|
||||||
@@ -448,10 +422,6 @@ async fn check_family(
|
|||||||
floor_plan.active_writers_current,
|
floor_plan.active_writers_current,
|
||||||
floor_plan.warm_writers_current,
|
floor_plan.warm_writers_current,
|
||||||
);
|
);
|
||||||
let live_writer_ids_by_addr = Arc::new(live_writer_ids_by_addr);
|
|
||||||
let writer_idle_since = Arc::new(writer_idle_since);
|
|
||||||
let bound_clients_by_writer = Arc::new(bound_clients_by_writer);
|
|
||||||
let mut reconnect_set = JoinSet::<FamilyReconnectOutcome>::new();
|
|
||||||
|
|
||||||
for (dc, endpoints) in dc_endpoints {
|
for (dc, endpoints) in dc_endpoints {
|
||||||
if endpoints.is_empty() {
|
if endpoints.is_empty() {
|
||||||
@@ -491,7 +461,7 @@ async fn check_family(
|
|||||||
required,
|
required,
|
||||||
outage_backoff,
|
outage_backoff,
|
||||||
outage_next_attempt,
|
outage_next_attempt,
|
||||||
&reconnect_sem,
|
&mut reconnect_budget,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
continue;
|
continue;
|
||||||
@@ -525,9 +495,9 @@ async fn check_family(
|
|||||||
&endpoints,
|
&endpoints,
|
||||||
alive,
|
alive,
|
||||||
required,
|
required,
|
||||||
live_writer_ids_by_addr.as_ref(),
|
&live_writer_ids_by_addr,
|
||||||
writer_idle_since.as_ref(),
|
&writer_idle_since,
|
||||||
bound_clients_by_writer.as_ref(),
|
&bound_clients_by_writer,
|
||||||
idle_refresh_next_attempt,
|
idle_refresh_next_attempt,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -540,8 +510,8 @@ async fn check_family(
|
|||||||
&endpoints,
|
&endpoints,
|
||||||
alive,
|
alive,
|
||||||
required,
|
required,
|
||||||
live_writer_ids_by_addr.as_ref(),
|
&live_writer_ids_by_addr,
|
||||||
bound_clients_by_writer.as_ref(),
|
&bound_clients_by_writer,
|
||||||
shadow_rotate_deadline,
|
shadow_rotate_deadline,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -551,8 +521,8 @@ async fn check_family(
|
|||||||
family_degraded = true;
|
family_degraded = true;
|
||||||
|
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
if reconnect_sem.available_permits() == 0 {
|
if reconnect_budget == 0 {
|
||||||
let base_ms = pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64;
|
let base_ms = pool.me_reconnect_backoff_base.as_millis() as u64;
|
||||||
let next_ms = (*backoff.get(&key).unwrap_or(&base_ms)).max(base_ms);
|
let next_ms = (*backoff.get(&key).unwrap_or(&base_ms)).max(base_ms);
|
||||||
let jitter = next_ms / JITTER_FRAC_NUM;
|
let jitter = next_ms / JITTER_FRAC_NUM;
|
||||||
let wait = Duration::from_millis(next_ms)
|
let wait = Duration::from_millis(next_ms)
|
||||||
@@ -575,10 +545,7 @@ async fn check_family(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let max_concurrent = pool
|
let max_concurrent = pool.me_reconnect_max_concurrent_per_dc.max(1) as usize;
|
||||||
.reconnect_runtime
|
|
||||||
.me_reconnect_max_concurrent_per_dc
|
|
||||||
.max(1) as usize;
|
|
||||||
if *inflight.get(&key).unwrap_or(&0) >= max_concurrent {
|
if *inflight.get(&key).unwrap_or(&0) >= max_concurrent {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -597,165 +564,117 @@ async fn check_family(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
*inflight.entry(key).or_insert(0) += 1;
|
*inflight.entry(key).or_insert(0) += 1;
|
||||||
let pool_for_reconnect = pool.clone();
|
|
||||||
let rng_for_reconnect = rng.clone();
|
let mut restored = 0usize;
|
||||||
let reconnect_sem_for_dc = reconnect_sem.clone();
|
for _ in 0..missing {
|
||||||
let endpoints_for_dc = endpoints.clone();
|
if reconnect_budget == 0 {
|
||||||
let live_writer_ids_by_addr_for_dc = live_writer_ids_by_addr.clone();
|
break;
|
||||||
let writer_idle_since_for_dc = writer_idle_since.clone();
|
}
|
||||||
let bound_clients_by_writer_for_dc = bound_clients_by_writer.clone();
|
reconnect_budget = reconnect_budget.saturating_sub(1);
|
||||||
let active_cap_effective_total = floor_plan.active_cap_effective_total;
|
if pool.active_contour_writer_count_total().await
|
||||||
reconnect_set.spawn(async move {
|
>= floor_plan.active_cap_effective_total
|
||||||
let mut restored = 0usize;
|
{
|
||||||
for _ in 0..missing {
|
let swapped = maybe_swap_idle_writer_for_cap(
|
||||||
let Ok(reconnect_permit) = reconnect_sem_for_dc.clone().try_acquire_owned() else {
|
pool,
|
||||||
break;
|
rng,
|
||||||
};
|
dc,
|
||||||
if pool_for_reconnect.active_contour_writer_count_total().await
|
family,
|
||||||
>= active_cap_effective_total
|
&endpoints,
|
||||||
{
|
&live_writer_ids_by_addr,
|
||||||
let swapped = maybe_swap_idle_writer_for_cap(
|
&writer_idle_since,
|
||||||
&pool_for_reconnect,
|
&bound_clients_by_writer,
|
||||||
&rng_for_reconnect,
|
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
&endpoints_for_dc,
|
|
||||||
live_writer_ids_by_addr_for_dc.as_ref(),
|
|
||||||
writer_idle_since_for_dc.as_ref(),
|
|
||||||
bound_clients_by_writer_for_dc.as_ref(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
if swapped {
|
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
|
||||||
.increment_me_floor_swap_idle_total();
|
|
||||||
restored += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
|
||||||
.increment_me_floor_cap_block_total();
|
|
||||||
pool_for_reconnect
|
|
||||||
.stats
|
|
||||||
.increment_me_floor_swap_idle_failed_total();
|
|
||||||
debug!(
|
|
||||||
dc = %dc,
|
|
||||||
?family,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
active_cap_effective_total,
|
|
||||||
"Adaptive floor cap reached, reconnect attempt blocked"
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let res = tokio::time::timeout(
|
|
||||||
pool_for_reconnect.reconnect_runtime.me_one_timeout,
|
|
||||||
pool_for_reconnect.connect_endpoints_round_robin(
|
|
||||||
dc,
|
|
||||||
&endpoints_for_dc,
|
|
||||||
rng_for_reconnect.as_ref(),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
match res {
|
if swapped {
|
||||||
Ok(true) => {
|
pool.stats.increment_me_floor_swap_idle_total();
|
||||||
restored += 1;
|
restored += 1;
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_success();
|
continue;
|
||||||
}
|
|
||||||
Ok(false) => {
|
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_attempt();
|
|
||||||
debug!(dc = %dc, ?family, "ME round-robin reconnect failed")
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
pool_for_reconnect.stats.increment_me_reconnect_attempt();
|
|
||||||
debug!(dc = %dc, ?family, "ME reconnect timed out");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
drop(reconnect_permit);
|
pool.stats.increment_me_floor_cap_block_total();
|
||||||
|
pool.stats.increment_me_floor_swap_idle_failed_total();
|
||||||
|
debug!(
|
||||||
|
dc = %dc,
|
||||||
|
?family,
|
||||||
|
alive,
|
||||||
|
required,
|
||||||
|
active_cap_effective_total = floor_plan.active_cap_effective_total,
|
||||||
|
"Adaptive floor cap reached, reconnect attempt blocked"
|
||||||
|
);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
let res = tokio::time::timeout(
|
||||||
|
pool.me_one_timeout,
|
||||||
|
pool.connect_endpoints_round_robin(dc, &endpoints, rng.as_ref()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
match res {
|
||||||
|
Ok(true) => {
|
||||||
|
restored += 1;
|
||||||
|
pool.stats.increment_me_reconnect_success();
|
||||||
|
}
|
||||||
|
Ok(false) => {
|
||||||
|
pool.stats.increment_me_reconnect_attempt();
|
||||||
|
debug!(dc = %dc, ?family, "ME round-robin reconnect failed")
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
pool.stats.increment_me_reconnect_attempt();
|
||||||
|
debug!(dc = %dc, ?family, "ME reconnect timed out");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
FamilyReconnectOutcome {
|
let now_alive = alive + restored;
|
||||||
key,
|
if now_alive >= required {
|
||||||
dc,
|
|
||||||
family,
|
|
||||||
alive,
|
|
||||||
required,
|
|
||||||
endpoint_count: endpoints_for_dc.len(),
|
|
||||||
restored,
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
while let Some(joined) = reconnect_set.join_next().await {
|
|
||||||
let outcome = match joined {
|
|
||||||
Ok(outcome) => outcome,
|
|
||||||
Err(join_error) => {
|
|
||||||
debug!(error = %join_error, "Health reconnect task failed");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let now = Instant::now();
|
|
||||||
let now_alive = outcome.alive + outcome.restored;
|
|
||||||
if now_alive >= outcome.required {
|
|
||||||
info!(
|
info!(
|
||||||
dc = %outcome.dc,
|
dc = %dc,
|
||||||
family = ?outcome.family,
|
?family,
|
||||||
alive = now_alive,
|
alive = now_alive,
|
||||||
required = outcome.required,
|
required,
|
||||||
endpoint_count = outcome.endpoint_count,
|
endpoint_count = endpoints.len(),
|
||||||
"ME writer floor restored for DC"
|
"ME writer floor restored for DC"
|
||||||
);
|
);
|
||||||
backoff.insert(
|
backoff.insert(key, pool.me_reconnect_backoff_base.as_millis() as u64);
|
||||||
outcome.key,
|
let jitter = pool.me_reconnect_backoff_base.as_millis() as u64 / JITTER_FRAC_NUM;
|
||||||
pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64,
|
let wait = pool.me_reconnect_backoff_base
|
||||||
);
|
|
||||||
let jitter = pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64
|
|
||||||
/ JITTER_FRAC_NUM;
|
|
||||||
let wait = pool.reconnect_runtime.me_reconnect_backoff_base
|
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
||||||
next_attempt.insert(outcome.key, now + wait);
|
next_attempt.insert(key, now + wait);
|
||||||
} else {
|
} else {
|
||||||
let curr = *backoff
|
let curr = *backoff
|
||||||
.get(&outcome.key)
|
.get(&key)
|
||||||
.unwrap_or(&(pool.reconnect_runtime.me_reconnect_backoff_base.as_millis() as u64));
|
.unwrap_or(&(pool.me_reconnect_backoff_base.as_millis() as u64));
|
||||||
let next_ms = (curr.saturating_mul(2))
|
let next_ms =
|
||||||
.min(pool.reconnect_runtime.me_reconnect_backoff_cap.as_millis() as u64);
|
(curr.saturating_mul(2)).min(pool.me_reconnect_backoff_cap.as_millis() as u64);
|
||||||
backoff.insert(outcome.key, next_ms);
|
backoff.insert(key, next_ms);
|
||||||
let jitter = next_ms / JITTER_FRAC_NUM;
|
let jitter = next_ms / JITTER_FRAC_NUM;
|
||||||
let wait = Duration::from_millis(next_ms)
|
let wait = Duration::from_millis(next_ms)
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
+ Duration::from_millis(rand::rng().random_range(0..=jitter.max(1)));
|
||||||
next_attempt.insert(outcome.key, now + wait);
|
next_attempt.insert(key, now + wait);
|
||||||
if pool.is_runtime_ready() {
|
if pool.is_runtime_ready() {
|
||||||
let warn_cooldown = pool.warn_rate_limit_duration();
|
let warn_cooldown = pool.warn_rate_limit_duration();
|
||||||
if should_emit_rate_limited_warn(
|
if should_emit_rate_limited_warn(floor_warn_next_allowed, key, now, warn_cooldown) {
|
||||||
floor_warn_next_allowed,
|
|
||||||
outcome.key,
|
|
||||||
now,
|
|
||||||
warn_cooldown,
|
|
||||||
) {
|
|
||||||
warn!(
|
warn!(
|
||||||
dc = %outcome.dc,
|
dc = %dc,
|
||||||
family = ?outcome.family,
|
?family,
|
||||||
alive = now_alive,
|
alive = now_alive,
|
||||||
required = outcome.required,
|
required,
|
||||||
endpoint_count = outcome.endpoint_count,
|
endpoint_count = endpoints.len(),
|
||||||
backoff_ms = next_ms,
|
backoff_ms = next_ms,
|
||||||
"DC writer floor is below required level, scheduled reconnect"
|
"DC writer floor is below required level, scheduled reconnect"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
info!(
|
info!(
|
||||||
dc = %outcome.dc,
|
dc = %dc,
|
||||||
family = ?outcome.family,
|
?family,
|
||||||
alive = now_alive,
|
alive = now_alive,
|
||||||
required = outcome.required,
|
required,
|
||||||
endpoint_count = outcome.endpoint_count,
|
endpoint_count = endpoints.len(),
|
||||||
backoff_ms = next_ms,
|
backoff_ms = next_ms,
|
||||||
"DC writer floor is below required level during startup, scheduled reconnect"
|
"DC writer floor is below required level during startup, scheduled reconnect"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(v) = inflight.get_mut(&outcome.key) {
|
if let Some(v) = inflight.get_mut(&key) {
|
||||||
*v = v.saturating_sub(1);
|
*v = v.saturating_sub(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -772,68 +691,6 @@ fn health_reconnect_budget(pool: &Arc<MePool>, dc_groups: usize) -> usize {
|
|||||||
.clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX)
|
.clamp(HEALTH_RECONNECT_BUDGET_MIN, HEALTH_RECONNECT_BUDGET_MAX)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn update_family_runtime_state(pool: &Arc<MePool>, family: IpFamily, degraded: bool) {
|
|
||||||
let now_epoch_secs = MePool::now_epoch_secs();
|
|
||||||
let previous_state = pool.family_runtime_state(family);
|
|
||||||
let mut state_since_epoch_secs = pool.family_runtime_state_since_epoch_secs(family);
|
|
||||||
let previous_suppressed_until_epoch_secs = pool.family_suppressed_until_epoch_secs(family);
|
|
||||||
let previous_fail_streak = pool.family_fail_streak(family);
|
|
||||||
let previous_recover_success_streak = pool.family_recover_success_streak(family);
|
|
||||||
|
|
||||||
let (next_state, suppressed_until_epoch_secs, fail_streak, recover_success_streak) =
|
|
||||||
if previous_suppressed_until_epoch_secs > now_epoch_secs {
|
|
||||||
let fail_streak = if degraded {
|
|
||||||
previous_fail_streak.saturating_add(1)
|
|
||||||
} else {
|
|
||||||
previous_fail_streak
|
|
||||||
};
|
|
||||||
(
|
|
||||||
MeFamilyRuntimeState::Suppressed,
|
|
||||||
previous_suppressed_until_epoch_secs,
|
|
||||||
fail_streak,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
} else if degraded {
|
|
||||||
let fail_streak = previous_fail_streak.saturating_add(1);
|
|
||||||
if fail_streak >= FAMILY_SUPPRESS_FAIL_STREAK_THRESHOLD {
|
|
||||||
(
|
|
||||||
MeFamilyRuntimeState::Suppressed,
|
|
||||||
now_epoch_secs.saturating_add(FAMILY_SUPPRESS_DURATION_SECS),
|
|
||||||
fail_streak,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
(MeFamilyRuntimeState::Degraded, 0, fail_streak, 0)
|
|
||||||
}
|
|
||||||
} else if matches!(previous_state, MeFamilyRuntimeState::Healthy) {
|
|
||||||
(MeFamilyRuntimeState::Healthy, 0, 0, 0)
|
|
||||||
} else {
|
|
||||||
let recover_success_streak = previous_recover_success_streak.saturating_add(1);
|
|
||||||
if recover_success_streak >= FAMILY_RECOVER_SUCCESS_STREAK_TARGET {
|
|
||||||
(MeFamilyRuntimeState::Healthy, 0, 0, 0)
|
|
||||||
} else {
|
|
||||||
(
|
|
||||||
MeFamilyRuntimeState::Recovering,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
recover_success_streak,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if next_state != previous_state || state_since_epoch_secs == 0 {
|
|
||||||
state_since_epoch_secs = now_epoch_secs;
|
|
||||||
}
|
|
||||||
pool.set_family_runtime_state(
|
|
||||||
family,
|
|
||||||
next_state,
|
|
||||||
state_since_epoch_secs,
|
|
||||||
suppressed_until_epoch_secs,
|
|
||||||
fail_streak,
|
|
||||||
recover_success_streak,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn should_emit_rate_limited_warn(
|
fn should_emit_rate_limited_warn(
|
||||||
next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
next_allowed: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
key: (i32, IpFamily),
|
key: (i32, IpFamily),
|
||||||
@@ -858,7 +715,6 @@ fn adaptive_floor_class_min(
|
|||||||
) -> usize {
|
) -> usize {
|
||||||
if endpoint_count <= 1 {
|
if endpoint_count <= 1 {
|
||||||
let min_single = (pool
|
let min_single = (pool
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_min_writers_single_endpoint
|
.me_adaptive_floor_min_writers_single_endpoint
|
||||||
.load(std::sync::atomic::Ordering::Relaxed) as usize)
|
.load(std::sync::atomic::Ordering::Relaxed) as usize)
|
||||||
.max(1);
|
.max(1);
|
||||||
@@ -1115,7 +971,7 @@ async fn maybe_swap_idle_writer_for_cap(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let connected = match tokio::time::timeout(
|
let connected = match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1221,7 +1077,7 @@ async fn maybe_refresh_idle_writer_for_dc(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let rotate_ok = match tokio::time::timeout(
|
let rotate_ok = match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1332,7 +1188,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
required: usize,
|
required: usize,
|
||||||
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
|
outage_backoff: &mut HashMap<(i32, IpFamily), u64>,
|
||||||
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
outage_next_attempt: &mut HashMap<(i32, IpFamily), Instant>,
|
||||||
reconnect_sem: &Arc<Semaphore>,
|
reconnect_budget: &mut usize,
|
||||||
) {
|
) {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
if let Some(ts) = outage_next_attempt.get(&key)
|
if let Some(ts) = outage_next_attempt.get(&key)
|
||||||
@@ -1342,7 +1198,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms();
|
let (min_backoff_ms, max_backoff_ms) = pool.single_endpoint_outage_backoff_bounds_ms();
|
||||||
if reconnect_sem.available_permits() == 0 {
|
if *reconnect_budget == 0 {
|
||||||
outage_next_attempt.insert(key, now + Duration::from_millis(min_backoff_ms.max(250)));
|
outage_next_attempt.insert(key, now + Duration::from_millis(min_backoff_ms.max(250)));
|
||||||
debug!(
|
debug!(
|
||||||
dc = %key.0,
|
dc = %key.0,
|
||||||
@@ -1353,17 +1209,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let Ok(_reconnect_permit) = reconnect_sem.clone().try_acquire_owned() else {
|
*reconnect_budget = (*reconnect_budget).saturating_sub(1);
|
||||||
outage_next_attempt.insert(key, now + Duration::from_millis(min_backoff_ms.max(250)));
|
|
||||||
debug!(
|
|
||||||
dc = %key.0,
|
|
||||||
family = ?key.1,
|
|
||||||
%endpoint,
|
|
||||||
required,
|
|
||||||
"Single-endpoint outage reconnect deferred by semaphore saturation"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
pool.stats
|
pool.stats
|
||||||
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
||||||
|
|
||||||
@@ -1372,7 +1218,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
pool.stats
|
pool.stats
|
||||||
.increment_me_single_endpoint_quarantine_bypass_total();
|
.increment_me_single_endpoint_quarantine_bypass_total();
|
||||||
match tokio::time::timeout(
|
match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_one_for_dc(endpoint, key.0, rng.as_ref()),
|
pool.connect_one_for_dc(endpoint, key.0, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1401,7 +1247,7 @@ async fn recover_single_endpoint_outage(
|
|||||||
} else {
|
} else {
|
||||||
let one_endpoint = [endpoint];
|
let one_endpoint = [endpoint];
|
||||||
match tokio::time::timeout(
|
match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_endpoints_round_robin(key.0, &one_endpoint, rng.as_ref()),
|
pool.connect_endpoints_round_robin(key.0, &one_endpoint, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1526,7 +1372,7 @@ async fn maybe_rotate_single_endpoint_shadow(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let rotate_ok = match tokio::time::timeout(
|
let rotate_ok = match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.me_one_timeout,
|
||||||
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
pool.connect_one_for_dc(endpoint, dc, rng.as_ref()),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1841,8 +1687,6 @@ mod tests {
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
MeRouteNoWriterMode::default(),
|
MeRouteNoWriterMode::default(),
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
)
|
)
|
||||||
|
|||||||
+407
-777
File diff suppressed because it is too large
Load Diff
@@ -72,7 +72,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
if changed {
|
if changed {
|
||||||
self.rebuild_endpoint_dc_map().await;
|
self.rebuild_endpoint_dc_map().await;
|
||||||
self.notify_writer_epoch();
|
self.writer_available.notify_waiters();
|
||||||
}
|
}
|
||||||
if changed {
|
if changed {
|
||||||
SnapshotApplyOutcome::AppliedChanged
|
SnapshotApplyOutcome::AppliedChanged
|
||||||
@@ -112,7 +112,7 @@ impl MePool {
|
|||||||
|
|
||||||
pub async fn reconnect_all(self: &Arc<Self>) {
|
pub async fn reconnect_all(self: &Arc<Self>) {
|
||||||
let ws = self.writers.read().await.clone();
|
let ws = self.writers.read().await.clone();
|
||||||
for w in ws.iter() {
|
for w in ws {
|
||||||
if let Ok(()) = self
|
if let Ok(()) = self
|
||||||
.connect_one_for_dc(w.addr, w.writer_dc, self.rng.as_ref())
|
.connect_one_for_dc(w.addr, w.writer_dc, self.rng.as_ref())
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -14,10 +14,7 @@ use super::pool::MePool;
|
|||||||
impl MePool {
|
impl MePool {
|
||||||
pub async fn init(self: &Arc<Self>, pool_size: usize, rng: &Arc<SecureRandom>) -> Result<()> {
|
pub async fn init(self: &Arc<Self>, pool_size: usize, rng: &Arc<SecureRandom>) -> Result<()> {
|
||||||
let family_order = self.family_order();
|
let family_order = self.family_order();
|
||||||
let connect_concurrency = self
|
let connect_concurrency = self.me_reconnect_max_concurrent_per_dc.max(1) as usize;
|
||||||
.reconnect_runtime
|
|
||||||
.me_reconnect_max_concurrent_per_dc
|
|
||||||
.max(1) as usize;
|
|
||||||
let ks = self.key_selector().await;
|
let ks = self.key_selector().await;
|
||||||
info!(
|
info!(
|
||||||
me_servers = self.proxy_map_v4.read().await.len(),
|
me_servers = self.proxy_map_v4.read().await.len(),
|
||||||
@@ -253,12 +250,10 @@ impl MePool {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.reconnect_runtime.me_warmup_stagger_enabled {
|
if self.me_warmup_stagger_enabled {
|
||||||
let jitter = rand::rng().random_range(
|
let jitter =
|
||||||
0..=self.reconnect_runtime.me_warmup_step_jitter.as_millis() as u64,
|
rand::rng().random_range(0..=self.me_warmup_step_jitter.as_millis() as u64);
|
||||||
);
|
let delay_ms = self.me_warmup_step_delay.as_millis() as u64 + jitter;
|
||||||
let delay_ms =
|
|
||||||
self.reconnect_runtime.me_warmup_step_delay.as_millis() as u64 + jitter;
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,10 +42,10 @@ pub async fn detect_public_ip() -> Option<IpAddr> {
|
|||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
fn configured_stun_servers(&self) -> Vec<String> {
|
fn configured_stun_servers(&self) -> Vec<String> {
|
||||||
if !self.nat_runtime.nat_stun_servers.is_empty() {
|
if !self.nat_stun_servers.is_empty() {
|
||||||
return self.nat_runtime.nat_stun_servers.clone();
|
return self.nat_stun_servers.clone();
|
||||||
}
|
}
|
||||||
if let Some(s) = &self.nat_runtime.nat_stun
|
if let Some(s) = &self.nat_stun
|
||||||
&& !s.trim().is_empty()
|
&& !s.trim().is_empty()
|
||||||
{
|
{
|
||||||
return vec![s.clone()];
|
return vec![s.clone()];
|
||||||
@@ -64,7 +64,7 @@ impl MePool {
|
|||||||
let mut next_idx = 0usize;
|
let mut next_idx = 0usize;
|
||||||
let mut live_servers = Vec::new();
|
let mut live_servers = Vec::new();
|
||||||
let mut best_by_ip: HashMap<IpAddr, (usize, std::net::SocketAddr)> = HashMap::new();
|
let mut best_by_ip: HashMap<IpAddr, (usize, std::net::SocketAddr)> = HashMap::new();
|
||||||
let concurrency = self.nat_runtime.nat_probe_concurrency.max(1);
|
let concurrency = self.nat_probe_concurrency.max(1);
|
||||||
|
|
||||||
while next_idx < servers.len() || !join_set.is_empty() {
|
while next_idx < servers.len() || !join_set.is_empty() {
|
||||||
while next_idx < servers.len() && join_set.len() < concurrency {
|
while next_idx < servers.len() && join_set.len() < concurrency {
|
||||||
@@ -137,13 +137,9 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn translate_ip_for_nat(&self, ip: IpAddr) -> IpAddr {
|
pub(super) fn translate_ip_for_nat(&self, ip: IpAddr) -> IpAddr {
|
||||||
let nat_ip = self.nat_runtime.nat_ip_cfg.or_else(|| {
|
let nat_ip = self
|
||||||
self.nat_runtime
|
.nat_ip_cfg
|
||||||
.nat_ip_detected
|
.or_else(|| self.nat_ip_detected.try_read().ok().and_then(|g| *g));
|
||||||
.try_read()
|
|
||||||
.ok()
|
|
||||||
.and_then(|g| *g)
|
|
||||||
});
|
|
||||||
|
|
||||||
let Some(nat_ip) = nat_ip else {
|
let Some(nat_ip) = nat_ip else {
|
||||||
return ip;
|
return ip;
|
||||||
@@ -167,7 +163,7 @@ impl MePool {
|
|||||||
addr: std::net::SocketAddr,
|
addr: std::net::SocketAddr,
|
||||||
reflected: Option<std::net::SocketAddr>,
|
reflected: Option<std::net::SocketAddr>,
|
||||||
) -> std::net::SocketAddr {
|
) -> std::net::SocketAddr {
|
||||||
let ip = if let Some(nat_ip) = self.nat_runtime.nat_ip_cfg {
|
let ip = if let Some(nat_ip) = self.nat_ip_cfg {
|
||||||
match (addr.ip(), nat_ip) {
|
match (addr.ip(), nat_ip) {
|
||||||
(IpAddr::V4(_), IpAddr::V4(dst)) => IpAddr::V4(dst),
|
(IpAddr::V4(_), IpAddr::V4(dst)) => IpAddr::V4(dst),
|
||||||
(IpAddr::V6(_), IpAddr::V6(dst)) => IpAddr::V6(dst),
|
(IpAddr::V6(_), IpAddr::V6(dst)) => IpAddr::V6(dst),
|
||||||
@@ -189,22 +185,22 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn maybe_detect_nat_ip(&self, local_ip: IpAddr) -> Option<IpAddr> {
|
pub(super) async fn maybe_detect_nat_ip(&self, local_ip: IpAddr) -> Option<IpAddr> {
|
||||||
if self.nat_runtime.nat_ip_cfg.is_some() {
|
if self.nat_ip_cfg.is_some() {
|
||||||
return self.nat_runtime.nat_ip_cfg;
|
return self.nat_ip_cfg;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !(is_bogon(local_ip) || local_ip.is_loopback() || local_ip.is_unspecified()) {
|
if !(is_bogon(local_ip) || local_ip.is_loopback() || local_ip.is_unspecified()) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ip) = *self.nat_runtime.nat_ip_detected.read().await {
|
if let Some(ip) = *self.nat_ip_detected.read().await {
|
||||||
return Some(ip);
|
return Some(ip);
|
||||||
}
|
}
|
||||||
|
|
||||||
match fetch_public_ipv4_with_retry().await {
|
match fetch_public_ipv4_with_retry().await {
|
||||||
Ok(Some(ip)) => {
|
Ok(Some(ip)) => {
|
||||||
{
|
{
|
||||||
let mut guard = self.nat_runtime.nat_ip_detected.write().await;
|
let mut guard = self.nat_ip_detected.write().await;
|
||||||
*guard = Some(IpAddr::V4(ip));
|
*guard = Some(IpAddr::V4(ip));
|
||||||
}
|
}
|
||||||
info!(public_ip = %ip, "Auto-detected public IP for NAT translation");
|
info!(public_ip = %ip, "Auto-detected public IP for NAT translation");
|
||||||
@@ -235,10 +231,10 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
// Backoff window
|
// Backoff window
|
||||||
if use_shared_cache
|
if use_shared_cache
|
||||||
&& let Some(until) = *self.nat_runtime.stun_backoff_until.read().await
|
&& let Some(until) = *self.stun_backoff_until.read().await
|
||||||
&& Instant::now() < until
|
&& Instant::now() < until
|
||||||
{
|
{
|
||||||
if let Ok(cache) = self.nat_runtime.nat_reflection_cache.try_lock() {
|
if let Ok(cache) = self.nat_reflection_cache.try_lock() {
|
||||||
let slot = match family {
|
let slot = match family {
|
||||||
IpFamily::V4 => cache.v4,
|
IpFamily::V4 => cache.v4,
|
||||||
IpFamily::V6 => cache.v6,
|
IpFamily::V6 => cache.v6,
|
||||||
@@ -248,8 +244,7 @@ impl MePool {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
if use_shared_cache && let Ok(mut cache) = self.nat_runtime.nat_reflection_cache.try_lock()
|
if use_shared_cache && let Ok(mut cache) = self.nat_reflection_cache.try_lock() {
|
||||||
{
|
|
||||||
let slot = match family {
|
let slot = match family {
|
||||||
IpFamily::V4 => &mut cache.v4,
|
IpFamily::V4 => &mut cache.v4,
|
||||||
IpFamily::V6 => &mut cache.v6,
|
IpFamily::V6 => &mut cache.v6,
|
||||||
@@ -263,18 +258,18 @@ impl MePool {
|
|||||||
|
|
||||||
let _singleflight_guard = if use_shared_cache {
|
let _singleflight_guard = if use_shared_cache {
|
||||||
Some(match family {
|
Some(match family {
|
||||||
IpFamily::V4 => self.nat_runtime.nat_reflection_singleflight_v4.lock().await,
|
IpFamily::V4 => self.nat_reflection_singleflight_v4.lock().await,
|
||||||
IpFamily::V6 => self.nat_runtime.nat_reflection_singleflight_v6.lock().await,
|
IpFamily::V6 => self.nat_reflection_singleflight_v6.lock().await,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
if use_shared_cache
|
if use_shared_cache
|
||||||
&& let Some(until) = *self.nat_runtime.stun_backoff_until.read().await
|
&& let Some(until) = *self.stun_backoff_until.read().await
|
||||||
&& Instant::now() < until
|
&& Instant::now() < until
|
||||||
{
|
{
|
||||||
if let Ok(cache) = self.nat_runtime.nat_reflection_cache.try_lock() {
|
if let Ok(cache) = self.nat_reflection_cache.try_lock() {
|
||||||
let slot = match family {
|
let slot = match family {
|
||||||
IpFamily::V4 => cache.v4,
|
IpFamily::V4 => cache.v4,
|
||||||
IpFamily::V6 => cache.v6,
|
IpFamily::V6 => cache.v6,
|
||||||
@@ -284,8 +279,7 @@ impl MePool {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
if use_shared_cache && let Ok(mut cache) = self.nat_runtime.nat_reflection_cache.try_lock()
|
if use_shared_cache && let Ok(mut cache) = self.nat_reflection_cache.try_lock() {
|
||||||
{
|
|
||||||
let slot = match family {
|
let slot = match family {
|
||||||
IpFamily::V4 => &mut cache.v4,
|
IpFamily::V4 => &mut cache.v4,
|
||||||
IpFamily::V6 => &mut cache.v6,
|
IpFamily::V6 => &mut cache.v6,
|
||||||
@@ -298,14 +292,13 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let attempt = if use_shared_cache {
|
let attempt = if use_shared_cache {
|
||||||
self.nat_runtime
|
self.nat_probe_attempts
|
||||||
.nat_probe_attempts
|
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||||
} else {
|
} else {
|
||||||
0
|
0
|
||||||
};
|
};
|
||||||
let configured_servers = self.configured_stun_servers();
|
let configured_servers = self.configured_stun_servers();
|
||||||
let live_snapshot = self.nat_runtime.nat_stun_live_servers.read().await.clone();
|
let live_snapshot = self.nat_stun_live_servers.read().await.clone();
|
||||||
let primary_servers = if live_snapshot.is_empty() {
|
let primary_servers = if live_snapshot.is_empty() {
|
||||||
configured_servers.clone()
|
configured_servers.clone()
|
||||||
} else {
|
} else {
|
||||||
@@ -329,15 +322,14 @@ impl MePool {
|
|||||||
|
|
||||||
let live_server_count = live_servers.len();
|
let live_server_count = live_servers.len();
|
||||||
if !live_servers.is_empty() {
|
if !live_servers.is_empty() {
|
||||||
*self.nat_runtime.nat_stun_live_servers.write().await = live_servers;
|
*self.nat_stun_live_servers.write().await = live_servers;
|
||||||
} else {
|
} else {
|
||||||
self.nat_runtime.nat_stun_live_servers.write().await.clear();
|
self.nat_stun_live_servers.write().await.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(reflected_addr) = selected_reflected {
|
if let Some(reflected_addr) = selected_reflected {
|
||||||
if use_shared_cache {
|
if use_shared_cache {
|
||||||
self.nat_runtime
|
self.nat_probe_attempts
|
||||||
.nat_probe_attempts
|
|
||||||
.store(0, std::sync::atomic::Ordering::Relaxed);
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
info!(
|
info!(
|
||||||
@@ -346,9 +338,7 @@ impl MePool {
|
|||||||
"STUN-Quorum reached, IP: {}",
|
"STUN-Quorum reached, IP: {}",
|
||||||
reflected_addr.ip()
|
reflected_addr.ip()
|
||||||
);
|
);
|
||||||
if use_shared_cache
|
if use_shared_cache && let Ok(mut cache) = self.nat_reflection_cache.try_lock() {
|
||||||
&& let Ok(mut cache) = self.nat_runtime.nat_reflection_cache.try_lock()
|
|
||||||
{
|
|
||||||
let slot = match family {
|
let slot = match family {
|
||||||
IpFamily::V4 => &mut cache.v4,
|
IpFamily::V4 => &mut cache.v4,
|
||||||
IpFamily::V6 => &mut cache.v6,
|
IpFamily::V6 => &mut cache.v6,
|
||||||
@@ -360,7 +350,7 @@ impl MePool {
|
|||||||
|
|
||||||
if use_shared_cache {
|
if use_shared_cache {
|
||||||
let backoff = Duration::from_secs(60 * 2u64.pow((attempt as u32).min(6)));
|
let backoff = Duration::from_secs(60 * 2u64.pow((attempt as u32).min(6)));
|
||||||
*self.nat_runtime.stun_backoff_until.write().await = Some(Instant::now() + backoff);
|
*self.stun_backoff_until.write().await = Some(Instant::now() + backoff);
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,40 +13,13 @@ use super::pool::{MePool, RefillDcKey, RefillEndpointKey, WriterContour};
|
|||||||
|
|
||||||
const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20;
|
const ME_FLAP_UPTIME_THRESHOLD_SECS: u64 = 20;
|
||||||
const ME_FLAP_QUARANTINE_SECS: u64 = 25;
|
const ME_FLAP_QUARANTINE_SECS: u64 = 25;
|
||||||
const ME_FLAP_MIN_UPTIME_MILLIS: u64 = 500;
|
|
||||||
const ME_REFILL_TOTAL_ATTEMPT_CAP: u32 = 20;
|
|
||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
pub(super) async fn sweep_endpoint_quarantine(&self) {
|
|
||||||
let configured = self
|
|
||||||
.endpoint_dc_map
|
|
||||||
.read()
|
|
||||||
.await
|
|
||||||
.keys()
|
|
||||||
.copied()
|
|
||||||
.collect::<HashSet<SocketAddr>>();
|
|
||||||
let now = Instant::now();
|
|
||||||
let mut guard = self.endpoint_quarantine.lock().await;
|
|
||||||
guard.retain(|addr, expiry| *expiry > now && configured.contains(addr));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) async fn maybe_quarantine_flapping_endpoint(
|
pub(super) async fn maybe_quarantine_flapping_endpoint(
|
||||||
&self,
|
&self,
|
||||||
addr: SocketAddr,
|
addr: SocketAddr,
|
||||||
uptime: Duration,
|
uptime: Duration,
|
||||||
reason: &'static str,
|
|
||||||
) {
|
) {
|
||||||
if uptime < Duration::from_millis(ME_FLAP_MIN_UPTIME_MILLIS) {
|
|
||||||
debug!(
|
|
||||||
%addr,
|
|
||||||
reason,
|
|
||||||
uptime_ms = uptime.as_millis(),
|
|
||||||
min_uptime_ms = ME_FLAP_MIN_UPTIME_MILLIS,
|
|
||||||
"Skipping flap quarantine for ultra-short writer lifetime"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if uptime > Duration::from_secs(ME_FLAP_UPTIME_THRESHOLD_SECS) {
|
if uptime > Duration::from_secs(ME_FLAP_UPTIME_THRESHOLD_SECS) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -58,7 +31,6 @@ impl MePool {
|
|||||||
self.stats.increment_me_endpoint_quarantine_total();
|
self.stats.increment_me_endpoint_quarantine_total();
|
||||||
warn!(
|
warn!(
|
||||||
%addr,
|
%addr,
|
||||||
reason,
|
|
||||||
uptime_ms = uptime.as_millis(),
|
uptime_ms = uptime.as_millis(),
|
||||||
quarantine_secs = ME_FLAP_QUARANTINE_SECS,
|
quarantine_secs = ME_FLAP_QUARANTINE_SECS,
|
||||||
"ME endpoint temporarily quarantined due to rapid writer flap"
|
"ME endpoint temporarily quarantined due to rapid writer flap"
|
||||||
@@ -233,16 +205,11 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn refill_writer_after_loss(self: &Arc<Self>, addr: SocketAddr, writer_dc: i32) -> bool {
|
async fn refill_writer_after_loss(self: &Arc<Self>, addr: SocketAddr, writer_dc: i32) -> bool {
|
||||||
let fast_retries = self.reconnect_runtime.me_reconnect_fast_retry_count.max(1);
|
let fast_retries = self.me_reconnect_fast_retry_count.max(1);
|
||||||
let mut total_attempts = 0u32;
|
|
||||||
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
let same_endpoint_quarantined = self.is_endpoint_quarantined(addr).await;
|
||||||
|
|
||||||
if !same_endpoint_quarantined {
|
if !same_endpoint_quarantined {
|
||||||
for attempt in 0..fast_retries {
|
for attempt in 0..fast_retries {
|
||||||
if total_attempts >= ME_REFILL_TOTAL_ATTEMPT_CAP {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
total_attempts = total_attempts.saturating_add(1);
|
|
||||||
self.stats.increment_me_reconnect_attempt();
|
self.stats.increment_me_reconnect_attempt();
|
||||||
match self
|
match self
|
||||||
.connect_one_for_dc(addr, writer_dc, self.rng.as_ref())
|
.connect_one_for_dc(addr, writer_dc, self.rng.as_ref())
|
||||||
@@ -283,10 +250,6 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for attempt in 0..fast_retries {
|
for attempt in 0..fast_retries {
|
||||||
if total_attempts >= ME_REFILL_TOTAL_ATTEMPT_CAP {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
total_attempts = total_attempts.saturating_add(1);
|
|
||||||
self.stats.increment_me_reconnect_attempt();
|
self.stats.increment_me_reconnect_attempt();
|
||||||
if self
|
if self
|
||||||
.connect_endpoints_round_robin(writer_dc, &dc_endpoints, self.rng.as_ref())
|
.connect_endpoints_round_robin(writer_dc, &dc_endpoints, self.rng.as_ref())
|
||||||
|
|||||||
@@ -37,23 +37,16 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn clear_pending_hardswap_state(&self) {
|
fn clear_pending_hardswap_state(&self) {
|
||||||
self.reinit
|
self.pending_hardswap_generation.store(0, Ordering::Relaxed);
|
||||||
.pending_hardswap_generation
|
self.pending_hardswap_started_at_epoch_secs
|
||||||
.store(0, Ordering::Relaxed);
|
.store(0, Ordering::Relaxed);
|
||||||
self.reinit
|
self.pending_hardswap_map_hash.store(0, Ordering::Relaxed);
|
||||||
.pending_hardswap_started_at_epoch_secs
|
self.warm_generation.store(0, Ordering::Relaxed);
|
||||||
.store(0, Ordering::Relaxed);
|
|
||||||
self.reinit
|
|
||||||
.pending_hardswap_map_hash
|
|
||||||
.store(0, Ordering::Relaxed);
|
|
||||||
self.reinit.warm_generation.store(0, Ordering::Relaxed);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn promote_warm_generation_to_active(&self, generation: u64) {
|
async fn promote_warm_generation_to_active(&self, generation: u64) {
|
||||||
self.reinit
|
self.active_generation.store(generation, Ordering::Relaxed);
|
||||||
.active_generation
|
self.warm_generation.store(0, Ordering::Relaxed);
|
||||||
.store(generation, Ordering::Relaxed);
|
|
||||||
self.reinit.warm_generation.store(0, Ordering::Relaxed);
|
|
||||||
|
|
||||||
let ws = self.writers.read().await;
|
let ws = self.writers.read().await;
|
||||||
for writer in ws.iter() {
|
for writer in ws.iter() {
|
||||||
@@ -191,14 +184,8 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn hardswap_warmup_connect_delay_ms(&self) -> u64 {
|
fn hardswap_warmup_connect_delay_ms(&self) -> u64 {
|
||||||
let min_ms = self
|
let min_ms = self.me_hardswap_warmup_delay_min_ms.load(Ordering::Relaxed);
|
||||||
.reinit
|
let max_ms = self.me_hardswap_warmup_delay_max_ms.load(Ordering::Relaxed);
|
||||||
.me_hardswap_warmup_delay_min_ms
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
let max_ms = self
|
|
||||||
.reinit
|
|
||||||
.me_hardswap_warmup_delay_max_ms
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
let (min_ms, max_ms) = if min_ms <= max_ms {
|
let (min_ms, max_ms) = if min_ms <= max_ms {
|
||||||
(min_ms, max_ms)
|
(min_ms, max_ms)
|
||||||
} else {
|
} else {
|
||||||
@@ -212,11 +199,9 @@ impl MePool {
|
|||||||
|
|
||||||
fn hardswap_warmup_backoff_ms(&self, pass_idx: usize) -> u64 {
|
fn hardswap_warmup_backoff_ms(&self, pass_idx: usize) -> u64 {
|
||||||
let base_ms = self
|
let base_ms = self
|
||||||
.reinit
|
|
||||||
.me_hardswap_warmup_pass_backoff_base_ms
|
.me_hardswap_warmup_pass_backoff_base_ms
|
||||||
.load(Ordering::Relaxed);
|
.load(Ordering::Relaxed);
|
||||||
let cap_ms =
|
let cap_ms = (self.me_reconnect_backoff_cap.as_millis() as u64).max(base_ms);
|
||||||
(self.reconnect_runtime.me_reconnect_backoff_cap.as_millis() as u64).max(base_ms);
|
|
||||||
let shift = (pass_idx as u32).min(20);
|
let shift = (pass_idx as u32).min(20);
|
||||||
let scaled = base_ms.saturating_mul(1u64 << shift);
|
let scaled = base_ms.saturating_mul(1u64 << shift);
|
||||||
let core = scaled.min(cap_ms);
|
let core = scaled.min(cap_ms);
|
||||||
@@ -259,7 +244,6 @@ impl MePool {
|
|||||||
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
||||||
) {
|
) {
|
||||||
let extra_passes = self
|
let extra_passes = self
|
||||||
.reinit
|
|
||||||
.me_hardswap_warmup_extra_passes
|
.me_hardswap_warmup_extra_passes
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
.min(10) as usize;
|
.min(10) as usize;
|
||||||
@@ -385,20 +369,13 @@ impl MePool {
|
|||||||
|
|
||||||
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
|
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
|
||||||
let previous_generation = self.current_generation();
|
let previous_generation = self.current_generation();
|
||||||
let hardswap = self.reinit.hardswap.load(Ordering::Relaxed);
|
let hardswap = self.hardswap.load(Ordering::Relaxed);
|
||||||
let generation = if hardswap {
|
let generation = if hardswap {
|
||||||
let pending_generation = self
|
let pending_generation = self.pending_hardswap_generation.load(Ordering::Relaxed);
|
||||||
.reinit
|
|
||||||
.pending_hardswap_generation
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
let pending_started_at = self
|
let pending_started_at = self
|
||||||
.reinit
|
|
||||||
.pending_hardswap_started_at_epoch_secs
|
.pending_hardswap_started_at_epoch_secs
|
||||||
.load(Ordering::Relaxed);
|
.load(Ordering::Relaxed);
|
||||||
let pending_map_hash = self
|
let pending_map_hash = self.pending_hardswap_map_hash.load(Ordering::Relaxed);
|
||||||
.reinit
|
|
||||||
.pending_hardswap_map_hash
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
let pending_age_secs = now_epoch_secs.saturating_sub(pending_started_at);
|
let pending_age_secs = now_epoch_secs.saturating_sub(pending_started_at);
|
||||||
let pending_ttl_expired =
|
let pending_ttl_expired =
|
||||||
pending_started_at > 0 && pending_age_secs > ME_HARDSWAP_PENDING_TTL_SECS;
|
pending_started_at > 0 && pending_age_secs > ME_HARDSWAP_PENDING_TTL_SECS;
|
||||||
@@ -428,30 +405,24 @@ impl MePool {
|
|||||||
"ME hardswap pending generation expired by TTL; starting fresh generation"
|
"ME hardswap pending generation expired by TTL; starting fresh generation"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let next_generation = self.reinit.generation.fetch_add(1, Ordering::Relaxed) + 1;
|
let next_generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
self.reinit
|
self.pending_hardswap_generation
|
||||||
.pending_hardswap_generation
|
|
||||||
.store(next_generation, Ordering::Relaxed);
|
.store(next_generation, Ordering::Relaxed);
|
||||||
self.reinit
|
self.pending_hardswap_started_at_epoch_secs
|
||||||
.pending_hardswap_started_at_epoch_secs
|
|
||||||
.store(now_epoch_secs, Ordering::Relaxed);
|
.store(now_epoch_secs, Ordering::Relaxed);
|
||||||
self.reinit
|
self.pending_hardswap_map_hash
|
||||||
.pending_hardswap_map_hash
|
|
||||||
.store(desired_map_hash, Ordering::Relaxed);
|
.store(desired_map_hash, Ordering::Relaxed);
|
||||||
self.reinit
|
self.warm_generation
|
||||||
.warm_generation
|
|
||||||
.store(next_generation, Ordering::Relaxed);
|
.store(next_generation, Ordering::Relaxed);
|
||||||
next_generation
|
next_generation
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
self.clear_pending_hardswap_state();
|
self.clear_pending_hardswap_state();
|
||||||
self.reinit.generation.fetch_add(1, Ordering::Relaxed) + 1
|
self.generation.fetch_add(1, Ordering::Relaxed) + 1
|
||||||
};
|
};
|
||||||
|
|
||||||
if hardswap {
|
if hardswap {
|
||||||
self.reinit
|
self.warm_generation.store(generation, Ordering::Relaxed);
|
||||||
.warm_generation
|
|
||||||
.store(generation, Ordering::Relaxed);
|
|
||||||
self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc)
|
self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc)
|
||||||
.await;
|
.await;
|
||||||
} else {
|
} else {
|
||||||
@@ -465,8 +436,7 @@ impl MePool {
|
|||||||
.map(|w| (w.writer_dc, w.addr))
|
.map(|w| (w.writer_dc, w.addr))
|
||||||
.collect();
|
.collect();
|
||||||
let min_ratio = Self::permille_to_ratio(
|
let min_ratio = Self::permille_to_ratio(
|
||||||
self.drain_runtime
|
self.me_pool_min_fresh_ratio_permille
|
||||||
.me_pool_min_fresh_ratio_permille
|
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
);
|
);
|
||||||
let (coverage_ratio, missing_dc) =
|
let (coverage_ratio, missing_dc) =
|
||||||
|
|||||||
@@ -94,9 +94,9 @@ impl MePool {
|
|||||||
|
|
||||||
pub(crate) async fn api_nat_stun_snapshot(&self) -> MeApiNatStunSnapshot {
|
pub(crate) async fn api_nat_stun_snapshot(&self) -> MeApiNatStunSnapshot {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut configured_servers = if !self.nat_runtime.nat_stun_servers.is_empty() {
|
let mut configured_servers = if !self.nat_stun_servers.is_empty() {
|
||||||
self.nat_runtime.nat_stun_servers.clone()
|
self.nat_stun_servers.clone()
|
||||||
} else if let Some(stun) = &self.nat_runtime.nat_stun {
|
} else if let Some(stun) = &self.nat_stun {
|
||||||
if stun.trim().is_empty() {
|
if stun.trim().is_empty() {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
} else {
|
} else {
|
||||||
@@ -108,11 +108,11 @@ impl MePool {
|
|||||||
configured_servers.sort();
|
configured_servers.sort();
|
||||||
configured_servers.dedup();
|
configured_servers.dedup();
|
||||||
|
|
||||||
let mut live_servers = self.nat_runtime.nat_stun_live_servers.read().await.clone();
|
let mut live_servers = self.nat_stun_live_servers.read().await.clone();
|
||||||
live_servers.sort();
|
live_servers.sort();
|
||||||
live_servers.dedup();
|
live_servers.dedup();
|
||||||
|
|
||||||
let reflection = self.nat_runtime.nat_reflection_cache.lock().await;
|
let reflection = self.nat_reflection_cache.lock().await;
|
||||||
let reflection_v4 = reflection.v4.map(|(ts, addr)| MeApiNatReflectionSnapshot {
|
let reflection_v4 = reflection.v4.map(|(ts, addr)| MeApiNatReflectionSnapshot {
|
||||||
addr,
|
addr,
|
||||||
age_secs: now.saturating_duration_since(ts).as_secs(),
|
age_secs: now.saturating_duration_since(ts).as_secs(),
|
||||||
@@ -123,19 +123,17 @@ impl MePool {
|
|||||||
});
|
});
|
||||||
drop(reflection);
|
drop(reflection);
|
||||||
|
|
||||||
let backoff_until = *self.nat_runtime.stun_backoff_until.read().await;
|
let backoff_until = *self.stun_backoff_until.read().await;
|
||||||
let stun_backoff_remaining_ms = backoff_until.and_then(|until| {
|
let stun_backoff_remaining_ms = backoff_until.and_then(|until| {
|
||||||
(until > now).then_some(until.duration_since(now).as_millis() as u64)
|
(until > now).then_some(until.duration_since(now).as_millis() as u64)
|
||||||
});
|
});
|
||||||
|
|
||||||
MeApiNatStunSnapshot {
|
MeApiNatStunSnapshot {
|
||||||
nat_probe_enabled: self.nat_runtime.nat_probe,
|
nat_probe_enabled: self.nat_probe,
|
||||||
nat_probe_disabled_runtime: self
|
nat_probe_disabled_runtime: self
|
||||||
.nat_runtime
|
|
||||||
.nat_probe_disabled
|
.nat_probe_disabled
|
||||||
.load(std::sync::atomic::Ordering::Relaxed),
|
.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
nat_probe_attempts: self
|
nat_probe_attempts: self
|
||||||
.nat_runtime
|
|
||||||
.nat_probe_attempts
|
.nat_probe_attempts
|
||||||
.load(std::sync::atomic::Ordering::Relaxed),
|
.load(std::sync::atomic::Ordering::Relaxed),
|
||||||
configured_servers,
|
configured_servers,
|
||||||
|
|||||||
@@ -160,7 +160,7 @@ impl MePool {
|
|||||||
|
|
||||||
let writers = self.writers.read().await.clone();
|
let writers = self.writers.read().await.clone();
|
||||||
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
||||||
for writer in writers.iter() {
|
for writer in writers {
|
||||||
if writer.draining.load(Ordering::Relaxed) {
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -197,7 +197,7 @@ impl MePool {
|
|||||||
|
|
||||||
let writers = self.writers.read().await.clone();
|
let writers = self.writers.read().await.clone();
|
||||||
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
let mut live_writers_by_dc = HashMap::<i16, usize>::new();
|
||||||
for writer in writers.iter() {
|
for writer in writers {
|
||||||
if writer.draining.load(Ordering::Relaxed) {
|
if writer.draining.load(Ordering::Relaxed) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -224,10 +224,7 @@ impl MePool {
|
|||||||
pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot {
|
pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot {
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
let active_generation = self.current_generation();
|
let active_generation = self.current_generation();
|
||||||
let drain_ttl_secs = self
|
let drain_ttl_secs = self.me_pool_drain_ttl_secs.load(Ordering::Relaxed);
|
||||||
.drain_runtime
|
|
||||||
.me_pool_drain_ttl_secs
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
|
|
||||||
let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
|
let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
|
||||||
if self.decision.ipv4_me {
|
if self.decision.ipv4_me {
|
||||||
@@ -258,7 +255,7 @@ impl MePool {
|
|||||||
let mut dc_rtt_agg = HashMap::<i16, (f64, u64)>::new();
|
let mut dc_rtt_agg = HashMap::<i16, (f64, u64)>::new();
|
||||||
let mut writer_rows = Vec::<MeApiWriterStatusSnapshot>::with_capacity(writers.len());
|
let mut writer_rows = Vec::<MeApiWriterStatusSnapshot>::with_capacity(writers.len());
|
||||||
|
|
||||||
for writer in writers.iter() {
|
for writer in writers {
|
||||||
let endpoint = writer.addr;
|
let endpoint = writer.addr;
|
||||||
let dc = i16::try_from(writer.writer_dc).ok();
|
let dc = i16::try_from(writer.writer_dc).ok();
|
||||||
let draining = writer.draining.load(Ordering::Relaxed);
|
let draining = writer.draining.load(Ordering::Relaxed);
|
||||||
@@ -339,7 +336,6 @@ impl MePool {
|
|||||||
let mut fresh_alive_writers = 0usize;
|
let mut fresh_alive_writers = 0usize;
|
||||||
let floor_mode = self.floor_mode();
|
let floor_mode = self.floor_mode();
|
||||||
let adaptive_cpu_cores = (self
|
let adaptive_cpu_cores = (self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_cpu_cores_effective
|
.me_adaptive_floor_cpu_cores_effective
|
||||||
.load(Ordering::Relaxed) as usize)
|
.load(Ordering::Relaxed) as usize)
|
||||||
.max(1);
|
.max(1);
|
||||||
@@ -354,26 +350,22 @@ impl MePool {
|
|||||||
self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
|
self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
|
||||||
let floor_min = if endpoint_count <= 1 {
|
let floor_min = if endpoint_count <= 1 {
|
||||||
(self
|
(self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_min_writers_single_endpoint
|
.me_adaptive_floor_min_writers_single_endpoint
|
||||||
.load(Ordering::Relaxed) as usize)
|
.load(Ordering::Relaxed) as usize)
|
||||||
.max(1)
|
.max(1)
|
||||||
.min(base_required.max(1))
|
.min(base_required.max(1))
|
||||||
} else {
|
} else {
|
||||||
(self
|
(self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_min_writers_multi_endpoint
|
.me_adaptive_floor_min_writers_multi_endpoint
|
||||||
.load(Ordering::Relaxed) as usize)
|
.load(Ordering::Relaxed) as usize)
|
||||||
.max(1)
|
.max(1)
|
||||||
.min(base_required.max(1))
|
.min(base_required.max(1))
|
||||||
};
|
};
|
||||||
let extra_per_core = if endpoint_count <= 1 {
|
let extra_per_core = if endpoint_count <= 1 {
|
||||||
self.floor_runtime
|
self.me_adaptive_floor_max_extra_writers_single_per_core
|
||||||
.me_adaptive_floor_max_extra_writers_single_per_core
|
|
||||||
.load(Ordering::Relaxed) as usize
|
.load(Ordering::Relaxed) as usize
|
||||||
} else {
|
} else {
|
||||||
self.floor_runtime
|
self.me_adaptive_floor_max_extra_writers_multi_per_core
|
||||||
.me_adaptive_floor_max_extra_writers_multi_per_core
|
|
||||||
.load(Ordering::Relaxed) as usize
|
.load(Ordering::Relaxed) as usize
|
||||||
};
|
};
|
||||||
let floor_max =
|
let floor_max =
|
||||||
@@ -444,7 +436,6 @@ impl MePool {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
let pending_started_at = self
|
let pending_started_at = self
|
||||||
.reinit
|
|
||||||
.pending_hardswap_started_at_epoch_secs
|
.pending_hardswap_started_at_epoch_secs
|
||||||
.load(Ordering::Relaxed);
|
.load(Ordering::Relaxed);
|
||||||
let pending_hardswap_age_secs =
|
let pending_hardswap_age_secs =
|
||||||
@@ -486,175 +477,119 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
MeApiRuntimeSnapshot {
|
MeApiRuntimeSnapshot {
|
||||||
active_generation: self.reinit.active_generation.load(Ordering::Relaxed),
|
active_generation: self.active_generation.load(Ordering::Relaxed),
|
||||||
warm_generation: self.reinit.warm_generation.load(Ordering::Relaxed),
|
warm_generation: self.warm_generation.load(Ordering::Relaxed),
|
||||||
pending_hardswap_generation: self
|
pending_hardswap_generation: self.pending_hardswap_generation.load(Ordering::Relaxed),
|
||||||
.reinit
|
|
||||||
.pending_hardswap_generation
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
pending_hardswap_age_secs,
|
pending_hardswap_age_secs,
|
||||||
hardswap_enabled: self.reinit.hardswap.load(Ordering::Relaxed),
|
hardswap_enabled: self.hardswap.load(Ordering::Relaxed),
|
||||||
floor_mode: floor_mode_label(self.floor_mode()),
|
floor_mode: floor_mode_label(self.floor_mode()),
|
||||||
adaptive_floor_idle_secs: self
|
adaptive_floor_idle_secs: self.me_adaptive_floor_idle_secs.load(Ordering::Relaxed),
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_idle_secs
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
adaptive_floor_min_writers_single_endpoint: self
|
adaptive_floor_min_writers_single_endpoint: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_min_writers_single_endpoint
|
.me_adaptive_floor_min_writers_single_endpoint
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_min_writers_multi_endpoint: self
|
adaptive_floor_min_writers_multi_endpoint: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_min_writers_multi_endpoint
|
.me_adaptive_floor_min_writers_multi_endpoint
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_recover_grace_secs: self
|
adaptive_floor_recover_grace_secs: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_recover_grace_secs
|
.me_adaptive_floor_recover_grace_secs
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_writers_per_core_total: self
|
adaptive_floor_writers_per_core_total: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_writers_per_core_total
|
.me_adaptive_floor_writers_per_core_total
|
||||||
.load(Ordering::Relaxed) as u16,
|
.load(Ordering::Relaxed) as u16,
|
||||||
adaptive_floor_cpu_cores_override: self
|
adaptive_floor_cpu_cores_override: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_cpu_cores_override
|
.me_adaptive_floor_cpu_cores_override
|
||||||
.load(Ordering::Relaxed) as u16,
|
.load(Ordering::Relaxed) as u16,
|
||||||
adaptive_floor_max_extra_writers_single_per_core: self
|
adaptive_floor_max_extra_writers_single_per_core: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_max_extra_writers_single_per_core
|
.me_adaptive_floor_max_extra_writers_single_per_core
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
as u16,
|
as u16,
|
||||||
adaptive_floor_max_extra_writers_multi_per_core: self
|
adaptive_floor_max_extra_writers_multi_per_core: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_max_extra_writers_multi_per_core
|
.me_adaptive_floor_max_extra_writers_multi_per_core
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
as u16,
|
as u16,
|
||||||
adaptive_floor_max_active_writers_per_core: self
|
adaptive_floor_max_active_writers_per_core: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_max_active_writers_per_core
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
as u16,
|
as u16,
|
||||||
adaptive_floor_max_warm_writers_per_core: self
|
adaptive_floor_max_warm_writers_per_core: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_max_warm_writers_per_core
|
.me_adaptive_floor_max_warm_writers_per_core
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
as u16,
|
as u16,
|
||||||
adaptive_floor_max_active_writers_global: self
|
adaptive_floor_max_active_writers_global: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_max_active_writers_global
|
.me_adaptive_floor_max_active_writers_global
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_max_warm_writers_global: self
|
adaptive_floor_max_warm_writers_global: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_max_warm_writers_global
|
.me_adaptive_floor_max_warm_writers_global
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_cpu_cores_detected: self
|
adaptive_floor_cpu_cores_detected: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_cpu_cores_detected
|
.me_adaptive_floor_cpu_cores_detected
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_cpu_cores_effective: self
|
adaptive_floor_cpu_cores_effective: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_cpu_cores_effective
|
.me_adaptive_floor_cpu_cores_effective
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_global_cap_raw: self
|
adaptive_floor_global_cap_raw: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_global_cap_raw
|
.me_adaptive_floor_global_cap_raw
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_global_cap_effective: self
|
adaptive_floor_global_cap_effective: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_global_cap_effective
|
.me_adaptive_floor_global_cap_effective
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_target_writers_total: self
|
adaptive_floor_target_writers_total: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_target_writers_total
|
.me_adaptive_floor_target_writers_total
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_active_cap_configured: self
|
adaptive_floor_active_cap_configured: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_active_cap_configured
|
.me_adaptive_floor_active_cap_configured
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_active_cap_effective: self
|
adaptive_floor_active_cap_effective: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_active_cap_effective
|
.me_adaptive_floor_active_cap_effective
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_warm_cap_configured: self
|
adaptive_floor_warm_cap_configured: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_warm_cap_configured
|
.me_adaptive_floor_warm_cap_configured
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_warm_cap_effective: self
|
adaptive_floor_warm_cap_effective: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_warm_cap_effective
|
.me_adaptive_floor_warm_cap_effective
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_active_writers_current: self
|
adaptive_floor_active_writers_current: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_active_writers_current
|
.me_adaptive_floor_active_writers_current
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
adaptive_floor_warm_writers_current: self
|
adaptive_floor_warm_writers_current: self
|
||||||
.floor_runtime
|
|
||||||
.me_adaptive_floor_warm_writers_current
|
.me_adaptive_floor_warm_writers_current
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_keepalive_enabled: self.writer_lifecycle.me_keepalive_enabled,
|
me_keepalive_enabled: self.me_keepalive_enabled,
|
||||||
me_keepalive_interval_secs: self.writer_lifecycle.me_keepalive_interval.as_secs(),
|
me_keepalive_interval_secs: self.me_keepalive_interval.as_secs(),
|
||||||
me_keepalive_jitter_secs: self.writer_lifecycle.me_keepalive_jitter.as_secs(),
|
me_keepalive_jitter_secs: self.me_keepalive_jitter.as_secs(),
|
||||||
me_keepalive_payload_random: self.writer_lifecycle.me_keepalive_payload_random,
|
me_keepalive_payload_random: self.me_keepalive_payload_random,
|
||||||
rpc_proxy_req_every_secs: self
|
rpc_proxy_req_every_secs: self.rpc_proxy_req_every_secs.load(Ordering::Relaxed),
|
||||||
.writer_lifecycle
|
me_reconnect_max_concurrent_per_dc: self.me_reconnect_max_concurrent_per_dc,
|
||||||
.rpc_proxy_req_every_secs
|
me_reconnect_backoff_base_ms: self.me_reconnect_backoff_base.as_millis() as u64,
|
||||||
.load(Ordering::Relaxed),
|
me_reconnect_backoff_cap_ms: self.me_reconnect_backoff_cap.as_millis() as u64,
|
||||||
me_reconnect_max_concurrent_per_dc: self
|
me_reconnect_fast_retry_count: self.me_reconnect_fast_retry_count,
|
||||||
.reconnect_runtime
|
me_pool_drain_ttl_secs: self.me_pool_drain_ttl_secs.load(Ordering::Relaxed),
|
||||||
.me_reconnect_max_concurrent_per_dc,
|
me_pool_force_close_secs: self.me_pool_force_close_secs.load(Ordering::Relaxed),
|
||||||
me_reconnect_backoff_base_ms: self
|
|
||||||
.reconnect_runtime
|
|
||||||
.me_reconnect_backoff_base
|
|
||||||
.as_millis() as u64,
|
|
||||||
me_reconnect_backoff_cap_ms: self.reconnect_runtime.me_reconnect_backoff_cap.as_millis()
|
|
||||||
as u64,
|
|
||||||
me_reconnect_fast_retry_count: self.reconnect_runtime.me_reconnect_fast_retry_count,
|
|
||||||
me_pool_drain_ttl_secs: self
|
|
||||||
.drain_runtime
|
|
||||||
.me_pool_drain_ttl_secs
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
me_pool_force_close_secs: self
|
|
||||||
.drain_runtime
|
|
||||||
.me_pool_force_close_secs
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
me_pool_min_fresh_ratio: Self::permille_to_ratio(
|
me_pool_min_fresh_ratio: Self::permille_to_ratio(
|
||||||
self.drain_runtime
|
self.me_pool_min_fresh_ratio_permille
|
||||||
.me_pool_min_fresh_ratio_permille
|
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
),
|
),
|
||||||
me_bind_stale_mode: bind_stale_mode_label(self.bind_stale_mode()),
|
me_bind_stale_mode: bind_stale_mode_label(self.bind_stale_mode()),
|
||||||
me_bind_stale_ttl_secs: self
|
me_bind_stale_ttl_secs: self.me_bind_stale_ttl_secs.load(Ordering::Relaxed),
|
||||||
.binding_policy
|
|
||||||
.me_bind_stale_ttl_secs
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
me_single_endpoint_shadow_writers: self
|
me_single_endpoint_shadow_writers: self
|
||||||
.single_endpoint_runtime
|
|
||||||
.me_single_endpoint_shadow_writers
|
.me_single_endpoint_shadow_writers
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_single_endpoint_outage_mode_enabled: self
|
me_single_endpoint_outage_mode_enabled: self
|
||||||
.single_endpoint_runtime
|
|
||||||
.me_single_endpoint_outage_mode_enabled
|
.me_single_endpoint_outage_mode_enabled
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_single_endpoint_outage_disable_quarantine: self
|
me_single_endpoint_outage_disable_quarantine: self
|
||||||
.single_endpoint_runtime
|
|
||||||
.me_single_endpoint_outage_disable_quarantine
|
.me_single_endpoint_outage_disable_quarantine
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_single_endpoint_outage_backoff_min_ms: self
|
me_single_endpoint_outage_backoff_min_ms: self
|
||||||
.single_endpoint_runtime
|
|
||||||
.me_single_endpoint_outage_backoff_min_ms
|
.me_single_endpoint_outage_backoff_min_ms
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_single_endpoint_outage_backoff_max_ms: self
|
me_single_endpoint_outage_backoff_max_ms: self
|
||||||
.single_endpoint_runtime
|
|
||||||
.me_single_endpoint_outage_backoff_max_ms
|
.me_single_endpoint_outage_backoff_max_ms
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_single_endpoint_shadow_rotate_every_secs: self
|
me_single_endpoint_shadow_rotate_every_secs: self
|
||||||
.single_endpoint_runtime
|
|
||||||
.me_single_endpoint_shadow_rotate_every_secs
|
.me_single_endpoint_shadow_rotate_every_secs
|
||||||
.load(Ordering::Relaxed),
|
.load(Ordering::Relaxed),
|
||||||
me_deterministic_writer_sort: self
|
me_deterministic_writer_sort: self.me_deterministic_writer_sort.load(Ordering::Relaxed),
|
||||||
.writer_selection_policy
|
|
||||||
.me_deterministic_writer_sort
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
me_writer_pick_mode: writer_pick_mode_label(self.writer_pick_mode()),
|
me_writer_pick_mode: writer_pick_mode_label(self.writer_pick_mode()),
|
||||||
me_writer_pick_sample_size: self.writer_pick_sample_size() as u8,
|
me_writer_pick_sample_size: self.writer_pick_sample_size() as u8,
|
||||||
me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()),
|
me_socks_kdf_policy: socks_kdf_policy_label(self.socks_kdf_policy()),
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
use std::collections::HashMap;
|
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -26,7 +25,6 @@ const ME_ACTIVE_PING_SECS: u64 = 25;
|
|||||||
const ME_ACTIVE_PING_JITTER_SECS: i64 = 5;
|
const ME_ACTIVE_PING_JITTER_SECS: i64 = 5;
|
||||||
const ME_IDLE_KEEPALIVE_MAX_SECS: u64 = 5;
|
const ME_IDLE_KEEPALIVE_MAX_SECS: u64 = 5;
|
||||||
const ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS: u64 = 700;
|
const ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS: u64 = 700;
|
||||||
const ME_PING_TRACKER_CLEANUP_EVERY: u32 = 32;
|
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
enum WriterTeardownMode {
|
enum WriterTeardownMode {
|
||||||
@@ -38,240 +36,6 @@ fn is_me_peer_closed_error(error: &ProxyError) -> bool {
|
|||||||
matches!(error, ProxyError::Io(ioe) if ioe.kind() == ErrorKind::UnexpectedEof)
|
matches!(error, ProxyError::Io(ioe) if ioe.kind() == ErrorKind::UnexpectedEof)
|
||||||
}
|
}
|
||||||
|
|
||||||
enum WriterLifecycleExit {
|
|
||||||
Reader(Result<()>),
|
|
||||||
Writer(Result<()>),
|
|
||||||
Ping,
|
|
||||||
Signal,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn writer_command_loop(
|
|
||||||
mut rx: mpsc::Receiver<WriterCommand>,
|
|
||||||
mut rpc_writer: RpcWriter,
|
|
||||||
cancel: CancellationToken,
|
|
||||||
) -> Result<()> {
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
cmd = rx.recv() => {
|
|
||||||
match cmd {
|
|
||||||
Some(WriterCommand::Data(payload)) => {
|
|
||||||
rpc_writer.send(&payload).await?;
|
|
||||||
}
|
|
||||||
Some(WriterCommand::DataAndFlush(payload)) => {
|
|
||||||
rpc_writer.send_and_flush(&payload).await?;
|
|
||||||
}
|
|
||||||
Some(WriterCommand::Close) | None => return Ok(()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ = cancel.cancelled() => return Ok(()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
async fn ping_loop(
|
|
||||||
pool_ping: std::sync::Weak<MePool>,
|
|
||||||
writer_id: u64,
|
|
||||||
tx_ping: mpsc::Sender<WriterCommand>,
|
|
||||||
ping_tracker_ping: Arc<tokio::sync::Mutex<HashMap<i64, Instant>>>,
|
|
||||||
stats_ping: Arc<crate::stats::Stats>,
|
|
||||||
keepalive_enabled: bool,
|
|
||||||
keepalive_interval: Duration,
|
|
||||||
keepalive_jitter: Duration,
|
|
||||||
cancel_ping_token: CancellationToken,
|
|
||||||
) {
|
|
||||||
let mut ping_id: i64 = rand::random::<i64>();
|
|
||||||
let mut cleanup_tick: u32 = 0;
|
|
||||||
let idle_interval_cap = Duration::from_secs(ME_IDLE_KEEPALIVE_MAX_SECS);
|
|
||||||
// Per-writer jittered start to avoid phase sync.
|
|
||||||
let startup_jitter = if keepalive_enabled {
|
|
||||||
let mut interval = keepalive_interval;
|
|
||||||
let Some(pool) = pool_ping.upgrade() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if pool.registry.is_writer_empty(writer_id).await {
|
|
||||||
interval = interval.min(idle_interval_cap);
|
|
||||||
}
|
|
||||||
let jitter_cap_ms = interval.as_millis() / 2;
|
|
||||||
let effective_jitter_ms = keepalive_jitter.as_millis().min(jitter_cap_ms).max(1);
|
|
||||||
Duration::from_millis(rand::rng().random_range(0..=effective_jitter_ms as u64))
|
|
||||||
} else {
|
|
||||||
let jitter =
|
|
||||||
rand::rng().random_range(-ME_ACTIVE_PING_JITTER_SECS..=ME_ACTIVE_PING_JITTER_SECS);
|
|
||||||
let wait = (ME_ACTIVE_PING_SECS as i64 + jitter).max(5) as u64;
|
|
||||||
Duration::from_secs(wait)
|
|
||||||
};
|
|
||||||
tokio::select! {
|
|
||||||
_ = cancel_ping_token.cancelled() => return,
|
|
||||||
_ = tokio::time::sleep(startup_jitter) => {}
|
|
||||||
}
|
|
||||||
loop {
|
|
||||||
let wait = if keepalive_enabled {
|
|
||||||
let mut interval = keepalive_interval;
|
|
||||||
let Some(pool) = pool_ping.upgrade() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if pool.registry.is_writer_empty(writer_id).await {
|
|
||||||
interval = interval.min(idle_interval_cap);
|
|
||||||
}
|
|
||||||
let jitter_cap_ms = interval.as_millis() / 2;
|
|
||||||
let effective_jitter_ms = keepalive_jitter.as_millis().min(jitter_cap_ms).max(1);
|
|
||||||
interval
|
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=effective_jitter_ms as u64))
|
|
||||||
} else {
|
|
||||||
let jitter =
|
|
||||||
rand::rng().random_range(-ME_ACTIVE_PING_JITTER_SECS..=ME_ACTIVE_PING_JITTER_SECS);
|
|
||||||
let secs = (ME_ACTIVE_PING_SECS as i64 + jitter).max(5) as u64;
|
|
||||||
Duration::from_secs(secs)
|
|
||||||
};
|
|
||||||
tokio::select! {
|
|
||||||
_ = cancel_ping_token.cancelled() => return,
|
|
||||||
_ = tokio::time::sleep(wait) => {}
|
|
||||||
}
|
|
||||||
let sent_id = ping_id;
|
|
||||||
let mut p = Vec::with_capacity(12);
|
|
||||||
p.extend_from_slice(&RPC_PING_U32.to_le_bytes());
|
|
||||||
p.extend_from_slice(&sent_id.to_le_bytes());
|
|
||||||
{
|
|
||||||
let mut tracker = ping_tracker_ping.lock().await;
|
|
||||||
cleanup_tick = cleanup_tick.wrapping_add(1);
|
|
||||||
if cleanup_tick.is_multiple_of(ME_PING_TRACKER_CLEANUP_EVERY) {
|
|
||||||
let before = tracker.len();
|
|
||||||
tracker.retain(|_, ts| ts.elapsed() < Duration::from_secs(120));
|
|
||||||
let expired = before.saturating_sub(tracker.len());
|
|
||||||
if expired > 0 {
|
|
||||||
stats_ping.increment_me_keepalive_timeout_by(expired as u64);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tracker.insert(sent_id, std::time::Instant::now());
|
|
||||||
}
|
|
||||||
ping_id = ping_id.wrapping_add(1);
|
|
||||||
stats_ping.increment_me_keepalive_sent();
|
|
||||||
if tx_ping
|
|
||||||
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
|
||||||
.await
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
stats_ping.increment_me_keepalive_failed();
|
|
||||||
debug!("ME ping failed, removing dead writer");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
async fn rpc_proxy_req_signal_loop(
|
|
||||||
pool_signal: std::sync::Weak<MePool>,
|
|
||||||
writer_id: u64,
|
|
||||||
tx_signal: mpsc::Sender<WriterCommand>,
|
|
||||||
stats_signal: Arc<crate::stats::Stats>,
|
|
||||||
cancel_signal: CancellationToken,
|
|
||||||
keepalive_jitter_signal: Duration,
|
|
||||||
rpc_proxy_req_every_secs: u64,
|
|
||||||
) {
|
|
||||||
if rpc_proxy_req_every_secs == 0 {
|
|
||||||
// Disabled service signal loop must stay parked until writer cancellation.
|
|
||||||
// Returning immediately here would complete `select!` and tear down writer lifecycle.
|
|
||||||
cancel_signal.cancelled().await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let interval = Duration::from_secs(rpc_proxy_req_every_secs);
|
|
||||||
let startup_jitter_ms = {
|
|
||||||
let jitter_cap_ms = interval.as_millis() / 2;
|
|
||||||
let effective_jitter_ms = keepalive_jitter_signal
|
|
||||||
.as_millis()
|
|
||||||
.min(jitter_cap_ms)
|
|
||||||
.max(1);
|
|
||||||
rand::rng().random_range(0..=effective_jitter_ms as u64)
|
|
||||||
};
|
|
||||||
|
|
||||||
tokio::select! {
|
|
||||||
_ = cancel_signal.cancelled() => return,
|
|
||||||
_ = tokio::time::sleep(Duration::from_millis(startup_jitter_ms)) => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
loop {
|
|
||||||
let wait = {
|
|
||||||
let jitter_cap_ms = interval.as_millis() / 2;
|
|
||||||
let effective_jitter_ms = keepalive_jitter_signal
|
|
||||||
.as_millis()
|
|
||||||
.min(jitter_cap_ms)
|
|
||||||
.max(1);
|
|
||||||
interval
|
|
||||||
+ Duration::from_millis(rand::rng().random_range(0..=effective_jitter_ms as u64))
|
|
||||||
};
|
|
||||||
|
|
||||||
tokio::select! {
|
|
||||||
_ = cancel_signal.cancelled() => return,
|
|
||||||
_ = tokio::time::sleep(wait) => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
let Some(pool) = pool_signal.upgrade() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
let Some(meta) = pool.registry.get_last_writer_meta(writer_id).await else {
|
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_skipped_no_meta_total();
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
|
|
||||||
let (conn_id, mut service_rx) = pool.registry.register().await;
|
|
||||||
// Service RPC_PROXY_REQ signal path is intentionally route-only:
|
|
||||||
// do not bind synthetic conn_id into regular writer/client accounting.
|
|
||||||
|
|
||||||
let payload = build_proxy_req_payload(
|
|
||||||
conn_id,
|
|
||||||
meta.client_addr,
|
|
||||||
meta.our_addr,
|
|
||||||
&[],
|
|
||||||
pool.proxy_tag.as_deref(),
|
|
||||||
meta.proto_flags,
|
|
||||||
);
|
|
||||||
|
|
||||||
if tx_signal
|
|
||||||
.send(WriterCommand::DataAndFlush(payload))
|
|
||||||
.await
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_failed_total();
|
|
||||||
let _ = pool.registry.unregister(conn_id).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_sent_total();
|
|
||||||
|
|
||||||
if matches!(
|
|
||||||
tokio::time::timeout(
|
|
||||||
Duration::from_millis(ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS),
|
|
||||||
service_rx.recv(),
|
|
||||||
)
|
|
||||||
.await,
|
|
||||||
Ok(Some(_))
|
|
||||||
) {
|
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_response_total();
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut close_payload = Vec::with_capacity(12);
|
|
||||||
close_payload.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
|
||||||
close_payload.extend_from_slice(&conn_id.to_le_bytes());
|
|
||||||
|
|
||||||
if tx_signal
|
|
||||||
.send(WriterCommand::DataAndFlush(Bytes::from(close_payload)))
|
|
||||||
.await
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_failed_total();
|
|
||||||
let _ = pool.registry.unregister(conn_id).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
stats_signal.increment_me_rpc_proxy_req_signal_close_sent_total();
|
|
||||||
let _ = pool.registry.unregister(conn_id).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
pub(crate) async fn prune_closed_writers(self: &Arc<Self>) {
|
pub(crate) async fn prune_closed_writers(self: &Arc<Self>) {
|
||||||
let closed_writer_ids: Vec<u64> = {
|
let closed_writer_ids: Vec<u64> = {
|
||||||
@@ -372,15 +136,46 @@ impl MePool {
|
|||||||
let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0));
|
let draining_started_at_epoch_secs = Arc::new(AtomicU64::new(0));
|
||||||
let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0));
|
let drain_deadline_epoch_secs = Arc::new(AtomicU64::new(0));
|
||||||
let allow_drain_fallback = Arc::new(AtomicBool::new(false));
|
let allow_drain_fallback = Arc::new(AtomicBool::new(false));
|
||||||
let (tx, rx) =
|
let (tx, mut rx) = mpsc::channel::<WriterCommand>(self.writer_cmd_channel_capacity);
|
||||||
mpsc::channel::<WriterCommand>(self.writer_lifecycle.writer_cmd_channel_capacity);
|
let mut rpc_writer = RpcWriter {
|
||||||
let rpc_writer = RpcWriter {
|
|
||||||
writer: hs.wr,
|
writer: hs.wr,
|
||||||
key: hs.write_key,
|
key: hs.write_key,
|
||||||
iv: hs.write_iv,
|
iv: hs.write_iv,
|
||||||
seq_no: 0,
|
seq_no: 0,
|
||||||
crc_mode: hs.crc_mode,
|
crc_mode: hs.crc_mode,
|
||||||
};
|
};
|
||||||
|
let cancel_wr = cancel.clone();
|
||||||
|
let cleanup_done = Arc::new(AtomicBool::new(false));
|
||||||
|
let cleanup_for_writer = cleanup_done.clone();
|
||||||
|
let pool_writer_task = Arc::downgrade(self);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
cmd = rx.recv() => {
|
||||||
|
match cmd {
|
||||||
|
Some(WriterCommand::Data(payload)) => {
|
||||||
|
if rpc_writer.send(&payload).await.is_err() { break; }
|
||||||
|
}
|
||||||
|
Some(WriterCommand::DataAndFlush(payload)) => {
|
||||||
|
if rpc_writer.send_and_flush(&payload).await.is_err() { break; }
|
||||||
|
}
|
||||||
|
Some(WriterCommand::Close) | None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = cancel_wr.cancelled() => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cleanup_for_writer
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
if let Some(pool) = pool_writer_task.upgrade() {
|
||||||
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
|
} else {
|
||||||
|
cancel_wr.cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
let writer = MeWriter {
|
let writer = MeWriter {
|
||||||
id: writer_id,
|
id: writer_id,
|
||||||
addr,
|
addr,
|
||||||
@@ -398,135 +193,329 @@ impl MePool {
|
|||||||
drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(),
|
drain_deadline_epoch_secs: drain_deadline_epoch_secs.clone(),
|
||||||
allow_drain_fallback: allow_drain_fallback.clone(),
|
allow_drain_fallback: allow_drain_fallback.clone(),
|
||||||
};
|
};
|
||||||
self.writers
|
self.writers.write().await.push(writer.clone());
|
||||||
.update(|writers| writers.push(writer.clone()))
|
|
||||||
.await;
|
|
||||||
self.registry.register_writer(writer_id, tx.clone()).await;
|
self.registry.register_writer(writer_id, tx.clone()).await;
|
||||||
self.registry.mark_writer_idle(writer_id).await;
|
self.registry.mark_writer_idle(writer_id).await;
|
||||||
self.conn_count.fetch_add(1, Ordering::Relaxed);
|
self.conn_count.fetch_add(1, Ordering::Relaxed);
|
||||||
self.notify_writer_epoch();
|
self.writer_available.notify_one();
|
||||||
|
|
||||||
let reg = self.registry.clone();
|
let reg = self.registry.clone();
|
||||||
let writers_arc = self.writers_arc();
|
let writers_arc = self.writers_arc();
|
||||||
let ping_tracker = Arc::new(tokio::sync::Mutex::new(HashMap::<i64, Instant>::new()));
|
let ping_tracker = self.ping_tracker.clone();
|
||||||
let ping_tracker_reader = ping_tracker.clone();
|
let ping_tracker_reader = ping_tracker.clone();
|
||||||
let ping_tracker_ping = ping_tracker.clone();
|
|
||||||
let rtt_stats = self.rtt_stats.clone();
|
let rtt_stats = self.rtt_stats.clone();
|
||||||
let stats_reader = self.stats.clone();
|
let stats_reader = self.stats.clone();
|
||||||
let stats_reader_close = self.stats.clone();
|
let stats_reader_close = self.stats.clone();
|
||||||
let stats_ping = self.stats.clone();
|
let stats_ping = self.stats.clone();
|
||||||
let stats_signal = self.stats.clone();
|
let pool = Arc::downgrade(self);
|
||||||
let pool_lifecycle = Arc::downgrade(self);
|
|
||||||
let pool_ping = Arc::downgrade(self);
|
|
||||||
let pool_signal = Arc::downgrade(self);
|
|
||||||
let tx_reader = tx.clone();
|
|
||||||
let tx_ping = tx.clone();
|
|
||||||
let tx_signal = tx.clone();
|
|
||||||
let keepalive_enabled = self.writer_lifecycle.me_keepalive_enabled;
|
|
||||||
let keepalive_interval = self.writer_lifecycle.me_keepalive_interval;
|
|
||||||
let keepalive_jitter = self.writer_lifecycle.me_keepalive_jitter;
|
|
||||||
let keepalive_jitter_signal = self.writer_lifecycle.me_keepalive_jitter;
|
|
||||||
let rpc_proxy_req_every_secs = self
|
|
||||||
.writer_lifecycle
|
|
||||||
.rpc_proxy_req_every_secs
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
let cancel_reader = cancel.clone();
|
|
||||||
let cancel_writer = cancel.clone();
|
|
||||||
let cancel_ping = cancel.clone();
|
let cancel_ping = cancel.clone();
|
||||||
|
let tx_ping = tx.clone();
|
||||||
|
let ping_tracker_ping = ping_tracker.clone();
|
||||||
|
let cleanup_for_reader = cleanup_done.clone();
|
||||||
|
let cleanup_for_ping = cleanup_done.clone();
|
||||||
|
let keepalive_enabled = self.me_keepalive_enabled;
|
||||||
|
let keepalive_interval = self.me_keepalive_interval;
|
||||||
|
let keepalive_jitter = self.me_keepalive_jitter;
|
||||||
|
let rpc_proxy_req_every_secs = self.rpc_proxy_req_every_secs.load(Ordering::Relaxed);
|
||||||
|
let tx_signal = tx.clone();
|
||||||
|
let stats_signal = self.stats.clone();
|
||||||
let cancel_signal = cancel.clone();
|
let cancel_signal = cancel.clone();
|
||||||
let cancel_select = cancel.clone();
|
let cleanup_for_signal = cleanup_done.clone();
|
||||||
let cancel_cleanup = cancel.clone();
|
let pool_signal = Arc::downgrade(self);
|
||||||
let reader_route_data_wait_ms = self.transport_policy.me_reader_route_data_wait_ms.clone();
|
let keepalive_jitter_signal = self.me_keepalive_jitter;
|
||||||
|
let cancel_reader_token = cancel.clone();
|
||||||
|
let cancel_ping_token = cancel_ping.clone();
|
||||||
|
let reader_route_data_wait_ms = self.me_reader_route_data_wait_ms.clone();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
// Reader MUST be the first branch in biased select! to avoid read starvation.
|
let res = reader_loop(
|
||||||
let exit = tokio::select! {
|
hs.rd,
|
||||||
biased;
|
hs.read_key,
|
||||||
|
hs.read_iv,
|
||||||
reader_res = reader_loop(
|
hs.crc_mode,
|
||||||
hs.rd,
|
reg.clone(),
|
||||||
hs.read_key,
|
BytesMut::new(),
|
||||||
hs.read_iv,
|
BytesMut::new(),
|
||||||
hs.crc_mode,
|
tx.clone(),
|
||||||
reg.clone(),
|
ping_tracker_reader,
|
||||||
BytesMut::new(),
|
rtt_stats.clone(),
|
||||||
BytesMut::new(),
|
stats_reader,
|
||||||
tx_reader,
|
writer_id,
|
||||||
ping_tracker_reader,
|
degraded.clone(),
|
||||||
rtt_stats,
|
rtt_ema_ms_x10.clone(),
|
||||||
stats_reader,
|
reader_route_data_wait_ms,
|
||||||
writer_id,
|
cancel_reader_token.clone(),
|
||||||
degraded,
|
)
|
||||||
rtt_ema_ms_x10,
|
.await;
|
||||||
reader_route_data_wait_ms,
|
let idle_close_by_peer = if let Err(e) = res.as_ref() {
|
||||||
cancel_reader,
|
is_me_peer_closed_error(e) && reg.is_writer_empty(writer_id).await
|
||||||
) => WriterLifecycleExit::Reader(reader_res),
|
} else {
|
||||||
writer_res = writer_command_loop(rx, rpc_writer, cancel_writer) => {
|
false
|
||||||
WriterLifecycleExit::Writer(writer_res)
|
};
|
||||||
|
if idle_close_by_peer {
|
||||||
|
stats_reader_close.increment_me_idle_close_by_peer_total();
|
||||||
|
info!(writer_id, "ME socket closed by peer on idle writer");
|
||||||
|
}
|
||||||
|
if cleanup_for_reader
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
if let Some(pool) = pool.upgrade() {
|
||||||
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
|
} else {
|
||||||
|
// Fallback for shutdown races: make writer task exit quickly so stale
|
||||||
|
// channels are observable by periodic prune.
|
||||||
|
cancel_reader_token.cancel();
|
||||||
}
|
}
|
||||||
_ = ping_loop(
|
}
|
||||||
pool_ping,
|
if let Err(e) = res
|
||||||
writer_id,
|
&& !idle_close_by_peer
|
||||||
tx_ping,
|
{
|
||||||
ping_tracker_ping,
|
warn!(error = %e, "ME reader ended");
|
||||||
stats_ping,
|
}
|
||||||
keepalive_enabled,
|
let remaining = writers_arc.read().await.len();
|
||||||
keepalive_interval,
|
debug!(writer_id, remaining, "ME reader task finished");
|
||||||
keepalive_jitter,
|
});
|
||||||
cancel_ping,
|
|
||||||
) => WriterLifecycleExit::Ping,
|
let pool_ping = Arc::downgrade(self);
|
||||||
_ = rpc_proxy_req_signal_loop(
|
tokio::spawn(async move {
|
||||||
pool_signal,
|
let mut ping_id: i64 = rand::random::<i64>();
|
||||||
writer_id,
|
let idle_interval_cap = Duration::from_secs(ME_IDLE_KEEPALIVE_MAX_SECS);
|
||||||
tx_signal,
|
// Per-writer jittered start to avoid phase sync.
|
||||||
stats_signal,
|
let startup_jitter = if keepalive_enabled {
|
||||||
cancel_signal,
|
let mut interval = keepalive_interval;
|
||||||
keepalive_jitter_signal,
|
if let Some(pool) = pool_ping.upgrade() {
|
||||||
rpc_proxy_req_every_secs,
|
if pool.registry.is_writer_empty(writer_id).await {
|
||||||
) => WriterLifecycleExit::Signal,
|
interval = interval.min(idle_interval_cap);
|
||||||
_ = cancel_select.cancelled() => WriterLifecycleExit::Cancelled,
|
}
|
||||||
|
} else {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let jitter_cap_ms = interval.as_millis() / 2;
|
||||||
|
let effective_jitter_ms = keepalive_jitter.as_millis().min(jitter_cap_ms).max(1);
|
||||||
|
Duration::from_millis(rand::rng().random_range(0..=effective_jitter_ms as u64))
|
||||||
|
} else {
|
||||||
|
let jitter = rand::rng()
|
||||||
|
.random_range(-ME_ACTIVE_PING_JITTER_SECS..=ME_ACTIVE_PING_JITTER_SECS);
|
||||||
|
let wait = (ME_ACTIVE_PING_SECS as i64 + jitter).max(5) as u64;
|
||||||
|
Duration::from_secs(wait)
|
||||||
|
};
|
||||||
|
tokio::select! {
|
||||||
|
_ = cancel_ping_token.cancelled() => return,
|
||||||
|
_ = tokio::time::sleep(startup_jitter) => {}
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
let wait = if keepalive_enabled {
|
||||||
|
let mut interval = keepalive_interval;
|
||||||
|
if let Some(pool) = pool_ping.upgrade() {
|
||||||
|
if pool.registry.is_writer_empty(writer_id).await {
|
||||||
|
interval = interval.min(idle_interval_cap);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let jitter_cap_ms = interval.as_millis() / 2;
|
||||||
|
let effective_jitter_ms =
|
||||||
|
keepalive_jitter.as_millis().min(jitter_cap_ms).max(1);
|
||||||
|
interval
|
||||||
|
+ Duration::from_millis(
|
||||||
|
rand::rng().random_range(0..=effective_jitter_ms as u64),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
let jitter = rand::rng()
|
||||||
|
.random_range(-ME_ACTIVE_PING_JITTER_SECS..=ME_ACTIVE_PING_JITTER_SECS);
|
||||||
|
let secs = (ME_ACTIVE_PING_SECS as i64 + jitter).max(5) as u64;
|
||||||
|
Duration::from_secs(secs)
|
||||||
|
};
|
||||||
|
tokio::select! {
|
||||||
|
_ = cancel_ping_token.cancelled() => {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ = tokio::time::sleep(wait) => {}
|
||||||
|
}
|
||||||
|
let sent_id = ping_id;
|
||||||
|
let mut p = Vec::with_capacity(12);
|
||||||
|
p.extend_from_slice(&RPC_PING_U32.to_le_bytes());
|
||||||
|
p.extend_from_slice(&sent_id.to_le_bytes());
|
||||||
|
{
|
||||||
|
let mut tracker = ping_tracker_ping.lock().await;
|
||||||
|
let now_epoch_ms = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis() as u64;
|
||||||
|
let mut run_cleanup = false;
|
||||||
|
if let Some(pool) = pool_ping.upgrade() {
|
||||||
|
let last_cleanup_ms = pool
|
||||||
|
.ping_tracker_last_cleanup_epoch_ms
|
||||||
|
.load(Ordering::Relaxed);
|
||||||
|
if now_epoch_ms.saturating_sub(last_cleanup_ms) >= 30_000
|
||||||
|
&& pool
|
||||||
|
.ping_tracker_last_cleanup_epoch_ms
|
||||||
|
.compare_exchange(
|
||||||
|
last_cleanup_ms,
|
||||||
|
now_epoch_ms,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Relaxed,
|
||||||
|
)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
run_cleanup = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if run_cleanup {
|
||||||
|
let before = tracker.len();
|
||||||
|
tracker.retain(|_, (ts, _)| ts.elapsed() < Duration::from_secs(120));
|
||||||
|
let expired = before.saturating_sub(tracker.len());
|
||||||
|
if expired > 0 {
|
||||||
|
stats_ping.increment_me_keepalive_timeout_by(expired as u64);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tracker.insert(sent_id, (std::time::Instant::now(), writer_id));
|
||||||
|
}
|
||||||
|
ping_id = ping_id.wrapping_add(1);
|
||||||
|
stats_ping.increment_me_keepalive_sent();
|
||||||
|
if tx_ping
|
||||||
|
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
stats_ping.increment_me_keepalive_failed();
|
||||||
|
debug!("ME ping failed, removing dead writer");
|
||||||
|
cancel_ping.cancel();
|
||||||
|
if cleanup_for_ping
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
&& let Some(pool) = pool_ping.upgrade()
|
||||||
|
{
|
||||||
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if rpc_proxy_req_every_secs == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let interval = Duration::from_secs(rpc_proxy_req_every_secs);
|
||||||
|
let startup_jitter_ms = {
|
||||||
|
let jitter_cap_ms = interval.as_millis() / 2;
|
||||||
|
let effective_jitter_ms = keepalive_jitter_signal
|
||||||
|
.as_millis()
|
||||||
|
.min(jitter_cap_ms)
|
||||||
|
.max(1);
|
||||||
|
rand::rng().random_range(0..=effective_jitter_ms as u64)
|
||||||
};
|
};
|
||||||
|
|
||||||
match exit {
|
tokio::select! {
|
||||||
WriterLifecycleExit::Reader(res) => {
|
_ = cancel_signal.cancelled() => return,
|
||||||
let idle_close_by_peer = if let Err(e) = res.as_ref() {
|
_ = tokio::time::sleep(Duration::from_millis(startup_jitter_ms)) => {}
|
||||||
is_me_peer_closed_error(e) && reg.is_writer_empty(writer_id).await
|
}
|
||||||
} else {
|
|
||||||
false
|
loop {
|
||||||
};
|
let wait = {
|
||||||
if idle_close_by_peer {
|
let jitter_cap_ms = interval.as_millis() / 2;
|
||||||
stats_reader_close.increment_me_idle_close_by_peer_total();
|
let effective_jitter_ms = keepalive_jitter_signal
|
||||||
info!(writer_id, "ME socket closed by peer on idle writer");
|
.as_millis()
|
||||||
}
|
.min(jitter_cap_ms)
|
||||||
if let Err(e) = res
|
.max(1);
|
||||||
&& !idle_close_by_peer
|
interval
|
||||||
|
+ Duration::from_millis(
|
||||||
|
rand::rng().random_range(0..=effective_jitter_ms as u64),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
tokio::select! {
|
||||||
|
_ = cancel_signal.cancelled() => break,
|
||||||
|
_ = tokio::time::sleep(wait) => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(pool) = pool_signal.upgrade() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(meta) = pool.registry.get_last_writer_meta(writer_id).await else {
|
||||||
|
stats_signal.increment_me_rpc_proxy_req_signal_skipped_no_meta_total();
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let (conn_id, mut service_rx) = pool.registry.register().await;
|
||||||
|
if !pool
|
||||||
|
.registry
|
||||||
|
.bind_writer(conn_id, writer_id, meta.clone())
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
let _ = pool.registry.unregister(conn_id).await;
|
||||||
|
stats_signal.increment_me_rpc_proxy_req_signal_skipped_no_meta_total();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload = build_proxy_req_payload(
|
||||||
|
conn_id,
|
||||||
|
meta.client_addr,
|
||||||
|
meta.our_addr,
|
||||||
|
&[],
|
||||||
|
pool.proxy_tag.as_deref(),
|
||||||
|
meta.proto_flags,
|
||||||
|
);
|
||||||
|
|
||||||
|
if tx_signal
|
||||||
|
.send(WriterCommand::DataAndFlush(payload))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
stats_signal.increment_me_rpc_proxy_req_signal_failed_total();
|
||||||
|
let _ = pool.registry.unregister(conn_id).await;
|
||||||
|
cancel_signal.cancel();
|
||||||
|
if cleanup_for_signal
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
{
|
{
|
||||||
warn!(error = %e, "ME reader ended");
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
WriterLifecycleExit::Writer(res) => {
|
|
||||||
if let Err(e) = res {
|
stats_signal.increment_me_rpc_proxy_req_signal_sent_total();
|
||||||
warn!(error = %e, "ME writer command loop ended");
|
|
||||||
|
if matches!(
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_millis(ME_RPC_PROXY_REQ_RESPONSE_WAIT_MS),
|
||||||
|
service_rx.recv(),
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
Ok(Some(_))
|
||||||
|
) {
|
||||||
|
stats_signal.increment_me_rpc_proxy_req_signal_response_total();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut close_payload = Vec::with_capacity(12);
|
||||||
|
close_payload.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
||||||
|
close_payload.extend_from_slice(&conn_id.to_le_bytes());
|
||||||
|
|
||||||
|
if tx_signal
|
||||||
|
.send(WriterCommand::DataAndFlush(Bytes::from(close_payload)))
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
stats_signal.increment_me_rpc_proxy_req_signal_failed_total();
|
||||||
|
let _ = pool.registry.unregister(conn_id).await;
|
||||||
|
cancel_signal.cancel();
|
||||||
|
if cleanup_for_signal
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
WriterLifecycleExit::Ping => {
|
|
||||||
debug!(writer_id, "ME ping loop finished");
|
|
||||||
}
|
|
||||||
WriterLifecycleExit::Signal => {
|
|
||||||
debug!(writer_id, "ME rpc_proxy_req signal loop finished");
|
|
||||||
}
|
|
||||||
WriterLifecycleExit::Cancelled => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(pool) = pool_lifecycle.upgrade() {
|
stats_signal.increment_me_rpc_proxy_req_signal_close_sent_total();
|
||||||
pool.remove_writer_and_close_clients(writer_id).await;
|
let _ = pool.registry.unregister(conn_id).await;
|
||||||
} else {
|
|
||||||
// Fallback for shutdown races: make lifecycle exit observable by prune.
|
|
||||||
cancel_cleanup.cancel();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let remaining = writers_arc.read().await.len();
|
|
||||||
debug!(writer_id, remaining, "ME writer lifecycle task finished");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -605,36 +594,23 @@ impl MePool {
|
|||||||
// The close command below is only a best-effort accelerator for task shutdown.
|
// The close command below is only a best-effort accelerator for task shutdown.
|
||||||
// Cleanup progress must never depend on command-channel availability.
|
// Cleanup progress must never depend on command-channel availability.
|
||||||
let _ = self.registry.writer_lost(writer_id).await;
|
let _ = self.registry.writer_lost(writer_id).await;
|
||||||
|
{
|
||||||
|
let mut tracker = self.ping_tracker.lock().await;
|
||||||
|
tracker.retain(|_, (_, wid)| *wid != writer_id);
|
||||||
|
}
|
||||||
self.rtt_stats.lock().await.remove(&writer_id);
|
self.rtt_stats.lock().await.remove(&writer_id);
|
||||||
if let Some(tx) = close_tx {
|
if let Some(tx) = close_tx {
|
||||||
// Keep teardown critical path non-blocking: close is best-effort only.
|
let _ = tx.send(WriterCommand::Close).await;
|
||||||
let _ = tx.try_send(WriterCommand::Close);
|
|
||||||
}
|
}
|
||||||
if let Some(addr) = removed_addr {
|
if let Some(addr) = removed_addr {
|
||||||
if let Some(uptime) = removed_uptime {
|
if let Some(uptime) = removed_uptime {
|
||||||
// Quarantine contract: only unexpected removals are considered endpoint flap.
|
// Quarantine flapping endpoints regardless of draining state.
|
||||||
if trigger_refill {
|
self.maybe_quarantine_flapping_endpoint(addr, uptime).await;
|
||||||
self.stats
|
|
||||||
.increment_me_endpoint_quarantine_unexpected_total();
|
|
||||||
self.maybe_quarantine_flapping_endpoint(addr, uptime, "unexpected")
|
|
||||||
.await;
|
|
||||||
} else {
|
|
||||||
self.stats
|
|
||||||
.increment_me_endpoint_quarantine_draining_suppressed_total();
|
|
||||||
debug!(
|
|
||||||
%addr,
|
|
||||||
uptime_ms = uptime.as_millis(),
|
|
||||||
"Skipping endpoint quarantine for draining writer removal"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if trigger_refill && let Some(writer_dc) = removed_dc {
|
if trigger_refill && let Some(writer_dc) = removed_dc {
|
||||||
self.trigger_immediate_refill_for_dc(addr, writer_dc);
|
self.trigger_immediate_refill_for_dc(addr, writer_dc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if removed {
|
|
||||||
self.notify_writer_epoch();
|
|
||||||
}
|
|
||||||
removed
|
removed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -700,10 +676,7 @@ impl MePool {
|
|||||||
MeBindStaleMode::Never => false,
|
MeBindStaleMode::Never => false,
|
||||||
MeBindStaleMode::Always => true,
|
MeBindStaleMode::Always => true,
|
||||||
MeBindStaleMode::Ttl => {
|
MeBindStaleMode::Ttl => {
|
||||||
let ttl_secs = self
|
let ttl_secs = self.me_bind_stale_ttl_secs.load(Ordering::Relaxed);
|
||||||
.binding_policy
|
|
||||||
.me_bind_stale_ttl_secs
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
if ttl_secs == 0 {
|
if ttl_secs == 0 {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,10 +32,10 @@ pub(crate) async fn reader_loop(
|
|||||||
enc_leftover: BytesMut,
|
enc_leftover: BytesMut,
|
||||||
mut dec: BytesMut,
|
mut dec: BytesMut,
|
||||||
tx: mpsc::Sender<WriterCommand>,
|
tx: mpsc::Sender<WriterCommand>,
|
||||||
ping_tracker: Arc<Mutex<HashMap<i64, Instant>>>,
|
ping_tracker: Arc<Mutex<HashMap<i64, (Instant, u64)>>>,
|
||||||
rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
rtt_stats: Arc<Mutex<HashMap<u64, (f64, f64)>>>,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
writer_id: u64,
|
_writer_id: u64,
|
||||||
degraded: Arc<AtomicBool>,
|
degraded: Arc<AtomicBool>,
|
||||||
writer_rtt_ema_ms_x10: Arc<AtomicU32>,
|
writer_rtt_ema_ms_x10: Arc<AtomicU32>,
|
||||||
reader_route_data_wait_ms: Arc<AtomicU64>,
|
reader_route_data_wait_ms: Arc<AtomicU64>,
|
||||||
@@ -45,7 +45,7 @@ pub(crate) async fn reader_loop(
|
|||||||
let mut expected_seq: i32 = 0;
|
let mut expected_seq: i32 = 0;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let mut tmp = [0u8; 65_536];
|
let mut tmp = [0u8; 16_384];
|
||||||
let n = tokio::select! {
|
let n = tokio::select! {
|
||||||
res = rd.read(&mut tmp) => res.map_err(ProxyError::Io)?,
|
res = rd.read(&mut tmp) => res.map_err(ProxyError::Io)?,
|
||||||
_ = cancel.cancelled() => return Ok(()),
|
_ = cancel.cancelled() => return Ok(()),
|
||||||
@@ -203,13 +203,13 @@ pub(crate) async fn reader_loop(
|
|||||||
} else if pt == RPC_PONG_U32 && body.len() >= 8 {
|
} else if pt == RPC_PONG_U32 && body.len() >= 8 {
|
||||||
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
stats.increment_me_keepalive_pong();
|
stats.increment_me_keepalive_pong();
|
||||||
if let Some(sent) = {
|
if let Some((sent, wid)) = {
|
||||||
let mut guard = ping_tracker.lock().await;
|
let mut guard = ping_tracker.lock().await;
|
||||||
guard.remove(&ping_id)
|
guard.remove(&ping_id)
|
||||||
} {
|
} {
|
||||||
let rtt = sent.elapsed().as_secs_f64() * 1000.0;
|
let rtt = sent.elapsed().as_secs_f64() * 1000.0;
|
||||||
let mut stats = rtt_stats.lock().await;
|
let mut stats = rtt_stats.lock().await;
|
||||||
let entry = stats.entry(writer_id).or_insert((rtt, rtt));
|
let entry = stats.entry(wid).or_insert((rtt, rtt));
|
||||||
entry.1 = entry.1 * 0.8 + rtt * 0.2;
|
entry.1 = entry.1 * 0.8 + rtt * 0.2;
|
||||||
if rtt < entry.0 {
|
if rtt < entry.0 {
|
||||||
entry.0 = rtt;
|
entry.0 = rtt;
|
||||||
@@ -224,7 +224,7 @@ pub(crate) async fn reader_loop(
|
|||||||
Ordering::Relaxed,
|
Ordering::Relaxed,
|
||||||
);
|
);
|
||||||
trace!(
|
trace!(
|
||||||
writer_id,
|
writer_id = wid,
|
||||||
rtt_ms = rtt,
|
rtt_ms = rtt,
|
||||||
ema_ms = entry.1,
|
ema_ms = entry.1,
|
||||||
base_ms = entry.0,
|
base_ms = entry.0,
|
||||||
|
|||||||
@@ -3,9 +3,8 @@ use std::net::SocketAddr;
|
|||||||
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use dashmap::DashMap;
|
|
||||||
use tokio::sync::mpsc::error::TrySendError;
|
use tokio::sync::mpsc::error::TrySendError;
|
||||||
use tokio::sync::{Mutex, mpsc};
|
use tokio::sync::{RwLock, mpsc};
|
||||||
|
|
||||||
use super::MeResponse;
|
use super::MeResponse;
|
||||||
use super::codec::WriterCommand;
|
use super::codec::WriterCommand;
|
||||||
@@ -51,15 +50,8 @@ pub(super) struct WriterActivitySnapshot {
|
|||||||
pub active_sessions_by_target_dc: HashMap<i16, usize>,
|
pub active_sessions_by_target_dc: HashMap<i16, usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct RoutingTable {
|
struct RegistryInner {
|
||||||
map: DashMap<u64, mpsc::Sender<MeResponse>>,
|
map: HashMap<u64, mpsc::Sender<MeResponse>>,
|
||||||
}
|
|
||||||
|
|
||||||
struct BindingState {
|
|
||||||
inner: Mutex<BindingInner>,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct BindingInner {
|
|
||||||
writers: HashMap<u64, mpsc::Sender<WriterCommand>>,
|
writers: HashMap<u64, mpsc::Sender<WriterCommand>>,
|
||||||
writer_for_conn: HashMap<u64, u64>,
|
writer_for_conn: HashMap<u64, u64>,
|
||||||
conns_for_writer: HashMap<u64, HashSet<u64>>,
|
conns_for_writer: HashMap<u64, HashSet<u64>>,
|
||||||
@@ -68,9 +60,10 @@ struct BindingInner {
|
|||||||
writer_idle_since_epoch_secs: HashMap<u64, u64>,
|
writer_idle_since_epoch_secs: HashMap<u64, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BindingInner {
|
impl RegistryInner {
|
||||||
fn new() -> Self {
|
fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
map: HashMap::new(),
|
||||||
writers: HashMap::new(),
|
writers: HashMap::new(),
|
||||||
writer_for_conn: HashMap::new(),
|
writer_for_conn: HashMap::new(),
|
||||||
conns_for_writer: HashMap::new(),
|
conns_for_writer: HashMap::new(),
|
||||||
@@ -82,8 +75,7 @@ impl BindingInner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct ConnRegistry {
|
pub struct ConnRegistry {
|
||||||
routing: RoutingTable,
|
inner: RwLock<RegistryInner>,
|
||||||
binding: BindingState,
|
|
||||||
next_id: AtomicU64,
|
next_id: AtomicU64,
|
||||||
route_channel_capacity: usize,
|
route_channel_capacity: usize,
|
||||||
route_backpressure_base_timeout_ms: AtomicU64,
|
route_backpressure_base_timeout_ms: AtomicU64,
|
||||||
@@ -102,12 +94,7 @@ impl ConnRegistry {
|
|||||||
pub fn with_route_channel_capacity(route_channel_capacity: usize) -> Self {
|
pub fn with_route_channel_capacity(route_channel_capacity: usize) -> Self {
|
||||||
let start = rand::random::<u64>() | 1;
|
let start = rand::random::<u64>() | 1;
|
||||||
Self {
|
Self {
|
||||||
routing: RoutingTable {
|
inner: RwLock::new(RegistryInner::new()),
|
||||||
map: DashMap::new(),
|
|
||||||
},
|
|
||||||
binding: BindingState {
|
|
||||||
inner: Mutex::new(BindingInner::new()),
|
|
||||||
},
|
|
||||||
next_id: AtomicU64::new(start),
|
next_id: AtomicU64::new(start),
|
||||||
route_channel_capacity: route_channel_capacity.max(1),
|
route_channel_capacity: route_channel_capacity.max(1),
|
||||||
route_backpressure_base_timeout_ms: AtomicU64::new(ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS),
|
route_backpressure_base_timeout_ms: AtomicU64::new(ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS),
|
||||||
@@ -143,14 +130,14 @@ impl ConnRegistry {
|
|||||||
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
|
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
|
||||||
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
|
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
|
||||||
let (tx, rx) = mpsc::channel(self.route_channel_capacity);
|
let (tx, rx) = mpsc::channel(self.route_channel_capacity);
|
||||||
self.routing.map.insert(id, tx);
|
self.inner.write().await.map.insert(id, tx);
|
||||||
(id, rx)
|
(id, rx)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn register_writer(&self, writer_id: u64, tx: mpsc::Sender<WriterCommand>) {
|
pub async fn register_writer(&self, writer_id: u64, tx: mpsc::Sender<WriterCommand>) {
|
||||||
let mut binding = self.binding.inner.lock().await;
|
let mut inner = self.inner.write().await;
|
||||||
binding.writers.insert(writer_id, tx);
|
inner.writers.insert(writer_id, tx);
|
||||||
binding
|
inner
|
||||||
.conns_for_writer
|
.conns_for_writer
|
||||||
.entry(writer_id)
|
.entry(writer_id)
|
||||||
.or_insert_with(HashSet::new);
|
.or_insert_with(HashSet::new);
|
||||||
@@ -158,18 +145,18 @@ impl ConnRegistry {
|
|||||||
|
|
||||||
/// Unregister connection, returning associated writer_id if any.
|
/// Unregister connection, returning associated writer_id if any.
|
||||||
pub async fn unregister(&self, id: u64) -> Option<u64> {
|
pub async fn unregister(&self, id: u64) -> Option<u64> {
|
||||||
self.routing.map.remove(&id);
|
let mut inner = self.inner.write().await;
|
||||||
let mut binding = self.binding.inner.lock().await;
|
inner.map.remove(&id);
|
||||||
binding.meta.remove(&id);
|
inner.meta.remove(&id);
|
||||||
if let Some(writer_id) = binding.writer_for_conn.remove(&id) {
|
if let Some(writer_id) = inner.writer_for_conn.remove(&id) {
|
||||||
let became_empty = if let Some(set) = binding.conns_for_writer.get_mut(&writer_id) {
|
let became_empty = if let Some(set) = inner.conns_for_writer.get_mut(&writer_id) {
|
||||||
set.remove(&id);
|
set.remove(&id);
|
||||||
set.is_empty()
|
set.is_empty()
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
};
|
};
|
||||||
if became_empty {
|
if became_empty {
|
||||||
binding
|
inner
|
||||||
.writer_idle_since_epoch_secs
|
.writer_idle_since_epoch_secs
|
||||||
.insert(writer_id, Self::now_epoch_secs());
|
.insert(writer_id, Self::now_epoch_secs());
|
||||||
}
|
}
|
||||||
@@ -180,7 +167,10 @@ impl ConnRegistry {
|
|||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult {
|
pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult {
|
||||||
let tx = self.routing.map.get(&id).map(|entry| entry.value().clone());
|
let tx = {
|
||||||
|
let inner = self.inner.read().await;
|
||||||
|
inner.map.get(&id).cloned()
|
||||||
|
};
|
||||||
|
|
||||||
let Some(tx) = tx else {
|
let Some(tx) = tx else {
|
||||||
return RouteResult::NoConn;
|
return RouteResult::NoConn;
|
||||||
@@ -233,7 +223,10 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn route_nowait(&self, id: u64, resp: MeResponse) -> RouteResult {
|
pub async fn route_nowait(&self, id: u64, resp: MeResponse) -> RouteResult {
|
||||||
let tx = self.routing.map.get(&id).map(|entry| entry.value().clone());
|
let tx = {
|
||||||
|
let inner = self.inner.read().await;
|
||||||
|
inner.map.get(&id).cloned()
|
||||||
|
};
|
||||||
|
|
||||||
let Some(tx) = tx else {
|
let Some(tx) = tx else {
|
||||||
return RouteResult::NoConn;
|
return RouteResult::NoConn;
|
||||||
@@ -256,7 +249,10 @@ impl ConnRegistry {
|
|||||||
return self.route_nowait(id, resp).await;
|
return self.route_nowait(id, resp).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let tx = self.routing.map.get(&id).map(|entry| entry.value().clone());
|
let tx = {
|
||||||
|
let inner = self.inner.read().await;
|
||||||
|
inner.map.get(&id).cloned()
|
||||||
|
};
|
||||||
|
|
||||||
let Some(tx) = tx else {
|
let Some(tx) = tx else {
|
||||||
return RouteResult::NoConn;
|
return RouteResult::NoConn;
|
||||||
@@ -295,39 +291,33 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn bind_writer(&self, conn_id: u64, writer_id: u64, meta: ConnMeta) -> bool {
|
pub async fn bind_writer(&self, conn_id: u64, writer_id: u64, meta: ConnMeta) -> bool {
|
||||||
let mut binding = self.binding.inner.lock().await;
|
let mut inner = self.inner.write().await;
|
||||||
// ROUTING IS THE SOURCE OF TRUTH:
|
if !inner.writers.contains_key(&writer_id) {
|
||||||
// never keep/attach writer binding for a connection that is already
|
|
||||||
// absent from the routing table.
|
|
||||||
if !self.routing.map.contains_key(&conn_id) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if !binding.writers.contains_key(&writer_id) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let previous_writer_id = binding.writer_for_conn.insert(conn_id, writer_id);
|
let previous_writer_id = inner.writer_for_conn.insert(conn_id, writer_id);
|
||||||
if let Some(previous_writer_id) = previous_writer_id
|
if let Some(previous_writer_id) = previous_writer_id
|
||||||
&& previous_writer_id != writer_id
|
&& previous_writer_id != writer_id
|
||||||
{
|
{
|
||||||
let became_empty =
|
let became_empty =
|
||||||
if let Some(set) = binding.conns_for_writer.get_mut(&previous_writer_id) {
|
if let Some(set) = inner.conns_for_writer.get_mut(&previous_writer_id) {
|
||||||
set.remove(&conn_id);
|
set.remove(&conn_id);
|
||||||
set.is_empty()
|
set.is_empty()
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
};
|
};
|
||||||
if became_empty {
|
if became_empty {
|
||||||
binding
|
inner
|
||||||
.writer_idle_since_epoch_secs
|
.writer_idle_since_epoch_secs
|
||||||
.insert(previous_writer_id, Self::now_epoch_secs());
|
.insert(previous_writer_id, Self::now_epoch_secs());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
binding.meta.insert(conn_id, meta.clone());
|
inner.meta.insert(conn_id, meta.clone());
|
||||||
binding.last_meta_for_writer.insert(writer_id, meta);
|
inner.last_meta_for_writer.insert(writer_id, meta);
|
||||||
binding.writer_idle_since_epoch_secs.remove(&writer_id);
|
inner.writer_idle_since_epoch_secs.remove(&writer_id);
|
||||||
binding
|
inner
|
||||||
.conns_for_writer
|
.conns_for_writer
|
||||||
.entry(writer_id)
|
.entry(writer_id)
|
||||||
.or_insert_with(HashSet::new)
|
.or_insert_with(HashSet::new)
|
||||||
@@ -336,32 +326,32 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn mark_writer_idle(&self, writer_id: u64) {
|
pub async fn mark_writer_idle(&self, writer_id: u64) {
|
||||||
let mut binding = self.binding.inner.lock().await;
|
let mut inner = self.inner.write().await;
|
||||||
binding
|
inner
|
||||||
.conns_for_writer
|
.conns_for_writer
|
||||||
.entry(writer_id)
|
.entry(writer_id)
|
||||||
.or_insert_with(HashSet::new);
|
.or_insert_with(HashSet::new);
|
||||||
binding
|
inner
|
||||||
.writer_idle_since_epoch_secs
|
.writer_idle_since_epoch_secs
|
||||||
.entry(writer_id)
|
.entry(writer_id)
|
||||||
.or_insert(Self::now_epoch_secs());
|
.or_insert(Self::now_epoch_secs());
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_last_writer_meta(&self, writer_id: u64) -> Option<ConnMeta> {
|
pub async fn get_last_writer_meta(&self, writer_id: u64) -> Option<ConnMeta> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
binding.last_meta_for_writer.get(&writer_id).cloned()
|
inner.last_meta_for_writer.get(&writer_id).cloned()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn writer_idle_since_snapshot(&self) -> HashMap<u64, u64> {
|
pub async fn writer_idle_since_snapshot(&self) -> HashMap<u64, u64> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
binding.writer_idle_since_epoch_secs.clone()
|
inner.writer_idle_since_epoch_secs.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn writer_idle_since_for_writer_ids(&self, writer_ids: &[u64]) -> HashMap<u64, u64> {
|
pub async fn writer_idle_since_for_writer_ids(&self, writer_ids: &[u64]) -> HashMap<u64, u64> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
let mut out = HashMap::<u64, u64>::with_capacity(writer_ids.len());
|
let mut out = HashMap::<u64, u64>::with_capacity(writer_ids.len());
|
||||||
for writer_id in writer_ids {
|
for writer_id in writer_ids {
|
||||||
if let Some(idle_since) = binding.writer_idle_since_epoch_secs.get(writer_id).copied() {
|
if let Some(idle_since) = inner.writer_idle_since_epoch_secs.get(writer_id).copied() {
|
||||||
out.insert(*writer_id, idle_since);
|
out.insert(*writer_id, idle_since);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -369,14 +359,14 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn writer_activity_snapshot(&self) -> WriterActivitySnapshot {
|
pub(super) async fn writer_activity_snapshot(&self) -> WriterActivitySnapshot {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
let mut bound_clients_by_writer = HashMap::<u64, usize>::new();
|
let mut bound_clients_by_writer = HashMap::<u64, usize>::new();
|
||||||
let mut active_sessions_by_target_dc = HashMap::<i16, usize>::new();
|
let mut active_sessions_by_target_dc = HashMap::<i16, usize>::new();
|
||||||
|
|
||||||
for (writer_id, conn_ids) in &binding.conns_for_writer {
|
for (writer_id, conn_ids) in &inner.conns_for_writer {
|
||||||
bound_clients_by_writer.insert(*writer_id, conn_ids.len());
|
bound_clients_by_writer.insert(*writer_id, conn_ids.len());
|
||||||
}
|
}
|
||||||
for conn_meta in binding.meta.values() {
|
for conn_meta in inner.meta.values() {
|
||||||
if conn_meta.target_dc == 0 {
|
if conn_meta.target_dc == 0 {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -392,39 +382,9 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_writer(&self, conn_id: u64) -> Option<ConnWriter> {
|
pub async fn get_writer(&self, conn_id: u64) -> Option<ConnWriter> {
|
||||||
let mut binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
// ROUTING IS THE SOURCE OF TRUTH:
|
let writer_id = inner.writer_for_conn.get(&conn_id).cloned()?;
|
||||||
// stale bindings are ignored and lazily cleaned when routing no longer
|
let writer = inner.writers.get(&writer_id).cloned()?;
|
||||||
// contains the connection.
|
|
||||||
if !self.routing.map.contains_key(&conn_id) {
|
|
||||||
binding.meta.remove(&conn_id);
|
|
||||||
if let Some(stale_writer_id) = binding.writer_for_conn.remove(&conn_id)
|
|
||||||
&& let Some(conns) = binding.conns_for_writer.get_mut(&stale_writer_id)
|
|
||||||
{
|
|
||||||
conns.remove(&conn_id);
|
|
||||||
if conns.is_empty() {
|
|
||||||
binding
|
|
||||||
.writer_idle_since_epoch_secs
|
|
||||||
.insert(stale_writer_id, Self::now_epoch_secs());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let writer_id = binding.writer_for_conn.get(&conn_id).copied()?;
|
|
||||||
let Some(writer) = binding.writers.get(&writer_id).cloned() else {
|
|
||||||
binding.writer_for_conn.remove(&conn_id);
|
|
||||||
binding.meta.remove(&conn_id);
|
|
||||||
if let Some(conns) = binding.conns_for_writer.get_mut(&writer_id) {
|
|
||||||
conns.remove(&conn_id);
|
|
||||||
if conns.is_empty() {
|
|
||||||
binding
|
|
||||||
.writer_idle_since_epoch_secs
|
|
||||||
.insert(writer_id, Self::now_epoch_secs());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return None;
|
|
||||||
};
|
|
||||||
Some(ConnWriter {
|
Some(ConnWriter {
|
||||||
writer_id,
|
writer_id,
|
||||||
tx: writer,
|
tx: writer,
|
||||||
@@ -432,16 +392,16 @@ impl ConnRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn active_conn_ids(&self) -> Vec<u64> {
|
pub async fn active_conn_ids(&self) -> Vec<u64> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
binding.writer_for_conn.keys().copied().collect()
|
inner.writer_for_conn.keys().copied().collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn writer_lost(&self, writer_id: u64) -> Vec<BoundConn> {
|
pub async fn writer_lost(&self, writer_id: u64) -> Vec<BoundConn> {
|
||||||
let mut binding = self.binding.inner.lock().await;
|
let mut inner = self.inner.write().await;
|
||||||
binding.writers.remove(&writer_id);
|
inner.writers.remove(&writer_id);
|
||||||
binding.last_meta_for_writer.remove(&writer_id);
|
inner.last_meta_for_writer.remove(&writer_id);
|
||||||
binding.writer_idle_since_epoch_secs.remove(&writer_id);
|
inner.writer_idle_since_epoch_secs.remove(&writer_id);
|
||||||
let conns = binding
|
let conns = inner
|
||||||
.conns_for_writer
|
.conns_for_writer
|
||||||
.remove(&writer_id)
|
.remove(&writer_id)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
@@ -450,11 +410,11 @@ impl ConnRegistry {
|
|||||||
|
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for conn_id in conns {
|
for conn_id in conns {
|
||||||
if binding.writer_for_conn.get(&conn_id).copied() != Some(writer_id) {
|
if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
binding.writer_for_conn.remove(&conn_id);
|
inner.writer_for_conn.remove(&conn_id);
|
||||||
if let Some(m) = binding.meta.get(&conn_id) {
|
if let Some(m) = inner.meta.get(&conn_id) {
|
||||||
out.push(BoundConn {
|
out.push(BoundConn {
|
||||||
conn_id,
|
conn_id,
|
||||||
meta: m.clone(),
|
meta: m.clone(),
|
||||||
@@ -466,13 +426,13 @@ impl ConnRegistry {
|
|||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn get_meta(&self, conn_id: u64) -> Option<ConnMeta> {
|
pub async fn get_meta(&self, conn_id: u64) -> Option<ConnMeta> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
binding.meta.get(&conn_id).cloned()
|
inner.meta.get(&conn_id).cloned()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn is_writer_empty(&self, writer_id: u64) -> bool {
|
pub async fn is_writer_empty(&self, writer_id: u64) -> bool {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
binding
|
inner
|
||||||
.conns_for_writer
|
.conns_for_writer
|
||||||
.get(&writer_id)
|
.get(&writer_id)
|
||||||
.map(|s| s.is_empty())
|
.map(|s| s.is_empty())
|
||||||
@@ -481,8 +441,8 @@ impl ConnRegistry {
|
|||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn unregister_writer_if_empty(&self, writer_id: u64) -> bool {
|
pub async fn unregister_writer_if_empty(&self, writer_id: u64) -> bool {
|
||||||
let mut binding = self.binding.inner.lock().await;
|
let mut inner = self.inner.write().await;
|
||||||
let Some(conn_ids) = binding.conns_for_writer.get(&writer_id) else {
|
let Some(conn_ids) = inner.conns_for_writer.get(&writer_id) else {
|
||||||
// Writer is already absent from the registry.
|
// Writer is already absent from the registry.
|
||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
@@ -490,19 +450,19 @@ impl ConnRegistry {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
binding.writers.remove(&writer_id);
|
inner.writers.remove(&writer_id);
|
||||||
binding.last_meta_for_writer.remove(&writer_id);
|
inner.last_meta_for_writer.remove(&writer_id);
|
||||||
binding.writer_idle_since_epoch_secs.remove(&writer_id);
|
inner.writer_idle_since_epoch_secs.remove(&writer_id);
|
||||||
binding.conns_for_writer.remove(&writer_id);
|
inner.conns_for_writer.remove(&writer_id);
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub(super) async fn non_empty_writer_ids(&self, writer_ids: &[u64]) -> HashSet<u64> {
|
pub(super) async fn non_empty_writer_ids(&self, writer_ids: &[u64]) -> HashSet<u64> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let inner = self.inner.read().await;
|
||||||
let mut out = HashSet::<u64>::with_capacity(writer_ids.len());
|
let mut out = HashSet::<u64>::with_capacity(writer_ids.len());
|
||||||
for writer_id in writer_ids {
|
for writer_id in writer_ids {
|
||||||
if let Some(conns) = binding.conns_for_writer.get(writer_id)
|
if let Some(conns) = inner.conns_for_writer.get(writer_id)
|
||||||
&& !conns.is_empty()
|
&& !conns.is_empty()
|
||||||
{
|
{
|
||||||
out.insert(*writer_id);
|
out.insert(*writer_id);
|
||||||
|
|||||||
@@ -26,9 +26,6 @@ use rand::seq::SliceRandom;
|
|||||||
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
|
const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
|
||||||
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
|
const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
|
||||||
const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
|
const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
|
||||||
const HYBRID_RECENT_SUCCESS_WINDOW_MS: u64 = 120_000;
|
|
||||||
const HYBRID_TIMEOUT_WARN_RATE_LIMIT_MS: u64 = 5_000;
|
|
||||||
const HYBRID_RECOVERY_TRIGGER_MIN_INTERVAL_MS: u64 = 5_000;
|
|
||||||
const PICK_PENALTY_WARM: u64 = 200;
|
const PICK_PENALTY_WARM: u64 = 200;
|
||||||
const PICK_PENALTY_DRAINING: u64 = 600;
|
const PICK_PENALTY_DRAINING: u64 = 600;
|
||||||
const PICK_PENALTY_STALE: u64 = 300;
|
const PICK_PENALTY_STALE: u64 = 300;
|
||||||
@@ -71,11 +68,8 @@ impl MePool {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let no_writer_mode = MeRouteNoWriterMode::from_u8(
|
let no_writer_mode =
|
||||||
self.route_runtime
|
MeRouteNoWriterMode::from_u8(self.me_route_no_writer_mode.load(Ordering::Relaxed));
|
||||||
.me_route_no_writer_mode
|
|
||||||
.load(Ordering::Relaxed),
|
|
||||||
);
|
|
||||||
let (routed_dc, unknown_target_dc) =
|
let (routed_dc, unknown_target_dc) =
|
||||||
self.resolve_target_dc_for_routing(target_dc as i32).await;
|
self.resolve_target_dc_for_routing(target_dc as i32).await;
|
||||||
let mut no_writer_deadline: Option<Instant> = None;
|
let mut no_writer_deadline: Option<Instant> = None;
|
||||||
@@ -83,11 +77,7 @@ impl MePool {
|
|||||||
let mut async_recovery_triggered = false;
|
let mut async_recovery_triggered = false;
|
||||||
let mut hybrid_recovery_round = 0u32;
|
let mut hybrid_recovery_round = 0u32;
|
||||||
let mut hybrid_last_recovery_at: Option<Instant> = None;
|
let mut hybrid_last_recovery_at: Option<Instant> = None;
|
||||||
let mut hybrid_total_deadline: Option<Instant> = None;
|
let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50));
|
||||||
let hybrid_wait_step = self
|
|
||||||
.route_runtime
|
|
||||||
.me_route_no_writer_wait
|
|
||||||
.max(Duration::from_millis(50));
|
|
||||||
let mut hybrid_wait_current = hybrid_wait_step;
|
let mut hybrid_wait_current = hybrid_wait_step;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
@@ -102,13 +92,9 @@ impl MePool {
|
|||||||
.tx
|
.tx
|
||||||
.try_send(WriterCommand::Data(current_payload.clone()))
|
.try_send(WriterCommand::Data(current_payload.clone()))
|
||||||
{
|
{
|
||||||
Ok(()) => {
|
Ok(()) => return Ok(()),
|
||||||
self.note_hybrid_route_success();
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
Err(TrySendError::Full(cmd)) => {
|
Err(TrySendError::Full(cmd)) => {
|
||||||
if current.tx.send(cmd).await.is_ok() {
|
if current.tx.send(cmd).await.is_ok() {
|
||||||
self.note_hybrid_route_success();
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
warn!(writer_id = current.writer_id, "ME writer channel closed");
|
warn!(writer_id = current.writer_id, "ME writer channel closed");
|
||||||
@@ -132,7 +118,7 @@ impl MePool {
|
|||||||
match no_writer_mode {
|
match no_writer_mode {
|
||||||
MeRouteNoWriterMode::AsyncRecoveryFailfast => {
|
MeRouteNoWriterMode::AsyncRecoveryFailfast => {
|
||||||
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
||||||
Instant::now() + self.route_runtime.me_route_no_writer_wait
|
Instant::now() + self.me_route_no_writer_wait
|
||||||
});
|
});
|
||||||
if !async_recovery_triggered && !unknown_target_dc {
|
if !async_recovery_triggered && !unknown_target_dc {
|
||||||
let triggered =
|
let triggered =
|
||||||
@@ -153,9 +139,7 @@ impl MePool {
|
|||||||
MeRouteNoWriterMode::InlineRecoveryLegacy => {
|
MeRouteNoWriterMode::InlineRecoveryLegacy => {
|
||||||
self.stats.increment_me_inline_recovery_total();
|
self.stats.increment_me_inline_recovery_total();
|
||||||
if !unknown_target_dc {
|
if !unknown_target_dc {
|
||||||
for _ in
|
for _ in 0..self.me_route_inline_recovery_attempts.max(1) {
|
||||||
0..self.route_runtime.me_route_inline_recovery_attempts.max(1)
|
|
||||||
{
|
|
||||||
for family in self.family_order() {
|
for family in self.family_order() {
|
||||||
let map = match family {
|
let map = match family {
|
||||||
IpFamily::V4 => self.proxy_map_v4.read().await.clone(),
|
IpFamily::V4 => self.proxy_map_v4.read().await.clone(),
|
||||||
@@ -184,7 +168,7 @@ impl MePool {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
||||||
Instant::now() + self.route_runtime.me_route_inline_recovery_wait
|
Instant::now() + self.me_route_inline_recovery_wait
|
||||||
});
|
});
|
||||||
if !self.wait_for_writer_until(deadline).await {
|
if !self.wait_for_writer_until(deadline).await {
|
||||||
if !self.writers.read().await.is_empty() {
|
if !self.writers.read().await.is_empty() {
|
||||||
@@ -198,15 +182,6 @@ impl MePool {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
||||||
let total_deadline = *hybrid_total_deadline.get_or_insert_with(|| {
|
|
||||||
Instant::now() + self.hybrid_total_wait_budget()
|
|
||||||
});
|
|
||||||
if Instant::now() >= total_deadline {
|
|
||||||
self.on_hybrid_timeout(total_deadline, routed_dc);
|
|
||||||
return Err(ProxyError::Proxy(
|
|
||||||
"ME writer not available within hybrid timeout".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if !unknown_target_dc {
|
if !unknown_target_dc {
|
||||||
self.maybe_trigger_hybrid_recovery(
|
self.maybe_trigger_hybrid_recovery(
|
||||||
routed_dc,
|
routed_dc,
|
||||||
@@ -239,9 +214,8 @@ impl MePool {
|
|||||||
let pick_mode = self.writer_pick_mode();
|
let pick_mode = self.writer_pick_mode();
|
||||||
match no_writer_mode {
|
match no_writer_mode {
|
||||||
MeRouteNoWriterMode::AsyncRecoveryFailfast => {
|
MeRouteNoWriterMode::AsyncRecoveryFailfast => {
|
||||||
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
let deadline = *no_writer_deadline
|
||||||
Instant::now() + self.route_runtime.me_route_no_writer_wait
|
.get_or_insert_with(|| Instant::now() + self.me_route_no_writer_wait);
|
||||||
});
|
|
||||||
if !async_recovery_triggered && !unknown_target_dc {
|
if !async_recovery_triggered && !unknown_target_dc {
|
||||||
let triggered =
|
let triggered =
|
||||||
self.trigger_async_recovery_for_target_dc(routed_dc).await;
|
self.trigger_async_recovery_for_target_dc(routed_dc).await;
|
||||||
@@ -264,7 +238,7 @@ impl MePool {
|
|||||||
self.stats.increment_me_inline_recovery_total();
|
self.stats.increment_me_inline_recovery_total();
|
||||||
if unknown_target_dc {
|
if unknown_target_dc {
|
||||||
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
let deadline = *no_writer_deadline.get_or_insert_with(|| {
|
||||||
Instant::now() + self.route_runtime.me_route_inline_recovery_wait
|
Instant::now() + self.me_route_inline_recovery_wait
|
||||||
});
|
});
|
||||||
if self.wait_for_candidate_until(routed_dc, deadline).await {
|
if self.wait_for_candidate_until(routed_dc, deadline).await {
|
||||||
continue;
|
continue;
|
||||||
@@ -276,9 +250,7 @@ impl MePool {
|
|||||||
"No ME writers available for target DC".into(),
|
"No ME writers available for target DC".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if emergency_attempts
|
if emergency_attempts >= self.me_route_inline_recovery_attempts.max(1) {
|
||||||
>= self.route_runtime.me_route_inline_recovery_attempts.max(1)
|
|
||||||
{
|
|
||||||
self.stats
|
self.stats
|
||||||
.increment_me_writer_pick_no_candidate_total(pick_mode);
|
.increment_me_writer_pick_no_candidate_total(pick_mode);
|
||||||
self.stats.increment_me_no_writer_failfast_total();
|
self.stats.increment_me_no_writer_failfast_total();
|
||||||
@@ -320,16 +292,6 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
MeRouteNoWriterMode::HybridAsyncPersistent => {
|
||||||
let total_deadline = *hybrid_total_deadline.get_or_insert_with(|| {
|
|
||||||
Instant::now() + self.hybrid_total_wait_budget()
|
|
||||||
});
|
|
||||||
if Instant::now() >= total_deadline {
|
|
||||||
self.on_hybrid_timeout(total_deadline, routed_dc);
|
|
||||||
return Err(ProxyError::Proxy(
|
|
||||||
"No ME writers available for target DC within hybrid timeout"
|
|
||||||
.into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if !unknown_target_dc {
|
if !unknown_target_dc {
|
||||||
self.maybe_trigger_hybrid_recovery(
|
self.maybe_trigger_hybrid_recovery(
|
||||||
routed_dc,
|
routed_dc,
|
||||||
@@ -370,11 +332,7 @@ impl MePool {
|
|||||||
pick_sample_size,
|
pick_sample_size,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
if self
|
if self.me_deterministic_writer_sort.load(Ordering::Relaxed) {
|
||||||
.writer_selection_policy
|
|
||||||
.me_deterministic_writer_sort
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
{
|
|
||||||
candidate_indices.sort_by(|lhs, rhs| {
|
candidate_indices.sort_by(|lhs, rhs| {
|
||||||
let left = &writers_snapshot[*lhs];
|
let left = &writers_snapshot[*lhs];
|
||||||
let right = &writers_snapshot[*rhs];
|
let right = &writers_snapshot[*rhs];
|
||||||
@@ -465,7 +423,6 @@ impl MePool {
|
|||||||
"Selected stale ME writer for fallback bind"
|
"Selected stale ME writer for fallback bind"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
self.note_hybrid_route_success();
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
Err(TrySendError::Full(_)) => {
|
Err(TrySendError::Full(_)) => {
|
||||||
@@ -496,19 +453,7 @@ impl MePool {
|
|||||||
.increment_me_writer_pick_blocking_fallback_total();
|
.increment_me_writer_pick_blocking_fallback_total();
|
||||||
let effective_our_addr = SocketAddr::new(w.source_ip, our_addr.port());
|
let effective_our_addr = SocketAddr::new(w.source_ip, our_addr.port());
|
||||||
let (payload, meta) = build_routed_payload(effective_our_addr);
|
let (payload, meta) = build_routed_payload(effective_our_addr);
|
||||||
let reserve_result =
|
match w.tx.clone().reserve_owned().await {
|
||||||
if let Some(timeout) = self.route_runtime.me_route_blocking_send_timeout {
|
|
||||||
match tokio::time::timeout(timeout, w.tx.clone().reserve_owned()).await {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(_) => {
|
|
||||||
self.stats.increment_me_writer_pick_full_total(pick_mode);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
w.tx.clone().reserve_owned().await
|
|
||||||
};
|
|
||||||
match reserve_result {
|
|
||||||
Ok(permit) => {
|
Ok(permit) => {
|
||||||
if !self.registry.bind_writer(conn_id, w.id, meta).await {
|
if !self.registry.bind_writer(conn_id, w.id, meta).await {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -526,7 +471,6 @@ impl MePool {
|
|||||||
if w.generation < self.current_generation() {
|
if w.generation < self.current_generation() {
|
||||||
self.stats.increment_pool_stale_pick_total();
|
self.stats.increment_pool_stale_pick_total();
|
||||||
}
|
}
|
||||||
self.note_hybrid_route_success();
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
@@ -539,7 +483,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_writer_until(&self, deadline: Instant) -> bool {
|
async fn wait_for_writer_until(&self, deadline: Instant) -> bool {
|
||||||
let mut rx = self.writer_epoch.subscribe();
|
let waiter = self.writer_available.notified();
|
||||||
if !self.writers.read().await.is_empty() {
|
if !self.writers.read().await.is_empty() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -548,14 +492,13 @@ impl MePool {
|
|||||||
return !self.writers.read().await.is_empty();
|
return !self.writers.read().await.is_empty();
|
||||||
}
|
}
|
||||||
let timeout = deadline.saturating_duration_since(now);
|
let timeout = deadline.saturating_duration_since(now);
|
||||||
if tokio::time::timeout(timeout, rx.changed()).await.is_ok() {
|
if tokio::time::timeout(timeout, waiter).await.is_ok() {
|
||||||
return !self.writers.read().await.is_empty();
|
return true;
|
||||||
}
|
}
|
||||||
!self.writers.read().await.is_empty()
|
!self.writers.read().await.is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_candidate_until(&self, routed_dc: i32, deadline: Instant) -> bool {
|
async fn wait_for_candidate_until(&self, routed_dc: i32, deadline: Instant) -> bool {
|
||||||
let mut rx = self.writer_epoch.subscribe();
|
|
||||||
loop {
|
loop {
|
||||||
if self.has_candidate_for_target_dc(routed_dc).await {
|
if self.has_candidate_for_target_dc(routed_dc).await {
|
||||||
return true;
|
return true;
|
||||||
@@ -566,6 +509,7 @@ impl MePool {
|
|||||||
return self.has_candidate_for_target_dc(routed_dc).await;
|
return self.has_candidate_for_target_dc(routed_dc).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let waiter = self.writer_available.notified();
|
||||||
if self.has_candidate_for_target_dc(routed_dc).await {
|
if self.has_candidate_for_target_dc(routed_dc).await {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -573,7 +517,7 @@ impl MePool {
|
|||||||
if remaining.is_zero() {
|
if remaining.is_zero() {
|
||||||
return self.has_candidate_for_target_dc(routed_dc).await;
|
return self.has_candidate_for_target_dc(routed_dc).await;
|
||||||
}
|
}
|
||||||
if tokio::time::timeout(remaining, rx.changed()).await.is_err() {
|
if tokio::time::timeout(remaining, waiter).await.is_err() {
|
||||||
return self.has_candidate_for_target_dc(routed_dc).await;
|
return self.has_candidate_for_target_dc(routed_dc).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -643,9 +587,6 @@ impl MePool {
|
|||||||
hybrid_last_recovery_at: &mut Option<Instant>,
|
hybrid_last_recovery_at: &mut Option<Instant>,
|
||||||
hybrid_wait_step: Duration,
|
hybrid_wait_step: Duration,
|
||||||
) {
|
) {
|
||||||
if !self.try_consume_hybrid_recovery_trigger_slot(HYBRID_RECOVERY_TRIGGER_MIN_INTERVAL_MS) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if let Some(last) = *hybrid_last_recovery_at
|
if let Some(last) = *hybrid_last_recovery_at
|
||||||
&& last.elapsed() < hybrid_wait_step
|
&& last.elapsed() < hybrid_wait_step
|
||||||
{
|
{
|
||||||
@@ -661,78 +602,6 @@ impl MePool {
|
|||||||
*hybrid_last_recovery_at = Some(Instant::now());
|
*hybrid_last_recovery_at = Some(Instant::now());
|
||||||
}
|
}
|
||||||
|
|
||||||
fn hybrid_total_wait_budget(&self) -> Duration {
|
|
||||||
let base = self
|
|
||||||
.route_runtime
|
|
||||||
.me_route_hybrid_max_wait
|
|
||||||
.max(Duration::from_millis(50));
|
|
||||||
let now_ms = Self::now_epoch_millis();
|
|
||||||
let last_success_ms = self
|
|
||||||
.route_runtime
|
|
||||||
.me_route_last_success_epoch_ms
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
if last_success_ms != 0
|
|
||||||
&& now_ms.saturating_sub(last_success_ms) <= HYBRID_RECENT_SUCCESS_WINDOW_MS
|
|
||||||
{
|
|
||||||
return base.saturating_mul(2);
|
|
||||||
}
|
|
||||||
base
|
|
||||||
}
|
|
||||||
|
|
||||||
fn note_hybrid_route_success(&self) {
|
|
||||||
self.route_runtime
|
|
||||||
.me_route_last_success_epoch_ms
|
|
||||||
.store(Self::now_epoch_millis(), Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn on_hybrid_timeout(&self, deadline: Instant, routed_dc: i32) {
|
|
||||||
self.stats.increment_me_hybrid_timeout_total();
|
|
||||||
let now_ms = Self::now_epoch_millis();
|
|
||||||
let mut last_warn_ms = self
|
|
||||||
.route_runtime
|
|
||||||
.me_route_hybrid_timeout_warn_epoch_ms
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
while now_ms.saturating_sub(last_warn_ms) >= HYBRID_TIMEOUT_WARN_RATE_LIMIT_MS {
|
|
||||||
match self
|
|
||||||
.route_runtime
|
|
||||||
.me_route_hybrid_timeout_warn_epoch_ms
|
|
||||||
.compare_exchange_weak(last_warn_ms, now_ms, Ordering::AcqRel, Ordering::Relaxed)
|
|
||||||
{
|
|
||||||
Ok(_) => {
|
|
||||||
warn!(
|
|
||||||
routed_dc,
|
|
||||||
budget_ms = self.hybrid_total_wait_budget().as_millis() as u64,
|
|
||||||
elapsed_ms = deadline.elapsed().as_millis() as u64,
|
|
||||||
"ME hybrid route timeout reached"
|
|
||||||
);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(actual) => last_warn_ms = actual,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn try_consume_hybrid_recovery_trigger_slot(&self, min_interval_ms: u64) -> bool {
|
|
||||||
let now_ms = Self::now_epoch_millis();
|
|
||||||
let mut last_trigger_ms = self
|
|
||||||
.route_runtime
|
|
||||||
.me_async_recovery_last_trigger_epoch_ms
|
|
||||||
.load(Ordering::Relaxed);
|
|
||||||
loop {
|
|
||||||
if now_ms.saturating_sub(last_trigger_ms) < min_interval_ms {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
match self
|
|
||||||
.route_runtime
|
|
||||||
.me_async_recovery_last_trigger_epoch_ms
|
|
||||||
.compare_exchange_weak(last_trigger_ms, now_ms, Ordering::AcqRel, Ordering::Relaxed)
|
|
||||||
{
|
|
||||||
Ok(_) => return true,
|
|
||||||
Err(actual) => last_trigger_ms = actual,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
||||||
if let Some(w) = self.registry.get_writer(conn_id).await {
|
if let Some(w) = self.registry.get_writer(conn_id).await {
|
||||||
let mut p = Vec::with_capacity(12);
|
let mut p = Vec::with_capacity(12);
|
||||||
@@ -880,7 +749,7 @@ impl MePool {
|
|||||||
(self.writer_idle_rank_for_selection(writer, idle_since_by_writer, now_epoch_secs)
|
(self.writer_idle_rank_for_selection(writer, idle_since_by_writer, now_epoch_secs)
|
||||||
as u64)
|
as u64)
|
||||||
* 100;
|
* 100;
|
||||||
let queue_cap = self.writer_lifecycle.writer_cmd_channel_capacity.max(1) as u64;
|
let queue_cap = self.writer_cmd_channel_capacity.max(1) as u64;
|
||||||
let queue_remaining = writer.tx.capacity() as u64;
|
let queue_remaining = writer.tx.capacity() as u64;
|
||||||
let queue_used = queue_cap.saturating_sub(queue_remaining.min(queue_cap));
|
let queue_used = queue_cap.saturating_sub(queue_remaining.min(queue_cap));
|
||||||
let queue_util_pct = queue_used.saturating_mul(100) / queue_cap;
|
let queue_util_pct = queue_used.saturating_mul(100) / queue_cap;
|
||||||
|
|||||||
@@ -113,8 +113,6 @@ async fn make_pool(
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
MeRouteNoWriterMode::default(),
|
MeRouteNoWriterMode::default(),
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -111,8 +111,6 @@ async fn make_pool(
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
MeRouteNoWriterMode::default(),
|
MeRouteNoWriterMode::default(),
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -106,8 +106,6 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc<MePool> {
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
MeRouteNoWriterMode::default(),
|
MeRouteNoWriterMode::default(),
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -95,8 +95,6 @@ async fn make_pool() -> Arc<MePool> {
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
MeRouteNoWriterMode::default(),
|
MeRouteNoWriterMode::default(),
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ async fn make_pool() -> Arc<MePool> {
|
|||||||
NetworkDecision::default(),
|
NetworkDecision::default(),
|
||||||
None,
|
None,
|
||||||
Arc::new(SecureRandom::new()),
|
Arc::new(SecureRandom::new()),
|
||||||
Arc::new(Stats::new()),
|
Arc::new(Stats::default()),
|
||||||
general.me_keepalive_enabled,
|
general.me_keepalive_enabled,
|
||||||
general.me_keepalive_interval_secs,
|
general.me_keepalive_interval_secs,
|
||||||
general.me_keepalive_jitter_secs,
|
general.me_keepalive_jitter_secs,
|
||||||
@@ -100,8 +100,6 @@ async fn make_pool() -> Arc<MePool> {
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
MeRouteNoWriterMode::default(),
|
MeRouteNoWriterMode::default(),
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
)
|
)
|
||||||
@@ -173,15 +171,10 @@ async fn bind_conn_to_writer(pool: &Arc<MePool>, writer_id: u64, port: u16) -> u
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn remove_draining_writer_does_not_quarantine_flapping_endpoint() {
|
async fn remove_draining_writer_still_quarantines_flapping_endpoint() {
|
||||||
let pool = make_pool().await;
|
let pool = make_pool().await;
|
||||||
let writer_id = 77;
|
let writer_id = 77;
|
||||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 12, 0, 77)), 443);
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 12, 0, 77)), 443);
|
||||||
let before_total = pool.stats.get_me_endpoint_quarantine_total();
|
|
||||||
let before_unexpected = pool.stats.get_me_endpoint_quarantine_unexpected_total();
|
|
||||||
let before_suppressed = pool
|
|
||||||
.stats
|
|
||||||
.get_me_endpoint_quarantine_draining_suppressed_total();
|
|
||||||
insert_writer(
|
insert_writer(
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
@@ -205,18 +198,8 @@ async fn remove_draining_writer_does_not_quarantine_flapping_endpoint() {
|
|||||||
"writer must be removed from pool after cleanup"
|
"writer must be removed from pool after cleanup"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
!pool.is_endpoint_quarantined(addr).await,
|
pool.is_endpoint_quarantined(addr).await,
|
||||||
"draining removals must not quarantine endpoint"
|
"draining removals must still quarantine flapping endpoints"
|
||||||
);
|
|
||||||
assert_eq!(pool.stats.get_me_endpoint_quarantine_total(), before_total);
|
|
||||||
assert_eq!(
|
|
||||||
pool.stats.get_me_endpoint_quarantine_unexpected_total(),
|
|
||||||
before_unexpected
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
pool.stats
|
|
||||||
.get_me_endpoint_quarantine_draining_suppressed_total(),
|
|
||||||
before_suppressed + 1
|
|
||||||
);
|
);
|
||||||
assert_eq!(pool.conn_count.load(Ordering::Relaxed), 0);
|
assert_eq!(pool.conn_count.load(Ordering::Relaxed), 0);
|
||||||
}
|
}
|
||||||
@@ -272,21 +255,16 @@ async fn edge_draining_only_detach_rejects_active_writer() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn adversarial_blackhat_single_unexpected_remove_establishes_single_quarantine_entry() {
|
async fn adversarial_blackhat_single_remove_establishes_single_quarantine_entry() {
|
||||||
let pool = make_pool().await;
|
let pool = make_pool().await;
|
||||||
let writer_id = 93;
|
let writer_id = 93;
|
||||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 12, 0, 93)), 443);
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 12, 0, 93)), 443);
|
||||||
let before_total = pool.stats.get_me_endpoint_quarantine_total();
|
|
||||||
let before_unexpected = pool.stats.get_me_endpoint_quarantine_unexpected_total();
|
|
||||||
let before_suppressed = pool
|
|
||||||
.stats
|
|
||||||
.get_me_endpoint_quarantine_draining_suppressed_total();
|
|
||||||
insert_writer(
|
insert_writer(
|
||||||
&pool,
|
&pool,
|
||||||
writer_id,
|
writer_id,
|
||||||
2,
|
2,
|
||||||
addr,
|
addr,
|
||||||
false,
|
true,
|
||||||
Instant::now() - Duration::from_secs(1),
|
Instant::now() - Duration::from_secs(1),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -294,49 +272,6 @@ async fn adversarial_blackhat_single_unexpected_remove_establishes_single_quaran
|
|||||||
pool.remove_writer_and_close_clients(writer_id).await;
|
pool.remove_writer_and_close_clients(writer_id).await;
|
||||||
assert!(pool.is_endpoint_quarantined(addr).await);
|
assert!(pool.is_endpoint_quarantined(addr).await);
|
||||||
assert_eq!(pool.endpoint_quarantine.lock().await.len(), 1);
|
assert_eq!(pool.endpoint_quarantine.lock().await.len(), 1);
|
||||||
assert_eq!(
|
|
||||||
pool.stats.get_me_endpoint_quarantine_total(),
|
|
||||||
before_total + 1
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
pool.stats.get_me_endpoint_quarantine_unexpected_total(),
|
|
||||||
before_unexpected + 1
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
pool.stats
|
|
||||||
.get_me_endpoint_quarantine_draining_suppressed_total(),
|
|
||||||
before_suppressed
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn remove_ultra_short_uptime_writer_skips_flap_quarantine() {
|
|
||||||
let pool = make_pool().await;
|
|
||||||
let writer_id = 931;
|
|
||||||
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 12, 0, 131)), 443);
|
|
||||||
let before_total = pool.stats.get_me_endpoint_quarantine_total();
|
|
||||||
let before_unexpected = pool.stats.get_me_endpoint_quarantine_unexpected_total();
|
|
||||||
insert_writer(
|
|
||||||
&pool,
|
|
||||||
writer_id,
|
|
||||||
2,
|
|
||||||
addr,
|
|
||||||
false,
|
|
||||||
Instant::now() - Duration::from_millis(50),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
pool.remove_writer_and_close_clients(writer_id).await;
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
!pool.is_endpoint_quarantined(addr).await,
|
|
||||||
"ultra-short unexpected lifetime must not quarantine endpoint"
|
|
||||||
);
|
|
||||||
assert_eq!(pool.stats.get_me_endpoint_quarantine_total(), before_total);
|
|
||||||
assert_eq!(
|
|
||||||
pool.stats.get_me_endpoint_quarantine_unexpected_total(),
|
|
||||||
before_unexpected + 1
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -106,8 +106,6 @@ async fn make_pool() -> (Arc<MePool>, Arc<SecureRandom>) {
|
|||||||
general.me_warn_rate_limit_ms,
|
general.me_warn_rate_limit_ms,
|
||||||
general.me_route_no_writer_mode,
|
general.me_route_no_writer_mode,
|
||||||
general.me_route_no_writer_wait_ms,
|
general.me_route_no_writer_wait_ms,
|
||||||
general.me_route_hybrid_max_wait_ms,
|
|
||||||
general.me_route_blocking_send_timeout_ms,
|
|
||||||
general.me_route_inline_recovery_attempts,
|
general.me_route_inline_recovery_attempts,
|
||||||
general.me_route_inline_recovery_wait_ms,
|
general.me_route_inline_recovery_wait_ms,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -317,7 +317,7 @@ pub struct UpstreamManager {
|
|||||||
connect_retry_attempts: u32,
|
connect_retry_attempts: u32,
|
||||||
connect_retry_backoff: Duration,
|
connect_retry_backoff: Duration,
|
||||||
connect_budget: Duration,
|
connect_budget: Duration,
|
||||||
/// Per-attempt TCP connect timeout to Telegram DC (`[general] tg_connect`, seconds).
|
/// Per-attempt TCP connect timeout to Telegram DC (`[timeouts] tg_connect`, seconds).
|
||||||
tg_connect_timeout_secs: u64,
|
tg_connect_timeout_secs: u64,
|
||||||
unhealthy_fail_threshold: u32,
|
unhealthy_fail_threshold: u32,
|
||||||
connect_failfast_hard_errors: bool,
|
connect_failfast_hard_errors: bool,
|
||||||
@@ -799,8 +799,8 @@ impl UpstreamManager {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let remaining_budget = self.connect_budget.saturating_sub(elapsed);
|
let remaining_budget = self.connect_budget.saturating_sub(elapsed);
|
||||||
let attempt_timeout =
|
let attempt_timeout = Duration::from_secs(self.tg_connect_timeout_secs)
|
||||||
Duration::from_secs(self.tg_connect_timeout_secs).min(remaining_budget);
|
.min(remaining_budget);
|
||||||
if attempt_timeout.is_zero() {
|
if attempt_timeout.is_zero() {
|
||||||
last_error = Some(ProxyError::ConnectionTimeout {
|
last_error = Some(ProxyError::ConnectionTimeout {
|
||||||
addr: target.to_string(),
|
addr: target.to_string(),
|
||||||
|
|||||||
Reference in New Issue
Block a user