astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-04-15 01:24:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

Format

Browse Source
This commit is contained in:
Alexey
2026-03-21 15:45:29 +03:00
parent 7a8f946029
commit d7bbb376c9
154 changed files with 6194 additions and 3775 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
src/config/hot_reload.rs
View File

@@ -31,11 +31,10 @@ use notify::{EventKind, RecursiveMode, Watcher, recommended_watcher};
use tokio::sync::{mpsc, watch};
use tracing::{error, info, warn};
use crate::config::{
LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
MeWriterPickMode,
};
use super::load::{LoadedConfig, ProxyConfig};
use crate::config::{
LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel, MeWriterPickMode,
};
const HOT_RELOAD_DEBOUNCE: Duration = Duration::from_millis(50);
@@ -44,16 +43,16 @@ const HOT_RELOAD_DEBOUNCE: Duration = Duration::from_millis(50);
/// Fields that are safe to swap without restarting listeners.
#[derive(Debug, Clone, PartialEq)]
pub struct HotFields {
pub log_level: LogLevel,
pub ad_tag: Option<String>,
pub dns_overrides: Vec<String>,
pub desync_all_full: bool,
pub update_every_secs: u64,
pub me_reinit_every_secs: u64,
pub me_reinit_singleflight: bool,
pub log_level: LogLevel,
pub ad_tag: Option<String>,
pub dns_overrides: Vec<String>,
pub desync_all_full: bool,
pub update_every_secs: u64,
pub me_reinit_every_secs: u64,
pub me_reinit_singleflight: bool,
pub me_reinit_coalesce_window_ms: u64,
pub hardswap: bool,
pub me_pool_drain_ttl_secs: u64,
pub hardswap: bool,
pub me_pool_drain_ttl_secs: u64,
pub me_instadrain: bool,
pub me_pool_drain_threshold: u64,
pub me_pool_min_fresh_ratio: f32,
@@ -113,12 +112,12 @@ pub struct HotFields {
pub me_health_interval_ms_healthy: u64,
pub me_admission_poll_ms: u64,
pub me_warn_rate_limit_ms: u64,
pub users: std::collections::HashMap<String, String>,
pub user_ad_tags: std::collections::HashMap<String, String>,
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
pub user_data_quota: std::collections::HashMap<String, u64>,
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
pub users: std::collections::HashMap<String, String>,
pub user_ad_tags: std::collections::HashMap<String, String>,
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
pub user_data_quota: std::collections::HashMap<String, u64>,
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
pub user_max_unique_ips_global_each: usize,
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
pub user_max_unique_ips_window_secs: u64,
@@ -127,16 +126,16 @@ pub struct HotFields {
impl HotFields {
pub fn from_config(cfg: &ProxyConfig) -> Self {
Self {
log_level: cfg.general.log_level.clone(),
ad_tag: cfg.general.ad_tag.clone(),
dns_overrides: cfg.network.dns_overrides.clone(),
desync_all_full: cfg.general.desync_all_full,
update_every_secs: cfg.general.effective_update_every_secs(),
me_reinit_every_secs: cfg.general.me_reinit_every_secs,
me_reinit_singleflight: cfg.general.me_reinit_singleflight,
log_level: cfg.general.log_level.clone(),
ad_tag: cfg.general.ad_tag.clone(),
dns_overrides: cfg.network.dns_overrides.clone(),
desync_all_full: cfg.general.desync_all_full,
update_every_secs: cfg.general.effective_update_every_secs(),
me_reinit_every_secs: cfg.general.me_reinit_every_secs,
me_reinit_singleflight: cfg.general.me_reinit_singleflight,
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
hardswap: cfg.general.hardswap,
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
hardswap: cfg.general.hardswap,
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
me_instadrain: cfg.general.me_instadrain,
me_pool_drain_threshold: cfg.general.me_pool_drain_threshold,
me_pool_min_fresh_ratio: cfg.general.me_pool_min_fresh_ratio,
@@ -189,15 +188,11 @@ impl HotFields {
me_adaptive_floor_min_writers_multi_endpoint: cfg
.general
.me_adaptive_floor_min_writers_multi_endpoint,
me_adaptive_floor_recover_grace_secs: cfg
.general
.me_adaptive_floor_recover_grace_secs,
me_adaptive_floor_recover_grace_secs: cfg.general.me_adaptive_floor_recover_grace_secs,
me_adaptive_floor_writers_per_core_total: cfg
.general
.me_adaptive_floor_writers_per_core_total,
me_adaptive_floor_cpu_cores_override: cfg
.general
.me_adaptive_floor_cpu_cores_override,
me_adaptive_floor_cpu_cores_override: cfg.general.me_adaptive_floor_cpu_cores_override,
me_adaptive_floor_max_extra_writers_single_per_core: cfg
.general
.me_adaptive_floor_max_extra_writers_single_per_core,
@@ -216,9 +211,15 @@ impl HotFields {
me_adaptive_floor_max_warm_writers_global: cfg
.general
.me_adaptive_floor_max_warm_writers_global,
me_route_backpressure_base_timeout_ms: cfg.general.me_route_backpressure_base_timeout_ms,
me_route_backpressure_high_timeout_ms: cfg.general.me_route_backpressure_high_timeout_ms,
me_route_backpressure_high_watermark_pct: cfg.general.me_route_backpressure_high_watermark_pct,
me_route_backpressure_base_timeout_ms: cfg
.general
.me_route_backpressure_base_timeout_ms,
me_route_backpressure_high_timeout_ms: cfg
.general
.me_route_backpressure_high_timeout_ms,
me_route_backpressure_high_watermark_pct: cfg
.general
.me_route_backpressure_high_watermark_pct,
me_reader_route_data_wait_ms: cfg.general.me_reader_route_data_wait_ms,
me_d2c_flush_batch_max_frames: cfg.general.me_d2c_flush_batch_max_frames,
me_d2c_flush_batch_max_bytes: cfg.general.me_d2c_flush_batch_max_bytes,
@@ -230,12 +231,12 @@ impl HotFields {
me_health_interval_ms_healthy: cfg.general.me_health_interval_ms_healthy,
me_admission_poll_ms: cfg.general.me_admission_poll_ms,
me_warn_rate_limit_ms: cfg.general.me_warn_rate_limit_ms,
users: cfg.access.users.clone(),
user_ad_tags: cfg.access.user_ad_tags.clone(),
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
user_expirations: cfg.access.user_expirations.clone(),
user_data_quota: cfg.access.user_data_quota.clone(),
user_max_unique_ips: cfg.access.user_max_unique_ips.clone(),
users: cfg.access.users.clone(),
user_ad_tags: cfg.access.user_ad_tags.clone(),
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
user_expirations: cfg.access.user_expirations.clone(),
user_data_quota: cfg.access.user_data_quota.clone(),
user_max_unique_ips: cfg.access.user_max_unique_ips.clone(),
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
@@ -334,7 +335,9 @@ struct ReloadState {
impl ReloadState {
fn new(applied_snapshot_hash: Option<u64>) -> Self {
Self { applied_snapshot_hash }
Self {
applied_snapshot_hash,
}
}
fn is_applied(&self, hash: u64) -> bool {
@@ -481,10 +484,14 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
new.general.me_adaptive_floor_writers_per_core_total;
cfg.general.me_adaptive_floor_cpu_cores_override =
new.general.me_adaptive_floor_cpu_cores_override;
cfg.general.me_adaptive_floor_max_extra_writers_single_per_core =
new.general.me_adaptive_floor_max_extra_writers_single_per_core;
cfg.general.me_adaptive_floor_max_extra_writers_multi_per_core =
new.general.me_adaptive_floor_max_extra_writers_multi_per_core;
cfg.general
.me_adaptive_floor_max_extra_writers_single_per_core = new
.general
.me_adaptive_floor_max_extra_writers_single_per_core;
cfg.general
.me_adaptive_floor_max_extra_writers_multi_per_core = new
.general
.me_adaptive_floor_max_extra_writers_multi_per_core;
cfg.general.me_adaptive_floor_max_active_writers_per_core =
new.general.me_adaptive_floor_max_active_writers_per_core;
cfg.general.me_adaptive_floor_max_warm_writers_per_core =
@@ -543,8 +550,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|| old.server.api.minimal_runtime_cache_ttl_ms
!= new.server.api.minimal_runtime_cache_ttl_ms
|| old.server.api.runtime_edge_enabled != new.server.api.runtime_edge_enabled
|| old.server.api.runtime_edge_cache_ttl_ms
!= new.server.api.runtime_edge_cache_ttl_ms
|| old.server.api.runtime_edge_cache_ttl_ms != new.server.api.runtime_edge_cache_ttl_ms
|| old.server.api.runtime_edge_top_n != new.server.api.runtime_edge_top_n
|| old.server.api.runtime_edge_events_capacity
!= new.server.api.runtime_edge_events_capacity
@@ -583,10 +589,8 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|| old.censorship.mask_shape_hardening != new.censorship.mask_shape_hardening
|| old.censorship.mask_shape_bucket_floor_bytes
!= new.censorship.mask_shape_bucket_floor_bytes
|| old.censorship.mask_shape_bucket_cap_bytes
!= new.censorship.mask_shape_bucket_cap_bytes
|| old.censorship.mask_shape_above_cap_blur
!= new.censorship.mask_shape_above_cap_blur
|| old.censorship.mask_shape_bucket_cap_bytes != new.censorship.mask_shape_bucket_cap_bytes
|| old.censorship.mask_shape_above_cap_blur != new.censorship.mask_shape_above_cap_blur
|| old.censorship.mask_shape_above_cap_blur_max_bytes
!= new.censorship.mask_shape_above_cap_blur_max_bytes
|| old.censorship.mask_timing_normalization_enabled
@@ -870,8 +874,7 @@ fn log_changes(
{
info!(
"config reload: me_bind_stale: mode={:?} ttl={}s",
new_hot.me_bind_stale_mode,
new_hot.me_bind_stale_ttl_secs
new_hot.me_bind_stale_mode, new_hot.me_bind_stale_ttl_secs
);
}
if old_hot.me_secret_atomic_snapshot != new_hot.me_secret_atomic_snapshot
@@ -951,8 +954,7 @@ fn log_changes(
if old_hot.me_socks_kdf_policy != new_hot.me_socks_kdf_policy {
info!(
"config reload: me_socks_kdf_policy: {:?} → {:?}",
old_hot.me_socks_kdf_policy,
new_hot.me_socks_kdf_policy,
old_hot.me_socks_kdf_policy, new_hot.me_socks_kdf_policy,
);
}
@@ -1006,8 +1008,7 @@ fn log_changes(
|| old_hot.me_route_backpressure_high_watermark_pct
!= new_hot.me_route_backpressure_high_watermark_pct
|| old_hot.me_reader_route_data_wait_ms != new_hot.me_reader_route_data_wait_ms
|| old_hot.me_health_interval_ms_unhealthy
!= new_hot.me_health_interval_ms_unhealthy
|| old_hot.me_health_interval_ms_unhealthy != new_hot.me_health_interval_ms_unhealthy
|| old_hot.me_health_interval_ms_healthy != new_hot.me_health_interval_ms_healthy
|| old_hot.me_admission_poll_ms != new_hot.me_admission_poll_ms
|| old_hot.me_warn_rate_limit_ms != new_hot.me_warn_rate_limit_ms
@@ -1044,19 +1045,27 @@ fn log_changes(
}
if old_hot.users != new_hot.users {
let mut added: Vec<&String> = new_hot.users.keys()
let mut added: Vec<&String> = new_hot
.users
.keys()
.filter(|u| !old_hot.users.contains_key(*u))
.collect();
added.sort();
let mut removed: Vec<&String> = old_hot.users.keys()
let mut removed: Vec<&String> = old_hot
.users
.keys()
.filter(|u| !new_hot.users.contains_key(*u))
.collect();
removed.sort();
let mut changed: Vec<&String> = new_hot.users.keys()
let mut changed: Vec<&String> = new_hot
.users
.keys()
.filter(|u| {
old_hot.users.get(*u)
old_hot
.users
.get(*u)
.map(|s| s != &new_hot.users[*u])
.unwrap_or(false)
})
@@ -1066,10 +1075,18 @@ fn log_changes(
if !added.is_empty() {
info!(
"config reload: users added: [{}]",
added.iter().map(|s| s.as_str()).collect::<Vec<_>>().join(", ")
added
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ")
);
let host = resolve_link_host(new_cfg, detected_ip_v4, detected_ip_v6);
let port = new_cfg.general.links.public_port.unwrap_or(new_cfg.server.port);
let port = new_cfg
.general
.links
.public_port
.unwrap_or(new_cfg.server.port);
for user in &added {
if let Some(secret) = new_hot.users.get(*user) {
print_user_links(user, secret, &host, port, new_cfg);
@@ -1079,13 +1096,21 @@ fn log_changes(
if !removed.is_empty() {
info!(
"config reload: users removed: [{}]",
removed.iter().map(|s| s.as_str()).collect::<Vec<_>>().join(", ")
removed
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ")
);
}
if !changed.is_empty() {
info!(
"config reload: users secret changed: [{}]",
changed.iter().map(|s| s.as_str()).collect::<Vec<_>>().join(", ")
changed
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ")
);
}
}
@@ -1116,8 +1141,7 @@ fn log_changes(
}
if old_hot.user_max_unique_ips_global_each != new_hot.user_max_unique_ips_global_each
|| old_hot.user_max_unique_ips_mode != new_hot.user_max_unique_ips_mode
|| old_hot.user_max_unique_ips_window_secs
!= new_hot.user_max_unique_ips_window_secs
|| old_hot.user_max_unique_ips_window_secs != new_hot.user_max_unique_ips_window_secs
{
info!(
"config reload: user_max_unique_ips policy global_each={} mode={:?} window={}s",
@@ -1152,7 +1176,10 @@ fn reload_config(
let next_manifest = WatchManifest::from_source_files(&source_files);
if let Err(e) = new_cfg.validate() {
error!("config reload: validation failed: {}; keeping old config", e);
error!(
"config reload: validation failed: {}; keeping old config",
e
);
return Some(next_manifest);
}
@@ -1217,7 +1244,7 @@ pub fn spawn_config_watcher(
) -> (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) {
let initial_level = initial.general.log_level.clone();
let (config_tx, config_rx) = watch::channel(initial);
let (log_tx, log_rx) = watch::channel(initial_level);
let (log_tx, log_rx) = watch::channel(initial_level);
let config_path = normalize_watch_path(&config_path);
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
@@ -1234,25 +1261,29 @@ pub fn spawn_config_watcher(
let tx_inotify = notify_tx.clone();
let manifest_for_inotify = manifest_state.clone();
let mut inotify_watcher = match recommended_watcher(move |res: notify::Result<notify::Event>| {
let Ok(event) = res else { return };
if !matches!(event.kind, EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)) {
return;
}
let is_our_file = manifest_for_inotify
.read()
.map(|manifest| manifest.matches_event_paths(&event.paths))
.unwrap_or(false);
if is_our_file {
let _ = tx_inotify.try_send(());
}
}) {
Ok(watcher) => Some(watcher),
Err(e) => {
warn!("config watcher: inotify unavailable: {}", e);
None
}
};
let mut inotify_watcher =
match recommended_watcher(move |res: notify::Result<notify::Event>| {
let Ok(event) = res else { return };
if !matches!(
event.kind,
EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)
) {
return;
}
let is_our_file = manifest_for_inotify
.read()
.map(|manifest| manifest.matches_event_paths(&event.paths))
.unwrap_or(false);
if is_our_file {
let _ = tx_inotify.try_send(());
}
}) {
Ok(watcher) => Some(watcher),
Err(e) => {
warn!("config watcher: inotify unavailable: {}", e);
None
}
};
apply_watch_manifest(
inotify_watcher.as_mut(),
Option::<&mut notify::poll::PollWatcher>::None,
@@ -1268,7 +1299,10 @@ pub fn spawn_config_watcher(
let mut poll_watcher = match notify::poll::PollWatcher::new(
move |res: notify::Result<notify::Event>| {
let Ok(event) = res else { return };
if !matches!(event.kind, EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)) {
if !matches!(
event.kind,
EventKind::Modify(_) | EventKind::Create(_) | EventKind::Remove(_)
) {
return;
}
let is_our_file = manifest_for_poll
@@ -1316,7 +1350,9 @@ pub fn spawn_config_watcher(
}
}
#[cfg(not(unix))]
if notify_rx.recv().await.is_none() { break; }
if notify_rx.recv().await.is_none() {
break;
}
// Debounce: drain extra events that arrive within a short quiet window.
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
@@ -1418,7 +1454,10 @@ mod tests {
new.server.port = old.server.port.saturating_add(1);
let applied = overlay_hot_fields(&old, &new);
assert_eq!(HotFields::from_config(&old), HotFields::from_config(&applied));
assert_eq!(
HotFields::from_config(&old),
HotFields::from_config(&applied)
);
assert_eq!(applied.server.port, old.server.port);
}
@@ -1437,7 +1476,10 @@ mod tests {
applied.general.me_bind_stale_mode,
new.general.me_bind_stale_mode
);
assert_ne!(HotFields::from_config(&old), HotFields::from_config(&applied));
assert_ne!(
HotFields::from_config(&old),
HotFields::from_config(&applied)
);
}
#[test]
@@ -1451,7 +1493,10 @@ mod tests {
applied.general.me_keepalive_interval_secs,
old.general.me_keepalive_interval_secs
);
assert_eq!(HotFields::from_config(&old), HotFields::from_config(&applied));
assert_eq!(
HotFields::from_config(&old),
HotFields::from_config(&applied)
);
}
#[test]
@@ -1463,7 +1508,10 @@ mod tests {
let applied = overlay_hot_fields(&old, &new);
assert_eq!(applied.general.hardswap, new.general.hardswap);
assert_eq!(applied.general.use_middle_proxy, old.general.use_middle_proxy);
assert_eq!(
applied.general.use_middle_proxy,
old.general.use_middle_proxy
);
assert!(!config_equal(&applied, &new));
}
@@ -1475,14 +1523,19 @@ mod tests {
write_reload_config(&path, Some(initial_tag), None);
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
let initial_hash = ProxyConfig::load_with_metadata(&path).unwrap().rendered_hash;
let initial_hash = ProxyConfig::load_with_metadata(&path)
.unwrap()
.rendered_hash;
let (config_tx, _config_rx) = watch::channel(initial_cfg.clone());
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
let mut reload_state = ReloadState::new(Some(initial_hash));
write_reload_config(&path, Some(final_tag), None);
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
assert_eq!(config_tx.borrow().general.ad_tag.as_deref(), Some(final_tag));
assert_eq!(
config_tx.borrow().general.ad_tag.as_deref(),
Some(final_tag)
);
let _ = std::fs::remove_file(path);
}
@@ -1495,7 +1548,9 @@ mod tests {
write_reload_config(&path, Some(initial_tag), None);
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
let initial_hash = ProxyConfig::load_with_metadata(&path).unwrap().rendered_hash;
let initial_hash = ProxyConfig::load_with_metadata(&path)
.unwrap()
.rendered_hash;
let (config_tx, _config_rx) = watch::channel(initial_cfg.clone());
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
let mut reload_state = ReloadState::new(Some(initial_hash));
@@ -1518,7 +1573,9 @@ mod tests {
write_reload_config(&path, Some(initial_tag), None);
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
let initial_hash = ProxyConfig::load_with_metadata(&path).unwrap().rendered_hash;
let initial_hash = ProxyConfig::load_with_metadata(&path)
.unwrap()
.rendered_hash;
let (config_tx, _config_rx) = watch::channel(initial_cfg.clone());
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
let mut reload_state = ReloadState::new(Some(initial_hash));
@@ -1532,7 +1589,10 @@ mod tests {
write_reload_config(&path, Some(final_tag), None);
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
assert_eq!(config_tx.borrow().general.ad_tag.as_deref(), Some(final_tag));
assert_eq!(
config_tx.borrow().general.ad_tag.as_deref(),
Some(final_tag)
);
let _ = std::fs::remove_file(path);
}