astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-04-15 09:34:10 +03:00
Code Issues Packages Projects Releases Wiki Activity

ME Pool Health + Rotation

Browse Source 
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-02-24 22:59:59 +03:00
parent 7d7ef84868
commit 4a95f6d195
7 changed files with 424 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
src/transport/middle_proxy/rotation.rs
View File

@@ -1,50 +1,87 @@
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use tokio::sync::watch;
use tracing::{info, warn};
use crate::config::ProxyConfig;
use crate::crypto::SecureRandom;
use super::MePool;
/// Periodically refresh ME connections to avoid long-lived degradation.
pub async fn me_rotation_task(pool: Arc<MePool>, rng: Arc<SecureRandom>, interval: Duration) {
let interval = interval.max(Duration::from_secs(600));
/// Periodically reinitialize ME generations and swap them after full warmup.
pub async fn me_rotation_task(
pool: Arc<MePool>,
rng: Arc<SecureRandom>,
mut config_rx: watch::Receiver<Arc<ProxyConfig>>,
) {
let mut interval_secs = config_rx
.borrow()
.general
.effective_me_reinit_every_secs()
.max(1);
let mut interval = Duration::from_secs(interval_secs);
let mut next_tick = tokio::time::Instant::now() + interval;
info!(interval_secs, "ME periodic reinit task started");
loop {
tokio::time::sleep(interval).await;
let sleep = tokio::time::sleep_until(next_tick);
tokio::pin!(sleep);
let candidate = {
let ws = pool.writers.read().await;
if ws.is_empty() {
None
} else {
let idx = (pool.rr.load(std::sync::atomic::Ordering::Relaxed) as usize) % ws.len();
ws.get(idx).cloned()
}
};
let Some(w) = candidate else {
continue;
};
info!(addr = %w.addr, writer_id = w.id, "Rotating ME connection");
match pool.connect_one(w.addr, rng.as_ref()).await {
Ok(()) => {
tokio::time::sleep(Duration::from_secs(2)).await;
let ws = pool.writers.read().await;
let new_alive = ws.iter().any(|nw|
nw.id != w.id && nw.addr == w.addr && !nw.degraded.load(Ordering::Relaxed) && !nw.draining.load(Ordering::Relaxed)
);
drop(ws);
if new_alive {
pool.mark_writer_draining(w.id).await;
} else {
warn!(addr = %w.addr, writer_id = w.id, "New writer died, keeping old");
tokio::select! {
_ = &mut sleep => {
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
let refreshed_secs = config_rx
.borrow()
.general
.effective_me_reinit_every_secs()
.max(1);
if refreshed_secs != interval_secs {
info!(
old_me_reinit_every_secs = interval_secs,
new_me_reinit_every_secs = refreshed_secs,
"ME periodic reinit interval changed"
);
interval_secs = refreshed_secs;
interval = Duration::from_secs(interval_secs);
}
next_tick = tokio::time::Instant::now() + interval;
}
Err(e) => {
warn!(addr = %w.addr, writer_id = w.id, error = %e, "ME rotation connect failed");
changed = config_rx.changed() => {
if changed.is_err() {
warn!("ME periodic reinit task stopped: config channel closed");
break;
}
let new_secs = config_rx
.borrow()
.general
.effective_me_reinit_every_secs()
.max(1);
if new_secs == interval_secs {
continue;
}
if new_secs < interval_secs {
info!(
old_me_reinit_every_secs = interval_secs,
new_me_reinit_every_secs = new_secs,
"ME periodic reinit interval decreased, running immediate reinit"
);
interval_secs = new_secs;
interval = Duration::from_secs(interval_secs);
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
next_tick = tokio::time::Instant::now() + interval;
} else {
info!(
old_me_reinit_every_secs = interval_secs,
new_me_reinit_every_secs = new_secs,
"ME periodic reinit interval increased"
);
interval_secs = new_secs;
interval = Duration::from_secs(interval_secs);
next_tick = tokio::time::Instant::now() + interval;
}
}
}
}