ME Writer Draining-state fixes

This commit is contained in:
Alexey
2026-03-17 13:58:01 +03:00
parent d78360982c
commit 2e8be87ccf
5 changed files with 309 additions and 52 deletions
+51 -1
View File
@@ -141,6 +141,38 @@ impl MePool {
out
}
pub(super) async fn has_non_draining_writer_per_desired_dc_group(&self) -> bool {
let desired_by_dc = self.desired_dc_endpoints().await;
let required_dcs: HashSet<i32> = desired_by_dc
.iter()
.filter_map(|(dc, endpoints)| {
if endpoints.is_empty() {
None
} else {
Some(*dc)
}
})
.collect();
if required_dcs.is_empty() {
return true;
}
let ws = self.writers.read().await;
let mut covered_dcs = HashSet::<i32>::with_capacity(required_dcs.len());
for writer in ws.iter() {
if writer.draining.load(Ordering::Relaxed) {
continue;
}
if required_dcs.contains(&writer.writer_dc) {
covered_dcs.insert(writer.writer_dc);
if covered_dcs.len() == required_dcs.len() {
return true;
}
}
}
false
}
fn hardswap_warmup_connect_delay_ms(&self) -> u64 {
let min_ms = self.me_hardswap_warmup_delay_min_ms.load(Ordering::Relaxed);
let max_ms = self.me_hardswap_warmup_delay_max_ms.load(Ordering::Relaxed);
@@ -475,12 +507,30 @@ impl MePool {
coverage_ratio = format_args!("{coverage_ratio:.3}"),
min_ratio = format_args!("{min_ratio:.3}"),
drain_timeout_secs,
"ME reinit cycle covered; draining stale writers"
"ME reinit cycle covered; processing stale writers"
);
self.stats.increment_pool_swap_total();
let can_drop_with_replacement = self
.has_non_draining_writer_per_desired_dc_group()
.await;
if can_drop_with_replacement {
info!(
stale_writers = stale_writer_ids.len(),
"ME reinit stale writers: replacement coverage ready, force-closing clients for fast rebind"
);
} else {
warn!(
stale_writers = stale_writer_ids.len(),
"ME reinit stale writers: replacement coverage incomplete, keeping draining fallback"
);
}
for writer_id in stale_writer_ids {
self.mark_writer_draining_with_timeout(writer_id, drain_timeout, !hardswap)
.await;
if can_drop_with_replacement {
self.stats.increment_pool_force_close_total();
self.remove_writer_and_close_clients(writer_id).await;
}
}
if hardswap {
self.clear_pending_hardswap_state();