Atomic Lifecycle for Writer Refresh + Hardswap

This commit is contained in:
Alexey
2026-09-12 20:14:41 +03:00
parent 54197e848a
commit 1ea5f7a1d6
44 changed files with 3020 additions and 581 deletions
+156
View File
@@ -0,0 +1,156 @@
use std::fmt::Write;
use crate::transport::middle_proxy::MeApiHardswapSnapshot;
pub(super) fn render(
out: &mut String,
snapshot: Option<&MeApiHardswapSnapshot>,
enabled: bool,
) {
let snapshot = enabled.then_some(snapshot).flatten();
let pending = snapshot.is_some_and(|value| value.pending);
let pending_age_secs = snapshot
.and_then(|value| value.pending_age_secs)
.unwrap_or(0);
let pending_writers_current = snapshot
.map(|value| value.pending_writers_current)
.unwrap_or(0);
let pending_writer_deficit = snapshot
.map(|value| value.pending_writer_deficit)
.unwrap_or(0);
let pending_missing_dc_groups = snapshot
.map(|value| value.pending_missing_dc_groups)
.unwrap_or(0);
let pending_map_current = snapshot
.and_then(|value| value.pending_map_current)
.is_some_and(|value| value);
let orphan_warm_writers_current = snapshot
.map(|value| value.orphan_warm_writers_current)
.unwrap_or(0);
let replacement_preparing_current = snapshot
.map(|value| value.replacement_preparing_current)
.unwrap_or(0);
let replacement_retiring_current = snapshot
.map(|value| value.replacement_retiring_current)
.unwrap_or(0);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending Whether an ME hardswap generation is pending"
);
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending gauge");
let _ = writeln!(out, "telemt_me_hardswap_pending {}", usize::from(pending));
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_age_seconds Age of the pending ME hardswap generation"
);
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_age_seconds gauge");
let _ = writeln!(
out,
"telemt_me_hardswap_pending_age_seconds {pending_age_secs}"
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_writers_current Authoritative warm writers in the pending generation"
);
let _ = writeln!(
out,
"# TYPE telemt_me_hardswap_pending_writers_current gauge"
);
let _ = writeln!(
out,
"telemt_me_hardswap_pending_writers_current {pending_writers_current}"
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_writer_deficit Writers missing from the pending generation floor"
);
let _ = writeln!(
out,
"# TYPE telemt_me_hardswap_pending_writer_deficit gauge"
);
let _ = writeln!(
out,
"telemt_me_hardswap_pending_writer_deficit {pending_writer_deficit}"
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_missing_dc_groups Desired DC groups missing pending-generation coverage"
);
let _ = writeln!(
out,
"# TYPE telemt_me_hardswap_pending_missing_dc_groups gauge"
);
let _ = writeln!(
out,
"telemt_me_hardswap_pending_missing_dc_groups {pending_missing_dc_groups}"
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_pending_map_current Whether the pending generation targets the current endpoint map"
);
let _ = writeln!(out, "# TYPE telemt_me_hardswap_pending_map_current gauge");
let _ = writeln!(
out,
"telemt_me_hardswap_pending_map_current {}",
usize::from(pending_map_current)
);
let _ = writeln!(
out,
"# HELP telemt_me_hardswap_orphan_warm_writers_current Warm writers not owned by the pending hardswap generation"
);
let _ = writeln!(
out,
"# TYPE telemt_me_hardswap_orphan_warm_writers_current gauge"
);
let _ = writeln!(
out,
"telemt_me_hardswap_orphan_warm_writers_current {orphan_warm_writers_current}"
);
let _ = writeln!(
out,
"# HELP telemt_me_writer_replacement_current ME writer replacements by transaction phase"
);
let _ = writeln!(out, "# TYPE telemt_me_writer_replacement_current gauge");
let _ = writeln!(
out,
"telemt_me_writer_replacement_current{{state=\"preparing\"}} {replacement_preparing_current}"
);
let _ = writeln!(
out,
"telemt_me_writer_replacement_current{{state=\"retiring\"}} {replacement_retiring_current}"
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn renders_bounded_hardswap_and_replacement_state() {
let snapshot = MeApiHardswapSnapshot {
pending: true,
pending_age_secs: Some(42),
pending_writers_current: 3,
pending_writer_deficit: 4,
pending_missing_dc_groups: 2,
pending_map_current: Some(true),
orphan_warm_writers_current: 1,
replacement_preparing_current: 5,
replacement_retiring_current: 6,
};
let mut out = String::new();
render(&mut out, Some(&snapshot), true);
assert!(out.contains("telemt_me_hardswap_pending 1"));
assert!(out.contains("telemt_me_hardswap_pending_age_seconds 42"));
assert!(out.contains("telemt_me_hardswap_pending_writer_deficit 4"));
assert!(out.contains(
"telemt_me_writer_replacement_current{state=\"preparing\"} 5"
));
assert!(out.contains(
"telemt_me_writer_replacement_current{state=\"retiring\"} 6"
));
}
}