astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-10-31 12:52:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Hardswap Invariants in tests + Quota fixes

Browse Source
This commit is contained in:
Alexey
2026-09-20 00:28:52 +03:00
parent 89dacbd17e
commit d706b3f3ba
66 changed files with 1821 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/transport/middle_proxy/pool/writer_admission.rs
+57 -56
View File
@@ -100,66 +100,64 @@ impl MePool {
contour: WriterContour,
intent: WriterOpenIntent,
writer_dc: i32,
family: IpFamily,
) -> bool {
if intent == WriterOpenIntent::Replacement {
return true;
}
let (active_writers, warm_writers, _) = self.non_draining_writer_counts_by_contour().await;
match contour {
WriterContour::Active => {
let active_cap = self.adaptive_floor_active_cap_configured_total();
if active_writers < active_cap {
return true;
}
if intent != WriterOpenIntent::Coverage {
return false;
}
let mut endpoints_len = 0;
let now_epoch = Self::now_epoch_secs();
let endpoint_snapshot = self.endpoint_snapshot.load();
if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch) {
if let Some(addrs) = endpoint_snapshot.map_v4.get(&writer_dc) {
endpoints_len += addrs.len();
}
}
if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch) {
if let Some(addrs) = endpoint_snapshot.map_v6.get(&writer_dc) {
endpoints_len += addrs.len();
}
}
if endpoints_len > 0 {
let base_req =
self.required_writers_for_dc_with_floor_mode(endpoints_len, false);
let active_generation = self.reinit.status.load().active_generation;
let active_for_dc = {
let ws = self.writers.read().await;
ws.iter()
.filter(|w| {
!w.draining.load(std::sync::atomic::Ordering::Relaxed)
&& w.writer_dc == writer_dc
&& w.generation == active_generation
&& matches!(
WriterContour::from_u8(
w.contour.load(std::sync::atomic::Ordering::Relaxed),
),
WriterContour::Active
)
})
.count()
};
if active_for_dc < base_req {
return true;
}
}
let coverage_required = self.active_coverage_required_total().await;
active_writers < coverage_required
}
WriterContour::Warm => warm_writers < self.adaptive_floor_warm_cap_configured_total(),
WriterContour::Draining => true,
let live = match contour {
WriterContour::Active => active_writers,
WriterContour::Warm => warm_writers,
WriterContour::Draining => return true,
};
let configured_cap = match contour {
WriterContour::Active => self.adaptive_floor_active_cap_configured_total(),
WriterContour::Warm => self.adaptive_floor_warm_cap_configured_total(),
WriterContour::Draining => usize::MAX,
};
if live < configured_cap {
return true;
}
if intent != WriterOpenIntent::Coverage {
return false;
}
let endpoint_snapshot = self.endpoint_snapshot.load();
let endpoint_count = match family {
IpFamily::V4 => endpoint_snapshot.map_v4.get(&writer_dc),
IpFamily::V6 => endpoint_snapshot.map_v6.get(&writer_dc),
}
.map(Vec::len)
.unwrap_or(0);
if endpoint_count == 0 {
return false;
}
let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
let status = self.reinit.status.load();
let generation = match contour {
WriterContour::Active => status.active_generation,
WriterContour::Warm => status.pending_hardswap_generation,
WriterContour::Draining => 0,
};
let family_count = {
let writers = self.writers.read().await;
writers
.iter()
.filter(|writer| {
!writer.draining.load(Ordering::Relaxed)
&& writer.writer_dc == writer_dc
&& writer.generation == generation
&& WriterContour::from_u8(writer.contour.load(Ordering::Relaxed)) == contour
&& writer.addr.is_ipv4() == (family == IpFamily::V4)
})
.count()
};
if family_count < required {
return true;
}
live < self.active_coverage_required_total().await
}
/// Reserves bounded transient capacity for a writer open attempt.
@@ -168,6 +166,7 @@ impl MePool {
contour: WriterContour,
intent: WriterOpenIntent,
writer_dc: i32,
family: IpFamily,
) -> Option<WriterOpenReservation<'_>> {
let counter = match contour {
WriterContour::Active => &self.writer_connect_active_reserved,
@@ -222,7 +221,7 @@ impl MePool {
loop {
if !self
.can_open_writer_for_contour(contour, intent, writer_dc)
.can_open_writer_for_contour(contour, intent, writer_dc, family)
.await
{
return None;
@@ -239,7 +238,9 @@ impl MePool {
WriterContour::Warm => self.adaptive_floor_warm_cap_configured_total(),
WriterContour::Draining => usize::MAX,
};
if contour == WriterContour::Active && intent == WriterOpenIntent::Coverage {
if intent == WriterOpenIntent::Coverage
&& matches!(contour, WriterContour::Active | WriterContour::Warm)
{
limit = limit
.max(self.active_coverage_required_total().await)
.saturating_add(
src/transport/middle_proxy/pool_reinit.rs
+15
View File
@@ -56,20 +56,35 @@ struct ReinitReservation {
struct ReinitCommitOutcome {
coverage_ratio: f32,
missing_dc: Vec<i32>,
missing_groups: Vec<DcFamilyGroup>,
stale_writer_ids: Vec<u64>,
force_close_writer_ids: Vec<u64>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct DcFamilyGroup {
dc: i32,
family: IpFamily,
}
struct HardswapCoverage {
ratio: f32,
missing_groups: Vec<DcFamilyGroup>,
writer_deficit: usize,
}
#[derive(Debug)]
enum ReinitCommitFailure {
Superseded,
Coverage {
coverage_ratio: f32,
missing_dc: Vec<i32>,
missing_groups: Vec<DcFamilyGroup>,
},
Redundancy {
coverage_ratio: f32,
missing_dc: Vec<i32>,
missing_groups: Vec<DcFamilyGroup>,
},
}
src/transport/middle_proxy/pool_reinit/coordination.rs
+87 -7
View File
@@ -138,22 +138,35 @@ impl MePool {
}
})
.map(|writer| (writer.writer_dc, writer.addr))
.collect::<HashSet<_>>();
let (coverage_ratio, missing_dc) =
Self::coverage_ratio(desired_by_dc, &authoritative_writer_addrs);
.collect::<Vec<_>>();
let (coverage_ratio, missing_dc, missing_groups) = if attempt.hardswap {
let coverage = self.hardswap_coverage(desired_by_dc, &authoritative_writer_addrs);
let missing_dc = Self::missing_group_dcs(&coverage.missing_groups);
(coverage.ratio, missing_dc, coverage.missing_groups)
} else {
let authoritative_writer_addrs = authoritative_writer_addrs
.iter()
.copied()
.collect::<HashSet<_>>();
let (coverage_ratio, missing_dc) =
Self::coverage_ratio(desired_by_dc, &authoritative_writer_addrs);
(coverage_ratio, missing_dc, Vec::new())
};
if coverage_ratio < min_ratio {
return Err(ReinitCommitFailure::Coverage {
coverage_ratio,
missing_dc,
missing_groups,
});
}
if attempt.hardswap
&& !missing_dc.is_empty()
&& !missing_groups.is_empty()
&& self.bind_stale_mode() == MeBindStaleMode::Never
{
return Err(ReinitCommitFailure::Redundancy {
coverage_ratio,
missing_dc,
missing_groups,
});
}
if !commit_reinit_state(
@@ -183,7 +196,7 @@ impl MePool {
.iter()
.flat_map(|(dc, endpoints)| endpoints.iter().copied().map(|addr| (*dc, addr)))
.collect::<HashSet<_>>();
let missing_dc_set = missing_dc.iter().copied().collect::<HashSet<_>>();
let missing_group_set = missing_groups.iter().copied().collect::<HashSet<_>>();
let mut stale_writer_ids = Vec::<u64>::new();
let mut force_close_writer_ids = Vec::<u64>::new();
for writer in writers.iter() {
@@ -199,8 +212,15 @@ impl MePool {
continue;
}
let preserve_fallback = attempt.hardswap
&& missing_dc_set.contains(&writer.writer_dc);
let writer_group = DcFamilyGroup {
dc: writer.writer_dc,
family: if writer.addr.is_ipv4() {
IpFamily::V4
} else {
IpFamily::V6
},
};
let preserve_fallback = attempt.hardswap && missing_group_set.contains(&writer_group);
if !preserve_fallback && attempt.hardswap {
registry_registration.retire(writer.id);
}
@@ -225,6 +245,7 @@ impl MePool {
Ok(ReinitCommitOutcome {
coverage_ratio,
missing_dc,
missing_groups,
stale_writer_ids,
force_close_writer_ids,
})
@@ -265,6 +286,65 @@ impl MePool {
(ratio, missing_dc)
}
/// Evaluates full writer-floor coverage independently for every DC and address family.
pub(super) fn hardswap_coverage(
&self,
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
writer_addrs: &[(i32, SocketAddr)],
) -> HardswapCoverage {
let mut covered = 0usize;
let mut total = 0usize;
let mut writer_deficit = 0usize;
let mut missing_groups = Vec::new();
for (dc, endpoints) in desired_by_dc {
for family in [IpFamily::V4, IpFamily::V6] {
let endpoint_count = endpoints
.iter()
.filter(|endpoint| endpoint.is_ipv4() == (family == IpFamily::V4))
.count();
if endpoint_count == 0 {
continue;
}
total = total.saturating_add(1);
let required = self.required_writers_for_dc(endpoint_count);
let alive = writer_addrs
.iter()
.filter(|(writer_dc, endpoint)| {
*writer_dc == *dc
&& endpoint.is_ipv4() == (family == IpFamily::V4)
&& endpoints.contains(endpoint)
})
.count();
if alive >= required {
covered = covered.saturating_add(1);
} else {
writer_deficit =
writer_deficit.saturating_add(required.saturating_sub(alive));
missing_groups.push(DcFamilyGroup { dc: *dc, family });
}
}
}
missing_groups.sort_unstable_by_key(|group| {
(group.dc, matches!(group.family, IpFamily::V6))
});
HardswapCoverage {
ratio: if total == 0 {
1.0
} else {
(covered as f32) / (total as f32)
},
missing_groups,
writer_deficit,
}
}
fn missing_group_dcs(groups: &[DcFamilyGroup]) -> Vec<i32> {
let mut dcs = groups.iter().map(|group| group.dc).collect::<Vec<_>>();
dcs.sort_unstable();
dcs.dedup();
dcs
}
/// Restores at least one active writer for every enabled desired DC group.
pub async fn reconcile_connections(self: &Arc<Self>, rng: &SecureRandom) {
let endpoint_snapshot = self.endpoint_snapshot.load_full();
src/transport/middle_proxy/pool_reinit/reconcile.rs
+110 -89
View File
@@ -15,102 +15,118 @@ impl MePool {
let total_passes = 1 + extra_passes;
for (dc, endpoints) in desired_by_dc {
if endpoints.is_empty() {
continue;
}
let mut endpoint_list: Vec<SocketAddr> = endpoints.iter().copied().collect();
endpoint_list.sort_unstable();
let required = self.required_writers_for_dc(endpoint_list.len());
let mut completed = false;
let mut last_fresh_count = self
.fresh_writer_count_for_dc_endpoints(generation, *dc, endpoints)
.await;
for pass_idx in 0..total_passes {
if last_fresh_count >= required {
completed = true;
break;
for family in [IpFamily::V4, IpFamily::V6] {
let family_endpoints = endpoints
.iter()
.copied()
.filter(|endpoint| endpoint.is_ipv4() == (family == IpFamily::V4))
.collect::<HashSet<_>>();
if family_endpoints.is_empty() {
continue;
}
let missing = required.saturating_sub(last_fresh_count);
debug!(
dc = *dc,
pass = pass_idx + 1,
total_passes,
fresh_count = last_fresh_count,
required,
missing,
endpoint_count = endpoint_list.len(),
"ME hardswap warmup pass started"
);
let mut endpoint_list = family_endpoints.iter().copied().collect::<Vec<_>>();
endpoint_list.sort_unstable();
let required = self.required_writers_for_dc(endpoint_list.len());
let mut completed = false;
let mut last_fresh_count = self
.fresh_writer_count_for_dc_endpoints(generation, *dc, &family_endpoints)
.await;
for attempt_idx in 0..missing {
let delay_ms = self.hardswap_warmup_connect_delay_ms();
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
for pass_idx in 0..total_passes {
if last_fresh_count >= required {
completed = true;
break;
}
let connected = self
.connect_endpoints_round_robin_with_generation_contour(
*dc,
&endpoint_list,
rng,
let missing = required.saturating_sub(last_fresh_count);
debug!(
dc = *dc,
family = ?family,
pass = pass_idx + 1,
total_passes,
fresh_count = last_fresh_count,
required,
missing,
endpoint_count = endpoint_list.len(),
"ME hardswap family warmup pass started"
);
for attempt_idx in 0..missing {
let delay_ms = self.hardswap_warmup_connect_delay_ms();
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
let connected = self
.connect_endpoints_round_robin_with_generation_contour(
*dc,
&endpoint_list,
rng,
generation,
WriterContour::Warm,
WriterOpenIntent::Coverage,
)
.await;
debug!(
dc = *dc,
family = ?family,
pass = pass_idx + 1,
total_passes,
attempt = attempt_idx + 1,
delay_ms,
connected,
"ME hardswap family warmup connect attempt finished"
);
}
last_fresh_count = self
.fresh_writer_count_for_dc_endpoints(
generation,
WriterContour::Warm,
WriterOpenIntent::Normal,
*dc,
&family_endpoints,
)
.await;
debug!(
dc = *dc,
pass = pass_idx + 1,
total_passes,
attempt = attempt_idx + 1,
delay_ms,
connected,
"ME hardswap warmup connect attempt finished"
);
if last_fresh_count >= required {
completed = true;
info!(
dc = *dc,
family = ?family,
pass = pass_idx + 1,
total_passes,
fresh_count = last_fresh_count,
required,
"ME hardswap writer floor reached for DC family"
);
break;
}
if pass_idx + 1 < total_passes {
let backoff_ms = self.hardswap_warmup_backoff_ms(pass_idx);
debug!(
dc = *dc,
family = ?family,
pass = pass_idx + 1,
total_passes,
fresh_count = last_fresh_count,
required,
backoff_ms,
"ME hardswap family warmup incomplete, delaying next pass"
);
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
}
}
last_fresh_count = self
.fresh_writer_count_for_dc_endpoints(generation, *dc, endpoints)
.await;
if last_fresh_count >= required {
completed = true;
info!(
if !completed {
warn!(
dc = *dc,
pass = pass_idx + 1,
total_passes,
family = ?family,
fresh_count = last_fresh_count,
required,
"ME hardswap warmup floor reached for DC"
);
break;
}
if pass_idx + 1 < total_passes {
let backoff_ms = self.hardswap_warmup_backoff_ms(pass_idx);
debug!(
dc = *dc,
pass = pass_idx + 1,
endpoint_count = endpoint_list.len(),
total_passes,
fresh_count = last_fresh_count,
required,
backoff_ms,
"ME hardswap warmup pass incomplete, delaying next pass"
"ME warmup stopped below the required DC-family writer floor"
);
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
}
}
if !completed {
warn!(
dc = *dc,
fresh_count = last_fresh_count,
required,
endpoint_count = endpoint_list.len(),
total_passes,
"ME warmup stopped: unable to reach required writer floor for DC"
);
}
}
}
@@ -220,27 +236,27 @@ impl MePool {
}
if hardswap {
let fresh_writer_addrs: HashSet<(i32, SocketAddr)> = writers
let fresh_writer_addrs: Vec<(i32, SocketAddr)> = writers
.iter()
.filter(|w| !w.draining.load(Ordering::Relaxed))
.filter(|w| w.generation == generation)
.map(|w| (w.writer_dc, w.addr))
.collect();
let (fresh_coverage_ratio, fresh_missing_dc) =
Self::coverage_ratio(&desired_by_dc, &fresh_writer_addrs);
if fresh_coverage_ratio < min_ratio {
let fresh_coverage = self.hardswap_coverage(&desired_by_dc, &fresh_writer_addrs);
if fresh_coverage.ratio < min_ratio {
self.set_last_drain_gate(
false,
fresh_missing_dc.is_empty(),
fresh_coverage.missing_groups.is_empty(),
MeDrainGateReason::CoverageQuorum,
now_epoch_secs,
);
warn!(
previous_generation,
generation,
fresh_coverage_ratio = format_args!("{fresh_coverage_ratio:.3}"),
missing_dc = ?fresh_missing_dc,
"ME hardswap pending: fresh generation DC coverage incomplete"
fresh_coverage_ratio = format_args!("{:.3}", fresh_coverage.ratio),
writer_deficit = fresh_coverage.writer_deficit,
missing_groups = ?fresh_coverage.missing_groups,
"ME hardswap pending: fresh generation DC-family floors incomplete"
);
return false;
}
@@ -263,6 +279,7 @@ impl MePool {
Err(ReinitCommitFailure::Coverage {
coverage_ratio,
missing_dc,
missing_groups,
}) => {
self.set_last_drain_gate(
false,
@@ -276,6 +293,7 @@ impl MePool {
coverage_ratio = format_args!("{coverage_ratio:.3}"),
min_ratio = format_args!("{min_ratio:.3}"),
missing_dc = ?missing_dc,
missing_groups = ?missing_groups,
"ME reinit coverage changed before commit; keeping current generation"
);
return false;
@@ -283,6 +301,7 @@ impl MePool {
Err(ReinitCommitFailure::Redundancy {
coverage_ratio,
missing_dc,
missing_groups,
}) => {
self.set_last_drain_gate(
true,
@@ -296,6 +315,7 @@ impl MePool {
coverage_ratio = format_args!("{coverage_ratio:.3}"),
min_ratio = format_args!("{min_ratio:.3}"),
missing_dc = ?missing_dc,
missing_groups = ?missing_groups,
"ME hardswap weighted quorum requires stale-binding fallback"
);
return false;
@@ -310,9 +330,10 @@ impl MePool {
if !outcome.missing_dc.is_empty() {
warn!(
missing_dc = ?outcome.missing_dc,
missing_groups = ?outcome.missing_groups,
coverage_ratio = format_args!("{:.3}", outcome.coverage_ratio),
min_ratio = format_args!("{min_ratio:.3}"),
"ME reinit committed with bounded stale fallback for uncovered DC groups"
"ME reinit committed with bounded stale fallback for uncovered DC-family groups"
);
}
src/transport/middle_proxy/pool_reinit/tests.rs
+108 -16
View File
@@ -1,5 +1,5 @@
use std::collections::{HashMap, HashSet};
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
use std::time::Instant;
@@ -7,7 +7,7 @@ use std::time::Instant;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use super::{MePool, ReinitCommitFailure, commit_reinit_state};
use super::{DcFamilyGroup, MePool, ReinitCommitFailure, commit_reinit_state};
use crate::config::MeBindStaleMode;
use crate::transport::middle_proxy::codec::WriterCommand;
use crate::transport::middle_proxy::pool::{
@@ -19,6 +19,13 @@ fn addr(octet: u8, port: u16) -> SocketAddr {
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, octet)), port)
}
fn addr_v6(segment: u16, port: u16) -> SocketAddr {
SocketAddr::new(
IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, segment)),
port,
)
}
async fn insert_writer(
pool: &Arc<MePool>,
writer_id: u64,
@@ -56,6 +63,32 @@ async fn insert_writer(
writer
}
async fn insert_writer_floor(
pool: &Arc<MePool>,
first_writer_id: u64,
writer_dc: i32,
endpoint: SocketAddr,
generation: u64,
contour: WriterContour,
) -> Vec<MeWriter> {
let required = pool.required_writers_for_dc(1);
let mut writers = Vec::with_capacity(required);
for offset in 0..required {
writers.push(
insert_writer(
pool,
first_writer_id + offset as u64,
writer_dc,
endpoint,
generation,
contour,
)
.await,
);
}
writers
}
fn desired_two_dcs() -> HashMap<i32, HashSet<SocketAddr>> {
HashMap::from([
(1, HashSet::from([addr(1, 2001)])),
@@ -181,7 +214,7 @@ async fn partial_hardswap_is_rejected_when_stale_binding_is_disabled() {
let reservation = pool
.reserve_reinit_attempt(true, map_hash, endpoint_revision, 100)
.expect("endpoint revision must remain current");
insert_writer(
insert_writer_floor(
&pool,
201,
1,
@@ -232,7 +265,7 @@ async fn partial_hardswap_preserves_fallback_only_for_missing_dc() {
let reservation = pool
.reserve_reinit_attempt(true, map_hash, endpoint_revision, 100)
.expect("endpoint revision must remain current");
let fresh_dc1 = insert_writer(
let fresh_dc1 = insert_writer_floor(
&pool,
401,
1,
@@ -255,11 +288,72 @@ async fn partial_hardswap_preserves_fallback_only_for_missing_dc() {
assert!(old_dc2.draining.load(Ordering::Acquire));
assert!(old_dc2.allow_drain_fallback.load(Ordering::Acquire));
assert_eq!(
WriterContour::from_u8(fresh_dc1.contour.load(Ordering::Acquire)),
WriterContour::from_u8(fresh_dc1[0].contour.load(Ordering::Acquire)),
WriterContour::Active
);
}
#[tokio::test]
async fn partial_hardswap_preserves_fallback_only_for_underfloor_family() {
let pool = make_pool().await;
pool.binding_policy
.me_bind_stale_mode
.store(MeBindStaleMode::Ttl.as_u8(), Ordering::Release);
let v4 = addr(1, 2001);
let v6 = addr_v6(1, 2001);
let desired_by_dc = HashMap::from([(1, HashSet::from([v4, v6]))]);
let active_generation = pool.current_generation();
let old_v4 = insert_writer(
&pool,
451,
1,
v4,
active_generation,
WriterContour::Active,
)
.await;
let old_v6 = insert_writer(
&pool,
452,
1,
v6,
active_generation,
WriterContour::Active,
)
.await;
let map_hash = MePool::desired_map_hash(&desired_by_dc);
let endpoint_revision = pool.endpoint_snapshot.load().revision;
let reservation = pool
.reserve_reinit_attempt(true, map_hash, endpoint_revision, 100)
.expect("endpoint revision must remain current");
insert_writer_floor(
&pool,
461,
1,
v4,
reservation.attempt.generation,
WriterContour::Warm,
)
.await;
let outcome = pool
.commit_reinit_attempt(&reservation.attempt, &desired_by_dc, 0.5)
.await
.expect("one covered family satisfies the configured weighted quorum");
assert_eq!(
outcome.missing_groups,
vec![DcFamilyGroup {
dc: 1,
family: crate::network::IpFamily::V6,
}]
);
assert!(outcome.force_close_writer_ids.contains(&old_v4.id));
assert!(!outcome.force_close_writer_ids.contains(&old_v6.id));
assert!(!old_v4.allow_drain_fallback.load(Ordering::Acquire));
assert!(old_v6.allow_drain_fallback.load(Ordering::Acquire));
}
#[tokio::test]
async fn complete_hardswap_promotes_fresh_generation_and_retires_old_writers() {
let pool = make_pool().await;
@@ -288,7 +382,7 @@ async fn complete_hardswap_promotes_fresh_generation_and_retires_old_writers() {
let reservation = pool
.reserve_reinit_attempt(true, map_hash, endpoint_revision, 100)
.expect("endpoint revision must remain current");
let fresh_dc1 = insert_writer(
let fresh_dc1 = insert_writer_floor(
&pool,
601,
1,
@@ -297,9 +391,9 @@ async fn complete_hardswap_promotes_fresh_generation_and_retires_old_writers() {
WriterContour::Warm,
)
.await;
let fresh_dc2 = insert_writer(
let fresh_dc2 = insert_writer_floor(
&pool,
602,
611,
2,
addr(2, 2002),
reservation.attempt.generation,
@@ -318,11 +412,11 @@ async fn complete_hardswap_promotes_fresh_generation_and_retires_old_writers() {
assert!(old_dc1.draining.load(Ordering::Acquire));
assert!(old_dc2.draining.load(Ordering::Acquire));
assert_eq!(
WriterContour::from_u8(fresh_dc1.contour.load(Ordering::Acquire)),
WriterContour::from_u8(fresh_dc1[0].contour.load(Ordering::Acquire)),
WriterContour::Active
);
assert_eq!(
WriterContour::from_u8(fresh_dc2.contour.load(Ordering::Acquire)),
WriterContour::from_u8(fresh_dc2[0].contour.load(Ordering::Acquire)),
WriterContour::Active
);
}
@@ -442,10 +536,8 @@ async fn generation_role_reconciliation_promotes_active_warm_and_drains_orphans(
);
assert!(orphan_warm.draining.load(Ordering::Acquire));
assert!(!orphan_warm.allow_drain_fallback.load(Ordering::Acquire));
assert_eq!(
pool.api_hardswap_snapshot()
.await
.orphan_warm_writers_current,
0
);
let snapshot = pool.api_hardswap_snapshot().await;
assert_eq!(snapshot.orphan_warm_writers_current, 0);
assert_eq!(snapshot.pending_writer_deficit, 2);
assert_eq!(snapshot.pending_missing_dc_groups, 1);
}
src/transport/middle_proxy/pool_status.rs
+2 -1
View File
@@ -6,6 +6,7 @@ use std::time::Instant;
use super::pool::{MePool, ReinitStatusSnapshot, WriterContour};
use crate::config::{MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy};
use crate::network::IpFamily;
use crate::transport::upstream::IpPreference;
// ME writer and DC coverage snapshots.
@@ -105,7 +106,7 @@ pub(crate) struct MeApiRuntimeSnapshot {
pub pending_writers_current: usize,
/// Number of writers still required to reach the pending generation floor.
pub pending_writer_deficit: usize,
/// Number of desired DC groups without pending-generation coverage.
/// Number of desired DC-family groups below their pending-generation floor.
pub pending_missing_dc_groups: usize,
/// Whether the pending generation targets the current desired endpoint map.
pub pending_map_current: Option<bool>,
src/transport/middle_proxy/pool_status/hardswap_snapshot.rs
+26 -12
View File
@@ -11,7 +11,7 @@ pub(crate) struct MeApiHardswapSnapshot {
pub pending_writers_current: usize,
/// Number of writers still required to reach the pending generation floor.
pub pending_writer_deficit: usize,
/// Number of desired DC groups without a pending-generation writer.
/// Number of desired DC-family groups below their pending-generation writer floor.
pub pending_missing_dc_groups: usize,
/// Whether the pending generation targets the current desired endpoint map.
pub pending_map_current: Option<bool>,
@@ -41,7 +41,7 @@ impl MePool {
let pending_generation = reinit.pending_hardswap_generation;
let pending = pending_generation != 0;
let mut pending_writers_current = 0usize;
let mut pending_by_dc = HashMap::<i32, usize>::new();
let mut pending_by_group = HashMap::<(i32, IpFamily), usize>::new();
let mut orphan_warm_writers_current = 0usize;
for writer in writers.iter() {
@@ -60,7 +60,14 @@ impl MePool {
.is_some_and(|endpoints| endpoints.contains(&writer.addr))
{
pending_writers_current = pending_writers_current.saturating_add(1);
*pending_by_dc.entry(writer.writer_dc).or_insert(0) += 1;
let family = if writer.addr.is_ipv4() {
IpFamily::V4
} else {
IpFamily::V6
};
*pending_by_group
.entry((writer.writer_dc, family))
.or_insert(0) += 1;
}
}
@@ -68,15 +75,22 @@ impl MePool {
let mut pending_missing_dc_groups = 0usize;
if pending {
for (dc, endpoints) in &desired_by_dc {
if endpoints.is_empty() {
continue;
}
let alive = pending_by_dc.get(dc).copied().unwrap_or(0);
let required = self.required_writers_for_dc(endpoints.len());
pending_writer_deficit = pending_writer_deficit
.saturating_add(required.saturating_sub(alive));
if alive == 0 {
pending_missing_dc_groups = pending_missing_dc_groups.saturating_add(1);
for family in [IpFamily::V4, IpFamily::V6] {
let endpoint_count = endpoints
.iter()
.filter(|endpoint| endpoint.is_ipv4() == (family == IpFamily::V4))
.count();
if endpoint_count == 0 {
continue;
}
let alive = pending_by_group.get(&(*dc, family)).copied().unwrap_or(0);
let required = self.required_writers_for_dc(endpoint_count);
let deficit = required.saturating_sub(alive);
pending_writer_deficit = pending_writer_deficit.saturating_add(deficit);
if deficit > 0 {
pending_missing_dc_groups =
pending_missing_dc_groups.saturating_add(1);
}
}
}
}
src/transport/middle_proxy/pool_writer/publication.rs
+2 -28
View File
@@ -102,28 +102,12 @@ impl MePool {
));
};
let required = match contour {
WriterContour::Active => self.required_writers_for_dc(
WriterContour::Active | WriterContour::Warm => self.required_writers_for_dc(
endpoints
.iter()
.filter(|endpoint| endpoint.is_ipv4() == writer.addr.is_ipv4())
.count(),
),
WriterContour::Warm => self.required_writers_for_dc(
endpoints
.iter()
.filter(|endpoint| {
let endpoint_family = if endpoint.is_ipv4() {
crate::network::IpFamily::V4
} else {
crate::network::IpFamily::V6
};
self.family_enabled_for_drain_coverage(
endpoint_family,
now_epoch_secs,
)
})
.count(),
),
WriterContour::Draining => 0,
};
let current = writers
@@ -134,18 +118,8 @@ impl MePool {
&& candidate.generation == writer.generation
&& WriterContour::from_u8(candidate.contour.load(Ordering::Acquire))
== contour
&& (contour == WriterContour::Warm
|| candidate.addr.is_ipv4() == writer.addr.is_ipv4())
&& candidate.addr.is_ipv4() == writer.addr.is_ipv4()
&& endpoints.contains(&candidate.addr)
&& (contour != WriterContour::Warm
|| self.family_enabled_for_drain_coverage(
if candidate.addr.is_ipv4() {
crate::network::IpFamily::V4
} else {
crate::network::IpFamily::V6
},
now_epoch_secs,
))
})
.count();
if current >= required {
src/transport/middle_proxy/pool_writer/replacement.rs
+5
View File
@@ -223,6 +223,11 @@ mod tests {
WriterContour::Active,
WriterOpenIntent::Replacement,
writer_dc,
if addr.is_ipv4() {
crate::network::IpFamily::V4
} else {
crate::network::IpFamily::V6
},
)
.await
.expect("replacement open must be admitted");
src/transport/middle_proxy/pool_writer/runtime.rs
+10 -1
View File
@@ -92,7 +92,16 @@ impl MePool {
intent: WriterOpenIntent,
) -> Result<PreparedWriter<'a>> {
let Some(writer_open_reservation) = self
.reserve_writer_open(contour, intent, writer_dc)
.reserve_writer_open(
contour,
intent,
writer_dc,
if addr.is_ipv4() {
crate::network::IpFamily::V4
} else {
crate::network::IpFamily::V6
},
)
.await
else {
return Err(ProxyError::Proxy(format!(