mirror of
https://github.com/telemt/telemt.git
synced 2026-11-02 13:52:03 +03:00
Atomic Lifecycle for Writer Refresh + Hardswap
This commit is contained in:
Alexey
@@ -1,7 +1,10 @@
|
||||
use super::*;
|
||||
|
||||
impl MePool {
|
||||
pub(super) fn desired_map_hash(desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>) -> u64 {
|
||||
/// Hashes the sorted desired endpoint map for generation authority checks.
|
||||
pub(in crate::transport::middle_proxy) fn desired_map_hash(
|
||||
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
||||
) -> u64 {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
let mut dcs: Vec<i32> = desired_by_dc.keys().copied().collect();
|
||||
dcs.sort_unstable();
|
||||
@@ -19,6 +22,7 @@ impl MePool {
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
/// Reserves one generation attempt and publishes its pending ownership snapshot.
|
||||
pub(super) fn reserve_reinit_attempt(
|
||||
self: &Arc<Self>,
|
||||
hardswap: bool,
|
||||
@@ -85,8 +89,62 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn commit_reinit_attempt(&self, attempt: &ReinitAttemptGuard) -> bool {
|
||||
/// Revalidates coverage and commits generation ownership under the publication barrier.
|
||||
pub(super) async fn commit_reinit_attempt(
|
||||
&self,
|
||||
attempt: &ReinitAttemptGuard,
|
||||
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
||||
min_ratio: f32,
|
||||
) -> std::result::Result<ReinitCommitOutcome, ReinitCommitFailure> {
|
||||
let writers = self.writers.write().await;
|
||||
let mut registry_registration = self.registry.prepare_writer_registration().await;
|
||||
let mut state = self.reinit.coordinator.lock();
|
||||
let Some(record) = state.attempts.get(&attempt.attempt_id).copied() else {
|
||||
return Err(ReinitCommitFailure::Superseded);
|
||||
};
|
||||
if record.generation != attempt.generation
|
||||
|| record.map_hash != state.desired_map_hash
|
||||
|| record.map_hash != attempt.map_hash
|
||||
|| (attempt.hardswap
|
||||
&& !state.pending.is_some_and(|pending| {
|
||||
pending.generation == attempt.generation
|
||||
&& pending.map_hash == attempt.map_hash
|
||||
}))
|
||||
{
|
||||
return Err(ReinitCommitFailure::Superseded);
|
||||
}
|
||||
|
||||
let authoritative_writer_addrs = writers
|
||||
.iter()
|
||||
.filter(|writer| !writer.draining.load(Ordering::Acquire))
|
||||
.filter(|writer| {
|
||||
if attempt.hardswap {
|
||||
writer.generation == attempt.generation
|
||||
} else {
|
||||
writer.generation == state.active_generation
|
||||
&& WriterContour::from_u8(writer.contour.load(Ordering::Acquire))
|
||||
== WriterContour::Active
|
||||
}
|
||||
})
|
||||
.map(|writer| (writer.writer_dc, writer.addr))
|
||||
.collect::<HashSet<_>>();
|
||||
let (coverage_ratio, missing_dc) =
|
||||
Self::coverage_ratio(desired_by_dc, &authoritative_writer_addrs);
|
||||
if coverage_ratio < min_ratio {
|
||||
return Err(ReinitCommitFailure::Coverage {
|
||||
coverage_ratio,
|
||||
missing_dc,
|
||||
});
|
||||
}
|
||||
if attempt.hardswap
|
||||
&& !missing_dc.is_empty()
|
||||
&& self.bind_stale_mode() == MeBindStaleMode::Never
|
||||
{
|
||||
return Err(ReinitCommitFailure::Redundancy {
|
||||
coverage_ratio,
|
||||
missing_dc,
|
||||
});
|
||||
}
|
||||
if !commit_reinit_state(
|
||||
&mut state,
|
||||
attempt.attempt_id,
|
||||
@@ -94,12 +152,12 @@ impl MePool {
|
||||
attempt.map_hash,
|
||||
attempt.hardswap,
|
||||
) {
|
||||
return false;
|
||||
return Err(ReinitCommitFailure::Superseded);
|
||||
}
|
||||
|
||||
if attempt.hardswap {
|
||||
let writers = self.writers.snapshot();
|
||||
for writer in writers.iter() {
|
||||
if !writer.draining.load(Ordering::Relaxed)
|
||||
if !writer.draining.load(Ordering::Acquire)
|
||||
&& writer.generation == attempt.generation
|
||||
{
|
||||
writer
|
||||
@@ -108,10 +166,59 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let desired_addrs = desired_by_dc
|
||||
.iter()
|
||||
.flat_map(|(dc, endpoints)| endpoints.iter().copied().map(|addr| (*dc, addr)))
|
||||
.collect::<HashSet<_>>();
|
||||
let missing_dc_set = missing_dc.iter().copied().collect::<HashSet<_>>();
|
||||
let mut stale_writer_ids = Vec::<u64>::new();
|
||||
let mut force_close_writer_ids = Vec::<u64>::new();
|
||||
for writer in writers.iter() {
|
||||
if writer.draining.load(Ordering::Acquire) {
|
||||
continue;
|
||||
}
|
||||
let stale = if attempt.hardswap {
|
||||
writer.generation < attempt.generation
|
||||
} else {
|
||||
!desired_addrs.contains(&(writer.writer_dc, writer.addr))
|
||||
};
|
||||
if !stale {
|
||||
continue;
|
||||
}
|
||||
|
||||
let preserve_fallback = attempt.hardswap
|
||||
&& missing_dc_set.contains(&writer.writer_dc);
|
||||
if !preserve_fallback && attempt.hardswap {
|
||||
registry_registration.retire(writer.id);
|
||||
}
|
||||
self.apply_writer_draining_state(
|
||||
writer,
|
||||
self.force_close_timeout(),
|
||||
preserve_fallback || !attempt.hardswap,
|
||||
);
|
||||
stale_writer_ids.push(writer.id);
|
||||
if (attempt.hardswap && !preserve_fallback)
|
||||
|| (!attempt.hardswap && missing_dc.is_empty())
|
||||
{
|
||||
force_close_writer_ids.push(writer.id);
|
||||
}
|
||||
}
|
||||
publish_reinit_state(self.reinit.as_ref(), &state);
|
||||
true
|
||||
drop(state);
|
||||
drop(registry_registration);
|
||||
drop(writers);
|
||||
self.notify_writer_epoch();
|
||||
|
||||
Ok(ReinitCommitOutcome {
|
||||
coverage_ratio,
|
||||
missing_dc,
|
||||
stale_writer_ids,
|
||||
force_close_writer_ids,
|
||||
})
|
||||
}
|
||||
|
||||
/// Computes desired DC-group coverage and returns missing groups in stable order.
|
||||
pub(super) fn coverage_ratio(
|
||||
desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
|
||||
active_writer_addrs: &HashSet<(i32, SocketAddr)>,
|
||||
@@ -146,6 +253,7 @@ impl MePool {
|
||||
(ratio, missing_dc)
|
||||
}
|
||||
|
||||
/// Restores at least one active writer for every enabled desired DC group.
|
||||
pub async fn reconcile_connections(self: &Arc<Self>, rng: &SecureRandom) {
|
||||
for family in self.family_order() {
|
||||
let map = self.proxy_map_for_family(family).await;
|
||||
@@ -175,7 +283,10 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn desired_dc_endpoints(&self) -> HashMap<i32, HashSet<SocketAddr>> {
|
||||
/// Returns the currently authoritative endpoint set for drain and coverage decisions.
|
||||
pub(in crate::transport::middle_proxy) async fn desired_dc_endpoints(
|
||||
&self,
|
||||
) -> HashMap<i32, HashSet<SocketAddr>> {
|
||||
let now_epoch_secs = Self::now_epoch_secs();
|
||||
let mut out: HashMap<i32, HashSet<SocketAddr>> = HashMap::new();
|
||||
|
||||
@@ -202,38 +313,63 @@ impl MePool {
|
||||
out
|
||||
}
|
||||
|
||||
pub(in crate::transport::middle_proxy) async fn has_non_draining_writer_per_desired_dc_group(
|
||||
&self,
|
||||
) -> bool {
|
||||
let desired_by_dc = self.desired_dc_endpoints().await;
|
||||
let required_dcs: HashSet<i32> = desired_by_dc
|
||||
.iter()
|
||||
.filter_map(|(dc, endpoints)| {
|
||||
if endpoints.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(*dc)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
if required_dcs.is_empty() {
|
||||
return true;
|
||||
}
|
||||
/// Promotes authoritative warm writers and drains warm or active generation orphans.
|
||||
pub(super) async fn reconcile_writer_generation_roles(&self) -> usize {
|
||||
let writers = self.writers.write().await;
|
||||
let mut registry_registration = self.registry.prepare_writer_registration().await;
|
||||
let state = self.reinit.coordinator.lock();
|
||||
let active_generation = state.active_generation;
|
||||
let pending_generation = state.pending.map(|pending| pending.generation);
|
||||
let preferred = self.preferred_endpoints_by_dc.load();
|
||||
let now_epoch_secs = Self::now_epoch_secs();
|
||||
let mut changed = 0usize;
|
||||
|
||||
let ws = self.writers.read().await;
|
||||
let mut covered_dcs = HashSet::<i32>::with_capacity(required_dcs.len());
|
||||
for writer in ws.iter() {
|
||||
if writer.draining.load(Ordering::Relaxed) {
|
||||
for writer in writers.iter() {
|
||||
if writer.draining.load(Ordering::Acquire) {
|
||||
continue;
|
||||
}
|
||||
if required_dcs.contains(&writer.writer_dc) {
|
||||
covered_dcs.insert(writer.writer_dc);
|
||||
if covered_dcs.len() == required_dcs.len() {
|
||||
return true;
|
||||
}
|
||||
let contour = WriterContour::from_u8(writer.contour.load(Ordering::Acquire));
|
||||
let family = if writer.addr.is_ipv4() {
|
||||
IpFamily::V4
|
||||
} else {
|
||||
IpFamily::V6
|
||||
};
|
||||
let endpoint_is_current = self
|
||||
.family_enabled_for_drain_coverage(family, now_epoch_secs)
|
||||
&& preferred
|
||||
.get(&writer.writer_dc)
|
||||
.is_some_and(|endpoints| endpoints.contains(&writer.addr));
|
||||
if contour == WriterContour::Warm
|
||||
&& writer.generation == active_generation
|
||||
&& endpoint_is_current
|
||||
{
|
||||
writer
|
||||
.contour
|
||||
.store(WriterContour::Active.as_u8(), Ordering::Release);
|
||||
changed = changed.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
let authoritative_warm = contour == WriterContour::Warm
|
||||
&& pending_generation == Some(writer.generation)
|
||||
&& endpoint_is_current;
|
||||
let stale_active = contour == WriterContour::Active
|
||||
&& writer.generation != active_generation;
|
||||
if authoritative_warm || (contour == WriterContour::Active && !stale_active) {
|
||||
continue;
|
||||
}
|
||||
|
||||
registry_registration.retire(writer.id);
|
||||
self.apply_writer_draining_state(writer, self.force_close_timeout(), false);
|
||||
changed = changed.saturating_add(1);
|
||||
}
|
||||
false
|
||||
drop(preferred);
|
||||
drop(state);
|
||||
drop(registry_registration);
|
||||
drop(writers);
|
||||
if changed > 0 {
|
||||
self.notify_writer_epoch();
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
pub(super) fn hardswap_warmup_connect_delay_ms(&self) -> u64 {
|
||||
@@ -270,6 +406,7 @@ impl MePool {
|
||||
core.saturating_add(rand::rng().random_range(0..=jitter))
|
||||
}
|
||||
|
||||
/// Counts non-draining writers owned by one generation and desired DC endpoint set.
|
||||
pub(super) async fn fresh_writer_count_for_dc_endpoints(
|
||||
&self,
|
||||
generation: u64,
|
||||
@@ -285,14 +422,21 @@ impl MePool {
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Counts authoritative active writers for one desired DC endpoint set.
|
||||
pub(in crate::transport::middle_proxy) async fn active_writer_count_for_dc_endpoints(
|
||||
&self,
|
||||
dc: i32,
|
||||
endpoints: &HashSet<SocketAddr>,
|
||||
) -> usize {
|
||||
let generation = self.current_generation();
|
||||
let ws = self.writers.read().await;
|
||||
ws.iter()
|
||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||
.filter(|w| w.generation == generation)
|
||||
.filter(|w| {
|
||||
WriterContour::from_u8(w.contour.load(Ordering::Acquire))
|
||||
== WriterContour::Active
|
||||
})
|
||||
.filter(|w| w.writer_dc == dc)
|
||||
.filter(|w| endpoints.contains(&w.addr))
|
||||
.count()
|
||||
|
||||
@@ -56,7 +56,7 @@ impl MePool {
|
||||
rng,
|
||||
generation,
|
||||
WriterContour::Warm,
|
||||
false,
|
||||
WriterOpenIntent::Normal,
|
||||
)
|
||||
.await;
|
||||
debug!(
|
||||
@@ -141,6 +141,15 @@ impl MePool {
|
||||
let attempt = reservation.attempt;
|
||||
let previous_generation = attempt.previous_generation;
|
||||
let generation = attempt.generation;
|
||||
let reconciled_roles = self.reconcile_writer_generation_roles().await;
|
||||
if reconciled_roles > 0 {
|
||||
info!(
|
||||
reconciled_roles,
|
||||
active_generation = previous_generation,
|
||||
pending_generation = generation,
|
||||
"ME writer generation roles reconciled"
|
||||
);
|
||||
}
|
||||
if reservation.pending_reused {
|
||||
self.stats.increment_me_hardswap_pending_reuse_total();
|
||||
debug!(
|
||||
@@ -180,14 +189,10 @@ impl MePool {
|
||||
);
|
||||
let (coverage_ratio, missing_dc) =
|
||||
Self::coverage_ratio(&desired_by_dc, &active_writer_addrs);
|
||||
let mut route_quorum_ok = coverage_ratio >= min_ratio;
|
||||
let mut redundancy_ok = missing_dc.is_empty();
|
||||
let mut redundancy_missing_dc = missing_dc.clone();
|
||||
let mut gate_coverage_ratio = coverage_ratio;
|
||||
if !hardswap && coverage_ratio < min_ratio {
|
||||
self.set_last_drain_gate(
|
||||
false,
|
||||
redundancy_ok,
|
||||
missing_dc.is_empty(),
|
||||
MeDrainGateReason::CoverageQuorum,
|
||||
now_epoch_secs,
|
||||
);
|
||||
@@ -211,14 +216,10 @@ impl MePool {
|
||||
.collect();
|
||||
let (fresh_coverage_ratio, fresh_missing_dc) =
|
||||
Self::coverage_ratio(&desired_by_dc, &fresh_writer_addrs);
|
||||
route_quorum_ok = fresh_coverage_ratio >= min_ratio;
|
||||
redundancy_ok = fresh_missing_dc.is_empty();
|
||||
redundancy_missing_dc = fresh_missing_dc.clone();
|
||||
gate_coverage_ratio = fresh_coverage_ratio;
|
||||
if fresh_coverage_ratio < min_ratio {
|
||||
self.set_last_drain_gate(
|
||||
false,
|
||||
redundancy_ok,
|
||||
fresh_missing_dc.is_empty(),
|
||||
MeDrainGateReason::CoverageQuorum,
|
||||
now_epoch_secs,
|
||||
);
|
||||
@@ -233,49 +234,77 @@ impl MePool {
|
||||
}
|
||||
}
|
||||
|
||||
drop(writers);
|
||||
let commit = self
|
||||
.commit_reinit_attempt(&attempt, &desired_by_dc, min_ratio)
|
||||
.await;
|
||||
let outcome = match commit {
|
||||
Ok(outcome) => outcome,
|
||||
Err(ReinitCommitFailure::Superseded) => {
|
||||
debug!(
|
||||
previous_generation,
|
||||
generation,
|
||||
"ME reinit result discarded after a newer desired-map attempt"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
Err(ReinitCommitFailure::Coverage {
|
||||
coverage_ratio,
|
||||
missing_dc,
|
||||
}) => {
|
||||
self.set_last_drain_gate(
|
||||
false,
|
||||
missing_dc.is_empty(),
|
||||
MeDrainGateReason::CoverageQuorum,
|
||||
now_epoch_secs,
|
||||
);
|
||||
warn!(
|
||||
previous_generation,
|
||||
generation,
|
||||
coverage_ratio = format_args!("{coverage_ratio:.3}"),
|
||||
min_ratio = format_args!("{min_ratio:.3}"),
|
||||
missing_dc = ?missing_dc,
|
||||
"ME reinit coverage changed before commit; keeping current generation"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
Err(ReinitCommitFailure::Redundancy {
|
||||
coverage_ratio,
|
||||
missing_dc,
|
||||
}) => {
|
||||
self.set_last_drain_gate(
|
||||
true,
|
||||
false,
|
||||
MeDrainGateReason::Redundancy,
|
||||
now_epoch_secs,
|
||||
);
|
||||
warn!(
|
||||
previous_generation,
|
||||
generation,
|
||||
coverage_ratio = format_args!("{coverage_ratio:.3}"),
|
||||
min_ratio = format_args!("{min_ratio:.3}"),
|
||||
missing_dc = ?missing_dc,
|
||||
"ME hardswap weighted quorum requires stale-binding fallback"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
self.set_last_drain_gate(
|
||||
route_quorum_ok,
|
||||
redundancy_ok,
|
||||
true,
|
||||
outcome.missing_dc.is_empty(),
|
||||
MeDrainGateReason::Open,
|
||||
now_epoch_secs,
|
||||
);
|
||||
if !redundancy_ok {
|
||||
if !outcome.missing_dc.is_empty() {
|
||||
warn!(
|
||||
missing_dc = ?redundancy_missing_dc,
|
||||
coverage_ratio = format_args!("{gate_coverage_ratio:.3}"),
|
||||
missing_dc = ?outcome.missing_dc,
|
||||
coverage_ratio = format_args!("{:.3}", outcome.coverage_ratio),
|
||||
min_ratio = format_args!("{min_ratio:.3}"),
|
||||
"ME reinit proceeds with weighted quorum while some DC groups remain uncovered"
|
||||
"ME reinit committed with bounded stale fallback for uncovered DC groups"
|
||||
);
|
||||
}
|
||||
|
||||
if !self.commit_reinit_attempt(&attempt) {
|
||||
debug!(
|
||||
previous_generation,
|
||||
generation, "ME reinit result discarded after a newer desired-map attempt"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
let desired_addrs: HashSet<(i32, SocketAddr)> = desired_by_dc
|
||||
.iter()
|
||||
.flat_map(|(dc, set)| set.iter().copied().map(|addr| (*dc, addr)))
|
||||
.collect();
|
||||
|
||||
let stale_writer_ids: Vec<u64> = writers
|
||||
.iter()
|
||||
.filter(|w| !w.draining.load(Ordering::Relaxed))
|
||||
.filter(|w| {
|
||||
if hardswap {
|
||||
w.generation < generation
|
||||
} else {
|
||||
!desired_addrs.contains(&(w.writer_dc, w.addr))
|
||||
}
|
||||
})
|
||||
.map(|w| w.id)
|
||||
.collect();
|
||||
drop(writers);
|
||||
|
||||
if stale_writer_ids.is_empty() {
|
||||
if outcome.stale_writer_ids.is_empty() {
|
||||
debug!("ME reinit cycle completed with no stale writers");
|
||||
return true;
|
||||
}
|
||||
@@ -283,35 +312,20 @@ impl MePool {
|
||||
let drain_timeout = self.force_close_timeout();
|
||||
let drain_timeout_secs = drain_timeout.map(|d| d.as_secs()).unwrap_or(0);
|
||||
info!(
|
||||
stale_writers = stale_writer_ids.len(),
|
||||
stale_writers = outcome.stale_writer_ids.len(),
|
||||
force_close_writers = outcome.force_close_writer_ids.len(),
|
||||
previous_generation,
|
||||
generation,
|
||||
hardswap,
|
||||
coverage_ratio = format_args!("{coverage_ratio:.3}"),
|
||||
coverage_ratio = format_args!("{:.3}", outcome.coverage_ratio),
|
||||
min_ratio = format_args!("{min_ratio:.3}"),
|
||||
drain_timeout_secs,
|
||||
"ME reinit cycle covered; processing stale writers"
|
||||
);
|
||||
self.stats.increment_pool_swap_total();
|
||||
let can_drop_with_replacement = self.has_non_draining_writer_per_desired_dc_group().await;
|
||||
if can_drop_with_replacement {
|
||||
info!(
|
||||
stale_writers = stale_writer_ids.len(),
|
||||
"ME reinit stale writers: replacement coverage ready, force-closing clients for fast rebind"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
stale_writers = stale_writer_ids.len(),
|
||||
"ME reinit stale writers: replacement coverage incomplete, keeping draining fallback"
|
||||
);
|
||||
}
|
||||
for writer_id in stale_writer_ids {
|
||||
self.mark_writer_draining_with_timeout(writer_id, drain_timeout, !hardswap)
|
||||
.await;
|
||||
if can_drop_with_replacement {
|
||||
self.stats.increment_pool_force_close_total();
|
||||
self.remove_writer_and_close_clients(writer_id).await;
|
||||
}
|
||||
for writer_id in outcome.force_close_writer_ids {
|
||||
self.stats.increment_pool_force_close_total();
|
||||
self.remove_writer_and_close_clients(writer_id).await;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
@@ -1,15 +1,68 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use super::{MePool, commit_reinit_state};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{MePool, ReinitCommitFailure, commit_reinit_state};
|
||||
use crate::config::MeBindStaleMode;
|
||||
use crate::transport::middle_proxy::codec::WriterCommand;
|
||||
use crate::transport::middle_proxy::pool::{
|
||||
ReinitAttemptState, ReinitCoordinatorState, ReinitPendingState,
|
||||
MeWriter, ReinitAttemptState, ReinitCoordinatorState, ReinitPendingState, WriterContour,
|
||||
};
|
||||
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool;
|
||||
|
||||
fn addr(octet: u8, port: u16) -> SocketAddr {
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, octet)), port)
|
||||
}
|
||||
|
||||
async fn insert_writer(
|
||||
pool: &Arc<MePool>,
|
||||
writer_id: u64,
|
||||
writer_dc: i32,
|
||||
endpoint: SocketAddr,
|
||||
generation: u64,
|
||||
contour: WriterContour,
|
||||
) -> MeWriter {
|
||||
let (tx, _rx) = mpsc::channel::<WriterCommand>(8);
|
||||
let byte_budget = pool.new_writer_byte_budget();
|
||||
let writer = MeWriter {
|
||||
id: writer_id,
|
||||
addr: endpoint,
|
||||
source_ip: endpoint.ip(),
|
||||
writer_dc,
|
||||
generation,
|
||||
contour: Arc::new(AtomicU8::new(contour.as_u8())),
|
||||
created_at: Instant::now(),
|
||||
tx: tx.clone(),
|
||||
byte_budget: byte_budget.clone(),
|
||||
cancel: CancellationToken::new(),
|
||||
degraded: Arc::new(AtomicBool::new(false)),
|
||||
rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)),
|
||||
draining: Arc::new(AtomicBool::new(false)),
|
||||
draining_started_at_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||
drain_deadline_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||
allow_drain_fallback: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
|
||||
pool.registry
|
||||
.register_writer(writer_id, tx, byte_budget)
|
||||
.await;
|
||||
pool.writers.write().await.push(writer.clone());
|
||||
pool.conn_count.fetch_add(1, Ordering::Relaxed);
|
||||
writer
|
||||
}
|
||||
|
||||
fn desired_two_dcs() -> HashMap<i32, HashSet<SocketAddr>> {
|
||||
HashMap::from([
|
||||
(1, HashSet::from([addr(1, 2001)])),
|
||||
(2, HashSet::from([addr(2, 2002)])),
|
||||
])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_ratio_counts_dc_coverage_not_floor() {
|
||||
let dc1 = addr(1, 2001);
|
||||
@@ -95,3 +148,226 @@ fn stale_concurrent_attempt_cannot_regress_active_generation() {
|
||||
assert_eq!(state.active_generation, 3);
|
||||
assert!(state.pending.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_hardswap_is_rejected_when_stale_binding_is_disabled() {
|
||||
let pool = make_pool().await;
|
||||
let desired_by_dc = desired_two_dcs();
|
||||
let active_generation = pool.current_generation();
|
||||
let old_dc1 = insert_writer(
|
||||
&pool,
|
||||
101,
|
||||
1,
|
||||
addr(1, 2001),
|
||||
active_generation,
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await;
|
||||
let old_dc2 = insert_writer(
|
||||
&pool,
|
||||
102,
|
||||
2,
|
||||
addr(2, 2002),
|
||||
active_generation,
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await;
|
||||
let map_hash = MePool::desired_map_hash(&desired_by_dc);
|
||||
let reservation = pool.reserve_reinit_attempt(true, map_hash, 100);
|
||||
insert_writer(
|
||||
&pool,
|
||||
201,
|
||||
1,
|
||||
addr(1, 2001),
|
||||
reservation.attempt.generation,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
|
||||
let result = pool
|
||||
.commit_reinit_attempt(&reservation.attempt, &desired_by_dc, 0.5)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(ReinitCommitFailure::Redundancy { .. })));
|
||||
assert_eq!(pool.current_generation(), active_generation);
|
||||
assert!(!old_dc1.draining.load(Ordering::Acquire));
|
||||
assert!(!old_dc2.draining.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_hardswap_preserves_fallback_only_for_missing_dc() {
|
||||
let pool = make_pool().await;
|
||||
pool.binding_policy
|
||||
.me_bind_stale_mode
|
||||
.store(MeBindStaleMode::Ttl.as_u8(), Ordering::Release);
|
||||
let desired_by_dc = desired_two_dcs();
|
||||
let active_generation = pool.current_generation();
|
||||
let old_dc1 = insert_writer(
|
||||
&pool,
|
||||
301,
|
||||
1,
|
||||
addr(1, 2001),
|
||||
active_generation,
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await;
|
||||
let old_dc2 = insert_writer(
|
||||
&pool,
|
||||
302,
|
||||
2,
|
||||
addr(9, 2999),
|
||||
active_generation,
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await;
|
||||
let map_hash = MePool::desired_map_hash(&desired_by_dc);
|
||||
let reservation = pool.reserve_reinit_attempt(true, map_hash, 100);
|
||||
let fresh_dc1 = insert_writer(
|
||||
&pool,
|
||||
401,
|
||||
1,
|
||||
addr(1, 2001),
|
||||
reservation.attempt.generation,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcome = pool
|
||||
.commit_reinit_attempt(&reservation.attempt, &desired_by_dc, 0.5)
|
||||
.await
|
||||
.expect("partial hardswap must commit when bounded stale fallback is enabled");
|
||||
|
||||
assert_eq!(pool.current_generation(), reservation.attempt.generation);
|
||||
assert_eq!(outcome.missing_dc, vec![2]);
|
||||
assert_eq!(outcome.force_close_writer_ids, vec![301]);
|
||||
assert!(old_dc1.draining.load(Ordering::Acquire));
|
||||
assert!(!old_dc1.allow_drain_fallback.load(Ordering::Acquire));
|
||||
assert!(old_dc2.draining.load(Ordering::Acquire));
|
||||
assert!(old_dc2.allow_drain_fallback.load(Ordering::Acquire));
|
||||
assert_eq!(
|
||||
WriterContour::from_u8(fresh_dc1.contour.load(Ordering::Acquire)),
|
||||
WriterContour::Active
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_hardswap_promotes_fresh_generation_and_retires_old_writers() {
|
||||
let pool = make_pool().await;
|
||||
let desired_by_dc = desired_two_dcs();
|
||||
let active_generation = pool.current_generation();
|
||||
let old_dc1 = insert_writer(
|
||||
&pool,
|
||||
501,
|
||||
1,
|
||||
addr(1, 2001),
|
||||
active_generation,
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await;
|
||||
let old_dc2 = insert_writer(
|
||||
&pool,
|
||||
502,
|
||||
2,
|
||||
addr(2, 2002),
|
||||
active_generation,
|
||||
WriterContour::Active,
|
||||
)
|
||||
.await;
|
||||
let map_hash = MePool::desired_map_hash(&desired_by_dc);
|
||||
let reservation = pool.reserve_reinit_attempt(true, map_hash, 100);
|
||||
let fresh_dc1 = insert_writer(
|
||||
&pool,
|
||||
601,
|
||||
1,
|
||||
addr(1, 2001),
|
||||
reservation.attempt.generation,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
let fresh_dc2 = insert_writer(
|
||||
&pool,
|
||||
602,
|
||||
2,
|
||||
addr(2, 2002),
|
||||
reservation.attempt.generation,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcome = pool
|
||||
.commit_reinit_attempt(&reservation.attempt, &desired_by_dc, 1.0)
|
||||
.await
|
||||
.expect("fully covered hardswap must commit");
|
||||
|
||||
assert_eq!(pool.current_generation(), reservation.attempt.generation);
|
||||
assert!(outcome.missing_dc.is_empty());
|
||||
assert_eq!(outcome.force_close_writer_ids, vec![501, 502]);
|
||||
assert!(old_dc1.draining.load(Ordering::Acquire));
|
||||
assert!(old_dc2.draining.load(Ordering::Acquire));
|
||||
assert_eq!(
|
||||
WriterContour::from_u8(fresh_dc1.contour.load(Ordering::Acquire)),
|
||||
WriterContour::Active
|
||||
);
|
||||
assert_eq!(
|
||||
WriterContour::from_u8(fresh_dc2.contour.load(Ordering::Acquire)),
|
||||
WriterContour::Active
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generation_role_reconciliation_promotes_active_warm_and_drains_orphans() {
|
||||
let pool = make_pool().await;
|
||||
let endpoint = addr(1, 2001);
|
||||
pool.preferred_endpoints_by_dc
|
||||
.store(Arc::new(HashMap::from([(1, vec![endpoint])])));
|
||||
let desired_by_dc = HashMap::from([(1, HashSet::from([endpoint]))]);
|
||||
let map_hash = MePool::desired_map_hash(&desired_by_dc);
|
||||
let reservation = pool.reserve_reinit_attempt(true, map_hash, 100);
|
||||
let active_warm = insert_writer(
|
||||
&pool,
|
||||
701,
|
||||
1,
|
||||
endpoint,
|
||||
pool.current_generation(),
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
let pending_warm = insert_writer(
|
||||
&pool,
|
||||
702,
|
||||
1,
|
||||
endpoint,
|
||||
reservation.attempt.generation,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
let orphan_warm = insert_writer(
|
||||
&pool,
|
||||
703,
|
||||
1,
|
||||
endpoint,
|
||||
reservation.attempt.generation + 10,
|
||||
WriterContour::Warm,
|
||||
)
|
||||
.await;
|
||||
|
||||
let changed = pool.reconcile_writer_generation_roles().await;
|
||||
|
||||
assert_eq!(changed, 2);
|
||||
assert_eq!(
|
||||
WriterContour::from_u8(active_warm.contour.load(Ordering::Acquire)),
|
||||
WriterContour::Active
|
||||
);
|
||||
assert_eq!(
|
||||
WriterContour::from_u8(pending_warm.contour.load(Ordering::Acquire)),
|
||||
WriterContour::Warm
|
||||
);
|
||||
assert!(orphan_warm.draining.load(Ordering::Acquire));
|
||||
assert!(!orphan_warm.allow_drain_fallback.load(Ordering::Acquire));
|
||||
assert_eq!(
|
||||
pool.api_hardswap_snapshot()
|
||||
.await
|
||||
.orphan_warm_writers_current,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user