This commit is contained in:
Alexey
2026-09-25 19:46:31 +03:00
parent 2d63fcf376
commit 326c0ecdb9
128 changed files with 914 additions and 1248 deletions
+3 -5
View File
@@ -76,11 +76,9 @@ pub fn parse_proxy_config_text(text: &str, http_status: u16) -> ProxyConfigData
pub async fn load_proxy_config_cache(path: &str) -> Result<ProxyConfigData> {
#[cfg(unix)]
let bytes = read_regular_limited_async(
Path::new(path).to_path_buf(),
HTTPS_RESPONSE_BODY_MAX_BYTES,
)
.await;
let bytes =
read_regular_limited_async(Path::new(path).to_path_buf(), HTTPS_RESPONSE_BODY_MAX_BYTES)
.await;
#[cfg(not(unix))]
let bytes = tokio::fs::read(path).await;
let bytes = bytes.map_err(|e| {
@@ -234,7 +234,8 @@ pub(super) async fn maybe_refresh_idle_writer_for_dc(
purpose,
&mut reservation,
);
let rotate_ok = match tokio::time::timeout(pool.reconnect_runtime.me_one_timeout, replace).await {
let rotate_ok = match tokio::time::timeout(pool.reconnect_runtime.me_one_timeout, replace).await
{
Ok(Ok(())) => true,
Ok(Err(error)) => {
debug!(
+2 -4
View File
@@ -270,10 +270,8 @@ async fn under_floor_idle_writer_still_enters_transactional_refresh() {
let writer = insert_active_writer_at(&pool, writer_id, 2, endpoint).await;
let key = (2, IpFamily::V4);
let live_writer_ids_by_addr = HashMap::from([((2, endpoint), vec![writer_id])]);
let writer_idle_since = HashMap::from([(
writer_id,
MePool::now_epoch_secs().saturating_sub(60),
)]);
let writer_idle_since =
HashMap::from([(writer_id, MePool::now_epoch_secs().saturating_sub(60))]);
let bound_clients_by_writer = HashMap::from([(writer_id, 0)]);
let mut next_attempt = HashMap::new();
let rng = Arc::new(SecureRandom::new());
+1 -1
View File
@@ -64,9 +64,9 @@ pub use ping::{
MePingFamily, MePingReport, MePingSample, format_me_route, format_sample_line, run_me_ping,
};
pub use pool::MePool;
pub(crate) use pool_status::MeApiHardswapSnapshot;
#[allow(unused_imports)]
pub use pool_nat::{detect_public_ip, stun_probe};
pub(crate) use pool_status::MeApiHardswapSnapshot;
pub(crate) use registry::ConnLease;
pub use registry::ConnRegistry;
pub use rotation::{MeReinitTrigger, me_reinit_scheduler, me_rotation_task};
@@ -120,12 +120,8 @@ impl MePool {
me_route_inline_recovery_wait_ms: u64,
me_connection_cleanup_capacity: usize,
) -> Arc<Self> {
let endpoint_snapshot = Self::build_endpoint_snapshot(
&decision,
proxy_map_v4,
proxy_map_v6,
1,
);
let endpoint_snapshot =
Self::build_endpoint_snapshot(&decision, proxy_map_v4, proxy_map_v6, 1);
let registry = Arc::new(ConnRegistry::with_route_and_cleanup_capacity(
me_route_channel_capacity,
me_connection_cleanup_capacity,
+1 -5
View File
@@ -198,11 +198,7 @@ impl MePool {
}
fn mirror_negative_dcs(map: &mut HashMap<i32, Vec<(IpAddr, u16)>>) {
let positive_dcs = map
.keys()
.copied()
.filter(|dc| *dc > 0)
.collect::<Vec<_>>();
let positive_dcs = map.keys().copied().filter(|dc| *dc > 0).collect::<Vec<_>>();
for dc in positive_dcs {
if !map.contains_key(&-dc)
&& let Some(endpoints) = map.get(&dc).cloned()
@@ -203,7 +203,9 @@ impl MePool {
.max(1)
.min(WRITER_REPLACEMENT_OPEN_LIMIT_MAX);
loop {
let reserved = self.writer_replacement_open_reserved.load(Ordering::Acquire);
let reserved = self
.writer_replacement_open_reserved
.load(Ordering::Acquire);
if reserved >= replacement_limit {
return None;
}
+5 -8
View File
@@ -252,12 +252,7 @@ impl MePool {
let addr = candidates[idx];
match self
.connect_one_with_generation_contour_for_dc_with_intent(
addr,
rng,
generation,
contour,
dc,
intent,
addr, rng, generation, contour, dc, intent,
)
.await
{
@@ -296,8 +291,10 @@ impl MePool {
let status = self.reinit.status.load();
let role_is_authoritative = match target.contour {
WriterContour::Active => target.generation == status.active_generation,
WriterContour::Warm => status.pending_hardswap_generation != 0
&& target.generation == status.pending_hardswap_generation,
WriterContour::Warm => {
status.pending_hardswap_generation != 0
&& target.generation == status.pending_hardswap_generation
}
WriterContour::Draining => false,
};
let pending_revision_matches = target.contour != WriterContour::Warm
+2 -3
View File
@@ -10,8 +10,8 @@ use rand::seq::SliceRandom;
use std::collections::hash_map::DefaultHasher;
use tracing::{debug, info, warn};
use crate::crypto::SecureRandom;
use crate::config::MeBindStaleMode;
use crate::crypto::SecureRandom;
use crate::network::IpFamily;
use super::pool::{
@@ -117,8 +117,7 @@ fn publish_reinit_state(reinit: &ReinitCore, state: &ReinitCoordinatorState) {
pending_hardswap_started_at_epoch_secs: pending
.map_or(0, |value| value.started_at_epoch_secs),
pending_hardswap_map_hash: pending.map_or(0, |value| value.map_hash),
pending_hardswap_endpoint_revision: pending
.map_or(0, |value| value.endpoint_revision),
pending_hardswap_endpoint_revision: pending.map_or(0, |value| value.endpoint_revision),
inflight: state.attempts.len(),
};
reinit
@@ -318,15 +318,13 @@ impl MePool {
if alive >= required {
covered = covered.saturating_add(1);
} else {
writer_deficit =
writer_deficit.saturating_add(required.saturating_sub(alive));
writer_deficit = writer_deficit.saturating_add(required.saturating_sub(alive));
missing_groups.push(DcFamilyGroup { dc: *dc, family });
}
}
}
missing_groups.sort_unstable_by_key(|group| {
(group.dc, matches!(group.family, IpFamily::V6))
});
missing_groups
.sort_unstable_by_key(|group| (group.dc, matches!(group.family, IpFamily::V6)));
HardswapCoverage {
ratio: if total == 0 {
1.0
@@ -453,8 +451,8 @@ impl MePool {
let authoritative_warm = contour == WriterContour::Warm
&& pending_generation == Some(writer.generation)
&& endpoint_is_current;
let stale_active = contour == WriterContour::Active
&& writer.generation != active_generation;
let stale_active =
contour == WriterContour::Active && writer.generation != active_generation;
if authoritative_warm || (contour == WriterContour::Active && !stale_active) {
continue;
}
@@ -535,8 +533,7 @@ impl MePool {
.filter(|w| !w.draining.load(Ordering::Relaxed))
.filter(|w| w.generation == generation)
.filter(|w| {
WriterContour::from_u8(w.contour.load(Ordering::Acquire))
== WriterContour::Active
WriterContour::from_u8(w.contour.load(Ordering::Acquire)) == WriterContour::Active
})
.filter(|w| w.writer_dc == dc)
.filter(|w| endpoints.contains(&w.addr))
@@ -34,11 +34,7 @@ impl MePool {
let total_passes = 1 + extra_passes;
for (dc, endpoints) in desired_by_dc {
if !self.hardswap_warmup_is_authoritative(
generation,
map_hash,
endpoint_revision,
) {
if !self.hardswap_warmup_is_authoritative(generation, map_hash, endpoint_revision) {
return;
}
for family in [IpFamily::V4, IpFamily::V6] {
@@ -119,11 +115,7 @@ impl MePool {
}
last_fresh_count = self
.fresh_writer_count_for_dc_endpoints(
generation,
*dc,
&family_endpoints,
)
.fresh_writer_count_for_dc_endpoints(generation, *dc, &family_endpoints)
.await;
if last_fresh_count >= required {
completed = true;
@@ -324,8 +316,7 @@ impl MePool {
Err(ReinitCommitFailure::Superseded) => {
debug!(
previous_generation,
generation,
"ME reinit result discarded after a newer desired-map attempt"
generation, "ME reinit result discarded after a newer desired-map attempt"
);
return false;
}
@@ -228,7 +228,10 @@ async fn partial_hardswap_is_rejected_when_stale_binding_is_disabled() {
.commit_reinit_attempt(&reservation.attempt, &desired_by_dc, 0.5)
.await;
assert!(matches!(result, Err(ReinitCommitFailure::Redundancy { .. })));
assert!(matches!(
result,
Err(ReinitCommitFailure::Redundancy { .. })
));
assert_eq!(pool.current_generation(), active_generation);
assert!(!old_dc1.draining.load(Ordering::Acquire));
assert!(!old_dc2.draining.load(Ordering::Acquire));
@@ -303,24 +306,8 @@ async fn partial_hardswap_preserves_fallback_only_for_underfloor_family() {
let v6 = addr_v6(1, 2001);
let desired_by_dc = HashMap::from([(1, HashSet::from([v4, v6]))]);
let active_generation = pool.current_generation();
let old_v4 = insert_writer(
&pool,
451,
1,
v4,
active_generation,
WriterContour::Active,
)
.await;
let old_v6 = insert_writer(
&pool,
452,
1,
v6,
active_generation,
WriterContour::Active,
)
.await;
let old_v4 = insert_writer(&pool, 451, 1, v4, active_generation, WriterContour::Active).await;
let old_v6 = insert_writer(&pool, 452, 1, v6, active_generation, WriterContour::Active).await;
let map_hash = MePool::desired_map_hash(&desired_by_dc);
let endpoint_revision = pool.endpoint_snapshot.load().revision;
let reservation = pool
@@ -81,8 +81,7 @@ impl MePool {
let (replacement_preparing_current, replacement_retiring_current) =
self.registry.writer_replacement_counts();
let pending_age_secs = pending.then(|| {
Self::now_epoch_secs()
.saturating_sub(reinit.pending_hardswap_started_at_epoch_secs)
Self::now_epoch_secs().saturating_sub(reinit.pending_hardswap_started_at_epoch_secs)
});
MeApiHardswapSnapshot {
@@ -83,12 +83,8 @@ impl MePool {
if endpoint_count == 0 {
continue;
}
let required =
self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
let alive = live_writers_by_group
.get(&(dc, ipv4))
.copied()
.unwrap_or(0);
let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
let alive = live_writers_by_group.get(&(dc, ipv4)).copied().unwrap_or(0);
if alive < required {
return false;
}
@@ -133,9 +129,7 @@ impl MePool {
.map(|endpoints| {
endpoint_family_counts(endpoints)
.into_iter()
.map(|(_, count)| {
self.required_writers_for_dc_with_floor_mode(count, false)
})
.map(|(_, count)| self.required_writers_for_dc_with_floor_mode(count, false))
.sum::<usize>()
})
.sum();
@@ -254,9 +248,7 @@ impl MePool {
let dc_required_writers = family_counts
.iter()
.filter(|(_, count)| *count > 0)
.map(|(_, count)| {
self.required_writers_for_dc_with_floor_mode(*count, false)
})
.map(|(_, count)| self.required_writers_for_dc_with_floor_mode(*count, false))
.sum::<usize>();
let floor_min = family_counts
.iter()
@@ -289,8 +281,7 @@ impl MePool {
.me_adaptive_floor_max_extra_writers_multi_per_core
.load(Ordering::Relaxed) as usize
};
family_base
.saturating_add(adaptive_cpu_cores.saturating_mul(extra_per_core))
family_base.saturating_add(adaptive_cpu_cores.saturating_mul(extra_per_core))
})
.sum::<usize>();
let floor_capped =
@@ -357,6 +348,9 @@ impl MePool {
}
fn endpoint_family_counts(endpoints: &BTreeSet<SocketAddr>) -> [(bool, usize); 2] {
let ipv4 = endpoints.iter().filter(|endpoint| endpoint.is_ipv4()).count();
let ipv4 = endpoints
.iter()
.filter(|endpoint| endpoint.is_ipv4())
.count();
[(true, ipv4), (false, endpoints.len().saturating_sub(ipv4))]
}
@@ -12,12 +12,7 @@ use crate::transport::middle_proxy::codec::WriterCommand;
use crate::transport::middle_proxy::pool::{MePool, MeWriter, WriterContour};
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool_with_decision;
fn writer(
pool: &Arc<MePool>,
id: u64,
dc: i32,
addr: SocketAddr,
) -> MeWriter {
fn writer(pool: &Arc<MePool>, id: u64, dc: i32, addr: SocketAddr) -> MeWriter {
let (tx, _rx) = mpsc::channel::<WriterCommand>(8);
MeWriter {
id,
@@ -60,12 +55,7 @@ async fn dual_family_status_reports_each_family_floor() {
let mut writers = pool.writers.write().await;
for (group, dc) in [2, -2].into_iter().enumerate() {
for offset in 0..required_per_family {
writers.push(writer(
&pool,
(group as u64 * 100) + offset as u64,
dc,
v4,
));
writers.push(writer(&pool, (group as u64 * 100) + offset as u64, dc, v4));
}
}
drop(writers);
@@ -122,11 +122,9 @@ impl MePool {
!candidate.draining.load(Ordering::Acquire)
&& candidate.writer_dc == writer.writer_dc
&& candidate.generation == writer.generation
&& WriterContour::from_u8(candidate.contour.load(Ordering::Acquire))
== contour
&& WriterContour::from_u8(candidate.contour.load(Ordering::Acquire)) == contour
&& candidate.addr.is_ipv4() == writer.addr.is_ipv4()
&& endpoint_snapshot
.contains_dc_endpoint(candidate.writer_dc, candidate.addr)
&& endpoint_snapshot.contains_dc_endpoint(candidate.writer_dc, candidate.addr)
})
.count();
if current >= required {
@@ -138,11 +138,7 @@ impl MePool {
writers.push(writer);
self.conn_count.fetch_add(1, Ordering::Relaxed);
writers.publish_current();
self.apply_writer_draining_state(
&writers[victim_pos],
self.force_close_timeout(),
false,
);
self.apply_writer_draining_state(&writers[victim_pos], self.force_close_timeout(), false);
self.lifecycle
.spawn_registered_writer(task_registration, writer_task);
reservation.mark_committed();
@@ -264,11 +260,8 @@ mod tests {
async fn replacement_commit_publishes_successor_before_draining_victim() {
let pool = make_pool().await;
let addr = endpoint(1);
pool.update_proxy_maps(
HashMap::from([(2, vec![(addr.ip(), addr.port())])]),
None,
)
.await;
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
.await;
let victim = install_writer(&pool, 1001, 2, addr).await;
let expected_role = WriterRole::from_writer(&victim);
let mut reservation = pool
@@ -307,11 +300,8 @@ mod tests {
async fn cancelled_replacement_waiting_for_publication_restores_all_reservations() {
let pool = make_pool().await;
let addr = endpoint(2);
pool.update_proxy_maps(
HashMap::from([(2, vec![(addr.ip(), addr.port())])]),
None,
)
.await;
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
.await;
let victim = install_writer(&pool, 2001, 2, addr).await;
let expected_role = WriterRole::from_writer(&victim);
let mut reservation = pool
@@ -336,10 +326,21 @@ mod tests {
assert!(result.is_err());
drop(writers_guard);
drop(reservation);
assert_eq!(pool.writer_replacement_open_reserved.load(Ordering::Acquire), 0);
assert_eq!(
pool.writer_replacement_open_reserved
.load(Ordering::Acquire),
0
);
assert_eq!(pool.registry.writer_replacement_counts(), (0, 0));
assert!(!victim.draining.load(Ordering::Acquire));
assert!(!pool.writers.read().await.iter().any(|writer| writer.id == 2002));
assert!(
!pool
.writers
.read()
.await
.iter()
.any(|writer| writer.id == 2002)
);
}
#[tokio::test]
@@ -379,7 +380,14 @@ mod tests {
assert!(result.is_err());
drop(reservation);
assert!(!victim.draining.load(Ordering::Acquire));
assert!(!pool.writers.read().await.iter().any(|writer| writer.id == 3002));
assert!(
!pool
.writers
.read()
.await
.iter()
.any(|writer| writer.id == 3002)
);
assert_eq!(pool.registry.writer_replacement_counts(), (0, 0));
}
}
@@ -92,12 +92,7 @@ impl MePool {
intent: WriterOpenIntent,
) -> Result<PreparedWriter<'a>> {
let Some(writer_open_reservation) = self
.reserve_writer_open(
contour,
intent,
writer_dc,
addr,
)
.reserve_writer_open(contour, intent, writer_dc, addr)
.await
else {
return Err(ProxyError::Proxy(format!(
@@ -76,10 +76,10 @@ impl WriterRegistrationGuard<'_> {
|| !Arc::ptr_eq(&route_state, reservation.state())
|| reservation.requires_idle()
&& self
.binding
.conns_for_writer
.get(&reservation.writer_id())
.is_none_or(|conn_ids| !conn_ids.is_empty())
.binding
.conns_for_writer
.get(&reservation.writer_id())
.is_none_or(|conn_ids| !conn_ids.is_empty())
{
return false;
}
@@ -98,9 +98,9 @@ impl ConnRegistry {
.map(|route| Arc::clone(&route.replacement_state))?;
if require_idle
&& binding
.conns_for_writer
.get(&writer_id)
.is_none_or(|conn_ids| !conn_ids.is_empty())
.conns_for_writer
.get(&writer_id)
.is_none_or(|conn_ids| !conn_ids.is_empty())
{
return None;
}
@@ -8,11 +8,11 @@ use tokio::sync::mpsc::error::TrySendError;
use super::super::codec::WriterCommand;
use super::super::{MeResponse, RouteBytePermit};
use super::replacement::WriterBindOutcome;
use super::{
BoundConn, ConnMeta, ConnRegistry, ConnWriter, HotConnBinding, RouteResult,
WriterActivitySnapshot,
};
use super::replacement::WriterBindOutcome;
impl ConnRegistry {
fn set_writer_bound_count(&self, writer_id: u64, count: usize) {
+1 -3
View File
@@ -169,8 +169,7 @@ impl MePool {
0..self.route_runtime.me_route_inline_recovery_attempts.max(1)
{
let endpoint_snapshot = self.endpoint_snapshot.load_full();
for (dc, addrs) in
&endpoint_snapshot.preferred_endpoints_by_dc
for (dc, addrs) in &endpoint_snapshot.preferred_endpoints_by_dc
{
for addr in addrs {
let _ = self
@@ -544,5 +543,4 @@ impl MePool {
return Ok(());
}
}
}
+2 -1
View File
@@ -31,7 +31,8 @@ impl MePool {
writer_reserved_bytes: usize,
payload_permit: Option<OwnedSemaphorePermit>,
) -> Result<BoundWriterSendOutcome> {
let Some((current, current_meta)) = self.registry.get_writer_with_meta(conn_id).await else {
let Some((current, current_meta)) = self.registry.get_writer_with_meta(conn_id).await
else {
return Ok(BoundWriterSendOutcome::Retry(payload_permit));
};
let deadline = writer_send_deadline(self.route_runtime.me_route_blocking_send_timeout);
+2 -2
View File
@@ -9,8 +9,8 @@ use super::super::MePool;
use super::super::codec::{ProxyReqCommand, WriterCommand};
use super::reservation::{
WriterByteReserveError, WriterCommandReserveError, proxy_req_payload_from_command,
proxy_req_resident_permits, proxy_tag_array, reserve_writer_bytes,
reserve_writer_command_slot, writer_send_deadline,
proxy_req_resident_permits, proxy_tag_array, reserve_writer_bytes, reserve_writer_command_slot,
writer_send_deadline,
};
use crate::error::{ProxyError, Result};
use crate::stream::PooledBuffer;
@@ -34,9 +34,7 @@ pub(super) fn proxy_req_payload_from_command(
}
}
pub(super) fn payload_permit_from_data_command(
cmd: WriterCommand,
) -> Option<OwnedSemaphorePermit> {
pub(super) fn payload_permit_from_data_command(cmd: WriterCommand) -> Option<OwnedSemaphorePermit> {
match cmd {
WriterCommand::Data { _permit, .. } => _permit,
_ => None,
+1 -5
View File
@@ -127,11 +127,7 @@ impl MePool {
WriterContour::Draining => PICK_PENALTY_DRAINING,
};
let stale = (writer.generation < current_generation) as usize;
let stale_penalty = if stale != 0 {
PICK_PENALTY_STALE
} else {
0
};
let stale_penalty = if stale != 0 { PICK_PENALTY_STALE } else { 0 };
let degraded = writer.degraded.load(Ordering::Relaxed) as usize;
let degraded_penalty = if degraded != 0 {
PICK_PENALTY_DEGRADED
@@ -10,9 +10,7 @@ use crate::network::IpFamily;
use crate::network::probe::NetworkDecision;
use crate::stats::Stats;
use super::pool::{
MePool, ReinitStatusSnapshot, WriterContour, WriterRole,
};
use super::pool::{MePool, ReinitStatusSnapshot, WriterContour, WriterRole};
use super::pool_writer_security_tests::make_pool_with_decision;
async fn make_pool() -> Arc<MePool> {
@@ -195,10 +193,7 @@ async fn refill_accepts_enabled_nonpreferred_family_without_multipath() {
let v6_addr = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443);
pool.update_proxy_maps(
HashMap::from([(2, vec![(v4_addr.ip(), v4_addr.port())])]),
Some(HashMap::from([(
2,
vec![(v6_addr.ip(), v6_addr.port())],
)])),
Some(HashMap::from([(2, vec![(v6_addr.ip(), v6_addr.port())])])),
)
.await;
@@ -98,13 +98,7 @@ async fn normal_active_publication_cannot_race_past_the_family_floor() {
let generation = pool.current_generation();
let writers = (1..=3)
.map(|writer_id| {
unregistered_writer(
&pool,
writer_id,
addr,
generation,
WriterContour::Active,
)
unregistered_writer(&pool, writer_id, addr, generation, WriterContour::Active)
})
.collect::<Vec<_>>();
let candidate = unregistered_writer(&pool, 4, addr, generation, WriterContour::Active);
@@ -135,10 +129,7 @@ async fn stale_same_family_writers_do_not_satisfy_current_endpoint_coverage() {
let current_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
let stale_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 2)), 443);
pool.update_proxy_maps(
std::collections::HashMap::from([(
2,
vec![(current_addr.ip(), current_addr.port())],
)]),
std::collections::HashMap::from([(2, vec![(current_addr.ip(), current_addr.port())])]),
None,
)
.await;
@@ -244,13 +235,7 @@ async fn normal_active_publication_allows_adaptive_growth_above_family_floor() {
let generation = pool.current_generation();
let writers = (1..=3)
.map(|writer_id| {
unregistered_writer(
&pool,
writer_id,
addr,
generation,
WriterContour::Active,
)
unregistered_writer(&pool, writer_id, addr, generation, WriterContour::Active)
})
.collect::<Vec<_>>();
let candidate = unregistered_writer(&pool, 4, addr, generation, WriterContour::Active);
@@ -287,13 +272,7 @@ async fn normal_active_publication_rejects_configured_contour_cap() {
let generation = pool.current_generation();
let writers = (1..=3)
.map(|writer_id| {
unregistered_writer(
&pool,
writer_id,
addr,
generation,
WriterContour::Active,
)
unregistered_writer(&pool, writer_id, addr, generation, WriterContour::Active)
})
.collect::<Vec<_>>();
let candidate = unregistered_writer(&pool, 4, addr, generation, WriterContour::Active);
@@ -238,10 +238,7 @@ async fn send_proxy_req_uses_live_same_dc_writer_while_preferred_endpoint_refill
.await;
assert!(pool.admission_ready_conditional_cast().await);
assert_eq!(
pool.preferred_endpoints_for_dc(2).await,
vec![new_addr]
);
assert_eq!(pool.preferred_endpoints_for_dc(2).await, vec![new_addr]);
let (conn_id, _rx) = pool.registry.register().await;
let result = pool