astelm
/
telemt
mirror of https://github.com/telemt/telemt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

RoutingTable + BindingState 

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey 2026-03-25 17:50:44 +03:00
parent 8bab3f70e1
commit 70c2f0f045
No known key found for this signature in database
2 changed files with 75 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/transport/middle_proxy/pool_writer.rs
View File

@ -425,15 +425,8 @@ impl MePool {
}; };
let (conn_id, mut service_rx) = pool.registry.register().await; let (conn_id, mut service_rx) = pool.registry.register().await;
if !pool // Service RPC_PROXY_REQ signal path is intentionally route-only:
.registry // do not bind synthetic conn_id into regular writer/client accounting.
.bind_writer(conn_id, writer_id, meta.clone())
.await
{
let _ = pool.registry.unregister(conn_id).await;
stats_signal.increment_me_rpc_proxy_req_signal_skipped_no_meta_total();
continue;
}
let payload = build_proxy_req_payload( let payload = build_proxy_req_payload(
conn_id, conn_id,

110
src/transport/middle_proxy/registry.rs
View File

@ -51,7 +51,15 @@ pub(super) struct WriterActivitySnapshot {
} }
struct RegistryInner { struct RegistryInner {
routing: RoutingTable,
binding: BindingState,
}
struct RoutingTable {
map: HashMap<u64, mpsc::Sender<MeResponse>>, map: HashMap<u64, mpsc::Sender<MeResponse>>,
}
struct BindingState {
writers: HashMap<u64, mpsc::Sender<WriterCommand>>, writers: HashMap<u64, mpsc::Sender<WriterCommand>>,
writer_for_conn: HashMap<u64, u64>, writer_for_conn: HashMap<u64, u64>,
conns_for_writer: HashMap<u64, HashSet<u64>>, conns_for_writer: HashMap<u64, HashSet<u64>>,
@ -63,13 +71,17 @@ struct RegistryInner {
impl RegistryInner { impl RegistryInner {
fn new() -> Self { fn new() -> Self {
Self { Self {
routing: RoutingTable {
map: HashMap::new(), map: HashMap::new(),
},
binding: BindingState {
writers: HashMap::new(), writers: HashMap::new(),
writer_for_conn: HashMap::new(), writer_for_conn: HashMap::new(),
conns_for_writer: HashMap::new(), conns_for_writer: HashMap::new(),
meta: HashMap::new(), meta: HashMap::new(),
last_meta_for_writer: HashMap::new(), last_meta_for_writer: HashMap::new(),
writer_idle_since_epoch_secs: HashMap::new(), writer_idle_since_epoch_secs: HashMap::new(),
},
} }
} }
} }
@ -130,14 +142,15 @@ impl ConnRegistry {
pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) { pub async fn register(&self) -> (u64, mpsc::Receiver<MeResponse>) {
let id = self.next_id.fetch_add(1, Ordering::Relaxed); let id = self.next_id.fetch_add(1, Ordering::Relaxed);
let (tx, rx) = mpsc::channel(self.route_channel_capacity); let (tx, rx) = mpsc::channel(self.route_channel_capacity);
self.inner.write().await.map.insert(id, tx); self.inner.write().await.routing.map.insert(id, tx);
(id, rx) (id, rx)
} }
pub async fn register_writer(&self, writer_id: u64, tx: mpsc::Sender<WriterCommand>) { pub async fn register_writer(&self, writer_id: u64, tx: mpsc::Sender<WriterCommand>) {
let mut inner = self.inner.write().await; let mut inner = self.inner.write().await;
inner.writers.insert(writer_id, tx); inner.binding.writers.insert(writer_id, tx);
inner inner
.binding
.conns_for_writer .conns_for_writer
.entry(writer_id) .entry(writer_id)
.or_insert_with(HashSet::new); .or_insert_with(HashSet::new);
@ -146,10 +159,10 @@ impl ConnRegistry {
/// Unregister connection, returning associated writer_id if any. /// Unregister connection, returning associated writer_id if any.
pub async fn unregister(&self, id: u64) -> Option<u64> { pub async fn unregister(&self, id: u64) -> Option<u64> {
let mut inner = self.inner.write().await; let mut inner = self.inner.write().await;
inner.map.remove(&id); inner.routing.map.remove(&id);
inner.meta.remove(&id); inner.binding.meta.remove(&id);
if let Some(writer_id) = inner.writer_for_conn.remove(&id) { if let Some(writer_id) = inner.binding.writer_for_conn.remove(&id) {
let became_empty = if let Some(set) = inner.conns_for_writer.get_mut(&writer_id) { let became_empty = if let Some(set) = inner.binding.conns_for_writer.get_mut(&writer_id) {
set.remove(&id); set.remove(&id);
set.is_empty() set.is_empty()
} else { } else {
@ -157,6 +170,7 @@ impl ConnRegistry {
}; };
if became_empty { if became_empty {
inner inner
.binding
.writer_idle_since_epoch_secs .writer_idle_since_epoch_secs
.insert(writer_id, Self::now_epoch_secs()); .insert(writer_id, Self::now_epoch_secs());
} }
@ -169,7 +183,7 @@ impl ConnRegistry {
pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult { pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult {
let tx = { let tx = {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.map.get(&id).cloned() inner.routing.map.get(&id).cloned()
}; };
let Some(tx) = tx else { let Some(tx) = tx else {
@ -225,7 +239,7 @@ impl ConnRegistry {
pub async fn route_nowait(&self, id: u64, resp: MeResponse) -> RouteResult { pub async fn route_nowait(&self, id: u64, resp: MeResponse) -> RouteResult {
let tx = { let tx = {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.map.get(&id).cloned() inner.routing.map.get(&id).cloned()
}; };
let Some(tx) = tx else { let Some(tx) = tx else {
@ -251,7 +265,7 @@ impl ConnRegistry {
let tx = { let tx = {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.map.get(&id).cloned() inner.routing.map.get(&id).cloned()
}; };
let Some(tx) = tx else { let Some(tx) = tx else {
@ -295,19 +309,19 @@ impl ConnRegistry {
// ROUTING IS THE SOURCE OF TRUTH: // ROUTING IS THE SOURCE OF TRUTH:
// never keep/attach writer binding for a connection that is already // never keep/attach writer binding for a connection that is already
// absent from the routing table. // absent from the routing table.
if !inner.map.contains_key(&conn_id) { if !inner.routing.map.contains_key(&conn_id) {
return false; return false;
} }
if !inner.writers.contains_key(&writer_id) { if !inner.binding.writers.contains_key(&writer_id) {
return false; return false;
} }
let previous_writer_id = inner.writer_for_conn.insert(conn_id, writer_id); let previous_writer_id = inner.binding.writer_for_conn.insert(conn_id, writer_id);
if let Some(previous_writer_id) = previous_writer_id if let Some(previous_writer_id) = previous_writer_id
&& previous_writer_id != writer_id && previous_writer_id != writer_id
{ {
let became_empty = let became_empty =
if let Some(set) = inner.conns_for_writer.get_mut(&previous_writer_id) { if let Some(set) = inner.binding.conns_for_writer.get_mut(&previous_writer_id) {
set.remove(&conn_id); set.remove(&conn_id);
set.is_empty() set.is_empty()
} else { } else {
@ -315,15 +329,17 @@ impl ConnRegistry {
}; };
if became_empty { if became_empty {
inner inner
.binding
.writer_idle_since_epoch_secs .writer_idle_since_epoch_secs
.insert(previous_writer_id, Self::now_epoch_secs()); .insert(previous_writer_id, Self::now_epoch_secs());
} }
} }
inner.meta.insert(conn_id, meta.clone()); inner.binding.meta.insert(conn_id, meta.clone());
inner.last_meta_for_writer.insert(writer_id, meta); inner.binding.last_meta_for_writer.insert(writer_id, meta);
inner.writer_idle_since_epoch_secs.remove(&writer_id); inner.binding.writer_idle_since_epoch_secs.remove(&writer_id);
inner inner
.binding
.conns_for_writer .conns_for_writer
.entry(writer_id) .entry(writer_id)
.or_insert_with(HashSet::new) .or_insert_with(HashSet::new)
@ -334,10 +350,12 @@ impl ConnRegistry {
pub async fn mark_writer_idle(&self, writer_id: u64) { pub async fn mark_writer_idle(&self, writer_id: u64) {
let mut inner = self.inner.write().await; let mut inner = self.inner.write().await;
inner inner
.binding
.conns_for_writer .conns_for_writer
.entry(writer_id) .entry(writer_id)
.or_insert_with(HashSet::new); .or_insert_with(HashSet::new);
inner inner
.binding
.writer_idle_since_epoch_secs .writer_idle_since_epoch_secs
.entry(writer_id) .entry(writer_id)
.or_insert(Self::now_epoch_secs()); .or_insert(Self::now_epoch_secs());
@ -345,19 +363,19 @@ impl ConnRegistry {
pub async fn get_last_writer_meta(&self, writer_id: u64) -> Option<ConnMeta> { pub async fn get_last_writer_meta(&self, writer_id: u64) -> Option<ConnMeta> {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.last_meta_for_writer.get(&writer_id).cloned() inner.binding.last_meta_for_writer.get(&writer_id).cloned()
} }
pub async fn writer_idle_since_snapshot(&self) -> HashMap<u64, u64> { pub async fn writer_idle_since_snapshot(&self) -> HashMap<u64, u64> {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.writer_idle_since_epoch_secs.clone() inner.binding.writer_idle_since_epoch_secs.clone()
} }
pub async fn writer_idle_since_for_writer_ids(&self, writer_ids: &[u64]) -> HashMap<u64, u64> { pub async fn writer_idle_since_for_writer_ids(&self, writer_ids: &[u64]) -> HashMap<u64, u64> {
let inner = self.inner.read().await; let inner = self.inner.read().await;
let mut out = HashMap::<u64, u64>::with_capacity(writer_ids.len()); let mut out = HashMap::<u64, u64>::with_capacity(writer_ids.len());
for writer_id in writer_ids { for writer_id in writer_ids {
if let Some(idle_since) = inner.writer_idle_since_epoch_secs.get(writer_id).copied() { if let Some(idle_since) = inner.binding.writer_idle_since_epoch_secs.get(writer_id).copied() {
out.insert(*writer_id, idle_since); out.insert(*writer_id, idle_since);
} }
} }
@ -369,10 +387,10 @@ impl ConnRegistry {
let mut bound_clients_by_writer = HashMap::<u64, usize>::new(); let mut bound_clients_by_writer = HashMap::<u64, usize>::new();
let mut active_sessions_by_target_dc = HashMap::<i16, usize>::new(); let mut active_sessions_by_target_dc = HashMap::<i16, usize>::new();
for (writer_id, conn_ids) in &inner.conns_for_writer { for (writer_id, conn_ids) in &inner.binding.conns_for_writer {
bound_clients_by_writer.insert(*writer_id, conn_ids.len()); bound_clients_by_writer.insert(*writer_id, conn_ids.len());
} }
for conn_meta in inner.meta.values() { for conn_meta in inner.binding.meta.values() {
if conn_meta.target_dc == 0 { if conn_meta.target_dc == 0 {
continue; continue;
} }
@ -392,14 +410,15 @@ impl ConnRegistry {
// ROUTING IS THE SOURCE OF TRUTH: // ROUTING IS THE SOURCE OF TRUTH:
// stale bindings are ignored and lazily cleaned when routing no longer // stale bindings are ignored and lazily cleaned when routing no longer
// contains the connection. // contains the connection.
if !inner.map.contains_key(&conn_id) { if !inner.routing.map.contains_key(&conn_id) {
inner.meta.remove(&conn_id); inner.binding.meta.remove(&conn_id);
if let Some(stale_writer_id) = inner.writer_for_conn.remove(&conn_id) if let Some(stale_writer_id) = inner.binding.writer_for_conn.remove(&conn_id)
&& let Some(conns) = inner.conns_for_writer.get_mut(&stale_writer_id) && let Some(conns) = inner.binding.conns_for_writer.get_mut(&stale_writer_id)
{ {
conns.remove(&conn_id); conns.remove(&conn_id);
if conns.is_empty() { if conns.is_empty() {
inner inner
.binding
.writer_idle_since_epoch_secs .writer_idle_since_epoch_secs
.insert(stale_writer_id, Self::now_epoch_secs()); .insert(stale_writer_id, Self::now_epoch_secs());
} }
@ -407,14 +426,15 @@ impl ConnRegistry {
return None; return None;
} }
let writer_id = inner.writer_for_conn.get(&conn_id).copied()?; let writer_id = inner.binding.writer_for_conn.get(&conn_id).copied()?;
let Some(writer) = inner.writers.get(&writer_id).cloned() else { let Some(writer) = inner.binding.writers.get(&writer_id).cloned() else {
inner.writer_for_conn.remove(&conn_id); inner.binding.writer_for_conn.remove(&conn_id);
inner.meta.remove(&conn_id); inner.binding.meta.remove(&conn_id);
if let Some(conns) = inner.conns_for_writer.get_mut(&writer_id) { if let Some(conns) = inner.binding.conns_for_writer.get_mut(&writer_id) {
conns.remove(&conn_id); conns.remove(&conn_id);
if conns.is_empty() { if conns.is_empty() {
inner inner
.binding
.writer_idle_since_epoch_secs .writer_idle_since_epoch_secs
.insert(writer_id, Self::now_epoch_secs()); .insert(writer_id, Self::now_epoch_secs());
} }
@ -429,15 +449,16 @@ impl ConnRegistry {
pub async fn active_conn_ids(&self) -> Vec<u64> { pub async fn active_conn_ids(&self) -> Vec<u64> {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.writer_for_conn.keys().copied().collect() inner.binding.writer_for_conn.keys().copied().collect()
} }
pub async fn writer_lost(&self, writer_id: u64) -> Vec<BoundConn> { pub async fn writer_lost(&self, writer_id: u64) -> Vec<BoundConn> {
let mut inner = self.inner.write().await; let mut inner = self.inner.write().await;
inner.writers.remove(&writer_id); inner.binding.writers.remove(&writer_id);
inner.last_meta_for_writer.remove(&writer_id); inner.binding.last_meta_for_writer.remove(&writer_id);
inner.writer_idle_since_epoch_secs.remove(&writer_id); inner.binding.writer_idle_since_epoch_secs.remove(&writer_id);
let conns = inner let conns = inner
.binding
.conns_for_writer .conns_for_writer
.remove(&writer_id) .remove(&writer_id)
.unwrap_or_default() .unwrap_or_default()
@ -446,11 +467,11 @@ impl ConnRegistry {
let mut out = Vec::new(); let mut out = Vec::new();
for conn_id in conns { for conn_id in conns {
if inner.writer_for_conn.get(&conn_id).copied() != Some(writer_id) { if inner.binding.writer_for_conn.get(&conn_id).copied() != Some(writer_id) {
continue; continue;
} }
inner.writer_for_conn.remove(&conn_id); inner.binding.writer_for_conn.remove(&conn_id);
if let Some(m) = inner.meta.get(&conn_id) { if let Some(m) = inner.binding.meta.get(&conn_id) {
out.push(BoundConn { out.push(BoundConn {
conn_id, conn_id,
meta: m.clone(), meta: m.clone(),
@ -463,12 +484,13 @@ impl ConnRegistry {
#[allow(dead_code)] #[allow(dead_code)]
pub async fn get_meta(&self, conn_id: u64) -> Option<ConnMeta> { pub async fn get_meta(&self, conn_id: u64) -> Option<ConnMeta> {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner.meta.get(&conn_id).cloned() inner.binding.meta.get(&conn_id).cloned()
} }
pub async fn is_writer_empty(&self, writer_id: u64) -> bool { pub async fn is_writer_empty(&self, writer_id: u64) -> bool {
let inner = self.inner.read().await; let inner = self.inner.read().await;
inner inner
.binding
.conns_for_writer .conns_for_writer
.get(&writer_id) .get(&writer_id)
.map(|s| s.is_empty()) .map(|s| s.is_empty())
@ -478,7 +500,7 @@ impl ConnRegistry {
#[allow(dead_code)] #[allow(dead_code)]
pub async fn unregister_writer_if_empty(&self, writer_id: u64) -> bool { pub async fn unregister_writer_if_empty(&self, writer_id: u64) -> bool {
let mut inner = self.inner.write().await; let mut inner = self.inner.write().await;
let Some(conn_ids) = inner.conns_for_writer.get(&writer_id) else { let Some(conn_ids) = inner.binding.conns_for_writer.get(&writer_id) else {
// Writer is already absent from the registry. // Writer is already absent from the registry.
return true; return true;
}; };
@ -486,10 +508,10 @@ impl ConnRegistry {
return false; return false;
} }
inner.writers.remove(&writer_id); inner.binding.writers.remove(&writer_id);
inner.last_meta_for_writer.remove(&writer_id); inner.binding.last_meta_for_writer.remove(&writer_id);
inner.writer_idle_since_epoch_secs.remove(&writer_id); inner.binding.writer_idle_since_epoch_secs.remove(&writer_id);
inner.conns_for_writer.remove(&writer_id); inner.binding.conns_for_writer.remove(&writer_id);
true true
} }
@ -498,7 +520,7 @@ impl ConnRegistry {
let inner = self.inner.read().await; let inner = self.inner.read().await;
let mut out = HashSet::<u64>::with_capacity(writer_ids.len()); let mut out = HashSet::<u64>::with_capacity(writer_ids.len());
for writer_id in writer_ids { for writer_id in writer_ids {
if let Some(conns) = inner.conns_for_writer.get(writer_id) if let Some(conns) = inner.binding.conns_for_writer.get(writer_id)
&& !conns.is_empty() && !conns.is_empty()
{ {
out.insert(*writer_id); out.insert(*writer_id);