HybridAsyncPersistent - new ME Route NoWriter Mode

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/transport/middle_proxy/pool_status.rs

@@ -100,6 +100,72 @@ pub(crate) struct MeApiRuntimeSnapshot {
 }
 
 impl MePool {
+    pub(crate) async fn admission_ready_full_floor(&self) -> bool {
+        let mut endpoints_by_dc = BTreeMap::<i16, BTreeSet<SocketAddr>>::new();
+        if self.decision.ipv4_me {
+            let map = self.proxy_map_v4.read().await.clone();
+            for (dc, addrs) in map {
+                let abs_dc = dc.abs();
+                if abs_dc == 0 {
+                    continue;
+                }
+                let Ok(dc_idx) = i16::try_from(abs_dc) else {
+                    continue;
+                };
+                let entry = endpoints_by_dc.entry(dc_idx).or_default();
+                for (ip, port) in addrs {
+                    entry.insert(SocketAddr::new(ip, port));
+                }
+            }
+        }
+        if self.decision.ipv6_me {
+            let map = self.proxy_map_v6.read().await.clone();
+            for (dc, addrs) in map {
+                let abs_dc = dc.abs();
+                if abs_dc == 0 {
+                    continue;
+                }
+                let Ok(dc_idx) = i16::try_from(abs_dc) else {
+                    continue;
+                };
+                let entry = endpoints_by_dc.entry(dc_idx).or_default();
+                for (ip, port) in addrs {
+                    entry.insert(SocketAddr::new(ip, port));
+                }
+            }
+        }
+
+        if endpoints_by_dc.is_empty() {
+            return false;
+        }
+
+        let writers = self.writers.read().await.clone();
+        let mut live_writers_by_endpoint = HashMap::<SocketAddr, usize>::new();
+        for writer in writers {
+            if writer.draining.load(Ordering::Relaxed) {
+                continue;
+            }
+            *live_writers_by_endpoint.entry(writer.addr).or_insert(0) += 1;
+        }
+
+        for endpoints in endpoints_by_dc.values() {
+            let endpoint_count = endpoints.len();
+            if endpoint_count == 0 {
+                return false;
+            }
+            let required = self.required_writers_for_dc_with_floor_mode(endpoint_count, false);
+            let alive: usize = endpoints
+                .iter()
+                .map(|endpoint| live_writers_by_endpoint.get(endpoint).copied().unwrap_or(0))
+                .sum();
+            if alive < required {
+                return false;
+            }
+        }
+
+        true
+    }
+
     pub(crate) async fn api_status_snapshot(&self) -> MeApiStatusSnapshot {
         let now_epoch_secs = Self::now_epoch_secs();
 

src/transport/middle_proxy/send.rs

@@ -22,6 +22,7 @@ use super::registry::ConnMeta;
 
 const IDLE_WRITER_PENALTY_MID_SECS: u64 = 45;
 const IDLE_WRITER_PENALTY_HIGH_SECS: u64 = 55;
+const HYBRID_GLOBAL_BURST_PERIOD_ROUNDS: u32 = 4;
 
 impl MePool {
     /// Send RPC_PROXY_REQ. `tag_override`: per-user ad_tag (from access.user_ad_tags); if None, uses pool default.
@@ -55,6 +56,9 @@ impl MePool {
         let mut no_writer_deadline: Option<Instant> = None;
         let mut emergency_attempts = 0u32;
         let mut async_recovery_triggered = false;
+        let mut hybrid_recovery_round = 0u32;
+        let mut hybrid_last_recovery_at: Option<Instant> = None;
+        let hybrid_wait_step = self.me_route_no_writer_wait.max(Duration::from_millis(50));
 
         loop {
             if let Some(current) = self.registry.get_writer(conn_id).await {
@@ -138,6 +142,18 @@ impl MePool {
                             }
                             continue;
                         }
+                        MeRouteNoWriterMode::HybridAsyncPersistent => {
+                            self.maybe_trigger_hybrid_recovery(
+                                target_dc,
+                                &mut hybrid_recovery_round,
+                                &mut hybrid_last_recovery_at,
+                                hybrid_wait_step,
+                            )
+                            .await;
+                            let deadline = Instant::now() + hybrid_wait_step;
+                            let _ = self.wait_for_writer_until(deadline).await;
+                            continue;
+                        }
                     }
                 }
                 ws.clone()
@@ -215,6 +231,18 @@ impl MePool {
                             return Err(ProxyError::Proxy("No ME writers available for target DC".into()));
                         }
                     }
+                    MeRouteNoWriterMode::HybridAsyncPersistent => {
+                        self.maybe_trigger_hybrid_recovery(
+                            target_dc,
+                            &mut hybrid_recovery_round,
+                            &mut hybrid_last_recovery_at,
+                            hybrid_wait_step,
+                        )
+                        .await;
+                        let deadline = Instant::now() + hybrid_wait_step;
+                        let _ = self.wait_for_candidate_until(target_dc, deadline).await;
+                        continue;
+                    }
                 }
             }
             let writer_idle_since = self.registry.writer_idle_since_snapshot().await;
@@ -459,6 +487,28 @@ impl MePool {
         preferred
     }
 
+    async fn maybe_trigger_hybrid_recovery(
+        self: &Arc<Self>,
+        target_dc: i16,
+        hybrid_recovery_round: &mut u32,
+        hybrid_last_recovery_at: &mut Option<Instant>,
+        hybrid_wait_step: Duration,
+    ) {
+        if let Some(last) = *hybrid_last_recovery_at
+            && last.elapsed() < hybrid_wait_step
+        {
+            return;
+        }
+
+        let round = *hybrid_recovery_round;
+        let target_triggered = self.trigger_async_recovery_for_target_dc(target_dc).await;
+        if !target_triggered || round % HYBRID_GLOBAL_BURST_PERIOD_ROUNDS == 0 {
+            self.trigger_async_recovery_global().await;
+        }
+        *hybrid_recovery_round = round.saturating_add(1);
+        *hybrid_last_recovery_at = Some(Instant::now());
+    }
+
     pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
         if let Some(w) = self.registry.get_writer(conn_id).await {
             let mut p = Vec::with_capacity(12);