Flush-response experiments

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/transport/middle_proxy/codec.rs

@@ -190,11 +190,26 @@ impl RpcWriter {
         self.writer.flush().await.map_err(ProxyError::Io)
     }
 
-    pub(crate) async fn send_keepalive(&mut self, payload: [u8; 4]) -> Result<()> {
-        // Keepalive is a frame with fl == 4 and 4 bytes payload.
-        let mut frame = Vec::with_capacity(8);
-        frame.extend_from_slice(&4u32.to_le_bytes());
-        frame.extend_from_slice(&payload);
-        self.send(&frame).await
+    /// Sends a 4-byte keepalive marker directly into the CBC stream.
+    /// This is not an RPC frame and must not consume sequence numbers.
+    pub(crate) async fn send_keepalive(&mut self) -> Result<()> {
+        let mut buf = [0u8; 16];
+        for i in 0..4 {
+            let start = i * 4;
+            let end = start + 4;
+            buf[start..end].copy_from_slice(&PADDING_FILLER);
+        }
+
+        let cipher = AesCbc::new(self.key, self.iv);
+        let mut v = buf.to_vec();
+        cipher
+            .encrypt_in_place(&mut v)
+            .map_err(|e| ProxyError::Crypto(format!("{e}")))?;
+
+        if v.len() >= 16 {
+            self.iv.copy_from_slice(&v[v.len() - 16..]);
+        }
+        self.writer.write_all(&v).await.map_err(ProxyError::Io)?;
+        self.writer.flush().await.map_err(ProxyError::Io)
     }
 }

src/transport/middle_proxy/pool.rs

@@ -23,7 +23,6 @@ use super::reader::reader_loop;
 use super::MeResponse;
 const ME_ACTIVE_PING_SECS: u64 = 25;
 const ME_ACTIVE_PING_JITTER_SECS: i64 = 5;
-const ME_KEEPALIVE_PAYLOAD_LEN: usize = 4;
 
 #[derive(Clone)]
 pub struct MeWriter {
@@ -361,7 +360,6 @@ impl MePool {
 
         // Additional connections up to pool_size total (round-robin across DCs), staggered to de-phase lifecycles.
         if self.me_warmup_stagger_enabled {
-            let mut delay_ms = 0u64;
             for (dc, addrs) in dc_addrs.iter() {
                 for (ip, port) in addrs {
                     if self.connection_count() >= pool_size {
@@ -369,7 +367,7 @@ impl MePool {
                     }
                     let addr = SocketAddr::new(*ip, *port);
                     let jitter = rand::rng().random_range(0..=self.me_warmup_step_jitter.as_millis() as u64);
-                    delay_ms = delay_ms.saturating_add(self.me_warmup_step_delay.as_millis() as u64 + jitter);
+                    let delay_ms = self.me_warmup_step_delay.as_millis() as u64 + jitter;
                     tokio::time::sleep(Duration::from_millis(delay_ms)).await;
                     if let Err(e) = self.connect_one(addr, rng.as_ref()).await {
                         debug!(%addr, dc = %dc, error = %e, "Extra ME connect failed (staggered)");
@@ -419,7 +417,6 @@ impl MePool {
         let draining = Arc::new(AtomicBool::new(false));
         let (tx, mut rx) = mpsc::channel::<WriterCommand>(4096);
         let tx_for_keepalive = tx.clone();
-        let keepalive_random = self.me_keepalive_payload_random;
         let stats = self.stats.clone();
         let mut rpc_writer = RpcWriter {
             writer: hs.wr,
@@ -440,11 +437,7 @@ impl MePool {
                                 if rpc_writer.send_and_flush(&payload).await.is_err() { break; }
                             }
                             Some(WriterCommand::Keepalive) => {
-                                let mut payload = [0u8; ME_KEEPALIVE_PAYLOAD_LEN];
-                                if keepalive_random {
-                                    rand::rng().fill(&mut payload);
-                                }
-                                match rpc_writer.send_keepalive(payload).await {
+                                match rpc_writer.send_keepalive().await {
                                     Ok(()) => {
                                         stats.increment_me_keepalive_sent();
                                     }

src/transport/middle_proxy/reader.rs

@@ -33,7 +33,6 @@ pub(crate) async fn reader_loop(
 ) -> Result<()> {
     let mut raw = enc_leftover;
     let mut expected_seq: i32 = 0;
-    let mut crc_errors = 0u32;
     let mut seq_mismatch = 0u32;
 
     loop {
@@ -80,13 +79,15 @@ pub(crate) async fn reader_loop(
             let frame = dec.split_to(fl);
             let pe = fl - 4;
             let ec = u32::from_le_bytes(frame[pe..pe + 4].try_into().unwrap());
-            if crc32(&frame[..pe]) != ec {
-                warn!("CRC mismatch in data frame");
-                crc_errors += 1;
-                if crc_errors > 3 {
-                    return Err(ProxyError::Proxy("Too many CRC mismatches".into()));
-                }
-                continue;
+            let actual_crc = crc32(&frame[..pe]);
+            if actual_crc != ec {
+                warn!(
+                    frame_len = fl,
+                    expected_crc = format_args!("0x{ec:08x}"),
+                    actual_crc = format_args!("0x{actual_crc:08x}"),
+                    "CRC mismatch — CBC crypto desync, aborting ME connection"
+                );
+                return Err(ProxyError::Proxy("CRC mismatch (crypto desync)".into()));
             }
 
             let seq_no = i32::from_le_bytes(frame[4..8].try_into().unwrap());