WEB: Lifecycle + Lane ownership + Diag fixes

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/web/manager/budget.rs

@@ -58,6 +58,21 @@ pub(crate) struct WebDataBudgetSnapshot {
     pub(crate) high_water_bytes: usize,
 }
 
+/// Bounded owner-usage view captured before WebSocket registry selection.
+pub(super) struct WebSocketFairnessSnapshot {
+    /// Equal byte share at the admission watermark for captured owners.
+    pub(super) fair_share: usize,
+    /// Captured shared-budget use indexed by profile owner.
+    pub(super) owner_bytes: HashMap<ProfileKey, usize>,
+}
+
+impl WebSocketFairnessSnapshot {
+    /// Returns the captured byte usage for one quota owner.
+    pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
+        self.owner_bytes.get(&owner).copied().unwrap_or(0)
+    }
+}
+
 impl WebDataBudget {
     pub(super) fn new(limits: WebLimitsConfig) -> Arc<Self> {
         Arc::new(Self {
@@ -220,16 +235,10 @@ impl WebDataBudget {
         self.pressured.store(true, Ordering::Release);
     }
 
-    pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
-        self.state
-            .lock()
-            .owner_bytes
-            .get(&owner)
-            .copied()
-            .unwrap_or(0)
-    }
-
-    pub(super) fn fair_share(&self, additional_owner: Option<ProfileKey>) -> usize {
+    pub(super) fn fairness_snapshot(
+        &self,
+        additional_owner: Option<ProfileKey>,
+    ) -> WebSocketFairnessSnapshot {
         let state = self.state.lock();
         let mut owners = state.owner_bytes.len();
         if additional_owner.is_some_and(|owner| !state.owner_bytes.contains_key(&owner)) {
@@ -239,7 +248,10 @@ impl WebDataBudget {
             self.limits.websocket_bytes_global,
             self.limits.websocket_admission_watermark_pct,
         );
-        admission / owners.max(1)
+        WebSocketFairnessSnapshot {
+            fair_share: admission / owners.max(1),
+            owner_bytes: state.owner_bytes.clone(),
+        }
     }
 
     pub(super) fn snapshot(&self) -> WebDataBudgetSnapshot {

src/web/manager/lifecycle.rs

@@ -2,13 +2,30 @@ use std::net::IpAddr;
 use std::sync::atomic::Ordering;
 use std::time::{Duration, Instant};
 
-use tracing::info;
+use tokio::time::Instant as TokioInstant;
+use tracing::{info, warn};
 
 use super::state::{
     decrement_map, remember_closed_token_locked, remove_bootstrap_locked, remove_expired_locked,
 };
 use super::{ProfileKey, TokenHash, WebProcessRuntime};
 
+/// Result of draining all process-owned WEB work under one absolute deadline.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum WebShutdownOutcome {
+    /// Every registered session and auxiliary task completed.
+    Drained,
+    /// Cancellation was asserted, but registered work remained at the deadline.
+    DeadlineExceeded,
+}
+
+/// Owned shutdown snapshot retained after sessions leave the live registry.
+pub(crate) struct WebShutdownDrain {
+    runtime: std::sync::Arc<WebProcessRuntime>,
+    sessions: Vec<std::sync::Arc<crate::web::session::WebSession>>,
+    started: TokioInstant,
+}
+
 impl WebProcessRuntime {
     /// Removes one closed session and retains a bounded host-bound replay marker.
     pub(crate) fn session_finished(
@@ -49,11 +66,20 @@ impl WebProcessRuntime {
         self.sessions_closed.fetch_add(1, Ordering::Relaxed);
     }
 
-    /// Stops issuance, closes all sessions, and joins bounded child work.
-    pub(crate) async fn shutdown(&self) {
+    /// Closes every WEB authority gate before any graceful wait begins.
+    pub(crate) fn begin_shutdown(self: &std::sync::Arc<Self>) -> WebShutdownDrain {
+        let started = TokioInstant::now();
         self.shutdown.cancel();
         self.close_websockets();
         self.data_budget.close();
+        self.http_connections.close();
+        self.http_handlers.close();
+        self.lane_polls.close();
+        self.lane_aux_polls.close();
+        self.body_readers.close();
+        self.body_bytes.close();
+        self.stream_handshakes.close();
+        self.websocket_connections.close();
         let sessions = {
             let mut state = self.state.lock();
             state.closed = true;
@@ -65,6 +91,24 @@ impl WebProcessRuntime {
         for session in &sessions {
             session.close();
         }
+        self.tasks.close();
+        WebShutdownDrain {
+            runtime: std::sync::Arc::clone(self),
+            sessions,
+            started,
+        }
+    }
+
+    /// Stops issuance and drains all WEB work until one absolute deadline.
+    pub(crate) async fn shutdown_until(
+        self: &std::sync::Arc<Self>,
+        deadline: TokioInstant,
+    ) -> WebShutdownOutcome {
+        self.begin_shutdown().wait_until(deadline).await
+    }
+
+    /// Stops issuance and drains WEB work under the currently configured budget.
+    pub(crate) async fn shutdown(self: &std::sync::Arc<Self>) -> WebShutdownOutcome {
         let timeout_secs = self
             .active_runtime
             .load()
@@ -72,34 +116,11 @@ impl WebProcessRuntime {
             .web
             .timeouts
             .shutdown_secs;
-        let waits = async {
-            for session in sessions {
-                session.wait().await;
-            }
-        };
-        let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), waits).await;
-        self.tasks.close();
-        let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), self.tasks.wait()).await;
-        let sessions_live = self.state.lock().sessions.len();
-        let streams_live = self.stream_admission.lock().streams_live;
-        let budget = self.data_budget.snapshot();
-        info!(
-            target: "telemt::web",
-            sessions_created = self.sessions_created.load(Ordering::Relaxed),
-            sessions_closed = self.sessions_closed.load(Ordering::Relaxed),
-            sessions_live,
-            streams_opened = self.streams_opened.load(Ordering::Relaxed),
-            streams_rejected = self.streams_rejected.load(Ordering::Relaxed),
-            streams_live,
-            pending_bytes = budget.queue_bytes,
-            pending_items = budget.queue_items,
-            websocket_bytes = budget.websocket_bytes,
-            data_high_water_bytes = budget.high_water_bytes,
-            bytes_up = self.bytes_up.load(Ordering::Relaxed),
-            bytes_down = self.bytes_down.load(Ordering::Relaxed),
-            limit_hits = self.limit_hits.load(Ordering::Relaxed),
-            "WEB runtime stopped"
-        );
+        let now = TokioInstant::now();
+        let deadline = now
+            .checked_add(Duration::from_secs(timeout_secs))
+            .unwrap_or(now);
+        self.shutdown_until(deadline).await
     }
 
     /// Expires credentials and closes idle sessions without holding locks across callbacks.
@@ -156,3 +177,228 @@ impl WebProcessRuntime {
         }
     }
 }
+
+impl WebShutdownDrain {
+    /// Waits for frozen session ownership and process auxiliary tasks concurrently.
+    pub(crate) async fn wait_until(self, deadline: TokioInstant) -> WebShutdownOutcome {
+        let sessions = &self.sessions;
+        let session_waits = async {
+            for session in sessions {
+                session.wait().await;
+            }
+        };
+        let outcome = wait_for_drain(deadline, session_waits, self.runtime.tasks.wait()).await;
+        self.log_outcome(outcome, deadline);
+        outcome
+    }
+
+    fn log_outcome(&self, outcome: WebShutdownOutcome, deadline: TokioInstant) {
+        let sessions_live = self.runtime.state.lock().sessions.len();
+        let streams_live = self.runtime.stream_admission.lock().streams_live;
+        let session_tasks_live = self.sessions.iter().fold(0usize, |total, session| {
+            total.saturating_add(session.tasks_live())
+        });
+        let sessions_pending = self
+            .sessions
+            .iter()
+            .filter(|session| session.tasks_live() != 0)
+            .count();
+        let auxiliary_tasks_live = self.runtime.tasks.len();
+        let budget = self.runtime.data_budget.snapshot();
+        let budget_ms = deadline
+            .saturating_duration_since(self.started)
+            .as_millis()
+            .min(u128::from(u64::MAX)) as u64;
+        let elapsed_ms = TokioInstant::now()
+            .saturating_duration_since(self.started)
+            .as_millis()
+            .min(u128::from(u64::MAX)) as u64;
+        match outcome {
+            WebShutdownOutcome::Drained => info!(
+                target: "telemt::web",
+                shutdown_drained = true,
+                shutdown_budget_ms = budget_ms,
+                shutdown_elapsed_ms = elapsed_ms,
+                sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
+                sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
+                sessions_live,
+                sessions_pending,
+                session_tasks_live,
+                auxiliary_tasks_live,
+                streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
+                streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
+                streams_live,
+                pending_bytes = budget.queue_bytes,
+                pending_items = budget.queue_items,
+                websocket_bytes = budget.websocket_bytes,
+                data_high_water_bytes = budget.high_water_bytes,
+                bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
+                bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
+                limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
+                "WEB runtime stopped"
+            ),
+            WebShutdownOutcome::DeadlineExceeded => warn!(
+                target: "telemt::web",
+                shutdown_drained = false,
+                shutdown_budget_ms = budget_ms,
+                shutdown_elapsed_ms = elapsed_ms,
+                sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
+                sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
+                sessions_live,
+                sessions_pending,
+                session_tasks_live,
+                auxiliary_tasks_live,
+                streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
+                streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
+                streams_live,
+                pending_bytes = budget.queue_bytes,
+                pending_items = budget.queue_items,
+                websocket_bytes = budget.websocket_bytes,
+                data_high_water_bytes = budget.high_water_bytes,
+                bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
+                bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
+                limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
+                "WEB runtime shutdown deadline exceeded"
+            ),
+        }
+    }
+}
+
+async fn wait_for_drain<S, T>(deadline: TokioInstant, sessions: S, tasks: T) -> WebShutdownOutcome
+where
+    S: std::future::Future<Output = ()>,
+    T: std::future::Future<Output = ()>,
+{
+    let waits = async {
+        tokio::join!(sessions, tasks);
+    };
+    if tokio::time::timeout_at(deadline, waits).await.is_ok() {
+        WebShutdownOutcome::Drained
+    } else {
+        WebShutdownOutcome::DeadlineExceeded
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::future::Future;
+    use std::pin::Pin;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::task::{Context, Poll};
+
+    use arc_swap::ArcSwap;
+    use tokio::sync::Notify;
+
+    use super::*;
+    use crate::config::ProxyConfig;
+    use crate::maestro::generation::test_runtime_generation;
+
+    struct DropProbe {
+        polls: Arc<AtomicUsize>,
+        drops: Arc<AtomicUsize>,
+    }
+
+    impl Future for DropProbe {
+        type Output = ();
+
+        fn poll(self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll<Self::Output> {
+            self.polls.fetch_add(1, Ordering::AcqRel);
+            Poll::Pending
+        }
+    }
+
+    impl Drop for DropProbe {
+        fn drop(&mut self) {
+            self.drops.fetch_add(1, Ordering::AcqRel);
+        }
+    }
+
+    fn runtime() -> (
+        Arc<WebProcessRuntime>,
+        Arc<crate::maestro::generation::RuntimeGeneration>,
+    ) {
+        let generation = test_runtime_generation(1, ProxyConfig::default());
+        let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
+        (runtime, generation)
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn drain_uses_one_absolute_deadline_for_both_wait_groups() {
+        let started = TokioInstant::now();
+        let outcome = wait_for_drain(
+            started + Duration::from_secs(5),
+            tokio::time::sleep(Duration::from_secs(4)),
+            tokio::time::sleep(Duration::from_secs(9)),
+        )
+        .await;
+
+        assert_eq!(outcome, WebShutdownOutcome::DeadlineExceeded);
+        assert_eq!(TokioInstant::now() - started, Duration::from_secs(5));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn drain_returns_when_both_wait_groups_finish() {
+        let started = TokioInstant::now();
+        let outcome = wait_for_drain(
+            started + Duration::from_secs(5),
+            tokio::time::sleep(Duration::from_secs(3)),
+            tokio::time::sleep(Duration::from_secs(2)),
+        )
+        .await;
+
+        assert_eq!(outcome, WebShutdownOutcome::Drained);
+        assert_eq!(TokioInstant::now() - started, Duration::from_secs(3));
+    }
+
+    #[tokio::test]
+    async fn post_shutdown_auxiliary_is_dropped_without_polling() {
+        let (runtime, generation) = runtime();
+        let polls = Arc::new(AtomicUsize::new(0));
+        let drops = Arc::new(AtomicUsize::new(0));
+        let drain = runtime.begin_shutdown();
+
+        runtime.spawn_auxiliary(DropProbe {
+            polls: Arc::clone(&polls),
+            drops: Arc::clone(&drops),
+        });
+
+        assert_eq!(polls.load(Ordering::Acquire), 0);
+        assert_eq!(drops.load(Ordering::Acquire), 1);
+        assert_eq!(
+            drain
+                .wait_until(TokioInstant::now() + Duration::from_secs(1))
+                .await,
+            WebShutdownOutcome::Drained
+        );
+        generation.stop_sessions().await;
+        generation.stop_background_tasks().await;
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn expired_deadline_still_closes_every_runtime_gate() {
+        let (runtime, generation) = runtime();
+        let existing = runtime.try_http_connection().unwrap();
+        let release = Arc::new(Notify::new());
+        let release_task = Arc::clone(&release);
+        runtime.tasks.spawn(async move {
+            release_task.notified().await;
+        });
+        tokio::task::yield_now().await;
+        let drain = runtime.begin_shutdown();
+
+        assert!(runtime.try_http_connection().is_none());
+        assert!(runtime.try_http_handler().is_none());
+        assert!(runtime.try_lane_poll(false).is_none());
+        assert_eq!(
+            drain.wait_until(TokioInstant::now()).await,
+            WebShutdownOutcome::DeadlineExceeded
+        );
+
+        drop(existing);
+        release.notify_waiters();
+        runtime.tasks.wait().await;
+        generation.stop_sessions().await;
+        generation.stop_background_tasks().await;
+    }
+}

src/web/manager/websocket.rs

@@ -9,6 +9,9 @@ use tokio_util::sync::CancellationToken;
 
 use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
 
+// Deterministic victim ordering remains isolated from registry mutation.
+mod policy;
+
 /// One process-owned WebSocket carrier class used for eviction priority.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub(crate) enum WebSocketKind {
@@ -25,6 +28,7 @@ struct WebSocketClaimKey {
 }
 
 #[repr(u8)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum WebSocketPhase {
     Claimed,
     Upgraded,
@@ -239,6 +243,8 @@ enum TryAdmitError {
     Closed,
 }
 
+// Admission inputs stay explicit so quota and cancellation ownership cannot drift.
+#[allow(clippy::too_many_arguments)]
 fn try_admit(
     runtime: &Arc<WebProcessRuntime>,
     owner: ProfileKey,
@@ -353,8 +359,7 @@ fn select_victim(
     excluded_id: Option<u64>,
     claim: bool,
 ) -> Option<Arc<WebSocketEntry>> {
-    let fair_share = runtime.data_budget.fair_share(Some(owner));
-    let requester_usage = runtime.data_budget.owner_usage(owner);
+    let fairness = runtime.data_budget.fairness_snapshot(Some(owner));
     let now = runtime.websocket_tick();
     let mut registry = runtime.websockets.lock();
     if claim && registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight {
@@ -366,31 +371,8 @@ fn select_victim(
         .filter(|entry| Some(entry.id) != excluded_id)
         .filter(|entry| !entry.closing.load(Ordering::Acquire))
         .filter_map(|entry| {
-            let owner_rank = if entry.session_id == session_id {
-                0
-            } else if entry.owner == owner {
-                1
-            } else if entry.client_ip == client_ip {
-                2
-            } else {
-                if requester_usage >= fair_share
-                    || runtime.data_budget.owner_usage(entry.owner) <= fair_share
-                {
-                    return None;
-                }
-                3
-            };
-            let priority = entry_priority(entry, now);
-            Some((
-                (
-                    owner_rank,
-                    priority,
-                    entry.last_progress_tick.load(Ordering::Acquire),
-                    entry.created_tick,
-                    entry.id,
-                ),
-                Arc::clone(entry),
-            ))
+            policy::admission_key(entry, now, owner, session_id, client_ip, &fairness)
+                .map(|key| (key, Arc::clone(entry)))
         })
         .min_by_key(|(key, _)| *key)
         .map(|(_, entry)| entry)?;
@@ -405,6 +387,7 @@ fn select_pressure_victim(
     now: u64,
     claim: bool,
 ) -> Option<Arc<WebSocketEntry>> {
+    let fairness = runtime.data_budget.fairness_snapshot(None);
     let mut registry = runtime.websockets.lock();
     if claim && registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight {
         return None;
@@ -415,12 +398,7 @@ fn select_pressure_victim(
         .filter(|entry| !entry.closing.load(Ordering::Acquire))
         .map(|entry| {
             (
-                (
-                    entry_priority(entry, now),
-                    entry.last_progress_tick.load(Ordering::Acquire),
-                    entry.created_tick,
-                    entry.id,
-                ),
+                policy::pressure_key(entry, now, &fairness),
                 Arc::clone(entry),
             )
         })
@@ -438,18 +416,23 @@ fn claim_stale_victims(runtime: &WebProcessRuntime, now: u64) -> Vec<Arc<WebSock
         .limits
         .max_websocket_evictions_in_flight
         .saturating_sub(registry.evictions_in_flight);
-    let candidates = registry
+    let mut candidates = registry
         .entries
         .values()
         .filter(|entry| !entry.closing.load(Ordering::Acquire))
-        .filter(|entry| {
-            now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
-        })
-        .take(available)
+        .filter(|entry| policy::victim_class(entry, now) == policy::VictimClass::Dead)
         .cloned()
         .collect::<Vec<_>>();
+    candidates.sort_unstable_by_key(|entry| {
+        (
+            entry.last_peer_tick.load(Ordering::Acquire),
+            entry.created_tick,
+            entry.id,
+        )
+    });
     candidates
         .into_iter()
+        .take(available)
         .filter(|entry| claim_entry(&mut registry, entry, runtime))
         .collect()
 }
@@ -474,18 +457,6 @@ fn claim_entry(
     true
 }
 
-fn entry_priority(entry: &WebSocketEntry, now: u64) -> u8 {
-    if entry.phase.load(Ordering::Acquire) < WebSocketPhase::Active as u8
-        || now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
-    {
-        0
-    } else if matches!(entry.kind, WebSocketKind::Lane(_)) {
-        1
-    } else {
-        2
-    }
-}
-
 fn dead_after(entry: &WebSocketEntry) -> u64 {
     entry.liveness_interval_ms.saturating_mul(2)
 }

src/web/manager/websocket/policy.rs

@@ -0,0 +1,113 @@
+use std::net::IpAddr;
+use std::sync::atomic::Ordering;
+
+use super::{WebSocketEntry, WebSocketKind, WebSocketPhase, dead_after};
+use crate::web::manager::ProfileKey;
+use crate::web::manager::budget::WebSocketFairnessSnapshot;
+
+/// Stable total-order key used by bounded victim selection.
+pub(super) type VictimKey = (u8, u8, u8, u64, u64, u64);
+
+/// Lifecycle class used before locality and least-recent-progress ordering.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum VictimClass {
+    /// Active connection whose peer-liveness deadline elapsed.
+    Dead,
+    /// Claimed or upgraded connection still bounded by its startup deadlines.
+    PreActive,
+    /// Active per-stream lane connection.
+    LiveLane,
+    /// Active multiplexed session connection.
+    LiveMultiplex,
+}
+
+impl VictimClass {
+    fn rank(self) -> u8 {
+        match self {
+            Self::Dead | Self::PreActive => 0,
+            Self::LiveLane => 1,
+            Self::LiveMultiplex => 2,
+        }
+    }
+}
+
+/// Classifies one non-closing connection without conflating startup with death.
+pub(super) fn victim_class(entry: &WebSocketEntry, now: u64) -> VictimClass {
+    let phase = entry.phase.load(Ordering::Acquire);
+    if phase == WebSocketPhase::Active as u8
+        && now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
+    {
+        VictimClass::Dead
+    } else if phase < WebSocketPhase::Active as u8 {
+        VictimClass::PreActive
+    } else if matches!(entry.kind, WebSocketKind::Lane(_)) {
+        VictimClass::LiveLane
+    } else {
+        VictimClass::LiveMultiplex
+    }
+}
+
+/// Returns one eligible admission-victim key with global dead-first ordering.
+pub(super) fn admission_key(
+    entry: &WebSocketEntry,
+    now: u64,
+    requester_owner: ProfileKey,
+    requester_session: u64,
+    requester_ip: IpAddr,
+    fairness: &WebSocketFairnessSnapshot,
+) -> Option<VictimKey> {
+    let class = victim_class(entry, now);
+    if class == VictimClass::Dead {
+        return Some((
+            0,
+            0,
+            0,
+            entry.last_peer_tick.load(Ordering::Acquire),
+            entry.created_tick,
+            entry.id,
+        ));
+    }
+    let locality = if entry.session_id == requester_session {
+        0
+    } else if entry.owner == requester_owner {
+        1
+    } else if entry.client_ip == requester_ip {
+        2
+    } else if fairness.owner_usage(requester_owner) < fairness.fair_share
+        && fairness.owner_usage(entry.owner) > fairness.fair_share
+    {
+        3
+    } else {
+        return None;
+    };
+    Some((
+        1,
+        locality,
+        class.rank(),
+        entry.last_progress_tick.load(Ordering::Acquire),
+        entry.created_tick,
+        entry.id,
+    ))
+}
+
+/// Returns one pressure-victim key preferring dead and over-share owners.
+pub(super) fn pressure_key(
+    entry: &WebSocketEntry,
+    now: u64,
+    fairness: &WebSocketFairnessSnapshot,
+) -> VictimKey {
+    let class = victim_class(entry, now);
+    let dead = class == VictimClass::Dead;
+    (
+        u8::from(!dead),
+        if dead {
+            0
+        } else {
+            u8::from(fairness.owner_usage(entry.owner) <= fairness.fair_share)
+        },
+        class.rank(),
+        entry.last_progress_tick.load(Ordering::Acquire),
+        entry.created_tick,
+        entry.id,
+    )
+}

src/web/manager/websocket/tests.rs

@@ -1,25 +1,39 @@
 use super::*;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::AtomicUsize;
 
-fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
-    let phase = if opened {
-        WebSocketPhase::Active
-    } else {
-        WebSocketPhase::Claimed
-    };
+use super::policy::{VictimClass, admission_key, pressure_key, victim_class};
+use crate::config::ProxyConfig;
+use crate::maestro::generation::test_runtime_generation;
+use crate::web::manager::budget::WebSocketFairnessSnapshot;
+use arc_swap::ArcSwap;
+
+#[allow(clippy::too_many_arguments)]
+fn entry(
+    id: u64,
+    owner: ProfileKey,
+    session_id: u64,
+    client_ip: &str,
+    kind: WebSocketKind,
+    phase: WebSocketPhase,
+    peer_tick: u64,
+    progress_tick: u64,
+) -> WebSocketEntry {
     WebSocketEntry {
-        id: 1,
-        owner: [0; 32],
-        session_id: 1,
+        id,
+        owner,
+        session_id,
         claim: WebSocketClaimKey {
             session_hash: [0; 32],
             kind,
         },
-        client_ip: "192.0.2.10".parse().unwrap(),
+        client_ip: client_ip.parse().unwrap(),
         kind,
         liveness_interval_ms: 10,
         created_tick: 1,
         last_peer_tick: AtomicU64::new(peer_tick),
-        last_progress_tick: AtomicU64::new(peer_tick),
+        last_progress_tick: AtomicU64::new(progress_tick),
         phase: AtomicU8::new(phase as u8),
         closing: AtomicBool::new(false),
         cancel: CancellationToken::new(),
@@ -27,25 +41,259 @@ fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
     }
 }
 
-#[test]
-fn preopen_and_dead_entries_precede_live_lane_and_multiplex_victims() {
-    let preopen = entry(WebSocketKind::Multiplex, false, 90);
-    let dead = entry(WebSocketKind::Multiplex, true, 1);
-    let lane = entry(WebSocketKind::Lane(7), true, 90);
-    let multiplex = entry(WebSocketKind::Multiplex, true, 90);
-
-    assert_eq!(entry_priority(&preopen, 100), 0);
-    assert_eq!(entry_priority(&dead, 100), 0);
-    assert_eq!(entry_priority(&lane, 100), 1);
-    assert_eq!(entry_priority(&multiplex, 100), 2);
+fn fairness(fair_share: usize, usages: &[(ProfileKey, usize)]) -> WebSocketFairnessSnapshot {
+    WebSocketFairnessSnapshot {
+        fair_share,
+        owner_bytes: usages.iter().copied().collect::<HashMap<_, _>>(),
+    }
 }
 
 #[test]
-fn dead_classification_keeps_each_connections_creation_time_interval() {
-    let short_interval = entry(WebSocketKind::Multiplex, true, 80);
-    let mut long_interval = entry(WebSocketKind::Multiplex, true, 80);
-    long_interval.liveness_interval_ms = 100;
+fn preactive_and_dead_are_distinct_lifecycle_classes() {
+    let preactive = entry(
+        1,
+        [1; 32],
+        1,
+        "192.0.2.10",
+        WebSocketKind::Multiplex,
+        WebSocketPhase::Claimed,
+        1,
+        1,
+    );
+    let dead = entry(
+        2,
+        [1; 32],
+        1,
+        "192.0.2.10",
+        WebSocketKind::Multiplex,
+        WebSocketPhase::Active,
+        1,
+        1,
+    );
 
-    assert_eq!(entry_priority(&short_interval, 100), 0);
-    assert_eq!(entry_priority(&long_interval, 100), 2);
+    assert_eq!(victim_class(&preactive, 100), VictimClass::PreActive);
+    assert_eq!(victim_class(&dead, 100), VictimClass::Dead);
+}
+
+#[test]
+fn dead_other_session_precedes_healthy_same_session() {
+    let requester_owner = [1; 32];
+    let usage = fairness(100, &[(requester_owner, 100), ([2; 32], 100)]);
+    let dead = entry(
+        2,
+        [2; 32],
+        2,
+        "198.51.100.10",
+        WebSocketKind::Multiplex,
+        WebSocketPhase::Active,
+        1,
+        1,
+    );
+    let healthy = entry(
+        1,
+        requester_owner,
+        1,
+        "192.0.2.10",
+        WebSocketKind::Lane(7),
+        WebSocketPhase::Active,
+        99,
+        99,
+    );
+
+    let dead_key = admission_key(
+        &dead,
+        100,
+        requester_owner,
+        1,
+        "192.0.2.10".parse().unwrap(),
+        &usage,
+    )
+    .unwrap();
+    let healthy_key = admission_key(
+        &healthy,
+        100,
+        requester_owner,
+        1,
+        "192.0.2.10".parse().unwrap(),
+        &usage,
+    )
+    .unwrap();
+
+    assert!(dead_key < healthy_key);
+}
+
+#[test]
+fn unrelated_live_victim_requires_opposite_fair_share_positions() {
+    let requester_owner = [1; 32];
+    let victim_owner = [2; 32];
+    let candidate = entry(
+        1,
+        victim_owner,
+        2,
+        "198.51.100.10",
+        WebSocketKind::Lane(7),
+        WebSocketPhase::Active,
+        99,
+        99,
+    );
+    let requester_ip = "192.0.2.10".parse().unwrap();
+
+    assert!(
+        admission_key(
+            &candidate,
+            100,
+            requester_owner,
+            1,
+            requester_ip,
+            &fairness(100, &[(requester_owner, 99), (victim_owner, 101)]),
+        )
+        .is_some()
+    );
+    assert!(
+        admission_key(
+            &candidate,
+            100,
+            requester_owner,
+            1,
+            requester_ip,
+            &fairness(100, &[(requester_owner, 100), (victim_owner, 101)]),
+        )
+        .is_none()
+    );
+}
+
+#[test]
+fn pressure_prefers_over_share_owner_then_lifecycle_and_id() {
+    let over_owner = [1; 32];
+    let under_owner = [2; 32];
+    let usage = fairness(100, &[(over_owner, 101), (under_owner, 99)]);
+    let over = entry(
+        9,
+        over_owner,
+        1,
+        "192.0.2.10",
+        WebSocketKind::Multiplex,
+        WebSocketPhase::Active,
+        99,
+        99,
+    );
+    let under = entry(
+        1,
+        under_owner,
+        2,
+        "198.51.100.10",
+        WebSocketKind::Lane(7),
+        WebSocketPhase::Active,
+        90,
+        90,
+    );
+
+    assert!(pressure_key(&over, 100, &usage) < pressure_key(&under, 100, &usage));
+
+    let equal_usage = fairness(100, &[(over_owner, 100), (under_owner, 100)]);
+    let preactive = entry(
+        2,
+        under_owner,
+        2,
+        "198.51.100.10",
+        WebSocketKind::Multiplex,
+        WebSocketPhase::Upgraded,
+        99,
+        99,
+    );
+    assert!(pressure_key(&preactive, 100, &equal_usage) < pressure_key(&under, 100, &equal_usage));
+
+    let lower_id = entry(
+        1,
+        under_owner,
+        2,
+        "198.51.100.10",
+        WebSocketKind::Lane(7),
+        WebSocketPhase::Active,
+        90,
+        90,
+    );
+    let higher_id = entry(
+        2,
+        under_owner,
+        2,
+        "198.51.100.10",
+        WebSocketKind::Lane(8),
+        WebSocketPhase::Active,
+        90,
+        90,
+    );
+
+    assert!(
+        pressure_key(&lower_id, 100, &equal_usage) < pressure_key(&higher_id, 100, &equal_usage)
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn concurrent_victim_claims_stay_bounded_and_return_to_zero() {
+    let config = ProxyConfig::default();
+    let generation = test_runtime_generation(1, config);
+    let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
+    let limit = runtime.limits.max_websocket_evictions_in_flight;
+    let entries = (0..limit.saturating_mul(2))
+        .map(|index| {
+            Arc::new(entry(
+                index as u64 + 1,
+                [index as u8; 32],
+                index as u64 + 1,
+                "192.0.2.10",
+                WebSocketKind::Lane(index as u32 + 1),
+                WebSocketPhase::Active,
+                1,
+                1,
+            ))
+        })
+        .collect::<Vec<_>>();
+    let connections = entries
+        .iter()
+        .map(|entry| WebSocketConnection {
+            runtime: Arc::downgrade(&runtime),
+            entry: Arc::clone(entry),
+            slot: None,
+            base_budget: None,
+        })
+        .collect::<Vec<_>>();
+    {
+        let mut registry = runtime.websockets.lock();
+        for entry in &entries {
+            registry.claims.insert(entry.claim, entry.id);
+            registry.entries.insert(entry.id, Arc::clone(entry));
+        }
+    }
+    let successes = Arc::new(AtomicUsize::new(0));
+    let mut tasks = Vec::new();
+    for task_id in 0..100usize {
+        let runtime = Arc::clone(&runtime);
+        let entries = entries.clone();
+        let successes = Arc::clone(&successes);
+        tasks.push(tokio::spawn(async move {
+            for attempt in 0..100usize {
+                let entry = &entries[(task_id * 100 + attempt) % entries.len()];
+                {
+                    let mut registry = runtime.websockets.lock();
+                    if claim_entry(&mut registry, entry, &runtime) {
+                        successes.fetch_add(1, Ordering::AcqRel);
+                    }
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+    for task in tasks {
+        task.await.unwrap();
+    }
+
+    assert_eq!(successes.load(Ordering::Acquire), limit);
+    assert_eq!(runtime.websockets.lock().evictions_in_flight, limit);
+
+    drop(connections);
+    assert_eq!(runtime.websockets.lock().evictions_in_flight, 0);
+    runtime.shutdown().await;
+    generation.stop_sessions().await;
+    generation.stop_background_tasks().await;
 }