WEB Session Lifecycle Observability + Bridge Recovery Drafts

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/web/manager/carrier_outcome.rs

@@ -99,7 +99,7 @@ impl WebProcessRuntime {
         identity: TraceIdentity,
     ) -> bool {
         let mut state = self.state.lock();
-        let scores = state.bootstraps.get_mut(&bootstrap_hash).and_then(|entry| {
+        let outcome = state.bootstraps.get_mut(&bootstrap_hash).and_then(|entry| {
             if entry.carrier_attempt == attempt
                 && entry
                     .session
@@ -108,13 +108,20 @@ impl WebProcessRuntime {
                 && entry.carrier_phase == CarrierChainPhase::Provisional
             {
                 entry.carrier_phase = CarrierChainPhase::CommittedPendingHealth;
-                Some(entry.carrier_scores)
+                Some((entry.carrier_scores, entry.recovery))
             } else {
                 None
             }
         });
         drop(state);
-        let Some(scores) = scores else { return false };
+        let Some((scores, recovery)) = outcome else {
+            return false;
+        };
+        if recovery {
+            self.telemetry.record_bridge_recovery(
+                crate::web::telemetry::WebBridgeRecoveryEvent::Committed,
+            );
+        }
         self.trace.record_carrier_lifecycle(
             client_ip,
             identity.clone(),

src/web/manager/control.rs

@@ -7,6 +7,7 @@ use serde::Serialize;
 
 use super::status::immutable_matches;
 use super::{SessionFilter, WebProcessRuntime};
+use crate::web::session::SessionCloseReason;
 
 const OPERATION_REF_VERSION: &str = "wo1";
 const OPERATION_RETENTION: usize = 32;
@@ -297,7 +298,7 @@ impl WebProcessRuntime {
                 status.matched = status.matched.saturating_add(direct.len());
             });
             for candidate in direct {
-                candidate.session.close();
+                candidate.session.close(SessionCloseReason::ApiClose);
                 self.update_operation(sequence, |status| {
                     status.close_signalled = status.close_signalled.saturating_add(1)
                 });
@@ -325,7 +326,7 @@ impl WebProcessRuntime {
                         current
                     };
                     if let Some(session) = session {
-                        session.close();
+                        session.close(SessionCloseReason::ApiClose);
                         self.update_operation(sequence, |status| {
                             status.matched = status.matched.saturating_add(1);
                             status.close_signalled = status.close_signalled.saturating_add(1);

src/web/manager/credentials.rs

@@ -7,13 +7,13 @@ use zeroize::Zeroizing;
 
 use super::state::{
     Bootstrap, CarrierChainPhase, allow_rate, evict_oldest_unused_bootstrap, matching_profile,
-    new_unique_token, remove_expired_locked,
+    new_unique_token, profile_key, remove_expired_locked,
 };
 use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
 use crate::config::WebRuntimeProfile;
 use crate::maestro::generation::RuntimeGeneration;
-use crate::web::session::WebSession;
-use crate::web::telemetry::WebRejectionReason;
+use crate::web::session::{SessionCloseReason, WebSession};
+use crate::web::telemetry::{WebBridgeRecoveryEvent, WebRejectionReason};
 
 impl WebProcessRuntime {
     /// Issues a one-use bootstrap credential for an active compatible profile.
@@ -24,7 +24,7 @@ impl WebProcessRuntime {
         client_ip: IpAddr,
     ) -> std::result::Result<BootstrapResult, ManagerError> {
         let generation = self.active_generation();
-        self.issue_bootstrap_inner(&generation, profile, client_ip, None)
+        self.issue_bootstrap_inner(&generation, profile, client_ip, None, false)
     }
 
     /// Issues one bootstrap against the generation that selected the bridge profile.
@@ -35,7 +35,7 @@ impl WebProcessRuntime {
         profile: Arc<WebRuntimeProfile>,
         client_ip: IpAddr,
     ) -> std::result::Result<BootstrapResult, ManagerError> {
-        self.issue_bootstrap_inner(generation, profile, client_ip, None)
+        self.issue_bootstrap_inner(generation, profile, client_ip, None, false)
     }
 
     /// Issues one bridge bootstrap with bounded non-secret request metadata.
@@ -46,7 +46,18 @@ impl WebProcessRuntime {
         client_ip: IpAddr,
         user_agent: Option<&str>,
     ) -> std::result::Result<BootstrapResult, ManagerError> {
-        self.issue_bootstrap_inner(generation, profile, client_ip, user_agent)
+        self.issue_bootstrap_inner(generation, profile, client_ip, user_agent, false)
+    }
+
+    /// Issues one recovery bootstrap with the same positive-only admission boundary.
+    pub(crate) fn issue_recovery_bootstrap_for_request(
+        &self,
+        generation: &Arc<RuntimeGeneration>,
+        profile: Arc<WebRuntimeProfile>,
+        client_ip: IpAddr,
+        user_agent: Option<&str>,
+    ) -> std::result::Result<BootstrapResult, ManagerError> {
+        self.issue_bootstrap_inner(generation, profile, client_ip, user_agent, true)
     }
 
     fn issue_bootstrap_inner(
@@ -55,6 +66,7 @@ impl WebProcessRuntime {
         profile: Arc<WebRuntimeProfile>,
         client_ip: IpAddr,
         user_agent: Option<&str>,
+        recovery: bool,
     ) -> std::result::Result<BootstrapResult, ManagerError> {
         let config = generation.config();
         let profile = config
@@ -76,7 +88,11 @@ impl WebProcessRuntime {
         let _operator_admission = self.try_operator_admission()?;
         let now = Instant::now();
         let mut state = self.state.lock();
-        remove_expired_locked(&mut state, now);
+        let expired_recoveries = remove_expired_locked(&mut state, now);
+        self.telemetry.record_bridge_recovery_count(
+            WebBridgeRecoveryEvent::ExpiredUnused,
+            expired_recoveries,
+        );
         state.apply_issuance_policy(generation.id, config.web.enabled);
         if state.closed || !state.issuance_enabled {
             self.record_limit_hit();
@@ -153,6 +169,7 @@ impl WebProcessRuntime {
                 session_client_ip: None,
                 session_ip_learning_eligible: false,
                 used: false,
+                recovery,
             },
         );
         *state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
@@ -162,6 +179,10 @@ impl WebProcessRuntime {
             .map(|entry| Arc::clone(&entry.profile))
             .ok_or(ManagerError::Closed)?;
         drop(state);
+        if recovery {
+            self.telemetry
+                .record_bridge_recovery(WebBridgeRecoveryEvent::BootstrapIssued);
+        }
         self.trace.record_profile_lifecycle(
             client_ip,
             Some(trace_session_id),
@@ -215,6 +236,24 @@ impl WebProcessRuntime {
             .ok_or(ManagerError::Authentication)
     }
 
+    /// Resolves a current bearer only when it belongs to the recovering profile.
+    pub(crate) fn bridge_recovery_session(
+        &self,
+        hash: TokenHash,
+        host: &str,
+        profile: &WebRuntimeProfile,
+    ) -> Option<Arc<WebSession>> {
+        let expected_profile = profile_key(profile);
+        self.state
+            .lock()
+            .sessions
+            .get(&hash)
+            .filter(|session| {
+                session.matches_host(host) && session.profile_key() == expected_profile
+            })
+            .cloned()
+    }
+
     /// Closes a live token and accepts bounded tombstone retries.
     pub(crate) fn close_token(
         &self,
@@ -256,7 +295,7 @@ impl WebProcessRuntime {
             .is_some_and(|closed| closed.host == host);
         drop(state);
         if let Some(session) = session {
-            if session.close()
+            if session.close(SessionCloseReason::ClientDelete).accepted()
                 && let (Some(failure), Some(phase)) = (failure, failure_phase)
             {
                 self.telemetry

src/web/manager/lifecycle.rs

@@ -11,6 +11,7 @@ use super::state::{
 };
 use super::{ProfileKey, TokenHash, WebProcessRuntime};
 use crate::maestro::generation::RuntimeGeneration;
+use crate::web::session::SessionCloseReason;
 
 /// Result of draining all process-owned WEB work under one absolute deadline.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -61,17 +62,29 @@ impl WebProcessRuntime {
         profile_key: ProfileKey,
         profile_host: &str,
         closed_token_lifetime: Duration,
+        reason: SessionCloseReason,
     ) {
         let mut state = self.state.lock();
         let Some(session) = state.sessions.remove(&hash) else {
             return;
         };
+        let recovery_closed_before_commit = state.bootstraps.values().any(|bootstrap| {
+            bootstrap.recovery
+                && bootstrap
+                    .session
+                    .as_ref()
+                    .is_some_and(|current| current.token_hash() == hash)
+                && !session.is_carrier_committed()
+        });
         decrement_map(&mut state.sessions_per_ip, &client_ip);
         decrement_map(&mut state.sessions_per_profile, &profile_key);
         remember_closed_token_locked(
             &mut state,
             hash,
             profile_host,
+            session.trace_session_id(),
+            session.carrier(),
+            reason,
             closed_token_lifetime,
             self.limits.max_sessions_global.saturating_mul(16),
         );
@@ -86,6 +99,8 @@ impl WebProcessRuntime {
                 &mut state,
                 trace_session_id,
                 session.carrier_attempt(),
+                session.carrier(),
+                reason,
                 closed_token_lifetime,
                 self.limits.max_sessions_global,
             );
@@ -104,7 +119,13 @@ impl WebProcessRuntime {
         for bootstrap_hash in bootstrap_hashes {
             remove_bootstrap_locked(&mut state, bootstrap_hash);
         }
-        self.telemetry.record_session_closed();
+        self.telemetry
+            .record_session_closed(session.carrier(), reason);
+        if recovery_closed_before_commit {
+            self.telemetry.record_bridge_recovery(
+                crate::web::telemetry::WebBridgeRecoveryEvent::ClosedBeforeCommit,
+            );
+        }
         drop(state);
         self.notify_operator_work_changed();
     }
@@ -135,7 +156,7 @@ impl WebProcessRuntime {
         };
         self.stream_admission.lock().closed = true;
         for session in &sessions {
-            session.close();
+            session.close(SessionCloseReason::RuntimeShutdown);
         }
         self.tasks.close();
         WebShutdownDrain {
@@ -215,14 +236,18 @@ impl WebProcessRuntime {
             for (hash, _) in expired {
                 remove_bootstrap_locked(&mut state, hash);
             }
-            remove_expired_locked(&mut state, now);
+            let expired_recoveries = remove_expired_locked(&mut state, now);
+            self.telemetry.record_bridge_recovery_count(
+                crate::web::telemetry::WebBridgeRecoveryEvent::ExpiredUnused,
+                expired_recoveries,
+            );
             (
                 state.sessions.values().cloned().collect::<Vec<_>>(),
                 expired_chains,
             )
         };
         for session in expired_chains {
-            session.close();
+            session.close(SessionCloseReason::NegotiationTimeout);
         }
         for session in sessions {
             session.close_if_due(now);

src/web/manager/operator_lifecycle.rs

@@ -9,6 +9,7 @@ use tokio::time::Instant as TokioInstant;
 use tokio_util::sync::CancellationToken;
 
 use super::WebProcessRuntime;
+use crate::web::session::SessionCloseReason;
 
 // Serialized state-machine values stay separate from synchronization mechanics.
 mod status;
@@ -447,7 +448,7 @@ impl WebProcessRuntime {
                     }
                     forced = true;
                     for session in sessions {
-                        session.close();
+                        session.close(SessionCloseReason::OperatorForce);
                     }
                 }
             }

src/web/manager/session_creation.rs

@@ -18,7 +18,7 @@ use super::{
 };
 use crate::config::{WebCarrier, WebRuntimeProfile};
 use crate::web::frame;
-use crate::web::session::WebSession;
+use crate::web::session::{SessionCloseReason, WebSession};
 use crate::web::telemetry::WebCarrierSelectionDisposition;
 use crate::web::trace::TraceLifecycleEvent;
 
@@ -56,7 +56,11 @@ impl WebProcessRuntime {
         let config = generation.config();
         let now = Instant::now();
         let mut state = self.state.lock();
-        remove_expired_locked(&mut state, now);
+        let expired_recoveries = remove_expired_locked(&mut state, now);
+        self.telemetry.record_bridge_recovery_count(
+            crate::web::telemetry::WebBridgeRecoveryEvent::ExpiredUnused,
+            expired_recoveries,
+        );
         state.apply_issuance_policy(generation.id, config.web.enabled);
         if state.closed || !state.issuance_enabled {
             return Err(ManagerError::Closed);
@@ -79,7 +83,7 @@ impl WebProcessRuntime {
                 let session = entry.session.clone();
                 drop(state);
                 if let Some(session) = session {
-                    session.close();
+                    session.close(SessionCloseReason::NegotiationTimeout);
                 }
                 return Err(ManagerError::Closed);
             }
@@ -335,7 +339,7 @@ impl WebProcessRuntime {
         state.sessions.insert(session_hash, Arc::clone(&session));
         *state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
         *state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
-        let (issuance_ip, candidate_count, user_agent, user_agent_id) = {
+        let (issuance_ip, candidate_count, user_agent, user_agent_id, recovery) = {
             let entry = state
                 .bootstraps
                 .get_mut(&bootstrap_hash)
@@ -362,10 +366,16 @@ impl WebProcessRuntime {
                 u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
                 entry.user_agent.clone(),
                 entry.user_agent_id,
+                entry.recovery,
             )
         };
         decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
         self.telemetry.record_session_created();
+        if recovery {
+            self.telemetry.record_bridge_recovery(
+                crate::web::telemetry::WebBridgeRecoveryEvent::SessionCreated,
+            );
+        }
         let identity = session.trace_identity();
         let result = CreateResult {
             token: session_token,

src/web/manager/session_creation/replacement.rs

@@ -20,7 +20,11 @@ impl WebProcessRuntime {
         let config = generation.config();
         let now = Instant::now();
         let mut state = self.state.lock();
-        remove_expired_locked(&mut state, now);
+        let expired_recoveries = remove_expired_locked(&mut state, now);
+        self.telemetry.record_bridge_recovery_count(
+            crate::web::telemetry::WebBridgeRecoveryEvent::ExpiredUnused,
+            expired_recoveries,
+        );
         state.apply_issuance_policy(generation.id, config.web.enabled);
         let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
             entry.carrier_transitioning
@@ -85,7 +89,7 @@ impl WebProcessRuntime {
         let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
             drop(state);
             self.cancel_replacement(bootstrap_hash, &replacement.old_session);
-            session.close();
+            session.close(crate::web::session::SessionCloseReason::Protocol);
             return Err(ManagerError::Closed);
         };
         let old_hash = replacement.old_session.token_hash();
@@ -94,6 +98,9 @@ impl WebProcessRuntime {
             &mut state,
             old_hash,
             &replacement.profile.host,
+            replacement.old_session.trace_session_id(),
+            replacement.old_session.carrier(),
+            crate::web::session::SessionCloseReason::CarrierSuperseded,
             Duration::from_secs(replacement.old_session.timeouts().bootstrap_lifetime_secs),
             self.limits.max_sessions_global.saturating_mul(16),
         );
@@ -115,7 +122,15 @@ impl WebProcessRuntime {
             *slot = Some(replacement.old_session.carrier());
         }
         self.telemetry.record_session_created();
-        self.telemetry.record_session_closed();
+        if entry.recovery {
+            self.telemetry.record_bridge_recovery(
+                crate::web::telemetry::WebBridgeRecoveryEvent::SessionCreated,
+            );
+        }
+        self.telemetry.record_session_closed(
+            replacement.old_session.carrier(),
+            crate::web::session::SessionCloseReason::CarrierSuperseded,
+        );
         let result = CreateResult {
             token: session_token,
             carrier: replacement.carrier,

src/web/manager/state.rs

@@ -86,6 +86,8 @@ pub(super) struct Bootstrap {
     pub(super) session_ip_learning_eligible: bool,
     /// Distinguishes unused issuance quota from completed creation replay state.
     pub(super) used: bool,
+    /// Whether this credential was issued by the post-commit recovery representation.
+    pub(super) recovery: bool,
 }
 
 /// Bounded replay marker for one explicitly or naturally closed session token.
@@ -94,6 +96,12 @@ pub(super) struct ClosedToken {
     pub(super) expires_at: Instant,
     /// Canonical host that owned the session.
     pub(super) host: String,
+    /// Non-secret logical trace owner retained for exact late-request diagnostics.
+    pub(super) trace_session_id: u64,
+    /// Carrier that owned the retired bearer.
+    pub(super) carrier: WebCarrier,
+    /// First-writer terminal cause for the retired bearer.
+    pub(super) reason: crate::web::session::SessionCloseReason,
 }
 
 /// Current logical-session owner stored without exposing bearer credentials.
@@ -116,6 +124,12 @@ pub(super) struct ClosedSession {
     pub(super) expires_at: Instant,
     /// Last carrier incarnation closed for this logical session.
     pub(super) attempt: u8,
+    /// Last carrier owning this logical session.
+    pub(super) carrier: WebCarrier,
+    /// First-writer terminal close cause.
+    pub(super) reason: crate::web::session::SessionCloseReason,
+    /// Monotonic instant used only to report bounded close age.
+    pub(super) closed_at: Instant,
 }
 
 /// Token-bucket state for one process-wide creation class.
@@ -303,13 +317,19 @@ pub(super) fn evict_oldest_unused_bootstrap(state: &mut ManagerState) -> bool {
 }
 
 /// Removes expired bootstrap and closed-token entries while the manager lock is held.
-pub(super) fn remove_expired_locked(state: &mut ManagerState, now: Instant) {
+pub(super) fn remove_expired_locked(state: &mut ManagerState, now: Instant) -> usize {
     let expired = state
         .bootstraps
         .iter()
-        .filter_map(|(hash, bootstrap)| (now > bootstrap.expires_at).then_some(*hash))
+        .filter_map(|(hash, bootstrap)| {
+            (now > bootstrap.expires_at).then_some((*hash, bootstrap.recovery && !bootstrap.used))
+        })
         .collect::<Vec<_>>();
-    for hash in expired {
+    let recovery_unused = expired
+        .iter()
+        .filter(|(_, recovery_unused)| *recovery_unused)
+        .count();
+    for (hash, _) in expired {
         remove_bootstrap_locked(state, hash);
     }
     state
@@ -325,6 +345,7 @@ pub(super) fn remove_expired_locked(state: &mut ManagerState, now: Instant) {
             state.closed_sessions.remove(&trace_session_id);
         }
     }
+    recovery_unused
 }
 
 /// Removes one bootstrap and releases its per-address issuance quota when unused.
@@ -342,6 +363,9 @@ pub(super) fn remember_closed_token_locked(
     state: &mut ManagerState,
     hash: TokenHash,
     host: &str,
+    trace_session_id: u64,
+    carrier: WebCarrier,
+    reason: crate::web::session::SessionCloseReason,
     lifetime: Duration,
     capacity: usize,
 ) {
@@ -350,6 +374,9 @@ pub(super) fn remember_closed_token_locked(
         ClosedToken {
             expires_at: Instant::now() + lifetime,
             host: host.to_string(),
+            trace_session_id,
+            carrier,
+            reason,
         },
     );
     while state.closed_tokens.len() > capacity {
@@ -370,6 +397,8 @@ pub(super) fn remember_closed_session_locked(
     state: &mut ManagerState,
     trace_session_id: u64,
     attempt: u8,
+    carrier: WebCarrier,
+    reason: crate::web::session::SessionCloseReason,
     lifetime: Duration,
     capacity: usize,
 ) {
@@ -380,6 +409,9 @@ pub(super) fn remember_closed_session_locked(
             ClosedSession {
                 expires_at: Instant::now() + lifetime,
                 attempt,
+                carrier,
+                reason,
+                closed_at: Instant::now(),
             },
         )
         .is_none()

src/web/manager/status.rs

@@ -198,7 +198,16 @@ pub(crate) enum SessionDetail {
     /// One exact live-session snapshot.
     Active(Box<SessionRow>),
     /// One bounded retained closed-session tombstone.
-    Gone { attempt: u8 },
+    Gone {
+        /// Last carrier incarnation number.
+        attempt: u8,
+        /// Carrier that owned the final incarnation.
+        carrier: WebCarrier,
+        /// First-writer terminal close cause.
+        reason: &'static str,
+        /// Monotonic age since final closure.
+        closed_age_ms: u64,
+    },
     /// A required short lock was contended.
     Busy,
     /// Neither a live session nor a retained tombstone exists.
@@ -500,13 +509,16 @@ impl WebProcessRuntime {
                 .map(|status| SessionDetail::Active(Box::new(self.row(candidate, status))))
                 .unwrap_or(SessionDetail::Busy);
         }
-        let closed = state
-            .closed_sessions
-            .get(&trace_session_id)
-            .map(|closed| closed.attempt);
-        closed.map_or(SessionDetail::NotFound, |attempt| SessionDetail::Gone {
-            attempt,
-        })
+        let now = Instant::now();
+        state.closed_sessions.get(&trace_session_id).map_or(
+            SessionDetail::NotFound,
+            |closed| SessionDetail::Gone {
+                attempt: closed.attempt,
+                carrier: closed.carrier,
+                reason: closed.reason.as_str(),
+                closed_age_ms: millis(now.saturating_duration_since(closed.closed_at)),
+            },
+        )
     }
 
     fn row(&self, candidate: Candidate, status: WebSessionStatus) -> SessionRow {
@@ -524,3 +536,7 @@ mod reference;
 pub(crate) use reference::SessionRefError;
 pub(super) use reference::immutable_matches;
 use reference::permits;
+
+fn millis(duration: std::time::Duration) -> u64 {
+    duration.as_millis().min(u128::from(u64::MAX)) as u64
+}