WEB Session Lifecycle Observability + Bridge Recovery Drafts

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/api/web_runtime.rs

@@ -21,6 +21,7 @@ mod request;
 mod observability;
 use observability::{
     WebCapacityStatus, WebCarrierNegotiationStatus, WebDecoyUpstreamStatus, WebIngressStatus,
+    WebLifecycleCountersStatus,
 };
 use request::{
     CloseRequest, DrainRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
@@ -92,12 +93,20 @@ pub(super) async fn handle(
             let trace_session_id = parse_session_ref(&runtime, session_ref)?;
             match runtime.session_detail(trace_session_id) {
                 SessionDetail::Active(row) => Ok(success_response(StatusCode::OK, row, revision)),
-                SessionDetail::Gone { attempt } => Ok(success_response(
+                SessionDetail::Gone {
+                    attempt,
+                    carrier,
+                    reason,
+                    closed_age_ms,
+                } => Ok(success_response(
                     StatusCode::GONE,
                     GoneSessionData {
                         session_ref: session_ref.to_string(),
                         state: "closed",
                         attempt,
+                        carrier,
+                        reason,
+                        closed_age_ms,
                     },
                     revision,
                 )),
@@ -272,6 +281,7 @@ struct WebStatusData {
     capacity: WebCapacityStatus,
     decoy_upstream: WebDecoyUpstreamStatus,
     carrier_negotiation: WebCarrierNegotiationStatus,
+    lifecycle_counters: WebLifecycleCountersStatus,
     #[serde(skip_serializing_if = "Option::is_none")]
     operator_lifecycle: Option<crate::web::manager::OperatorLifecycleStatus>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -306,6 +316,7 @@ impl WebStatusData {
         let capacity = WebCapacityStatus::new(&publication, runtime, config);
         let decoy_upstream = WebDecoyUpstreamStatus::new(&publication);
         let carrier_negotiation = WebCarrierNegotiationStatus::new(&publication);
+        let lifecycle_counters = WebLifecycleCountersStatus::new(&publication, config);
         Self {
             lifecycle: publication.lifecycle.as_str(),
             lifecycle_epoch: publication.epoch,
@@ -322,6 +333,7 @@ impl WebStatusData {
             capacity,
             decoy_upstream,
             carrier_negotiation,
+            lifecycle_counters,
             operator_lifecycle,
             runtime: runtime.map(WebProcessRuntime::try_status),
         }
@@ -333,6 +345,9 @@ struct GoneSessionData {
     session_ref: String,
     state: &'static str,
     attempt: u8,
+    carrier: crate::config::WebCarrier,
+    reason: &'static str,
+    closed_age_ms: u64,
 }
 
 #[derive(Serialize)]

src/api/web_runtime/observability.rs

@@ -5,7 +5,8 @@ use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
 use crate::web::manager::{WebCapacityResourceStatus, WebCapacitySnapshot, WebProcessRuntime};
 use crate::web::telemetry::{WebOutcomeCounter, WebRejectionCounter};
 use crate::web::telemetry::{
-    WebCarrierFailureCounter, WebCarrierLearningCounter, WebCarrierSelectionCounter,
+    WebBridgeRecoveryCounter, WebCarrierFailureCounter, WebCarrierLearningCounter,
+    WebCarrierSelectionCounter, WebSessionCloseCounter, WebSessionLifecycleObservationCounter,
 };
 
 /// Private WEB ingress state owned by this Telemt process.
@@ -139,6 +140,27 @@ impl WebCarrierNegotiationStatus {
     }
 }
 
+/// Fixed-cardinality process-lifetime WEB lifecycle counters.
+#[derive(Serialize)]
+pub(super) struct WebLifecycleCountersStatus {
+    bridge_recovery_secs: u64,
+    session_closures: Vec<WebSessionCloseCounter>,
+    session_observations: Vec<WebSessionLifecycleObservationCounter>,
+    bridge_recovery_events: Vec<WebBridgeRecoveryCounter>,
+}
+
+impl WebLifecycleCountersStatus {
+    /// Builds a complete counter set from process-owned telemetry.
+    pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
+        Self {
+            bridge_recovery_secs: config.web.timeouts.bridge_recovery_secs,
+            session_closures: publication.telemetry.session_close_counters(),
+            session_observations: publication.telemetry.session_observation_counters(),
+            bridge_recovery_events: publication.telemetry.bridge_recovery_counters(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::config::ProxyConfig;
@@ -167,6 +189,11 @@ mod tests {
         let decoy = serde_json::to_value(super::WebDecoyUpstreamStatus::new(&publication)).unwrap();
         let carrier =
             serde_json::to_value(super::WebCarrierNegotiationStatus::new(&publication)).unwrap();
+        let lifecycle = serde_json::to_value(super::WebLifecycleCountersStatus::new(
+            &publication,
+            &config,
+        ))
+        .unwrap();
 
         assert_eq!(
             capacity["rejections"].as_array().unwrap().len(),
@@ -200,5 +227,22 @@ mod tests {
             crate::config::WebCarrier::ALL.len()
                 * crate::web::telemetry::WebCarrierLearningOutcome::ALL.len()
         );
+        assert_eq!(
+            lifecycle["session_closures"].as_array().unwrap().len(),
+            crate::config::WebCarrier::ALL.len()
+                * crate::web::session::SessionCloseReason::ALL.len()
+        );
+        assert_eq!(
+            lifecycle["session_observations"].as_array().unwrap().len(),
+            crate::config::WebCarrier::ALL.len()
+                * crate::web::telemetry::WebSessionLifecycleObservation::ALL.len()
+        );
+        assert_eq!(
+            lifecycle["bridge_recovery_events"]
+                .as_array()
+                .unwrap()
+                .len(),
+            crate::web::telemetry::WebBridgeRecoveryEvent::ALL.len()
+        );
     }
 }