API for WEB: bounded lifecycle and overload observability added

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/web/http/decoy.rs

@@ -15,6 +15,7 @@ use super::{
 };
 use crate::config::{WebRuntimeDecoy, WebRuntimeVhost};
 use crate::web::manager::WebProcessRuntime;
+use crate::web::telemetry::WebDecoyUpstreamOutcome;
 
 /// Serves the configured ordinary site after optionally removing carrier material.
 /// Transport-sanitized static fallbacks remain uncacheable after query removal.
@@ -195,16 +196,18 @@ async fn proxy_to_upstream(
         .map(|value| value.as_str())
         .unwrap_or("/");
     let Ok(uri) = path_and_query.parse::<Uri>() else {
-        return bad_gateway();
+        return decoy_failure(runtime, WebDecoyUpstreamOutcome::RequestError);
     };
     *request.uri_mut() = uri;
     let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
         Ok(lease) => lease,
-        Err(()) => return bad_gateway(),
+        Err(()) => {
+            return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
+        }
     };
-    let stream = match tokio::time::timeout(header_timeout, TcpStream::connect(addr)).await {
-        Ok(Ok(stream)) => stream,
-        _ => return bad_gateway(),
+    let stream = match connect_upstream(addr, header_timeout).await {
+        Ok(stream) => stream,
+        Err(outcome) => return decoy_failure(runtime, outcome),
     };
     drop(_deadline_lease);
     let max_header_bytes = runtime
@@ -217,12 +220,19 @@ async fn proxy_to_upstream(
     builder.max_buf_size(max_header_bytes);
     let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
         Ok(lease) => lease,
-        Err(()) => return bad_gateway(),
+        Err(()) => {
+            return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
+        }
     };
     let (mut sender, connection) =
         match tokio::time::timeout(header_timeout, builder.handshake(TokioIo::new(stream))).await {
             Ok(Ok(parts)) => parts,
-            _ => return bad_gateway(),
+            Ok(Err(_)) => {
+                return decoy_failure(runtime, WebDecoyUpstreamOutcome::HttpHandshakeError);
+            }
+            Err(_) => {
+                return decoy_failure(runtime, WebDecoyUpstreamOutcome::HttpHandshakeTimeout);
+            }
         };
     drop(_deadline_lease);
     runtime.spawn_auxiliary(async move {
@@ -230,14 +240,24 @@ async fn proxy_to_upstream(
     });
     let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
         Ok(lease) => lease,
-        Err(()) => return bad_gateway(),
+        Err(()) => {
+            return decoy_failure(runtime, WebDecoyUpstreamOutcome::DeadlineExhausted);
+        }
     };
     let mut response =
         match tokio::time::timeout(header_timeout, sender.send_request(request)).await {
             Ok(Ok(response)) => response,
-            _ => return bad_gateway(),
+            Ok(Err(_)) => {
+                return decoy_failure(runtime, WebDecoyUpstreamOutcome::RequestError);
+            }
+            Err(_) => {
+                return decoy_failure(runtime, WebDecoyUpstreamOutcome::ResponseHeadTimeout);
+            }
         };
     drop(_deadline_lease);
+    runtime
+        .telemetry()
+        .record_decoy(WebDecoyUpstreamOutcome::Success);
     remove_hop_by_hop(response.headers_mut());
     response.map(|body| {
         body.map_err(|error| -> BoxError { Box::new(error) })
@@ -245,6 +265,25 @@ async fn proxy_to_upstream(
     })
 }
 
+async fn connect_upstream(
+    addr: SocketAddr,
+    timeout: Duration,
+) -> Result<TcpStream, WebDecoyUpstreamOutcome> {
+    match tokio::time::timeout(timeout, TcpStream::connect(addr)).await {
+        Ok(Ok(stream)) => Ok(stream),
+        Ok(Err(error)) if error.kind() == std::io::ErrorKind::ConnectionRefused => {
+            Err(WebDecoyUpstreamOutcome::ConnectRefused)
+        }
+        Ok(Err(_)) => Err(WebDecoyUpstreamOutcome::ConnectError),
+        Err(_) => Err(WebDecoyUpstreamOutcome::ConnectTimeout),
+    }
+}
+
+fn decoy_failure(runtime: &WebProcessRuntime, outcome: WebDecoyUpstreamOutcome) -> HttpResponse {
+    runtime.telemetry().record_decoy(outcome);
+    bad_gateway()
+}
+
 fn lease_deadline(
     deadline: Option<&super::activity::RequestDeadlineHandle>,
     timeout: Duration,
@@ -315,4 +354,16 @@ mod tests {
         assert!(resolve_static_path("/../index.html", &site).is_none());
         assert!(resolve_static_path("//index.html", &site).is_none());
     }
+
+    #[tokio::test]
+    async fn closed_loopback_origin_is_classified_as_connect_refused() {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        drop(listener);
+
+        assert_eq!(
+            connect_upstream(addr, Duration::from_secs(1)).await.err(),
+            Some(WebDecoyUpstreamOutcome::ConnectRefused)
+        );
+    }
 }

src/web/http/operator_lifecycle_tests.rs

@@ -46,10 +46,7 @@ async fn live_runtime() -> (
     Arc<crate::maestro::generation::RuntimeGeneration>,
     TcpListener,
 ) {
-    let generation = test_runtime_generation(
-        1,
-        runtime_config([71; 32], WebCarrier::Https),
-    );
+    let generation = test_runtime_generation(1, runtime_config([71; 32], WebCarrier::Https));
     let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
     let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
     (runtime, generation, listener)
@@ -100,12 +97,7 @@ async fn pause_preserves_decoy_retry_and_exact_session_replay() {
     let hello = frame::encode(FrameType::Hello, 0, &[1]);
 
     runtime.pause_operator().await.unwrap();
-    let paused_create = request(
-        &listener,
-        &runtime,
-        create_request(&bootstrap, &hello),
-    )
-    .await;
+    let paused_create = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
     let (paused_headers, _) = split_response(&paused_create);
     assert!(paused_headers.starts_with(b"HTTP/1.1 503"));
     assert_eq!(response_header(paused_headers, "retry-after"), "1");
@@ -131,26 +123,20 @@ async fn pause_preserves_decoy_retry_and_exact_session_replay() {
     let decoy = request(&listener, &runtime, decoy).await;
     let (decoy_headers, decoy_body) = split_response(&decoy);
     assert!(decoy_headers.starts_with(b"HTTP/1.1 200"));
-    assert!(!decoy_body.windows(11).any(|window| window == b"bootstrap=\""));
+    assert!(
+        !decoy_body
+            .windows(11)
+            .any(|window| window == b"bootstrap=\"")
+    );
 
     runtime.resume_operator().await.unwrap();
-    let created = request(
-        &listener,
-        &runtime,
-        create_request(&bootstrap, &hello),
-    )
-    .await;
+    let created = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
     let (created_headers, _) = split_response(&created);
     assert!(created_headers.starts_with(b"HTTP/1.1 200"));
     let token = response_header(created_headers, "x-session-token").to_string();
 
     runtime.pause_operator().await.unwrap();
-    let replay = request(
-        &listener,
-        &runtime,
-        create_request(&bootstrap, &hello),
-    )
-    .await;
+    let replay = request(&listener, &runtime, create_request(&bootstrap, &hello)).await;
     let (replay_headers, _) = split_response(&replay);
     assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
     assert_eq!(response_header(replay_headers, "x-session-token"), token);
@@ -230,7 +216,10 @@ async fn paused_replacement_preserves_old_session_and_attempt_replay() {
     let replay = request(&listener, &runtime, first_request).await;
     let (replay_headers, _) = split_response(&replay);
     assert!(replay_headers.starts_with(b"HTTP/1.1 200"));
-    assert_eq!(response_header(replay_headers, "x-session-token"), first_token);
+    assert_eq!(
+        response_header(replay_headers, "x-session-token"),
+        first_token
+    );
 
     runtime.resume_operator().await.unwrap();
     let replacement = request(&listener, &runtime, replacement_request).await;
@@ -249,7 +238,10 @@ async fn client_delete_during_drain_completes_naturally() {
     let (runtime, generation, listener) = live_runtime().await;
     let bootstrap = issue_bootstrap(&runtime);
     let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
-    runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
+    runtime
+        .drain_operator(Duration::from_secs(30))
+        .await
+        .unwrap();
 
     let delete = format!(
         "DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {token}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
@@ -281,8 +273,14 @@ async fn deadline_force_closes_all_sessions_and_requires_explicit_resume() {
     let bootstrap = issue_bootstrap(&runtime);
     let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
 
-    let accepted = runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
-    assert_eq!(serde_json::to_value(&accepted).unwrap()["state"], "draining");
+    let accepted = runtime
+        .drain_operator(Duration::from_secs(30))
+        .await
+        .unwrap();
+    assert_eq!(
+        serde_json::to_value(&accepted).unwrap()["state"],
+        "draining"
+    );
     assert!(matches!(
         runtime.drain_operator(Duration::from_secs(30)).await,
         Err(OperatorLifecycleError::OperationInProgress)
@@ -323,7 +321,10 @@ async fn resume_before_deadline_cancels_drain_without_closing_existing_session()
     let bootstrap = issue_bootstrap(&runtime);
     let (_, token) = create_session(&listener, &runtime, &bootstrap).await;
 
-    runtime.drain_operator(Duration::from_secs(30)).await.unwrap();
+    runtime
+        .drain_operator(Duration::from_secs(30))
+        .await
+        .unwrap();
     let still_draining = runtime.pause_operator().await.unwrap();
     assert_eq!(
         serde_json::to_value(&still_draining).unwrap()["state"],

src/web/http/session.rs

@@ -190,9 +190,9 @@ pub(super) async fn handle_session(
         }
         Err(
             error @ (ManagerError::Limit
-                | ManagerError::Backpressure
-                | ManagerError::Concurrent
-                | ManagerError::AdmissionPaused),
+            | ManagerError::Backpressure
+            | ManagerError::Concurrent
+            | ManagerError::AdmissionPaused),
         ) => {
             runtime.trace().record_profile_lifecycle(
                 client_ip,

src/web/http/websocket/driver/io.rs

@@ -46,7 +46,7 @@ pub(super) async fn reserve_data(
     cancellation: &CancellationToken,
     timeout: Duration,
 ) -> Result<WebSocketBudgetLease, ()> {
-    tokio::time::timeout(timeout, async {
+    match tokio::time::timeout(timeout, async {
         loop {
             if cancellation.is_cancelled() {
                 return Err(());
@@ -63,7 +63,15 @@ pub(super) async fn reserve_data(
         }
     })
     .await
-    .map_err(|_| ())?
+    {
+        Ok(result) => result,
+        Err(_) => {
+            runtime.telemetry().record_rejection(
+                crate::web::telemetry::WebRejectionReason::WebSocketBytesCapacity,
+            );
+            Err(())
+        }
+    }
 }
 
 pub(super) async fn process_multiplex(