WEB Debug + Trace

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/web/http/activity.rs

@@ -8,6 +8,7 @@ use hyper::body::{Body, Frame, SizeHint};
 use parking_lot::Mutex;
 
 use super::{BoxError, HttpBody};
+use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
 
 /// Request lifecycle guard that refreshes HTTP connection activity on completion.
 pub(super) struct RequestActivity {
@@ -32,12 +33,33 @@ impl Drop for RequestActivity {
 pub(super) struct ActivityBody {
     inner: HttpBody,
     activity: RequestActivity,
+    trace: Option<Arc<HttpTraceExchange>>,
+    terminal: bool,
 }
 
 impl ActivityBody {
     /// Binds one response body to its request activity guard.
-    pub(super) fn new(inner: HttpBody, activity: RequestActivity) -> Self {
-        Self { inner, activity }
+    pub(super) fn new(
+        inner: HttpBody,
+        activity: RequestActivity,
+        trace: Option<Arc<HttpTraceExchange>>,
+    ) -> Self {
+        Self {
+            inner,
+            activity,
+            trace,
+            terminal: false,
+        }
+    }
+
+    fn finish(&mut self, state: TraceBodyState) {
+        if self.terminal {
+            return;
+        }
+        self.terminal = true;
+        if let Some(trace) = &self.trace {
+            trace.body_finished(TraceDirection::Response, state);
+        }
     }
 }
 
@@ -50,6 +72,21 @@ impl Body for ActivityBody {
         context: &mut Context<'_>,
     ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
         let result = Pin::new(&mut self.inner).poll_frame(context);
+        match &result {
+            Poll::Ready(Some(Ok(frame))) => {
+                if let Some(data) = frame.data_ref()
+                    && let Some(trace) = &self.trace
+                {
+                    trace.body_data(TraceDirection::Response, data);
+                }
+                if self.inner.is_end_stream() {
+                    self.finish(TraceBodyState::Complete);
+                }
+            }
+            Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
+            Poll::Ready(None) => self.finish(TraceBodyState::Complete),
+            Poll::Pending => {}
+        }
         if result.is_ready() {
             *self.activity.last_activity.lock() = Instant::now();
         }
@@ -64,3 +101,9 @@ impl Body for ActivityBody {
         self.inner.size_hint()
     }
 }
+
+impl Drop for ActivityBody {
+    fn drop(&mut self) {
+        self.finish(TraceBodyState::Aborted);
+    }
+}

src/web/http/body.rs

@@ -1,11 +1,87 @@
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
 use std::time::Duration;
 
 use bytes::Bytes;
 use http_body_util::{BodyExt, Empty, Limited};
 use hyper::Request;
-use hyper::body::{Body as _, Incoming};
+use hyper::body::{Body, Frame, Incoming, SizeHint};
 
 use crate::web::manager::WebProcessRuntime;
+use crate::web::trace::{
+    HttpTraceExchange, TraceBodyState, TraceDirection,
+};
+
+/// Incoming request body wrapper that observes frames without changing streaming semantics.
+pub(super) struct RequestBody {
+    inner: Incoming,
+    trace: Option<Arc<HttpTraceExchange>>,
+    terminal: bool,
+}
+
+impl RequestBody {
+    /// Wraps one Hyper request body with optional enabled-only capture state.
+    pub(super) fn new(inner: Incoming, trace: Option<Arc<HttpTraceExchange>>) -> Self {
+        Self {
+            inner,
+            trace,
+            terminal: false,
+        }
+    }
+
+    fn finish(&mut self, state: TraceBodyState) {
+        if self.terminal {
+            return;
+        }
+        self.terminal = true;
+        if let Some(trace) = &self.trace {
+            trace.body_finished(TraceDirection::Request, state);
+        }
+    }
+}
+
+impl Body for RequestBody {
+    type Data = Bytes;
+    type Error = hyper::Error;
+
+    fn poll_frame(
+        mut self: Pin<&mut Self>,
+        context: &mut Context<'_>,
+    ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
+        let result = Pin::new(&mut self.inner).poll_frame(context);
+        match &result {
+            Poll::Ready(Some(Ok(frame))) => {
+                if let Some(data) = frame.data_ref()
+                    && let Some(trace) = &self.trace
+                {
+                    trace.body_data(TraceDirection::Request, data);
+                }
+                if self.inner.is_end_stream() {
+                    self.finish(TraceBodyState::Complete);
+                }
+            }
+            Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
+            Poll::Ready(None) => self.finish(TraceBodyState::Complete),
+            Poll::Pending => {}
+        }
+        result
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.inner.is_end_stream()
+    }
+
+    fn size_hint(&self) -> SizeHint {
+        self.inner.size_hint()
+    }
+}
+
+impl Drop for RequestBody {
+    fn drop(&mut self) {
+        self.finish(TraceBodyState::Aborted);
+    }
+}
 
 /// Collected carrier request retaining its process-wide body reservation.
 pub(super) struct CollectedBody {
@@ -29,7 +105,7 @@ pub(super) enum CollectBodyError {
 
 /// Collects one bounded carrier body under reader, byte, and deadline ownership.
 pub(super) async fn collect_body(
-    request: Request<Incoming>,
+    request: Request<RequestBody>,
     runtime: &WebProcessRuntime,
     limit: usize,
     allow_empty: bool,

src/web/http/decoy.rs

@@ -28,6 +28,7 @@ where
     B: hyper::body::Body<Data = Bytes> + Send + 'static,
     B::Error: Error + Send + Sync + 'static,
 {
+    super::set_trace_route(&request, crate::web::trace::TraceRoute::Decoy);
     if sanitize_transport {
         sanitize_transport_request(&mut request);
     }

src/web/http/down.rs

@@ -0,0 +1,106 @@
+use std::sync::Arc;
+
+use hyper::header::{self, HeaderName, HeaderValue};
+use hyper::{Request, StatusCode};
+
+use super::body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
+use super::decoy::serve_decoy;
+use super::request::canonical_u64_header;
+use super::{
+    HttpResponse, carrier_empty, carrier_headers, carrier_lane, full_response, insert_header,
+    request_trace, service_unavailable,
+};
+use crate::config::WebRuntimeVhost;
+use crate::web::manager::{ManagerError, TokenHash, WebProcessRuntime};
+use crate::web::trace::{TraceDirection, TraceRoute};
+
+/// Handles one authenticated long-poll downlink exchange.
+pub(super) async fn handle_down(
+    request: Request<RequestBody>,
+    runtime: Arc<WebProcessRuntime>,
+    vhost: Arc<WebRuntimeVhost>,
+    token_hash: TokenHash,
+) -> HttpResponse {
+    if request.method() != hyper::Method::POST
+        || request.headers().contains_key(header::CONTENT_TYPE)
+    {
+        return serve_decoy(request, vhost, true, &runtime).await;
+    }
+    let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
+        return serve_decoy(request, vhost, true, &runtime).await;
+    };
+    let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
+        return serve_decoy(request, vhost, true, &runtime).await;
+    };
+    if let Some(trace) = request_trace(&request) {
+        trace.set_route(TraceRoute::Downlink);
+        trace.bind_identity(session.trace_identity());
+    }
+    let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
+        return serve_decoy(request, vhost, true, &runtime).await;
+    };
+    let CollectedBody {
+        request,
+        body,
+        _body_budget,
+    } = match collect_body(request, &runtime, 1, true).await {
+        Ok(result) => result,
+        Err(CollectBodyError::Limit) => return service_unavailable(),
+        Err(CollectBodyError::Invalid(request)) => {
+            return serve_decoy(request, vhost, true, &runtime).await;
+        }
+    };
+    if !body.is_empty() {
+        return serve_decoy(request, vhost, true, &runtime).await;
+    }
+    let _lane_poll = if lane_id.is_some() {
+        let Some(permit) = runtime.try_lane_poll() else {
+            return service_unavailable();
+        };
+        Some(permit)
+    } else {
+        None
+    };
+    let result = match lane_id {
+        Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
+        None => session.poll_down(cursor).await,
+    };
+    match result {
+        Ok(result) if result.body.is_empty() => {
+            let mut response = carrier_empty(StatusCode::NO_CONTENT);
+            insert_header(
+                &mut response,
+                HeaderName::from_static("x-down-cursor"),
+                &result.next_cursor.to_string(),
+            );
+            if result.lane_closed {
+                response.headers_mut().insert(
+                    HeaderName::from_static("x-lane-closed"),
+                    HeaderValue::from_static("1"),
+                );
+            }
+            response
+        }
+        Ok(result) => {
+            if let Some(trace) = request_trace(&request) {
+                trace.record_frames(
+                    TraceDirection::Response,
+                    &result.body,
+                    &runtime.active_generation().config().web.limits,
+                );
+            }
+            let mut response = full_response(StatusCode::OK, result.body);
+            carrier_headers(&mut response);
+            insert_header(
+                &mut response,
+                HeaderName::from_static("x-down-cursor"),
+                &result.next_cursor.to_string(),
+            );
+            response
+        }
+        Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
+            service_unavailable()
+        }
+        Err(_) => serve_decoy(request, vhost, true, &runtime).await,
+    }
+}

src/web/http/response.rs

@@ -0,0 +1,88 @@
+use bytes::Bytes;
+use http_body_util::{BodyExt, Full};
+use hyper::header::{self, HeaderName, HeaderValue};
+use hyper::{Request, Response, StatusCode};
+
+use super::request::canonical_u64_header;
+use super::{BoxError, HttpResponse};
+use crate::config::WebCarrier;
+use crate::web::frame;
+
+/// Validates and resolves the optional carrier lane header.
+pub(super) fn carrier_lane<B>(
+    request: &Request<B>,
+    carrier: WebCarrier,
+) -> Option<Option<u32>> {
+    match carrier {
+        WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
+        WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
+            .and_then(|value| u32::try_from(value).ok())
+            .filter(|value| *value <= frame::MAX_STREAM_ID)
+            .map(Some),
+    }
+}
+
+/// Applies common binary carrier response headers.
+pub(super) fn carrier_headers(response: &mut HttpResponse) {
+    response.headers_mut().insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static("application/octet-stream"),
+    );
+    response
+        .headers_mut()
+        .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
+}
+
+/// Builds an empty no-store carrier response.
+pub(super) fn carrier_empty(status: StatusCode) -> HttpResponse {
+    let mut response = empty_response(status);
+    response
+        .headers_mut()
+        .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
+    response
+}
+
+/// Builds the bounded retryable carrier saturation response.
+pub(super) fn service_unavailable() -> HttpResponse {
+    let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
+    response
+        .headers_mut()
+        .insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
+    response
+}
+
+/// Builds the ordinary decoy upstream failure response.
+pub(super) fn bad_gateway() -> HttpResponse {
+    full_response(
+        StatusCode::BAD_GATEWAY,
+        Bytes::from_static(b"site unavailable\n"),
+    )
+}
+
+/// Builds the ordinary unmatched-host response.
+pub(super) fn generic_not_found() -> HttpResponse {
+    full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
+}
+
+/// Builds one in-memory response with an exact content length.
+pub(super) fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
+    let length = body.len();
+    let body = Full::new(body)
+        .map_err(|never| -> BoxError { match never {} })
+        .boxed_unsync();
+    let mut response = Response::new(body);
+    *response.status_mut() = status;
+    insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
+    response
+}
+
+fn empty_response(status: StatusCode) -> HttpResponse {
+    full_response(status, Bytes::new())
+}
+
+/// Inserts one validated dynamic response header value.
+pub(super) fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
+    if let Ok(value) = HeaderValue::from_str(value) {
+        response.headers_mut().insert(name, value);
+    }
+}

src/web/http/tests.rs

@@ -17,7 +17,7 @@ use crate::maestro::generation::test_runtime_generation;
 use crate::web::frame::{self, FrameType};
 use crate::web::manager::WebProcessRuntime;
 
-fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
+pub(super) fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
     let profile = Arc::new(WebRuntimeProfile {
         host: "proxy.example.com".to_string(),
         public_addr: "203.0.113.10:443".parse().unwrap(),
@@ -25,6 +25,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
         secret_mode: WebSecretMode::Plain,
         carrier,
         capability,
+        key_fingerprint: "0000000000000000".to_string(),
         max_sessions: 4,
         max_streams: 16,
         max_streams_per_session: 4,
@@ -71,7 +72,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
     config
 }
 
-async fn request(
+pub(super) async fn request(
     listener: &TcpListener,
     runtime: &Arc<WebProcessRuntime>,
     request: Vec<u8>,
@@ -97,7 +98,7 @@ async fn request(
     response
 }
 
-fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
+pub(super) fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
     let separator = response
         .windows(4)
         .position(|window| window == b"\r\n\r\n")

src/web/http/trace_tests.rs

@@ -0,0 +1,75 @@
+use std::sync::Arc;
+
+use arc_swap::ArcSwap;
+use base64::Engine as _;
+use tokio::net::TcpListener;
+
+use super::tests::{request, runtime_config, split_response};
+use crate::config::{WebCarrier, WebDebugBodyCapture};
+use crate::maestro::generation::test_runtime_generation;
+use crate::web::manager::WebProcessRuntime;
+use crate::web::trace::{TraceBodyState, TraceRecordKind, TraceRoute};
+
+#[tokio::test]
+async fn enabled_debug_records_bridge_request_response_without_credentials() {
+    let capability = [18u8; 32];
+    let mut config = runtime_config(capability, WebCarrier::Https);
+    config.web.debug.enabled = true;
+    config.web.debug.body_capture = WebDebugBodyCapture::Prefix;
+    config.web.debug.body_prefix_bytes = 4096;
+    let generation = test_runtime_generation(1, config);
+    let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
+    let runtime = WebProcessRuntime::start(active_runtime);
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
+    let root = format!(
+        "GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.50\r\nUser-Agent: debug-client/1\r\nConnection: close\r\n\r\n"
+    )
+    .into_bytes();
+
+    let response = request(&listener, &runtime, root).await;
+    let (_, body) = split_response(&response);
+    let body = std::str::from_utf8(body).unwrap();
+    let bootstrap = body
+        .split_once("bootstrap=\"")
+        .and_then(|(_, suffix)| suffix.split_once('"'))
+        .map(|(token, _)| token)
+        .unwrap();
+    let records = runtime
+        .trace()
+        .snapshot_matching(|record| matches!(&record.kind, TraceRecordKind::Http(_)));
+    assert_eq!(records.len(), 1);
+    let record = &records[0].record;
+    assert_eq!(record.effective_ip, Some("192.0.2.50".parse().unwrap()));
+    assert_eq!(record.user_agent.as_deref(), Some("debug-client/1"));
+    assert!(record.identity.session_id.is_some());
+    assert_eq!(record.identity.user.as_deref(), Some("alice"));
+    let TraceRecordKind::Http(http) = &record.kind else {
+        panic!("expected HTTP debug record");
+    };
+    assert_eq!(http.method, "GET");
+    assert_eq!(http.path, "/");
+    assert_eq!(http.route, TraceRoute::Bridge);
+    assert_eq!(http.status, Some(200));
+    assert_eq!(
+        http.response_body.as_ref().unwrap().state,
+        TraceBodyState::Complete
+    );
+    let captured_response = &http.response_body.as_ref().unwrap().captured;
+    assert!(
+        !captured_response
+            .windows(bootstrap.len())
+            .any(|value| value == bootstrap.as_bytes())
+    );
+    assert!(
+        captured_response
+            .windows(bootstrap.len())
+            .any(|value| value.iter().all(|byte| *byte == b'*'))
+    );
+    assert!(http.timings.as_ref().unwrap().response_ready_us.is_some());
+    assert!(http.timings.as_ref().unwrap().response_body_us.is_some());
+
+    runtime.shutdown().await;
+    generation.stop_sessions().await;
+    generation.stop_background_tasks().await;
+}