WEB Debug + Trace

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/maestro/listeners/control.rs

@@ -14,6 +14,7 @@ use super::plan::{ListenerBindSpec, listener_bind_plan};
 #[cfg(unix)]
 use super::unix::UnixAcceptHandle;
 use crate::web::manager::WebProcessRuntime;
+use crate::web::trace::WebTraceStore;
 
 /// Process-owned listener inventory and accept-task lifecycle controller.
 pub(crate) struct ListenerManager {
@@ -43,12 +44,14 @@ impl ListenerManager {
     pub(crate) fn start(
         bound: BoundListeners,
         active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
+        trace: Arc<WebTraceStore>,
     ) -> Self {
         let has_web = bound
             .listeners
             .iter()
             .any(|listener| listener.spec.transport == ListenerTransport::Web);
-        let web_runtime = has_web.then(|| WebProcessRuntime::start(active_runtime.clone()));
+        let web_runtime =
+            has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
         let mut slots = BTreeMap::new();
         for listener in bound.listeners {
             let addr = listener.spec.addr;
@@ -307,7 +310,11 @@ mod tests {
             #[cfg(unix)]
             unix_listener: None,
         };
-        let mut manager = ListenerManager::start(bound, active_runtime);
+        let trace = WebTraceStore::new(
+            runtime.config().web.debug.clone(),
+            &runtime.config().web.limits,
+        );
+        let mut manager = ListenerManager::start(bound, active_runtime, trace);
         let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let blocked_addr = blocker.local_addr().unwrap();
         let mut desired = ProxyConfig::default();
@@ -330,7 +337,11 @@ mod tests {
             #[cfg(unix)]
             unix_listener: None,
         };
-        let mut manager = ListenerManager::start(bound, active_runtime);
+        let trace = WebTraceStore::new(
+            runtime.config().web.debug.clone(),
+            &runtime.config().web.limits,
+        );
+        let mut manager = ListenerManager::start(bound, active_runtime, trace);
         let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let new_addr = reservation.local_addr().unwrap();
         drop(reservation);

src/maestro/orchestrator.rs

@@ -17,6 +17,7 @@ use crate::stats::{QuotaStore, Stats};
 use crate::synlimit_control;
 use crate::transport::UpstreamManager;
 use crate::transport::middle_proxy::MePool;
+use crate::web::trace::WebTraceStore;
 
 use super::{
     bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
@@ -104,6 +105,7 @@ pub(super) async fn run_telemt_core(
         config.access.user_rate_limits.clone(),
         config.access.cidr_rate_limits.clone(),
     );
+    let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
 
     let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
     let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
@@ -154,6 +156,7 @@ pub(super) async fn run_telemt_core(
             let reload_control_api = reload_control.clone();
             let active_runtime_rx_api = active_runtime_rx.clone();
             let runtime_watch_rx_api = runtime_watch_rx.clone();
+            let web_trace_api = web_trace.clone();
             tokio::spawn(async move {
                 api::serve(
                     listen,
@@ -171,6 +174,7 @@ pub(super) async fn run_telemt_core(
                     reload_control_api,
                     active_runtime_rx_api,
                     runtime_watch_rx_api,
+                    web_trace_api,
                 )
                 .await;
             });
@@ -314,7 +318,11 @@ pub(super) async fn run_telemt_core(
     active_runtime_tx.send_replace(Some(active_runtime.clone()));
     runtime_tasks::mark_runtime_ready(&startup_tracker).await;
 
-    let listener_manager = listeners::ListenerManager::start(bound, active_runtime.clone());
+    let listener_manager = listeners::ListenerManager::start(
+        bound,
+        active_runtime.clone(),
+        web_trace.clone(),
+    );
     let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
         active_runtime.clone(),
         reload_control,
@@ -325,6 +333,7 @@ pub(super) async fn run_telemt_core(
         runtime_log_filter,
         runtime_watch_tx,
         listener_manager,
+        web_trace,
     );
 
     shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);

src/maestro/reload_supervisor.rs

@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 
 use crate::stats::QuotaStore;
+use crate::web::trace::WebTraceStore;
 
 use super::generation::{RuntimeGeneration, RuntimeWatchState};
 use super::listeners::{ListenerManager, PreparedListenerTransition};
@@ -29,6 +30,7 @@ pub(crate) struct ReloadSupervisor {
     runtime_log_filter: RuntimeLogFilter,
     runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
     listener_manager: Arc<Mutex<ListenerManager>>,
+    web_trace: Arc<WebTraceStore>,
 }
 
 /// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
@@ -105,6 +107,7 @@ impl ReloadSupervisor {
         runtime_log_filter: RuntimeLogFilter,
         runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
         listener_manager: ListenerManager,
+        web_trace: Arc<WebTraceStore>,
     ) -> ReloadSupervisorHandle {
         let listener_manager = Arc::new(Mutex::new(listener_manager));
         let supervisor = Self {
@@ -117,6 +120,7 @@ impl ReloadSupervisor {
             runtime_log_filter,
             runtime_watch_tx,
             listener_manager: listener_manager.clone(),
+            web_trace,
         };
         let control = supervisor.control.clone();
         let shutdown = CancellationToken::new();
@@ -309,6 +313,7 @@ impl ReloadSupervisor {
         };
         old_runtime.stop_accepting_sessions();
         let replaced = self.active_runtime.swap(new_runtime.clone());
+        self.web_trace.apply_policy(&new_runtime.config().web.debug);
         if let Some(pending) = pending_listener_transition {
             self.listener_manager
                 .lock()

src/maestro/reload_supervisor_tests.rs

@@ -34,6 +34,10 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
     let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
     let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
     let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
+    let web_trace = crate::web::trace::WebTraceStore::new(
+        old_runtime.config().web.debug.clone(),
+        &old_runtime.config().web.limits,
+    );
     let supervisor = Arc::new(ReloadSupervisor {
         active_runtime,
         control: control.clone(),
@@ -44,6 +48,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
         runtime_log_filter: runtime_log_filter(),
         runtime_watch_tx,
         listener_manager,
+        web_trace,
     });
     let command = ReloadCommand {
         reload_id: accepted.reload_id,
@@ -306,6 +311,10 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
         runtime_log_filter(),
         runtime_watch_tx,
         listener_manager,
+        crate::web::trace::WebTraceStore::new(
+            runtime.config().web.debug.clone(),
+            &runtime.config().web.limits,
+        ),
     );
 
     tokio::time::timeout(Duration::from_secs(1), handle.quiesce())

src/maestro/runtime_build.rs

@@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use tokio::sync::{RwLock, Semaphore, watch};
 
-use crate::config::{ProxyConfig, ServerConfig};
+use crate::config::{ProxyConfig, ServerConfig, web_debug_fits_limits};
 use crate::crypto::SecureRandom;
 use crate::ip_tracker::UserIpTracker;
 use crate::network::probe::{decide_network_capabilities, run_probe};
@@ -415,6 +415,10 @@ pub(crate) fn resolve_reload_config(
             effective.web = old.web.clone();
         }
     }
+    if !web_debug_fits_limits(&effective.web.debug, &effective.web.limits) {
+        fields.push("web.debug".to_string());
+        effective.web.debug = old.web.debug.clone();
+    }
     let runtime_changed = !configs_equal(old, &effective);
     ResolvedReloadConfig {
         effective,

src/maestro/runtime_build_tests.rs

@@ -170,6 +170,27 @@ fn web_allocation_limits_are_deferred_until_restart() {
     assert!(!resolved.runtime_changed);
 }
 
+#[test]
+fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
+    let mut old = ProxyConfig::default();
+    old.rebuild_runtime_user_auth().unwrap();
+    old.rebuild_runtime_web().unwrap();
+    let mut desired = old.clone();
+    desired.web.limits.max_body_bytes = 4 * 1024 * 1024;
+    desired.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
+
+    let resolved = resolve_reload_config(&old, &desired);
+
+    assert_eq!(
+        resolved.deferred_process_fields,
+        vec!["web.limits".to_string(), "web.debug".to_string()]
+    );
+    assert_eq!(
+        resolved.effective.web.debug.body_prefix_bytes,
+        old.web.debug.body_prefix_bytes
+    );
+}
+
 #[test]
 fn strict_middle_proxy_requires_a_prepared_pool() {
     assert!(strict_middle_proxy_unavailable(true, false, false));