diff --git a/src/api/config_edit/tests.rs b/src/api/config_edit/tests.rs index 1143a31..6979f47 100644 --- a/src/api/config_edit/tests.rs +++ b/src/api/config_edit/tests.rs @@ -107,14 +107,24 @@ async fn read_managed_config_exposes_web_without_runtime_or_access_secrets() { #[tokio::test] async fn patch_web_debug_is_hot_and_limits_are_process_deferred() { let (path, _directory) = temp_config("[web]\nenabled = false\n"); + let active = ProxyConfig::load(&path).unwrap(); let debug_patch: Json = serde_json::json!({ - "web": {"debug": {"enabled": true, "capture_headers": false}} + "web": {"debug": { + "enabled": true, + "sideband": true, + "capture_headers": false + }} }); - let debug = apply_patch_to_path(&path, &debug_patch, None) + let mut debug = apply_patch_to_path(&path, &debug_patch, None) .await .unwrap(); + let desired = ProxyConfig::load(&path).unwrap(); + reconcile_runtime_effect(&mut debug, &active, &desired).unwrap(); + assert!(!debug.restart_required); + assert!(debug.runtime_reload_required); assert!(!debug.process_restart_required); assert!(debug.changed.iter().any(|section| section == "web")); + assert!(desired.web.debug.sideband); let limits_patch: Json = serde_json::json!({ "web": {"limits": {"max_http_connections": 2049}} diff --git a/src/api/web_status.rs b/src/api/web_status.rs index d162e0d..e11d1a4 100644 --- a/src/api/web_status.rs +++ b/src/api/web_status.rs @@ -67,6 +67,11 @@ pub(super) async fn render( push_filter_form(&mut html, &query); html.push_str("

Store

"); summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled)); + summary_row( + &mut html, + "sideband", + yes_no(status.policy.bridge_diagnostics_enabled()), + ); summary_row(&mut html, "body capture", body_mode(&status.policy)); summary_row(&mut html, "window seconds", &query.window_secs.to_string()); summary_row( diff --git a/src/api/web_status/tests.rs b/src/api/web_status/tests.rs index 52ef582..1314104 100644 --- a/src/api/web_status/tests.rs +++ b/src/api/web_status/tests.rs @@ -22,6 +22,7 @@ fn html_escaping_covers_active_markup_characters() { async fn renderer_filters_groups_and_sets_control_plane_security_headers() { let policy = WebDebugConfig { enabled: true, + sideband: true, ..Default::default() }; let limits = crate::config::WebLimitsConfig { @@ -58,6 +59,7 @@ async fn renderer_filters_groups_and_sets_control_plane_security_headers() { let body = response.into_body().collect().await.unwrap().to_bytes(); let body = std::str::from_utf8(&body).unwrap(); assert!(body.contains("session_created")); + assert!(body.contains("")); assert!(body.contains("0123456789abcdef")); assert!(body.contains("192.0.2.40")); } diff --git a/src/config/hot_reload/tests.rs b/src/config/hot_reload/tests.rs index 4741f10..f81bbd7 100644 --- a/src/config/hot_reload/tests.rs +++ b/src/config/hot_reload/tests.rs @@ -144,11 +144,13 @@ fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() { let old = sample_config(); let mut new = old.clone(); new.web.debug.enabled = true; + new.web.debug.sideband = true; new.web.debug.default_window_secs = 60; new.web.limits.debug_records_capacity += 1; let applied = overlay_hot_fields(&old, &new); assert!(applied.web.debug.enabled); + assert!(applied.web.debug.sideband); assert_eq!(applied.web.debug.default_window_secs, 60); assert_eq!( applied.web.limits.debug_records_capacity, diff --git a/src/config/load/strict_keys.rs b/src/config/load/strict_keys.rs index 69dc383..30f2480 100644 --- a/src/config/load/strict_keys.rs +++ b/src/config/load/strict_keys.rs @@ -326,6 +326,7 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[ const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[ "enabled", + "sideband", "capture_lifecycle", "capture_headers", "capture_timings", diff --git a/src/config/tests/load_basic_tests/web_tests.rs b/src/config/tests/load_basic_tests/web_tests.rs index b18d004..0e078b1 100644 --- a/src/config/tests/load_basic_tests/web_tests.rs +++ b/src/config/tests/load_basic_tests/web_tests.rs @@ -301,17 +301,31 @@ fn web_carrier_learning_capacity_must_remain_nonzero() { #[test] fn web_debug_table_uses_debug_name_and_bounded_defaults() { + assert!(!crate::config::WebDebugConfig::default().sideband); + let mut ineffective = crate::config::WebDebugConfig { + enabled: true, + sideband: true, + ..Default::default() + }; + ineffective.capture_lifecycle = false; + assert!(!ineffective.bridge_diagnostics_enabled()); + let configured = WEB_CONFIG.replace( "[[web.vhosts]]", - "[web.debug]\nenabled = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]", + "[web.debug]\nenabled = true\nsideband = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]", ); let config = load_config_from_temp_toml(&configured); assert!(config.web.debug.enabled); + assert!(config.web.debug.sideband); + assert!(config.web.debug.bridge_diagnostics_enabled()); assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix); assert_eq!(config.web.debug.body_prefix_bytes, 2048); assert_eq!(config.web.debug.default_window_secs, 180); assert_eq!(config.web.debug.max_window_secs, 900); + let strict = format!("[general]\nconfig_strict = true\n{configured}"); + assert!(load_config_from_temp_toml(&strict).web.debug.sideband); + let old_name = format!( "[general]\nconfig_strict = true\n{}", WEB_CONFIG.replace( @@ -321,6 +335,16 @@ fn web_debug_table_uses_debug_name_and_bounded_defaults() { ); let error = load_config_error_from_temp_toml(&old_name); assert!(error.contains("web.trace")); + + let old_parameter = format!( + "[general]\nconfig_strict = true\n{}", + WEB_CONFIG.replace( + "[[web.vhosts]]", + "[web.debug]\nbridge_diagnostics = true\n\n[[web.vhosts]]", + ) + ); + let error = load_config_error_from_temp_toml(&old_parameter); + assert!(error.contains("web.debug.bridge_diagnostics")); } #[test] diff --git a/src/config/types/web_debug.rs b/src/config/types/web_debug.rs index 4bc0df9..63cf7b2 100644 --- a/src/config/types/web_debug.rs +++ b/src/config/types/web_debug.rs @@ -23,6 +23,9 @@ pub struct WebDebugConfig { /// Enables process-owned WEB debug collection. #[serde(default)] pub enabled: bool, + /// Enables generated-bridge diagnostic reports over the HTTPS sideband. + #[serde(default)] + pub sideband: bool, /// Records typed bridge, session, stream, handshake, and relay events. #[serde(default = "default_true")] pub capture_lifecycle: bool, @@ -56,6 +59,7 @@ impl Default for WebDebugConfig { fn default() -> Self { Self { enabled: false, + sideband: false, capture_lifecycle: true, capture_headers: true, capture_timings: true, @@ -69,6 +73,13 @@ impl Default for WebDebugConfig { } } +impl WebDebugConfig { + /// Returns whether newly issued bridges may report diagnostic lifecycle events. + pub(crate) const fn bridge_diagnostics_enabled(&self) -> bool { + self.enabled && self.sideband && self.capture_lifecycle + } +} + fn default_true() -> bool { true } diff --git a/src/web/bridge.rs b/src/web/bridge.rs index 68f1b74..7c637b4 100644 --- a/src/web/bridge.rs +++ b/src/web/bridge.rs @@ -32,17 +32,92 @@ pub(crate) fn render( websocket_open_secs: u64, reconnect_grace_secs: u64, carrier_probe_coalesce_ms: u64, + bridge_diagnostics_enabled: bool, rng: &SecureRandom, ) -> BridgePage { let mut nonce = [0u8; 18]; rng.fill(&mut nonce); let nonce = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(nonce); + let diagnostic_script = if bridge_diagnostics_enabled { + format!("\n") + } else { + String::new() + }; + let diagnostic_hook = |method: &str| { + bridge_diagnostics_enabled + .then(|| { + format!("if(clientDiagnostics)try{{clientDiagnostics.{method}()}}catch(error){{}}") + }) + .unwrap_or_default() + }; + let diagnostic_runtime_started = if bridge_diagnostics_enabled { + format!("{}\n", diagnostic_hook("runtimeStarted")) + } else { + String::new() + }; let body = DOCUMENT + .replace("__DIAGNOSTIC_RUNTIME__\n", &diagnostic_script) .replace("__RESPONSE_RUNTIME__", RESPONSE_RUNTIME) .replace("__REQUEST_RUNTIME__", REQUEST_RUNTIME) .replace("__BUFFER_RUNTIME__", BUFFER_RUNTIME) .replace("__RECOVERY_RUNTIME__", RECOVERY_RUNTIME) .replace("__RUNTIME__", RUNTIME) + .replace( + "__DIAGNOSTIC_BINDING__;\n", + if bridge_diagnostics_enabled { + "const clientDiagnostics=globalThis.TelemtBridgeDiagnostics;\n" + } else { + "" + }, + ) + .replace( + "__DIAGNOSTIC_RUNTIME_STARTED__;\n", + &diagnostic_runtime_started, + ) + .replace( + "__DIAGNOSTIC_BOUNDARY_ACTIVATED__;", + &diagnostic_hook("boundaryActivated"), + ) + .replace( + "__STATUS_FUNCTION__", + if bridge_diagnostics_enabled { + "state=>{if(port&&!closed){port.postMessage({t:'status',state});if(clientDiagnostics)try{clientDiagnostics.statusPosted()}catch(error){}}}" + } else { + "state=>{if(port&&!closed)port.postMessage({t:'status',state})}" + }, + ) + .replace( + "__DIAGNOSTIC_HELLO_RECEIVED__;", + &diagnostic_hook("helloReceived"), + ) + .replace( + "__HELLO_TIMEOUT_CALLBACK__", + if bridge_diagnostics_enabled { + "()=>{if(clientDiagnostics)try{clientDiagnostics.helloTimeout()}catch(error){}fail('timeout')}" + } else { + "()=>fail('timeout')" + }, + ) + .replace( + "__DIAGNOSTIC_CLIENT_CLOSE__;", + &diagnostic_hook("clientCloseBeforeHello"), + ) + .replace( + "__PAGEHIDE_CALLBACK__", + if bridge_diagnostics_enabled { + "()=>{if(clientDiagnostics)try{clientDiagnostics.documentUnloadedBeforeHello()}catch(error){}fail('navigation')}" + } else { + "()=>fail('navigation')" + }, + ) + .replace( + "__DIAGNOSTIC_BOOTSTRAP_REPLACED__;", + if bridge_diagnostics_enabled { + "if(clientDiagnostics)try{clientDiagnostics.setBootstrap(bootstrap)}catch(error){}" + } else { + "" + }, + ) .replace("__NONCE__", &nonce) .replace("__HOST__", host) .replace("__BOOTSTRAP__", bootstrap) @@ -88,6 +163,7 @@ pub(crate) fn render( } const DOCUMENT: &str = include_str!("bridge/document.html"); +const DIAGNOSTIC_RUNTIME: &str = include_str!("bridge/diagnostic.js"); const RESPONSE_RUNTIME: &str = include_str!("bridge/response.js"); const REQUEST_RUNTIME: &str = include_str!("bridge/request.js"); const BUFFER_RUNTIME: &str = include_str!("bridge/buffers.js"); diff --git a/src/web/bridge/diagnostic.js b/src/web/bridge/diagnostic.js new file mode 100644 index 0000000..101301b --- /dev/null +++ b/src/web/bridge/diagnostic.js @@ -0,0 +1,36 @@ +(()=>{'use strict'; +let bootstrap="__BOOTSTRAP__",hello=false,emitted=0,boundaryTimer=null; +const relayOrigin='https://__HOST__',requestMs=__BRIDGE_REQUEST_SECS__*1000; +function eventBit(event){ + if(event==='runtime_started')return 1;if(event==='status_posted')return 2;if(event==='hello_received')return 4;if(event==='boundary_timeout')return 8; + if(event==='hello_timeout')return 16;if(event==='client_close_before_hello')return 32;if(event==='document_unloaded_before_hello')return 64; + return event==='runtime_error_before_hello'?128:0; +} +function discard(response){try{if(response.body)response.body.cancel().catch(()=>{})}catch(error){}} +function report(event){ + if(hello&&event!=='hello_received')return;const bit=eventBit(event);if(!bit||(emitted&bit))return;emitted|=bit; + let timer=null; + try{ + const controller=new AbortController(),body=JSON.stringify({v:1,event});timer=setTimeout(()=>controller.abort(),requestMs); + fetch(relayOrigin+'/api/v1/diagnostic',{method:'POST',body,signal:controller.signal,keepalive:true,mode:'same-origin',credentials:'omit',cache:'no-store',redirect:'error',referrerPolicy:'no-referrer',headers:{Authorization:'Bearer '+bootstrap,'Content-Type':'application/json'}}) + .then(discard,()=>{}).then(()=>clearTimeout(timer),()=>clearTimeout(timer)); + }catch(error){if(timer)clearTimeout(timer)} +} +const boundaryWall=Date.now()+requestMs,boundaryMonotonic=performance.now()+requestMs; +function waitBoundary(){ + boundaryTimer=null;if(hello||(emitted&2))return;const remaining=Math.min(boundaryWall-Date.now(),boundaryMonotonic-performance.now()); + if(remaining>0){boundaryTimer=setTimeout(waitBoundary,remaining);return}report('boundary_timeout'); +} +function clearBoundary(){try{if(boundaryTimer)clearTimeout(boundaryTimer)}catch(error){}boundaryTimer=null} +function runtimeStarted(){report('runtime_started');try{boundaryTimer=setTimeout(waitBoundary,Math.max(1,Math.min(boundaryWall-Date.now(),boundaryMonotonic-performance.now())))}catch(error){}} +function boundaryActivated(){clearBoundary()} +function statusPosted(){clearBoundary();report('status_posted')} +function helloReceived(){clearBoundary();hello=true;report('hello_received')} +function helloTimeout(){report('hello_timeout')} +function clientCloseBeforeHello(){report('client_close_before_hello')} +function documentUnloadedBeforeHello(){report('document_unloaded_before_hello')} +function setBootstrap(value){if(/^[A-Za-z0-9_-]{43}$/.test(value))bootstrap=value} +addEventListener('error',()=>report('runtime_error_before_hello')); +addEventListener('unhandledrejection',()=>report('runtime_error_before_hello')); +globalThis.TelemtBridgeDiagnostics=Object.freeze({runtimeStarted,boundaryActivated,statusPosted,helloReceived,helloTimeout,clientCloseBeforeHello,documentUnloadedBeforeHello,setBootstrap}); +})(); diff --git a/src/web/bridge/document.html b/src/web/bridge/document.html index ea68ec7..7c98b5d 100644 --- a/src/web/bridge/document.html +++ b/src/web/bridge/document.html @@ -6,6 +6,7 @@ Connection +__DIAGNOSTIC_RUNTIME__ diff --git a/src/web/bridge/runtime.js b/src/web/bridge/runtime.js index 8933e1e..2055afb 100644 --- a/src/web/bridge/runtime.js +++ b/src/web/bridge/runtime.js @@ -1,6 +1,8 @@ (()=>{'use strict'; let bootstrap="__BOOTSTRAP__"; const relayOrigin='https://__HOST__',carrierCapabilities='https,https-lanes,websocket,websocket-lanes'; +__DIAGNOSTIC_BINDING__; +__DIAGNOSTIC_RUNTIME_STARTED__; const responseBody=globalThis.TelemtBridgeResponse;if(!responseBody)throw new Error('missing response runtime'); const requestSupport=globalThis.TelemtBridgeRequest;if(!requestSupport)throw new Error('missing request runtime'); const bufferSupport=globalThis.TelemtBridgeBuffers;if(!bufferSupport)throw new Error('missing buffer runtime'); @@ -23,7 +25,7 @@ const pending=[],upPending=[],recoveryPending=[],lanes=new Map(),closedLanes=new const canonicalFailures=['timeout','network','upgrade','http','protocol']; const failure=(reason,message)=>Object.assign(new Error(message||reason),{telemtReason:reason}); const failureReason=(error,fallback)=>error&&canonicalFailures.includes(error.telemtReason)?error.telemtReason:fallback; -const status=state=>{if(port&&!closed)port.postMessage({t:'status',state})}; +const status=__STATUS_FUNCTION__; const socketURL=()=>relayOrigin.replace(/^https:/,'wss:')+'/api/v1/ws'; const requestClient=requestSupport.create({ origin:()=>relayOrigin,closed:()=>closed,retryMs:()=>bridgeRetryMs,longPollMs:()=>longPollMs,requestMs:()=>bridgeRequestMs, @@ -62,7 +64,7 @@ function retireCarrier(policy){ } lanes.clear();closedLanes.clear();closedLaneOrder.length=0;releasePending(pending,null);releasePending(recoveryPending,null); for(const id of retireAllStreams())if(port){const frame=closeFrame(id);port.postMessage(frame,[frame])} - bootstrap=policy.bootstrap;batchLimit=policy.limits.carrier_batch_bytes;queueLimit=policy.limits.pending_bytes_per_session; + bootstrap=policy.bootstrap;__DIAGNOSTIC_BOOTSTRAP_REPLACED__;batchLimit=policy.limits.carrier_batch_bytes;queueLimit=policy.limits.pending_bytes_per_session; queueItemLimit=policy.limits.pending_items_per_session;maxStreams=policy.limits.max_streams_per_session; laneQueueLimit=Math.min(queueLimit,8388608);laneItemLimit=Math.min(queueItemLimit,1024); longPollMs=policy.timeouts.long_poll_secs*1000;bridgeRequestMs=policy.timeouts.bridge_request_secs*1000; @@ -487,20 +489,20 @@ function close(notifyServer){ buffers.assertEmpty(); } function activatePort(nextPort){ - initialized=true;port=nextPort; + initialized=true;port=nextPort;__DIAGNOSTIC_BOUNDARY_ACTIVATED__; port.onmessage=message=>{ observeResumeTrigger(); if(message.data instanceof ArrayBuffer){ - if(!createStarted){createStarted=true;if(helloTimer)clearTimeout(helloTimer);helloTimer=null;helloFrame=message.data;if(negotiationEnabled){negotiationStartedAt=Date.now();armCarrierDeadline(attemptEpoch)}createSession(attemptEpoch)} + if(!createStarted){__DIAGNOSTIC_HELLO_RECEIVED__;createStarted=true;if(helloTimer)clearTimeout(helloTimer);helloTimer=null;helloFrame=message.data;if(negotiationEnabled){negotiationStartedAt=Date.now();armCarrierDeadline(attemptEpoch)}createSession(attemptEpoch)} else{ let data;try{data=acceptNativeFrames(message.data)}catch(error){fail(error&&error.telemtReason==='capacity'?'capacity':'protocol');return}if(!data)return; if(recoveryController.active()&&!recoveryReplaced){if(!reserve(data,null)){fail('capacity');return}recoveryPending.push(data)} else if(!carrierCommitted){if(!reserve(data,null)){fail('capacity');return}pending.push(data);maybeStartCandidate()} else queueCarrier(data); } - }else if(message.data&&message.data.t==='close'){status('failed');close(true)} + }else if(message.data&&message.data.t==='close'){__DIAGNOSTIC_CLIENT_CLOSE__;status('failed');close(true)} }; - port.start();status('connecting');helloTimer=setTimeout(()=>fail('timeout'),bridgeRequestMs); + port.start();status('connecting');helloTimer=setTimeout(__HELLO_TIMEOUT_CALLBACK__,bridgeRequestMs); } recoveryController=recoverySupport.create({ budgetMs:()=>bridgeRecoveryMs,requestMs:()=>bridgeRequestMs,url:()=>relayOrigin+recoveryPath,token:()=>cleanupToken||sessionToken, @@ -539,5 +541,5 @@ function discoverAndroid(){ discoverAndroid(); addEventListener('online',observeResumeTrigger); if(globalThis.document&&typeof globalThis.document.addEventListener==='function')globalThis.document.addEventListener('visibilitychange',()=>{if(globalThis.document.visibilityState==='visible')observeResumeTrigger()}); -addEventListener('pagehide',()=>fail('navigation'),{once:true}); +addEventListener('pagehide',__PAGEHIDE_CALLBACK__,{once:true}); })(); diff --git a/src/web/bridge/tests.rs b/src/web/bridge/tests.rs index 4ebab32..b71b967 100644 --- a/src/web/bridge/tests.rs +++ b/src/web/bridge/tests.rs @@ -18,6 +18,30 @@ fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage { 15, 120, 0, + false, + &SecureRandom::new(), + ) +} + +fn render_diagnostic_page(bootstrap: &str) -> BridgePage { + render( + "proxy.example.com", + bootstrap, + 2 * 1024 * 1024, + 32 * 1024 * 1024, + 16 * 1024, + 1024, + true, + 4, + [3, 5, 8, 12], + 25, + 10, + 90, + 15, + 15, + 120, + 0, + true, &SecureRandom::new(), ) } @@ -77,6 +101,7 @@ fn rendered_page_embeds_the_configured_bridge_timing_policy() { 11, 119, 4, + false, &SecureRandom::new(), ); @@ -128,6 +153,7 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() { 15, 120, 0, + false, &SecureRandom::new(), ); assert!(page.body.contains( @@ -190,3 +216,87 @@ fn rendered_page_preserves_exact_v1_status_control_envelope() { ); assert!(!page.body.contains("port.postMessage({t:'status',state,")); } + +#[test] +fn bridge_diagnostic_sideband_is_absent_by_default() { + let page = render_page("IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII", 4); + + assert!(page.body.contains("\n
sidebandyes